@opengeni/worker-bundle 0.20.1 → 0.22.0-canary.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn/admission.d.ts +52 -0
- package/dist/activities/agent-turn/agent-build.d.ts +89 -0
- package/dist/activities/agent-turn/claim.d.ts +67 -0
- package/dist/activities/agent-turn/codex-capacity.d.ts +40 -0
- package/dist/activities/agent-turn/codex.d.ts +32 -0
- package/dist/activities/agent-turn/compaction-prep.d.ts +88 -0
- package/dist/activities/agent-turn/credential-leases.d.ts +56 -0
- package/dist/activities/agent-turn/errors.d.ts +253 -0
- package/dist/activities/agent-turn/failure-settlement.d.ts +33 -0
- package/dist/activities/agent-turn/file-resources.d.ts +37 -0
- package/dist/activities/agent-turn/finalization.d.ts +42 -0
- package/dist/activities/agent-turn/github-repository-bindings.d.ts +105 -0
- package/dist/activities/agent-turn/governance-model.d.ts +66 -0
- package/dist/activities/agent-turn/history-sink.d.ts +52 -0
- package/dist/activities/agent-turn/history.d.ts +63 -0
- package/dist/activities/agent-turn/media-artifacts.d.ts +145 -0
- package/dist/activities/agent-turn/model-usage.d.ts +218 -0
- package/dist/activities/agent-turn/quiescence.d.ts +135 -0
- package/dist/activities/agent-turn/run-credentials.d.ts +84 -0
- package/dist/activities/agent-turn/run.d.ts +3 -0
- package/dist/activities/agent-turn/sandbox-establish.d.ts +93 -0
- package/dist/activities/agent-turn/sandbox-provision.d.ts +89 -0
- package/dist/activities/agent-turn/sandbox-route.d.ts +151 -0
- package/dist/activities/agent-turn/sandbox-runtime.d.ts +44 -0
- package/dist/activities/agent-turn/stream-attempt.d.ts +107 -0
- package/dist/activities/agent-turn/tool-environment.d.ts +142 -0
- package/dist/activities/agent-turn/tool-policy.d.ts +127 -0
- package/dist/activities/agent-turn/tool-result-spill.d.ts +59 -0
- package/dist/activities/agent-turn/turn-context.d.ts +144 -0
- package/dist/activities/agent-turn/xai-capacity.d.ts +2 -0
- package/dist/activities/agent-turn.d.ts +12 -954
- package/dist/activities/automations.d.ts +16 -0
- package/dist/activities/common.d.ts +1 -1
- package/dist/activities/connector-attachments.d.ts +13 -0
- package/dist/activities/documents.d.ts +1 -0
- package/dist/activities/environment.d.ts +26 -15
- package/dist/activities/generated-images.d.ts +1 -1
- package/dist/activities/git-credential-renewal.d.ts +3 -1
- package/dist/activities/goals.d.ts +4 -1
- package/dist/activities/google-drive-provider.d.ts +29 -0
- package/dist/activities/google-drive-publication.d.ts +66 -0
- package/dist/activities/image-generation-references.d.ts +2 -0
- package/dist/activities/incident-telemetry-authority.d.ts +52 -0
- package/dist/activities/incident-telemetry-preflight.d.ts +71 -0
- package/dist/activities/knowledge-source-sync-driver.d.ts +21 -0
- package/dist/activities/knowledge-source-sync.d.ts +33 -1
- package/dist/activities/mcp-credentials.d.ts +9 -3
- package/dist/activities/open-suffix-resume.d.ts +77 -0
- package/dist/activities/packs.d.ts +3 -11
- package/dist/activities/parent-wake.d.ts +19 -0
- package/dist/activities/retained-screenshot-reaper.d.ts +1 -1
- package/dist/activities/retained-screenshots.d.ts +1 -1
- package/dist/activities/rig-verification.d.ts +1 -1
- package/dist/activities/run-input.d.ts +16 -7
- package/dist/activities/sandbox-images.d.ts +1 -14
- package/dist/activities/sandbox-lease.d.ts +21 -3
- package/dist/activities/scheduled-tasks.d.ts +47 -0
- package/dist/activities/types.d.ts +23 -2
- package/dist/activities/video-media-validation.d.ts +1 -1
- package/dist/activities/video-output-retention.d.ts +1 -2
- package/dist/activities/xai-auth.d.ts +3 -0
- package/dist/{activities-control-SMKF2OBE.js → activities-control-DKG22ONJ.js} +2946 -549
- package/dist/activities-control-DKG22ONJ.js.map +1 -0
- package/dist/activities-control.d.ts +6 -2
- package/dist/activities-turn-4IMC3MAA.js +21326 -0
- package/dist/activities-turn-4IMC3MAA.js.map +1 -0
- package/dist/activities.d.ts +7 -3
- package/dist/artifact-materializer-entry.js +41 -22
- package/dist/artifact-materializer-entry.js.map +1 -1
- package/dist/{chunk-N5TJIFCE.js → chunk-CE7TPKBS.js} +468 -1
- package/dist/chunk-CE7TPKBS.js.map +1 -0
- package/dist/chunk-CNYGVRKF.js +4698 -0
- package/dist/chunk-CNYGVRKF.js.map +1 -0
- package/dist/connection-provider-use.d.ts +18 -0
- package/dist/editable-artifact-materializer.d.ts +4 -3
- package/dist/github-rest-mcp.d.ts +26 -0
- package/dist/index.d.ts +10 -2
- package/dist/index.js +58 -14
- package/dist/index.js.map +1 -1
- package/dist/knowledge-source-sync-activity-policy.d.ts +2 -0
- package/dist/model-context-contributions.d.ts +8 -4
- package/dist/observability-metrics.d.ts +63 -4
- package/dist/opensandbox-kubernetes-inventory.d.ts +21 -0
- package/dist/personal-github-git-credentials.d.ts +9 -0
- package/dist/pr-review-credentials.d.ts +6 -0
- package/dist/sandbox-archive-storage.d.ts +45 -0
- package/dist/sandbox-resume.d.ts +65 -6
- package/dist/sandbox-routing.d.ts +38 -2
- package/dist/scheduled-alert-occurrence.d.ts +33 -0
- package/dist/workflow-bundle.js +87 -28
- package/dist/workflows/activities.d.ts +1 -0
- package/dist/workflows/automations.d.ts +6 -0
- package/dist/workflows/document-index.d.ts +1 -0
- package/dist/workflows.d.ts +1 -0
- package/package.json +17 -17
- package/src/activities/agent-turn/admission.ts +292 -0
- package/src/activities/agent-turn/agent-build.ts +802 -0
- package/src/activities/agent-turn/claim.ts +590 -0
- package/src/activities/agent-turn/codex-capacity.ts +720 -0
- package/src/activities/agent-turn/codex.ts +94 -0
- package/src/activities/agent-turn/compaction-prep.ts +722 -0
- package/src/activities/agent-turn/credential-leases.ts +240 -0
- package/src/activities/agent-turn/errors.ts +1068 -0
- package/src/activities/agent-turn/failure-settlement.ts +1436 -0
- package/src/activities/agent-turn/file-resources.ts +227 -0
- package/src/activities/agent-turn/finalization.ts +754 -0
- package/src/activities/agent-turn/github-repository-bindings.ts +300 -0
- package/src/activities/agent-turn/governance-model.ts +465 -0
- package/src/activities/agent-turn/history-sink.ts +139 -0
- package/src/activities/agent-turn/history.ts +200 -0
- package/src/activities/agent-turn/media-artifacts.ts +359 -0
- package/src/activities/agent-turn/model-usage.ts +823 -0
- package/src/activities/agent-turn/quiescence.ts +353 -0
- package/src/activities/agent-turn/run-credentials.ts +761 -0
- package/src/activities/agent-turn/run.ts +1530 -0
- package/src/activities/agent-turn/sandbox-establish.ts +1207 -0
- package/src/activities/agent-turn/sandbox-provision.ts +569 -0
- package/src/activities/agent-turn/sandbox-route.ts +358 -0
- package/src/activities/agent-turn/sandbox-runtime.ts +731 -0
- package/src/activities/agent-turn/stream-attempt.ts +1670 -0
- package/src/activities/agent-turn/tool-environment.ts +729 -0
- package/src/activities/agent-turn/tool-policy.ts +216 -0
- package/src/activities/agent-turn/tool-result-spill.ts +265 -0
- package/src/activities/agent-turn/turn-context.ts +281 -0
- package/src/activities/agent-turn/xai-capacity.ts +229 -0
- package/src/activities/agent-turn.ts +12 -12575
- package/src/activities/api-integrations.ts +3 -0
- package/src/activities/automations.ts +214 -0
- package/src/activities/codemode-dispatcher.ts +12 -1
- package/src/activities/common.ts +1 -5
- package/src/activities/connector-attachments.ts +168 -0
- package/src/activities/context-compaction.ts +18 -0
- package/src/activities/documents.ts +6 -4
- package/src/activities/environment.ts +151 -55
- package/src/activities/generated-images.ts +26 -11
- package/src/activities/git-credential-renewal.ts +13 -3
- package/src/activities/goals.ts +85 -19
- package/src/activities/google-drive-provider.ts +178 -0
- package/src/activities/google-drive-publication.ts +938 -0
- package/src/activities/image-generation-references.ts +11 -2
- package/src/activities/incident-telemetry-authority.ts +327 -0
- package/src/activities/incident-telemetry-preflight.ts +317 -0
- package/src/activities/knowledge-source-sync-driver.ts +23 -0
- package/src/activities/knowledge-source-sync.ts +556 -40
- package/src/activities/mcp-credentials.ts +191 -17
- package/src/activities/open-suffix-resume.ts +422 -0
- package/src/activities/packs.ts +12 -60
- package/src/activities/parent-wake.ts +64 -14
- package/src/activities/retained-screenshot-reaper.ts +6 -3
- package/src/activities/retained-screenshots.ts +15 -47
- package/src/activities/rig-verification.ts +1 -1
- package/src/activities/run-input.ts +128 -136
- package/src/activities/sandbox-images.ts +7 -166
- package/src/activities/sandbox-lease.ts +406 -15
- package/src/activities/scheduled-tasks.ts +2103 -411
- package/src/activities/session-state.ts +10 -2
- package/src/activities/types.ts +32 -2
- package/src/activities/video-media-validation.ts +3 -3
- package/src/activities/video-output-retention.ts +2 -57
- package/src/activities/xai-auth.ts +8 -0
- package/src/activities-control.ts +2 -0
- package/src/activities.ts +3 -0
- package/src/activity-services.ts +13 -4
- package/src/connection-provider-use.ts +36 -0
- package/src/editable-artifact-materializer-subprocess.ts +25 -14
- package/src/editable-artifact-materializer.ts +29 -18
- package/src/github-rest-mcp.ts +713 -0
- package/src/http.ts +16 -6
- package/src/index.ts +40 -5
- package/src/knowledge-source-sync-activity-policy.ts +2 -0
- package/src/model-context-contributions.ts +33 -10
- package/src/observability-metrics.ts +357 -0
- package/src/opensandbox-kubernetes-inventory.ts +292 -0
- package/src/personal-github-git-credentials.ts +331 -0
- package/src/pr-review-credentials.ts +163 -0
- package/src/sandbox-archive-storage.ts +159 -0
- package/src/sandbox-resume.ts +449 -63
- package/src/sandbox-routing.ts +314 -16
- package/src/scheduled-alert-occurrence.ts +176 -0
- package/src/workflows/activities.ts +18 -2
- package/src/workflows/automations.ts +11 -0
- package/src/workflows/session.ts +13 -3
- package/src/workflows.ts +1 -0
- package/dist/activities-control-SMKF2OBE.js.map +0 -1
- package/dist/activities-turn-IDS3WEHA.js +0 -16532
- package/dist/activities-turn-IDS3WEHA.js.map +0 -1
- package/dist/chunk-N5TJIFCE.js.map +0 -1
- package/dist/chunk-P6Z2SLAA.js +0 -3092
- package/dist/chunk-P6Z2SLAA.js.map +0 -1
|
@@ -6,16 +6,26 @@ import {
|
|
|
6
6
|
sandboxDrainTiming
|
|
7
7
|
} from "./chunk-GRFQTTSQ.js";
|
|
8
8
|
import {
|
|
9
|
+
INCIDENT_TELEMETRY_AUTHORITY_FENCE_LINEAGE_KEY,
|
|
10
|
+
MAX_AUTOMATIC_PROVIDER_RECOVERIES,
|
|
9
11
|
assertReusableSessionRevivable,
|
|
12
|
+
collectWorkspaceArchiveObjectKeys,
|
|
10
13
|
createCodexCapacityActivities,
|
|
11
14
|
createGoalActivities,
|
|
12
15
|
createSharedActivityServices,
|
|
13
16
|
currentActivityContext,
|
|
17
|
+
deleteUnpublishedWorkspaceArchiveObject,
|
|
18
|
+
deleteWorkspaceArchiveObjectKeys,
|
|
14
19
|
deliverFailedChildTurnToParent,
|
|
20
|
+
evaluateIncidentTelemetryPreflight,
|
|
21
|
+
incidentTelemetryAuthorityFence,
|
|
22
|
+
incidentTelemetryPreflightDeclaration,
|
|
15
23
|
notifyParentOfChildIdle,
|
|
16
|
-
|
|
24
|
+
providerRecoveryCountFromMetadata,
|
|
25
|
+
putVersion1TarArchiveOrInline,
|
|
17
26
|
reconcilePendingParentSystemUpdates,
|
|
18
27
|
reconcilePendingSessionWorkflowWakes,
|
|
28
|
+
resolveIncidentTelemetryResponderMetadata,
|
|
19
29
|
resolveWorkspacePackRuntime,
|
|
20
30
|
rigProviderImageSourceImage,
|
|
21
31
|
scheduledUserMessagePayload,
|
|
@@ -23,11 +33,13 @@ import {
|
|
|
23
33
|
settingsWithRigImage,
|
|
24
34
|
withFirstPartyTools,
|
|
25
35
|
workflowIdForSession
|
|
26
|
-
} from "./chunk-
|
|
36
|
+
} from "./chunk-CNYGVRKF.js";
|
|
27
37
|
import {
|
|
38
|
+
inspectOpenSandboxKubernetesInventory,
|
|
28
39
|
recordCreditBalanceGauges,
|
|
29
40
|
recordCreditMicros,
|
|
30
41
|
recordExpiredDrainingSandboxLeaseGauges,
|
|
42
|
+
recordOpenSandboxKubernetesInventoryGauges,
|
|
31
43
|
recordRetainedProcessInventoryGauges,
|
|
32
44
|
recordRetainedProcessReconciliation,
|
|
33
45
|
recordSandboxCheckpointArtifactGauges,
|
|
@@ -38,8 +50,9 @@ import {
|
|
|
38
50
|
recordSandboxLeaseGauges,
|
|
39
51
|
recordSandboxOrphansTerminated,
|
|
40
52
|
recordSandboxRotationBacklogGauges,
|
|
41
|
-
recordTurnsQueuedGauge
|
|
42
|
-
|
|
53
|
+
recordTurnsQueuedGauge,
|
|
54
|
+
runtimeMetricsHooksForObservability
|
|
55
|
+
} from "./chunk-CE7TPKBS.js";
|
|
43
56
|
|
|
44
57
|
// src/activities/browser-state-artifact-reaper.ts
|
|
45
58
|
import {
|
|
@@ -125,14 +138,255 @@ function cleanupErrorCategory(error) {
|
|
|
125
138
|
return typeof code === "string" && SAFE_PROVIDER_ERROR_CODES.has(code) ? code : "unknown";
|
|
126
139
|
}
|
|
127
140
|
|
|
128
|
-
// src/activities/
|
|
141
|
+
// src/activities/automations.ts
|
|
142
|
+
import { resolveTurnExecutionPolicyV1 } from "@opengeni/config";
|
|
143
|
+
import {
|
|
144
|
+
assertWorkspaceModelPolicyAllows,
|
|
145
|
+
canonicalConfiguredModel,
|
|
146
|
+
createAndStartSessionWithOutcome,
|
|
147
|
+
recordWorkspaceUsage
|
|
148
|
+
} from "@opengeni/core";
|
|
149
|
+
import {
|
|
150
|
+
AutomationAuthorityRevokedError,
|
|
151
|
+
assertAutomationRunAuthorityInTransaction,
|
|
152
|
+
claimAutomationRun,
|
|
153
|
+
settleAutomationRun
|
|
154
|
+
} from "@opengeni/db";
|
|
155
|
+
|
|
156
|
+
// src/activities/agent-run-admission.ts
|
|
129
157
|
import { configuredStaticUsageLimits } from "@opengeni/config";
|
|
158
|
+
import { getBillingBalance, isCodexBilledTurn, sumUsageQuantity } from "@opengeni/db";
|
|
159
|
+
async function agentRunAdmissionDenial(services, input) {
|
|
160
|
+
const externallyBilled = await isCodexBilledTurn({
|
|
161
|
+
db: services.db,
|
|
162
|
+
settings: services.settings,
|
|
163
|
+
workspaceId: input.workspaceId,
|
|
164
|
+
model: input.model
|
|
165
|
+
});
|
|
166
|
+
if (!externallyBilled && (services.settings.billingMode === "stripe" || services.settings.usageLimitsMode === "managed")) {
|
|
167
|
+
if (services.entitlements) {
|
|
168
|
+
const decision = await services.entitlements.admitRun({
|
|
169
|
+
accountId: input.accountId,
|
|
170
|
+
workspaceId: input.workspaceId,
|
|
171
|
+
action: "agent_run:create",
|
|
172
|
+
quantity: input.requestedAgentRuns
|
|
173
|
+
});
|
|
174
|
+
if (!decision.allowed) return "insufficient_credits";
|
|
175
|
+
} else {
|
|
176
|
+
const balance = await getBillingBalance(services.db, input.accountId);
|
|
177
|
+
if (balance.balanceMicros <= 0) return "insufficient_credits";
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
if (services.settings.usageLimitsMode !== "static" && services.settings.usageLimitsMode !== "managed") {
|
|
181
|
+
return null;
|
|
182
|
+
}
|
|
183
|
+
const limits = configuredStaticUsageLimits(services.settings);
|
|
184
|
+
if (!externallyBilled && limits.maxMonthlyCostMicrosPerAccount) {
|
|
185
|
+
const used = await sumUsageQuantity(services.db, {
|
|
186
|
+
accountId: input.accountId,
|
|
187
|
+
eventType: "model.cost",
|
|
188
|
+
since: startOfUtcMonth()
|
|
189
|
+
});
|
|
190
|
+
if (used >= limits.maxMonthlyCostMicrosPerAccount) {
|
|
191
|
+
return "monthly_model_cost_limit";
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
if (limits.maxMonthlyAgentRunsPerWorkspace) {
|
|
195
|
+
const used = await sumUsageQuantity(services.db, {
|
|
196
|
+
workspaceId: input.workspaceId,
|
|
197
|
+
eventType: "agent_run.created",
|
|
198
|
+
since: startOfUtcMonth()
|
|
199
|
+
});
|
|
200
|
+
if (used + input.requestedAgentRuns > limits.maxMonthlyAgentRunsPerWorkspace) {
|
|
201
|
+
return "monthly_agent_run_limit";
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
return null;
|
|
205
|
+
}
|
|
206
|
+
function startOfUtcMonth() {
|
|
207
|
+
const now = /* @__PURE__ */ new Date();
|
|
208
|
+
return new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), 1));
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
// src/activities/automations.ts
|
|
212
|
+
function createAutomationActivities(services, options = {}) {
|
|
213
|
+
const claim = options.claim ?? claimAutomationRun;
|
|
214
|
+
const settle = options.settle ?? settleAutomationRun;
|
|
215
|
+
const assertAuthority = options.assertAuthority ?? assertAutomationRunAuthorityInTransaction;
|
|
216
|
+
const createSession2 = options.createSession ?? createAndStartSessionWithOutcome;
|
|
217
|
+
const admit = options.admit ?? agentRunAdmissionDenial;
|
|
218
|
+
const assertModelPolicy = options.assertModelPolicy ?? assertWorkspaceModelPolicyAllows;
|
|
219
|
+
const recordUsage = options.recordUsage ?? recordWorkspaceUsage;
|
|
220
|
+
return {
|
|
221
|
+
dispatchAutomationRun: async (input) => {
|
|
222
|
+
const service = await services();
|
|
223
|
+
const run = await claim(service.db, input);
|
|
224
|
+
if (!run) return { action: "not_found" };
|
|
225
|
+
if (run.status === "dispatched") {
|
|
226
|
+
if (!run.sessionId) throw new Error("dispatched automation run is missing its session");
|
|
227
|
+
return { action: "already_dispatched", sessionId: run.sessionId };
|
|
228
|
+
}
|
|
229
|
+
if (run.status === "skipped") {
|
|
230
|
+
return { action: "skipped", reason: run.errorCode ?? "authority_revoked" };
|
|
231
|
+
}
|
|
232
|
+
const accepted = run.acceptedExecution;
|
|
233
|
+
if (accepted.accountId !== input.accountId || accepted.workspaceId !== input.workspaceId || accepted.sourceId !== run.sourceId || accepted.triggerId !== run.triggerId || accepted.triggerRevision !== run.triggerRevision || accepted.eventId !== run.eventId) {
|
|
234
|
+
await settle(service.db, {
|
|
235
|
+
workspaceId: input.workspaceId,
|
|
236
|
+
runId: run.id,
|
|
237
|
+
status: "skipped",
|
|
238
|
+
errorCode: "accepted_execution_mismatch"
|
|
239
|
+
});
|
|
240
|
+
return { action: "skipped", reason: "accepted_execution_mismatch" };
|
|
241
|
+
}
|
|
242
|
+
const template = accepted.sessionTemplate;
|
|
243
|
+
const model = canonicalConfiguredModel(
|
|
244
|
+
service.settings,
|
|
245
|
+
template.model ?? service.settings.openaiModel
|
|
246
|
+
);
|
|
247
|
+
if (!model) throw new Error("automation has no configured model");
|
|
248
|
+
await assertModelPolicy(service.db, service.settings, input.workspaceId, model);
|
|
249
|
+
const denial = await admit(service, {
|
|
250
|
+
accountId: input.accountId,
|
|
251
|
+
workspaceId: input.workspaceId,
|
|
252
|
+
model,
|
|
253
|
+
requestedAgentRuns: 1
|
|
254
|
+
});
|
|
255
|
+
if (denial) {
|
|
256
|
+
await settle(service.db, {
|
|
257
|
+
workspaceId: input.workspaceId,
|
|
258
|
+
runId: run.id,
|
|
259
|
+
status: "failed",
|
|
260
|
+
errorCode: denial
|
|
261
|
+
});
|
|
262
|
+
throw new Error(`automation admission denied: ${denial}`);
|
|
263
|
+
}
|
|
264
|
+
const reasoningEffort = template.reasoningEffort ?? service.settings.openaiReasoningEffort;
|
|
265
|
+
const sandboxBackend = template.sandboxBackend ?? service.settings.sandboxBackend;
|
|
266
|
+
if (sandboxBackend === "selfhosted") {
|
|
267
|
+
await settle(service.db, {
|
|
268
|
+
workspaceId: input.workspaceId,
|
|
269
|
+
runId: run.id,
|
|
270
|
+
status: "skipped",
|
|
271
|
+
errorCode: "interactive_compute_not_allowed"
|
|
272
|
+
});
|
|
273
|
+
return { action: "skipped", reason: "interactive_compute_not_allowed" };
|
|
274
|
+
}
|
|
275
|
+
const turnExecutionPolicy = resolveTurnExecutionPolicyV1(service.settings, {
|
|
276
|
+
modelId: model,
|
|
277
|
+
requestedModelId: template.model,
|
|
278
|
+
modelSource: template.model ? "explicit" : "deployment",
|
|
279
|
+
reasoningEffort,
|
|
280
|
+
reasoningSource: template.reasoningEffort ? "explicit" : "deployment",
|
|
281
|
+
latencyMode: "standard",
|
|
282
|
+
latencyModeSource: "deployment"
|
|
283
|
+
});
|
|
284
|
+
try {
|
|
285
|
+
const created = await createSession2({
|
|
286
|
+
db: service.db,
|
|
287
|
+
bus: service.bus,
|
|
288
|
+
workflowClient: {
|
|
289
|
+
wakeSessionWorkflow: async (wake) => {
|
|
290
|
+
await service.wakeSessionWorkflow?.(wake);
|
|
291
|
+
}
|
|
292
|
+
},
|
|
293
|
+
beforeCreateCommit: async (tx, sessionId) => {
|
|
294
|
+
await assertAuthority(tx, {
|
|
295
|
+
workspaceId: input.workspaceId,
|
|
296
|
+
runId: run.id,
|
|
297
|
+
triggerId: accepted.triggerId,
|
|
298
|
+
triggerRevision: accepted.triggerRevision,
|
|
299
|
+
sourceId: accepted.sourceId,
|
|
300
|
+
sourceVersion: accepted.sourceVersion,
|
|
301
|
+
sessionId
|
|
302
|
+
});
|
|
303
|
+
},
|
|
304
|
+
accountId: input.accountId,
|
|
305
|
+
workspaceId: input.workspaceId,
|
|
306
|
+
initialMessage: accepted.initialMessage,
|
|
307
|
+
resources: template.resources,
|
|
308
|
+
skills: template.skills,
|
|
309
|
+
tools: template.tools,
|
|
310
|
+
toolPolicy: { mode: "explicit", inheritedFromSessionId: null },
|
|
311
|
+
model,
|
|
312
|
+
reasoningEffort,
|
|
313
|
+
turnExecutionPolicy,
|
|
314
|
+
sandboxBackend,
|
|
315
|
+
metadata: {
|
|
316
|
+
...template.metadata,
|
|
317
|
+
automationRunId: run.id,
|
|
318
|
+
automationSourceId: run.sourceId,
|
|
319
|
+
automationTriggerId: run.triggerId,
|
|
320
|
+
automationTriggerRevision: run.triggerRevision
|
|
321
|
+
},
|
|
322
|
+
createdBy: {
|
|
323
|
+
kind: "service",
|
|
324
|
+
subjectId: accepted.serviceSubjectId,
|
|
325
|
+
label: accepted.serviceLabel
|
|
326
|
+
},
|
|
327
|
+
createdByContext: accepted.provenance,
|
|
328
|
+
instructions: template.instructions,
|
|
329
|
+
policyRole: template.policyRole,
|
|
330
|
+
firstPartyMcpPermissions: template.firstPartyMcpPermissions,
|
|
331
|
+
firstPartyMcpTools: template.firstPartyMcpTools,
|
|
332
|
+
createIdempotencyKey: `automation-run:${run.id}`,
|
|
333
|
+
subjectId: accepted.serviceSubjectId
|
|
334
|
+
});
|
|
335
|
+
await recordUsage(
|
|
336
|
+
{ db: service.db, settings: service.settings },
|
|
337
|
+
{
|
|
338
|
+
accountId: input.accountId,
|
|
339
|
+
workspaceId: input.workspaceId,
|
|
340
|
+
subjectId: accepted.serviceSubjectId,
|
|
341
|
+
eventType: "agent_run.created",
|
|
342
|
+
quantity: 1,
|
|
343
|
+
unit: "run",
|
|
344
|
+
sourceResourceType: "automation_run",
|
|
345
|
+
sourceResourceId: run.id,
|
|
346
|
+
sessionId: created.session.id,
|
|
347
|
+
initiator: created.session.createdBy,
|
|
348
|
+
initiatorContext: created.session.createdByContext,
|
|
349
|
+
origin: "system",
|
|
350
|
+
idempotencyKey: `agent_run.created:automation:${run.id}`
|
|
351
|
+
}
|
|
352
|
+
).catch((error) => {
|
|
353
|
+
service.observability.warn("automation usage recording failed", {
|
|
354
|
+
workspaceId: input.workspaceId,
|
|
355
|
+
runId: run.id,
|
|
356
|
+
errorCategory: error instanceof Error ? error.name : "unknown"
|
|
357
|
+
});
|
|
358
|
+
});
|
|
359
|
+
return { action: "started", sessionId: created.session.id };
|
|
360
|
+
} catch (error) {
|
|
361
|
+
if (error instanceof AutomationAuthorityRevokedError) {
|
|
362
|
+
await settle(service.db, {
|
|
363
|
+
workspaceId: input.workspaceId,
|
|
364
|
+
runId: run.id,
|
|
365
|
+
status: "skipped",
|
|
366
|
+
errorCode: "authority_revoked"
|
|
367
|
+
});
|
|
368
|
+
return { action: "skipped", reason: "authority_revoked" };
|
|
369
|
+
}
|
|
370
|
+
await settle(service.db, {
|
|
371
|
+
workspaceId: input.workspaceId,
|
|
372
|
+
runId: run.id,
|
|
373
|
+
status: "failed",
|
|
374
|
+
errorCode: "dispatch_failed"
|
|
375
|
+
});
|
|
376
|
+
throw error;
|
|
377
|
+
}
|
|
378
|
+
}
|
|
379
|
+
};
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
// src/activities/documents.ts
|
|
383
|
+
import { configuredStaticUsageLimits as configuredStaticUsageLimits2 } from "@opengeni/config";
|
|
130
384
|
import {
|
|
131
|
-
getBillingBalance,
|
|
385
|
+
getBillingBalance as getBillingBalance2,
|
|
132
386
|
recordUsageEvent,
|
|
133
387
|
resolveDocumentIndexAuthority,
|
|
134
388
|
rlsContextForWorkspace,
|
|
135
|
-
sumUsageQuantity,
|
|
389
|
+
sumUsageQuantity as sumUsageQuantity2,
|
|
136
390
|
withWorkspaceUsageLock
|
|
137
391
|
} from "@opengeni/db";
|
|
138
392
|
function createDocumentActivities(services, resolveDocumentServices) {
|
|
@@ -145,7 +399,7 @@ function createDocumentActivities(services, resolveDocumentServices) {
|
|
|
145
399
|
if (!resolveDocumentServices) {
|
|
146
400
|
throw new Error("document services are not configured");
|
|
147
401
|
}
|
|
148
|
-
const [{
|
|
402
|
+
const [{ getDocumentForIndexing, indexDocumentNow }, documentServices] = await Promise.all([
|
|
149
403
|
import("@opengeni/documents"),
|
|
150
404
|
resolveDocumentServices()
|
|
151
405
|
]);
|
|
@@ -173,9 +427,11 @@ function createDocumentActivities(services, resolveDocumentServices) {
|
|
|
173
427
|
if (suppliedCount !== 0 && suppliedCount !== suppliedAuthorityFields.length) {
|
|
174
428
|
throw new Error("document authority tuple is partial");
|
|
175
429
|
}
|
|
176
|
-
const claimedDocument = await
|
|
177
|
-
|
|
178
|
-
|
|
430
|
+
const claimedDocument = await getDocumentForIndexing(
|
|
431
|
+
lockedDb,
|
|
432
|
+
input.workspaceId,
|
|
433
|
+
input.documentId
|
|
434
|
+
);
|
|
179
435
|
if (!claimedDocument || claimedDocument.authorityKind !== storedAuthority.authorityKind || claimedDocument.authorityWorkspaceId !== storedAuthority.authorityWorkspaceId || claimedDocument.authoritySubjectId !== storedAuthority.authoritySubjectId) {
|
|
180
436
|
throw new Error("document authority changed before indexing");
|
|
181
437
|
}
|
|
@@ -188,7 +444,7 @@ function createDocumentActivities(services, resolveDocumentServices) {
|
|
|
188
444
|
{
|
|
189
445
|
beforeEmbed: async ({ chunkCount }) => {
|
|
190
446
|
if (settings.billingMode === "stripe" || settings.usageLimitsMode === "managed") {
|
|
191
|
-
const balance = await
|
|
447
|
+
const balance = await getBillingBalance2(lockedDb, input.accountId);
|
|
192
448
|
if (balance.balanceMicros <= 0) {
|
|
193
449
|
throw new Error("insufficient OpenGeni credits");
|
|
194
450
|
}
|
|
@@ -196,14 +452,14 @@ function createDocumentActivities(services, resolveDocumentServices) {
|
|
|
196
452
|
if (settings.usageLimitsMode !== "static" && settings.usageLimitsMode !== "managed") {
|
|
197
453
|
return;
|
|
198
454
|
}
|
|
199
|
-
const limit =
|
|
455
|
+
const limit = configuredStaticUsageLimits2(settings).maxDocumentIndexedChunksPerWorkspace;
|
|
200
456
|
if (!limit) {
|
|
201
457
|
return;
|
|
202
458
|
}
|
|
203
|
-
const used = await
|
|
459
|
+
const used = await sumUsageQuantity2(lockedDb, {
|
|
204
460
|
workspaceId: input.workspaceId,
|
|
205
461
|
eventType: "document.indexed",
|
|
206
|
-
since:
|
|
462
|
+
since: startOfUtcMonth2()
|
|
207
463
|
});
|
|
208
464
|
if (used + chunkCount > limit) {
|
|
209
465
|
throw new Error(`monthly document indexing limit reached (${limit} chunks)`);
|
|
@@ -232,7 +488,7 @@ function createDocumentActivities(services, resolveDocumentServices) {
|
|
|
232
488
|
}
|
|
233
489
|
};
|
|
234
490
|
}
|
|
235
|
-
function
|
|
491
|
+
function startOfUtcMonth2(date = /* @__PURE__ */ new Date()) {
|
|
236
492
|
return new Date(Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), 1));
|
|
237
493
|
}
|
|
238
494
|
|
|
@@ -405,6 +661,9 @@ function createFileUploadReaperActivities(services, options = {}) {
|
|
|
405
661
|
// src/activities/knowledge-source-sync.ts
|
|
406
662
|
import { createHash as createHash2 } from "crypto";
|
|
407
663
|
import { heartbeat } from "@temporalio/activity";
|
|
664
|
+
import {
|
|
665
|
+
googleDriveProviderRetryOptions
|
|
666
|
+
} from "@opengeni/config";
|
|
408
667
|
import {
|
|
409
668
|
KnowledgeSourceSyncRunSummary
|
|
410
669
|
} from "@opengeni/contracts";
|
|
@@ -431,6 +690,7 @@ import {
|
|
|
431
690
|
appendKnowledgeDocumentVersion,
|
|
432
691
|
appendKnowledgeSourceAclVersion,
|
|
433
692
|
beginKnowledgeSyncRun,
|
|
693
|
+
beginGoogleDriveObjectAclRefresh,
|
|
434
694
|
buildConnectionTokenResolver,
|
|
435
695
|
checkpointKnowledgeSourceSync,
|
|
436
696
|
claimKnowledgeSourceSyncLease,
|
|
@@ -452,6 +712,7 @@ import {
|
|
|
452
712
|
KnowledgeSourceSyncObservationFenceError,
|
|
453
713
|
reconcileKnowledgeSourceSyncCompleteScan,
|
|
454
714
|
reconcileKnowledgeSourceSyncLiveGeneration,
|
|
715
|
+
recordGoogleDriveObjectAclEvidence,
|
|
455
716
|
recordKnowledgeSourceSyncItemOutcomes,
|
|
456
717
|
recordKnowledgeSourceSyncObjectObservations,
|
|
457
718
|
recordUsageEvent as recordUsageEvent2,
|
|
@@ -467,6 +728,121 @@ import {
|
|
|
467
728
|
} from "@opengeni/db";
|
|
468
729
|
import { readResponseJsonBounded } from "@opengeni/network";
|
|
469
730
|
|
|
731
|
+
// src/activities/google-drive-provider.ts
|
|
732
|
+
var GoogleDriveProviderTransportError = class extends Error {
|
|
733
|
+
constructor() {
|
|
734
|
+
super("google_drive_provider_transport_failed");
|
|
735
|
+
this.name = "GoogleDriveProviderTransportError";
|
|
736
|
+
}
|
|
737
|
+
};
|
|
738
|
+
async function fetchGoogleDriveProvider(input) {
|
|
739
|
+
const now = input.now ?? Date.now;
|
|
740
|
+
const sleep = input.sleep ?? Bun.sleep;
|
|
741
|
+
let retryDelaySpentMs = 0;
|
|
742
|
+
for (let attempt = 1; attempt <= input.policy.attempts; attempt += 1) {
|
|
743
|
+
let response;
|
|
744
|
+
input.heartbeat?.({ operation: input.operation, attempt, phase: "request_start" });
|
|
745
|
+
try {
|
|
746
|
+
response = await input.fetchImpl(input.url, {
|
|
747
|
+
...input.init,
|
|
748
|
+
redirect: "error",
|
|
749
|
+
signal: AbortSignal.timeout(input.policy.requestTimeoutMs)
|
|
750
|
+
});
|
|
751
|
+
} catch {
|
|
752
|
+
input.heartbeat?.({ operation: input.operation, attempt, phase: "request_end" });
|
|
753
|
+
recordRequest(input.observability, input.operation, "retryable_error");
|
|
754
|
+
const delayMs2 = retryDelayMs(input.policy, attempt, null);
|
|
755
|
+
if (!canRetry(input.policy, attempt, retryDelaySpentMs, delayMs2)) {
|
|
756
|
+
throw new GoogleDriveProviderTransportError();
|
|
757
|
+
}
|
|
758
|
+
recordRetry(input.observability, input.operation, "transport", attempt, delayMs2);
|
|
759
|
+
retryDelaySpentMs += delayMs2;
|
|
760
|
+
input.heartbeat?.({ operation: input.operation, attempt, phase: "retry_delay_start" });
|
|
761
|
+
await sleep(delayMs2);
|
|
762
|
+
input.heartbeat?.({ operation: input.operation, attempt, phase: "retry_delay_end" });
|
|
763
|
+
continue;
|
|
764
|
+
}
|
|
765
|
+
input.heartbeat?.({ operation: input.operation, attempt, phase: "request_end" });
|
|
766
|
+
if (!isRetryableStatus(response.status)) {
|
|
767
|
+
recordRequest(
|
|
768
|
+
input.observability,
|
|
769
|
+
input.operation,
|
|
770
|
+
response.ok ? "succeeded" : "terminal_error"
|
|
771
|
+
);
|
|
772
|
+
return response;
|
|
773
|
+
}
|
|
774
|
+
recordRequest(input.observability, input.operation, "retryable_error");
|
|
775
|
+
const delayMs = retryDelayMs(
|
|
776
|
+
input.policy,
|
|
777
|
+
attempt,
|
|
778
|
+
retryAfterDelayMs(response.headers.get("retry-after"), now())
|
|
779
|
+
);
|
|
780
|
+
if (!canRetry(input.policy, attempt, retryDelaySpentMs, delayMs)) return response;
|
|
781
|
+
await response.body?.cancel().catch(() => void 0);
|
|
782
|
+
recordRetry(
|
|
783
|
+
input.observability,
|
|
784
|
+
input.operation,
|
|
785
|
+
response.status === 429 ? "rate_limited" : "provider_unavailable",
|
|
786
|
+
attempt,
|
|
787
|
+
delayMs
|
|
788
|
+
);
|
|
789
|
+
retryDelaySpentMs += delayMs;
|
|
790
|
+
input.heartbeat?.({ operation: input.operation, attempt, phase: "retry_delay_start" });
|
|
791
|
+
await sleep(delayMs);
|
|
792
|
+
input.heartbeat?.({ operation: input.operation, attempt, phase: "retry_delay_end" });
|
|
793
|
+
}
|
|
794
|
+
throw new GoogleDriveProviderTransportError();
|
|
795
|
+
}
|
|
796
|
+
function canRetry(policy, attempt, retryDelaySpentMs, delayMs) {
|
|
797
|
+
return attempt < policy.attempts && retryDelaySpentMs + delayMs <= policy.budgetMs;
|
|
798
|
+
}
|
|
799
|
+
function retryDelayMs(policy, attempt, retryAfterMs) {
|
|
800
|
+
const exponential = Math.min(
|
|
801
|
+
policy.maxDelayMs,
|
|
802
|
+
policy.initialDelayMs * 2 ** Math.max(0, attempt - 1)
|
|
803
|
+
);
|
|
804
|
+
return Math.min(policy.maxDelayMs, Math.max(exponential, retryAfterMs ?? 0));
|
|
805
|
+
}
|
|
806
|
+
function retryAfterDelayMs(value, nowMs) {
|
|
807
|
+
if (!value) return null;
|
|
808
|
+
const seconds = Number(value);
|
|
809
|
+
if (Number.isFinite(seconds) && seconds >= 0) return Math.round(seconds * 1e3);
|
|
810
|
+
const timestamp = Date.parse(value);
|
|
811
|
+
return Number.isFinite(timestamp) ? Math.max(0, timestamp - nowMs) : null;
|
|
812
|
+
}
|
|
813
|
+
function isRetryableStatus(status) {
|
|
814
|
+
return status === 429 || status >= 500;
|
|
815
|
+
}
|
|
816
|
+
function recordRequest(observability, operation, outcome) {
|
|
817
|
+
observability.incrementCounter({
|
|
818
|
+
name: "opengeni_google_drive_provider_requests_total",
|
|
819
|
+
help: "Google Drive provider request attempts by operation and bounded outcome.",
|
|
820
|
+
labels: { operation, outcome }
|
|
821
|
+
});
|
|
822
|
+
}
|
|
823
|
+
function recordRetry(observability, operation, reason, attempt, delayMs) {
|
|
824
|
+
observability.incrementCounter({
|
|
825
|
+
name: "opengeni_google_drive_provider_retries_total",
|
|
826
|
+
help: "Bounded Google Drive provider retries by operation and reason.",
|
|
827
|
+
labels: { operation, reason }
|
|
828
|
+
});
|
|
829
|
+
observability.observeHistogram({
|
|
830
|
+
name: "opengeni_google_drive_provider_retry_delay_seconds",
|
|
831
|
+
help: "Delay applied before a bounded Google Drive provider retry.",
|
|
832
|
+
labels: { operation, reason },
|
|
833
|
+
value: delayMs / 1e3,
|
|
834
|
+
buckets: [0.1, 0.25, 0.5, 1, 2, 5, 10, 30, 60]
|
|
835
|
+
});
|
|
836
|
+
observability.warn("Google Drive provider request retry scheduled", {
|
|
837
|
+
provider: "google_drive",
|
|
838
|
+
operation,
|
|
839
|
+
outcome: "retry_scheduled",
|
|
840
|
+
reason,
|
|
841
|
+
attempt,
|
|
842
|
+
delayMs
|
|
843
|
+
});
|
|
844
|
+
}
|
|
845
|
+
|
|
470
846
|
// src/activities/google-drive-changes.ts
|
|
471
847
|
import { createHash } from "crypto";
|
|
472
848
|
import {
|
|
@@ -747,6 +1123,8 @@ var DRIVE_JSON_MAX_BYTES = 2 * 1024 * 1024;
|
|
|
747
1123
|
var GOOGLE_DRIVE_FULL_RECONCILIATION_INTERVAL_MS = 24 * 60 * 60 * 1e3;
|
|
748
1124
|
var GOOGLE_DRIVE_SYNC_INVOCATION_SLICE_MS = 3e4;
|
|
749
1125
|
var GOOGLE_DRIVE_EXECUTION_CHECKPOINT_MAX_BYTES = 2 * 1024 * 1024;
|
|
1126
|
+
var GOOGLE_DRIVE_ACL_FRESHNESS_MS = 26 * 60 * 60 * 1e3;
|
|
1127
|
+
var GOOGLE_DRIVE_ACL_MAX_PRINCIPALS = 1e3;
|
|
750
1128
|
var ATLASSIAN_API_BASE = "https://api.atlassian.com";
|
|
751
1129
|
var ATLASSIAN_JSON_MAX_BYTES = 4 * 1024 * 1024;
|
|
752
1130
|
function knowledgeSyncObservationPolicy(kind) {
|
|
@@ -799,6 +1177,10 @@ function createKnowledgeSourceSyncActivities(services, resolveDocumentServices)
|
|
|
799
1177
|
reconnectRequired: false,
|
|
800
1178
|
failures: []
|
|
801
1179
|
};
|
|
1180
|
+
observability.info("Knowledge source sync started", {
|
|
1181
|
+
provider: providerLabel,
|
|
1182
|
+
outcome: "started"
|
|
1183
|
+
});
|
|
802
1184
|
try {
|
|
803
1185
|
summary.phase = "inventory";
|
|
804
1186
|
const resolved = await getKnowledgeSourceForSyncAuthority(db, {
|
|
@@ -879,6 +1261,7 @@ function createKnowledgeSourceSyncActivities(services, resolveDocumentServices)
|
|
|
879
1261
|
const provider = await resolveKnowledgeSyncProvider({
|
|
880
1262
|
db,
|
|
881
1263
|
settings,
|
|
1264
|
+
observability,
|
|
882
1265
|
accountId: input.accountId,
|
|
883
1266
|
workspaceId: input.workspaceId,
|
|
884
1267
|
action,
|
|
@@ -1008,7 +1391,26 @@ function createKnowledgeSourceSyncActivities(services, resolveDocumentServices)
|
|
|
1008
1391
|
if (obligation.documentId !== details.version.documentId) {
|
|
1009
1392
|
throw new SyncFailure("authority_changed", false);
|
|
1010
1393
|
}
|
|
1011
|
-
if (obligation.status === "indexed") return obligation;
|
|
1394
|
+
if (obligation.status === "indexed" && !details.indexRequired) return obligation;
|
|
1395
|
+
if (obligation.status === "indexed" && details.indexRequired) {
|
|
1396
|
+
summary.phase = "index";
|
|
1397
|
+
try {
|
|
1398
|
+
const indexedDocument = await documentActivities.indexDocument({
|
|
1399
|
+
accountId: input.accountId,
|
|
1400
|
+
workspaceId: input.workspaceId,
|
|
1401
|
+
documentId: details.version.documentId,
|
|
1402
|
+
authorityKind: action.destination.kind,
|
|
1403
|
+
authorityWorkspaceId: action.destination.workspaceId,
|
|
1404
|
+
authoritySubjectId: action.destination.subjectId
|
|
1405
|
+
});
|
|
1406
|
+
if (indexedDocument.status !== "ready") {
|
|
1407
|
+
throw new Error("document indexing did not complete");
|
|
1408
|
+
}
|
|
1409
|
+
} catch {
|
|
1410
|
+
throw new SyncFailure("indexing_failed", true);
|
|
1411
|
+
}
|
|
1412
|
+
return obligation;
|
|
1413
|
+
}
|
|
1012
1414
|
if (obligation.status === "failed") {
|
|
1013
1415
|
const retried = await retryKnowledgeSourceSyncIndexObligation(db, {
|
|
1014
1416
|
accountId: input.accountId,
|
|
@@ -1061,6 +1463,64 @@ function createKnowledgeSourceSyncActivities(services, resolveDocumentServices)
|
|
|
1061
1463
|
}
|
|
1062
1464
|
return { ...obligation, status: "indexed" };
|
|
1063
1465
|
};
|
|
1466
|
+
const recordEntryAcl = async (details) => {
|
|
1467
|
+
if (!driver.readAcl || !provider.aclAuthority) {
|
|
1468
|
+
return { aclEligibility: "pending", aclEvidence: null };
|
|
1469
|
+
}
|
|
1470
|
+
await beginGoogleDriveObjectAclRefresh(db, {
|
|
1471
|
+
accountId: input.accountId,
|
|
1472
|
+
workspaceId: input.workspaceId,
|
|
1473
|
+
obligationId: details.obligation.id,
|
|
1474
|
+
sourceSyncGeneration: details.obligation.sourceSyncGeneration,
|
|
1475
|
+
sourceConfigGeneration: action.sourceConfigGeneration,
|
|
1476
|
+
sourceLifecycleGeneration: action.sourceLifecycleGeneration,
|
|
1477
|
+
objectLifecycleGeneration: details.object.lifecycleGeneration,
|
|
1478
|
+
objectVersionGeneration: details.version.versionGeneration
|
|
1479
|
+
});
|
|
1480
|
+
const remainingProviderRequests = action.limits.maxProviderRequests - summary.providerRequests;
|
|
1481
|
+
if (remainingProviderRequests < 1) {
|
|
1482
|
+
throw new SyncFailure("resource_limit", false);
|
|
1483
|
+
}
|
|
1484
|
+
const evidence = await driver.readAcl(details.entry, remainingProviderRequests);
|
|
1485
|
+
summary.providerRequests += evidence.providerRequests;
|
|
1486
|
+
if (summary.providerRequests > action.limits.maxProviderRequests) {
|
|
1487
|
+
throw new SyncFailure("resource_limit", false);
|
|
1488
|
+
}
|
|
1489
|
+
const recorded = await recordGoogleDriveObjectAclEvidence(db, {
|
|
1490
|
+
accountId: input.accountId,
|
|
1491
|
+
workspaceId: input.workspaceId,
|
|
1492
|
+
obligationId: details.obligation.id,
|
|
1493
|
+
connectionId: provider.aclAuthority.connectionId,
|
|
1494
|
+
connectionVersion: provider.aclAuthority.connectionVersion,
|
|
1495
|
+
sourceGooglePermissionId: provider.aclAuthority.sourceGooglePermissionId,
|
|
1496
|
+
sourceSyncGeneration: details.obligation.sourceSyncGeneration,
|
|
1497
|
+
sourceConfigGeneration: action.sourceConfigGeneration,
|
|
1498
|
+
sourceLifecycleGeneration: action.sourceLifecycleGeneration,
|
|
1499
|
+
objectLifecycleGeneration: details.object.lifecycleGeneration,
|
|
1500
|
+
objectVersionGeneration: details.version.versionGeneration,
|
|
1501
|
+
providerRevision: evidence.providerRevision,
|
|
1502
|
+
driveId: evidence.driveId,
|
|
1503
|
+
aclRevision: evidence.aclRevision,
|
|
1504
|
+
eligibility: evidence.eligibility,
|
|
1505
|
+
observedAt: evidence.observedAt,
|
|
1506
|
+
expiresAt: evidence.expiresAt,
|
|
1507
|
+
citationLocator: driver.citationLocator(details.entry),
|
|
1508
|
+
operationId: `drive-acl:${details.obligation.id}:${evidence.aclRevision}`,
|
|
1509
|
+
principals: evidence.principals
|
|
1510
|
+
});
|
|
1511
|
+
return {
|
|
1512
|
+
aclEligibility: evidence.eligibility,
|
|
1513
|
+
aclEvidence: {
|
|
1514
|
+
version: 1,
|
|
1515
|
+
provider: "google_drive",
|
|
1516
|
+
evidenceId: recorded.evidenceId,
|
|
1517
|
+
aclRevision: evidence.aclRevision,
|
|
1518
|
+
aclHash: recorded.aclHash,
|
|
1519
|
+
observedAt: evidence.observedAt,
|
|
1520
|
+
expiresAt: evidence.expiresAt
|
|
1521
|
+
}
|
|
1522
|
+
};
|
|
1523
|
+
};
|
|
1064
1524
|
const outcomes = [];
|
|
1065
1525
|
for (const entry of entriesToProcess) {
|
|
1066
1526
|
heartbeat({ sourceId: input.sourceId, externalObjectId: entry.externalObjectId });
|
|
@@ -1158,9 +1618,15 @@ function createKnowledgeSourceSyncActivities(services, resolveDocumentServices)
|
|
|
1158
1618
|
entry,
|
|
1159
1619
|
object,
|
|
1160
1620
|
version: currentVersion,
|
|
1161
|
-
indexRequired:
|
|
1621
|
+
indexRequired: (currentObservation?.documentChunkCount ?? 0) === 0,
|
|
1162
1622
|
obligation: currentObligation
|
|
1163
1623
|
});
|
|
1624
|
+
const objectAcl2 = await recordEntryAcl({
|
|
1625
|
+
entry,
|
|
1626
|
+
object,
|
|
1627
|
+
version: currentVersion,
|
|
1628
|
+
obligation: obligation2
|
|
1629
|
+
});
|
|
1164
1630
|
summary.unchanged += 1;
|
|
1165
1631
|
outcomes.push({
|
|
1166
1632
|
externalObjectId: entry.externalObjectId,
|
|
@@ -1168,7 +1634,8 @@ function createKnowledgeSourceSyncActivities(services, resolveDocumentServices)
|
|
|
1168
1634
|
contentSha256: currentVersion.contentSha256,
|
|
1169
1635
|
providerRevision: entry.externalVersionId,
|
|
1170
1636
|
metadataHash,
|
|
1171
|
-
aclEligibility:
|
|
1637
|
+
aclEligibility: objectAcl2.aclEligibility,
|
|
1638
|
+
aclEvidence: objectAcl2.aclEvidence,
|
|
1172
1639
|
indexObligationId: obligation2.id
|
|
1173
1640
|
});
|
|
1174
1641
|
continue;
|
|
@@ -1249,9 +1716,10 @@ function createKnowledgeSourceSyncActivities(services, resolveDocumentServices)
|
|
|
1249
1716
|
version: version2,
|
|
1250
1717
|
indexRequired: (currentObservation?.documentChunkCount ?? 0) === 0
|
|
1251
1718
|
});
|
|
1719
|
+
const objectAcl2 = await recordEntryAcl({ entry, object, version: version2, obligation: obligation2 });
|
|
1252
1720
|
summary.unchanged += 1;
|
|
1253
1721
|
summary.indexed += 1;
|
|
1254
|
-
summary.aclPending += 1;
|
|
1722
|
+
if (objectAcl2.aclEligibility === "pending") summary.aclPending += 1;
|
|
1255
1723
|
outcomes.push({
|
|
1256
1724
|
externalObjectId: entry.externalObjectId,
|
|
1257
1725
|
outcome: "unchanged",
|
|
@@ -1259,7 +1727,8 @@ function createKnowledgeSourceSyncActivities(services, resolveDocumentServices)
|
|
|
1259
1727
|
sizeBytes: bytes.byteLength,
|
|
1260
1728
|
providerRevision: entry.externalVersionId,
|
|
1261
1729
|
metadataHash,
|
|
1262
|
-
aclEligibility:
|
|
1730
|
+
aclEligibility: objectAcl2.aclEligibility,
|
|
1731
|
+
aclEvidence: objectAcl2.aclEvidence,
|
|
1263
1732
|
indexObligationId: obligation2.id
|
|
1264
1733
|
});
|
|
1265
1734
|
continue;
|
|
@@ -1349,9 +1818,10 @@ function createKnowledgeSourceSyncActivities(services, resolveDocumentServices)
|
|
|
1349
1818
|
version,
|
|
1350
1819
|
indexRequired: true
|
|
1351
1820
|
});
|
|
1821
|
+
const objectAcl = await recordEntryAcl({ entry, object, version, obligation });
|
|
1352
1822
|
summary.indexed += 1;
|
|
1353
1823
|
summary.imported += 1;
|
|
1354
|
-
summary.aclPending += 1;
|
|
1824
|
+
if (objectAcl.aclEligibility === "pending") summary.aclPending += 1;
|
|
1355
1825
|
summary.bytes += bytes.byteLength;
|
|
1356
1826
|
outcomes.push({
|
|
1357
1827
|
externalObjectId: entry.externalObjectId,
|
|
@@ -1360,7 +1830,8 @@ function createKnowledgeSourceSyncActivities(services, resolveDocumentServices)
|
|
|
1360
1830
|
sizeBytes: bytes.byteLength,
|
|
1361
1831
|
providerRevision: entry.externalVersionId,
|
|
1362
1832
|
metadataHash,
|
|
1363
|
-
aclEligibility:
|
|
1833
|
+
aclEligibility: objectAcl.aclEligibility,
|
|
1834
|
+
aclEvidence: objectAcl.aclEvidence,
|
|
1364
1835
|
indexObligationId: obligation.id
|
|
1365
1836
|
});
|
|
1366
1837
|
} catch (error) {
|
|
@@ -1520,6 +1991,7 @@ function createKnowledgeSourceSyncActivities(services, resolveDocumentServices)
|
|
|
1520
1991
|
labels: { provider: providerLabel },
|
|
1521
1992
|
amount: summary.bytes
|
|
1522
1993
|
});
|
|
1994
|
+
recordSyncHealthTelemetry(observability, providerLabel, summary, "succeeded");
|
|
1523
1995
|
return {
|
|
1524
1996
|
action: "complete",
|
|
1525
1997
|
bufferedWake: settled.bufferedWake,
|
|
@@ -1604,6 +2076,7 @@ function createKnowledgeSourceSyncActivities(services, resolveDocumentServices)
|
|
|
1604
2076
|
labels: { provider: providerLabel },
|
|
1605
2077
|
amount: summary.bytes
|
|
1606
2078
|
});
|
|
2079
|
+
recordSyncHealthTelemetry(observability, providerLabel, summary, "failed", failure);
|
|
1607
2080
|
return {
|
|
1608
2081
|
action: "failed",
|
|
1609
2082
|
bufferedWake: settled.bufferedWake,
|
|
@@ -1771,6 +2244,11 @@ async function resolveKnowledgeSyncProvider(input) {
|
|
|
1771
2244
|
return {
|
|
1772
2245
|
kind: "google_drive",
|
|
1773
2246
|
selectedDestination: source.destination,
|
|
2247
|
+
aclAuthority: {
|
|
2248
|
+
connectionId: resolvedConnection.id,
|
|
2249
|
+
connectionVersion: resolvedConnection.version,
|
|
2250
|
+
sourceGooglePermissionId: metadata.googlePermissionId
|
|
2251
|
+
},
|
|
1774
2252
|
driver: googleDriveSyncDriver({
|
|
1775
2253
|
actionProviderCoordinationKey: action.providerCoordinationKey,
|
|
1776
2254
|
metadata,
|
|
@@ -1779,6 +2257,8 @@ async function resolveKnowledgeSyncProvider(input) {
|
|
|
1779
2257
|
workspaceId: input.workspaceId,
|
|
1780
2258
|
initiatingSubjectId: action.initiatingSubjectId,
|
|
1781
2259
|
authorization,
|
|
2260
|
+
retry: googleDriveProviderRetryOptions(input.settings),
|
|
2261
|
+
observability: input.observability,
|
|
1782
2262
|
limits: action.limits,
|
|
1783
2263
|
connectionId: resolvedConnection.id,
|
|
1784
2264
|
fullReconciliationIntervalMs: GOOGLE_DRIVE_FULL_RECONCILIATION_INTERVAL_MS,
|
|
@@ -1803,6 +2283,7 @@ async function resolveKnowledgeSyncProvider(input) {
|
|
|
1803
2283
|
return {
|
|
1804
2284
|
kind: "atlassian",
|
|
1805
2285
|
selectedDestination: source.destination,
|
|
2286
|
+
aclAuthority: null,
|
|
1806
2287
|
driver: atlassianSyncDriver({
|
|
1807
2288
|
actionProviderCoordinationKey: action.providerCoordinationKey,
|
|
1808
2289
|
metadata,
|
|
@@ -1863,10 +2344,18 @@ function atlassianSyncDriver(input) {
|
|
|
1863
2344
|
function googleDriveSyncDriver(input) {
|
|
1864
2345
|
const maxElapsedMs = input.limits.maxElapsedSeconds * 1e3;
|
|
1865
2346
|
const provider = input.provider ?? {
|
|
1866
|
-
getStartPageToken: async (driveId) => await getDriveStartPageToken(driveId, input.authorization),
|
|
1867
|
-
listChanges: async (pageToken, driveId, pageSize) => await listDriveChanges(
|
|
1868
|
-
|
|
1869
|
-
|
|
2347
|
+
getStartPageToken: async (driveId) => await getDriveStartPageToken(driveId, input.authorization, input.retry, input.observability),
|
|
2348
|
+
listChanges: async (pageToken, driveId, pageSize) => await listDriveChanges(
|
|
2349
|
+
pageToken,
|
|
2350
|
+
driveId,
|
|
2351
|
+
pageSize,
|
|
2352
|
+
input.authorization,
|
|
2353
|
+
input.retry,
|
|
2354
|
+
input.observability
|
|
2355
|
+
),
|
|
2356
|
+
getFile: async (fileId) => await getDriveFile(fileId, input.authorization, input.retry, input.observability),
|
|
2357
|
+
listChildren: async (request) => await listDriveChildren(request, input.authorization, input.retry, input.observability),
|
|
2358
|
+
listPermissions: async (request) => await listDrivePermissions(request, input.authorization, input.retry, input.observability)
|
|
1870
2359
|
};
|
|
1871
2360
|
const clock = provider.now ?? Date.now;
|
|
1872
2361
|
const budgetLimits = {
|
|
@@ -2177,12 +2666,26 @@ function googleDriveSyncDriver(input) {
|
|
|
2177
2666
|
};
|
|
2178
2667
|
}
|
|
2179
2668
|
},
|
|
2180
|
-
fetchContent: async (entry, maxBytes) => await fetchDriveBytes(entry, input.authorization, maxBytes),
|
|
2669
|
+
fetchContent: async (entry, maxBytes) => await fetchDriveBytes(entry, input.authorization, maxBytes, input.retry, input.observability),
|
|
2670
|
+
readAcl: async (entry, maxProviderRequests) => {
|
|
2671
|
+
if (!provider.listPermissions) {
|
|
2672
|
+
throw new SyncFailure("provider_rejected", false);
|
|
2673
|
+
}
|
|
2674
|
+
return await collectGoogleDriveAclEvidence({
|
|
2675
|
+
entry,
|
|
2676
|
+
maxProviderRequests,
|
|
2677
|
+
listPermissions: provider.listPermissions,
|
|
2678
|
+
getFile: provider.getFile,
|
|
2679
|
+
now: clock
|
|
2680
|
+
});
|
|
2681
|
+
},
|
|
2181
2682
|
citationLocator: (entry) => ({
|
|
2182
|
-
version:
|
|
2683
|
+
version: 2,
|
|
2183
2684
|
providerKey: "google-drive",
|
|
2184
2685
|
providerCoordinationKey: input.actionProviderCoordinationKey,
|
|
2185
2686
|
externalObjectId: entry.externalObjectId,
|
|
2687
|
+
providerRevision: entry.externalVersionId,
|
|
2688
|
+
driveId: entry.driveId,
|
|
2186
2689
|
sourceUri: entry.sourceUri
|
|
2187
2690
|
})
|
|
2188
2691
|
};
|
|
@@ -2351,16 +2854,18 @@ function googleDriveInventoryStopIsHard(reason, budget, limits) {
|
|
|
2351
2854
|
}
|
|
2352
2855
|
return reason === "elapsed_time_limit" && budget.elapsedMs >= limits.maxElapsedMs;
|
|
2353
2856
|
}
|
|
2354
|
-
async function getDriveStartPageToken(driveId, authorization) {
|
|
2857
|
+
async function getDriveStartPageToken(driveId, authorization, retry, observability) {
|
|
2355
2858
|
if (!authorization) throw new SyncFailure("connection_reconnect_required", true, true);
|
|
2356
2859
|
const url = new URL(`${DRIVE_API_BASE}/changes/startPageToken`);
|
|
2357
2860
|
url.searchParams.set("supportsAllDrives", "true");
|
|
2358
2861
|
if (driveId) url.searchParams.set("driveId", driveId);
|
|
2359
|
-
const response = await
|
|
2360
|
-
|
|
2361
|
-
|
|
2362
|
-
|
|
2363
|
-
|
|
2862
|
+
const response = await requestGoogleDrive(
|
|
2863
|
+
url,
|
|
2864
|
+
{ headers: { authorization, accept: "application/json" } },
|
|
2865
|
+
"start_page_token",
|
|
2866
|
+
retry,
|
|
2867
|
+
observability
|
|
2868
|
+
);
|
|
2364
2869
|
if (!response.ok) throwGoogleDriveResponse(response);
|
|
2365
2870
|
const payload = await readResponseJsonBounded(
|
|
2366
2871
|
response,
|
|
@@ -2369,7 +2874,7 @@ async function getDriveStartPageToken(driveId, authorization) {
|
|
|
2369
2874
|
);
|
|
2370
2875
|
return requiredString(payload.startPageToken, "start_page_token");
|
|
2371
2876
|
}
|
|
2372
|
-
async function listDriveChanges(pageToken, driveId, pageSize, authorization) {
|
|
2877
|
+
async function listDriveChanges(pageToken, driveId, pageSize, authorization, retry, observability) {
|
|
2373
2878
|
if (!authorization) throw new SyncFailure("connection_reconnect_required", true, true);
|
|
2374
2879
|
const url = new URL(`${DRIVE_API_BASE}/changes`);
|
|
2375
2880
|
url.searchParams.set("pageToken", pageToken);
|
|
@@ -2382,11 +2887,13 @@ async function listDriveChanges(pageToken, driveId, pageSize, authorization) {
|
|
|
2382
2887
|
"nextPageToken,newStartPageToken,changes(fileId,removed,time,driveId,file(id,name,mimeType,driveId,parents,modifiedTime,createdTime,version,md5Checksum,size,webViewLink,trashed))"
|
|
2383
2888
|
);
|
|
2384
2889
|
if (driveId) url.searchParams.set("driveId", driveId);
|
|
2385
|
-
const response = await
|
|
2386
|
-
|
|
2387
|
-
|
|
2388
|
-
|
|
2389
|
-
|
|
2890
|
+
const response = await requestGoogleDrive(
|
|
2891
|
+
url,
|
|
2892
|
+
{ headers: { authorization, accept: "application/json" } },
|
|
2893
|
+
"list_changes",
|
|
2894
|
+
retry,
|
|
2895
|
+
observability
|
|
2896
|
+
);
|
|
2390
2897
|
if (response.status === 410) throw new GoogleDriveCursorInvalidError();
|
|
2391
2898
|
if (!response.ok) throwGoogleDriveResponse(response);
|
|
2392
2899
|
const payload = await readResponseJsonBounded(
|
|
@@ -2409,7 +2916,7 @@ async function listDriveChanges(pageToken, driveId, pageSize, authorization) {
|
|
|
2409
2916
|
newStartPageToken: optionalString(payload.newStartPageToken)
|
|
2410
2917
|
};
|
|
2411
2918
|
}
|
|
2412
|
-
async function getDriveFile(fileId, authorization) {
|
|
2919
|
+
async function getDriveFile(fileId, authorization, retry, observability) {
|
|
2413
2920
|
if (!authorization) throw new SyncFailure("connection_reconnect_required", true, true);
|
|
2414
2921
|
const url = new URL(`${DRIVE_API_BASE}/files/${encodeURIComponent(fileId)}`);
|
|
2415
2922
|
url.searchParams.set("supportsAllDrives", "true");
|
|
@@ -2417,65 +2924,276 @@ async function getDriveFile(fileId, authorization) {
|
|
|
2417
2924
|
"fields",
|
|
2418
2925
|
"id,name,mimeType,driveId,parents,modifiedTime,createdTime,version,md5Checksum,size,webViewLink,trashed"
|
|
2419
2926
|
);
|
|
2420
|
-
const response = await
|
|
2421
|
-
|
|
2422
|
-
|
|
2423
|
-
|
|
2424
|
-
|
|
2927
|
+
const response = await requestGoogleDrive(
|
|
2928
|
+
url,
|
|
2929
|
+
{ headers: { authorization, accept: "application/json" } },
|
|
2930
|
+
"get_file",
|
|
2931
|
+
retry,
|
|
2932
|
+
observability
|
|
2933
|
+
);
|
|
2425
2934
|
if (response.status === 404) return null;
|
|
2426
2935
|
if (!response.ok) throwGoogleDriveResponse(response);
|
|
2427
2936
|
return parseDriveProviderItem(
|
|
2428
2937
|
await readResponseJsonBounded(response, DRIVE_JSON_MAX_BYTES, "Google Drive file metadata")
|
|
2429
2938
|
);
|
|
2430
2939
|
}
|
|
2431
|
-
function
|
|
2432
|
-
if (response.status === 401 || response.status === 403) {
|
|
2433
|
-
throw new SyncFailure("connection_reconnect_required", true, true);
|
|
2434
|
-
}
|
|
2435
|
-
throw new SyncFailure(
|
|
2436
|
-
response.status >= 500 || response.status === 429 ? "provider_unavailable" : "provider_rejected",
|
|
2437
|
-
response.status >= 500 || response.status === 429
|
|
2438
|
-
);
|
|
2439
|
-
}
|
|
2440
|
-
function selectedDriveSource(metadata, externalSourceId) {
|
|
2441
|
-
const sources = metadata.selectedSources ?? (metadata.selectedSource ? [metadata.selectedSource] : []);
|
|
2442
|
-
return sources.find((source) => source.id === externalSourceId) ?? null;
|
|
2443
|
-
}
|
|
2444
|
-
function googleDriveSyncProviderAccessAllowed(input) {
|
|
2445
|
-
return input.resolvedConnectionVersion !== void 0 && input.connectionVersion === input.resolvedConnectionVersion && input.connectionStatus === "active" && (input.lifecycleState === void 0 || input.lifecycleState === "active") && googleDriveScopesAllowCapability(input.grantedScopes, "recursive_source_sync");
|
|
2446
|
-
}
|
|
2447
|
-
async function listDriveChildren(input, authorization) {
|
|
2940
|
+
async function listDrivePermissions(input, authorization, retry, observability) {
|
|
2448
2941
|
if (!authorization) throw new SyncFailure("connection_reconnect_required", true, true);
|
|
2449
|
-
const url = new URL(`${DRIVE_API_BASE}/files`);
|
|
2450
|
-
url.searchParams.set(
|
|
2451
|
-
"q",
|
|
2452
|
-
`'${input.folderId.replaceAll("'", "\\'")}' in parents and trashed = false`
|
|
2453
|
-
);
|
|
2454
|
-
url.searchParams.set("pageSize", String(input.pageSize));
|
|
2942
|
+
const url = new URL(`${DRIVE_API_BASE}/files/${encodeURIComponent(input.fileId)}/permissions`);
|
|
2455
2943
|
url.searchParams.set("supportsAllDrives", "true");
|
|
2456
|
-
url.searchParams.set("
|
|
2944
|
+
url.searchParams.set("pageSize", String(input.pageSize));
|
|
2457
2945
|
url.searchParams.set(
|
|
2458
2946
|
"fields",
|
|
2459
|
-
"nextPageToken,
|
|
2947
|
+
"nextPageToken,permissions(id,type,emailAddress,domain,role,allowFileDiscovery,expirationTime,deleted,permissionDetails(inherited))"
|
|
2460
2948
|
);
|
|
2461
2949
|
if (input.pageToken) url.searchParams.set("pageToken", input.pageToken);
|
|
2462
|
-
|
|
2463
|
-
url
|
|
2464
|
-
|
|
2950
|
+
const response = await requestGoogleDrive(
|
|
2951
|
+
url,
|
|
2952
|
+
{ headers: { authorization, accept: "application/json" } },
|
|
2953
|
+
"list_permissions",
|
|
2954
|
+
retry,
|
|
2955
|
+
observability
|
|
2956
|
+
);
|
|
2957
|
+
if (response.status === 403 || response.status === 404) {
|
|
2958
|
+
return { permissions: [], nextPageToken: null, denied: true };
|
|
2465
2959
|
}
|
|
2466
|
-
|
|
2467
|
-
|
|
2468
|
-
|
|
2469
|
-
|
|
2470
|
-
|
|
2471
|
-
|
|
2472
|
-
|
|
2473
|
-
|
|
2474
|
-
|
|
2475
|
-
|
|
2476
|
-
|
|
2477
|
-
|
|
2478
|
-
|
|
2960
|
+
if (!response.ok) throwGoogleDriveResponse(response);
|
|
2961
|
+
const payload = await readResponseJsonBounded(
|
|
2962
|
+
response,
|
|
2963
|
+
DRIVE_JSON_MAX_BYTES,
|
|
2964
|
+
"Google Drive permission list"
|
|
2965
|
+
);
|
|
2966
|
+
const rawPermissions = Array.isArray(payload.permissions) ? payload.permissions : null;
|
|
2967
|
+
if (!rawPermissions) throw new SyncFailure("provider_payload_invalid", false);
|
|
2968
|
+
return {
|
|
2969
|
+
permissions: rawPermissions.map(parseDrivePermission),
|
|
2970
|
+
nextPageToken: optionalString(payload.nextPageToken),
|
|
2971
|
+
denied: false
|
|
2972
|
+
};
|
|
2973
|
+
}
|
|
2974
|
+
async function collectGoogleDriveAclEvidence(input) {
|
|
2975
|
+
if (!Number.isSafeInteger(input.maxProviderRequests) || input.maxProviderRequests < 1) {
|
|
2976
|
+
throw new SyncFailure("resource_limit", false);
|
|
2977
|
+
}
|
|
2978
|
+
const now = input.now ?? Date.now;
|
|
2979
|
+
const observedAt = new Date(now());
|
|
2980
|
+
if (!Number.isFinite(observedAt.getTime())) {
|
|
2981
|
+
throw new SyncFailure("provider_payload_invalid", false);
|
|
2982
|
+
}
|
|
2983
|
+
const permissions = [];
|
|
2984
|
+
const seenTokens = /* @__PURE__ */ new Set();
|
|
2985
|
+
let pageToken = null;
|
|
2986
|
+
let providerRequests = 0;
|
|
2987
|
+
let permissionListDenied = false;
|
|
2988
|
+
do {
|
|
2989
|
+
if (providerRequests >= input.maxProviderRequests) {
|
|
2990
|
+
throw new SyncFailure("resource_limit", false);
|
|
2991
|
+
}
|
|
2992
|
+
providerRequests += 1;
|
|
2993
|
+
const page = await input.listPermissions({
|
|
2994
|
+
fileId: input.entry.externalObjectId,
|
|
2995
|
+
pageToken,
|
|
2996
|
+
pageSize: Math.min(100, GOOGLE_DRIVE_ACL_MAX_PRINCIPALS - permissions.length)
|
|
2997
|
+
});
|
|
2998
|
+
if (page.denied) {
|
|
2999
|
+
permissionListDenied = true;
|
|
3000
|
+
break;
|
|
3001
|
+
}
|
|
3002
|
+
permissions.push(...page.permissions);
|
|
3003
|
+
if (permissions.length > GOOGLE_DRIVE_ACL_MAX_PRINCIPALS) {
|
|
3004
|
+
throw new SyncFailure("resource_limit", false);
|
|
3005
|
+
}
|
|
3006
|
+
pageToken = page.nextPageToken;
|
|
3007
|
+
if (pageToken) {
|
|
3008
|
+
if (pageToken.length > 4096 || seenTokens.has(pageToken)) {
|
|
3009
|
+
throw new SyncFailure("provider_payload_invalid", false);
|
|
3010
|
+
}
|
|
3011
|
+
seenTokens.add(pageToken);
|
|
3012
|
+
}
|
|
3013
|
+
} while (pageToken);
|
|
3014
|
+
if (permissionListDenied) {
|
|
3015
|
+
if (!input.getFile || providerRequests >= input.maxProviderRequests) {
|
|
3016
|
+
return googleDriveDeniedAclEvidence(input.entry, observedAt, providerRequests);
|
|
3017
|
+
}
|
|
3018
|
+
providerRequests += 1;
|
|
3019
|
+
const current = await input.getFile(input.entry.externalObjectId);
|
|
3020
|
+
if (!current || current.trashed) {
|
|
3021
|
+
return googleDriveDeniedAclEvidence(input.entry, observedAt, providerRequests);
|
|
3022
|
+
}
|
|
3023
|
+
const aclRevision2 = createHash2("sha256").update(
|
|
3024
|
+
JSON.stringify({
|
|
3025
|
+
version: 1,
|
|
3026
|
+
mode: "source_owner_only",
|
|
3027
|
+
externalObjectId: input.entry.externalObjectId,
|
|
3028
|
+
providerRevision: input.entry.externalVersionId
|
|
3029
|
+
})
|
|
3030
|
+
).digest("hex");
|
|
3031
|
+
return {
|
|
3032
|
+
eligibility: "eligible",
|
|
3033
|
+
providerRevision: input.entry.externalVersionId,
|
|
3034
|
+
driveId: input.entry.driveId,
|
|
3035
|
+
aclRevision: aclRevision2,
|
|
3036
|
+
observedAt: observedAt.toISOString(),
|
|
3037
|
+
expiresAt: new Date(observedAt.getTime() + GOOGLE_DRIVE_ACL_FRESHNESS_MS).toISOString(),
|
|
3038
|
+
providerRequests,
|
|
3039
|
+
principals: []
|
|
3040
|
+
};
|
|
3041
|
+
}
|
|
3042
|
+
const canonicalPermissions = permissions.map((permission) => ({
|
|
3043
|
+
id: permission.id,
|
|
3044
|
+
type: permission.type,
|
|
3045
|
+
role: permission.role,
|
|
3046
|
+
emailAddress: permission.emailAddress,
|
|
3047
|
+
domain: permission.domain,
|
|
3048
|
+
allowFileDiscovery: permission.allowFileDiscovery,
|
|
3049
|
+
expirationTime: permission.expirationTime,
|
|
3050
|
+
inherited: permission.inherited
|
|
3051
|
+
})).sort((left, right) => JSON.stringify(left).localeCompare(JSON.stringify(right)));
|
|
3052
|
+
const aclRevision = createHash2("sha256").update(
|
|
3053
|
+
JSON.stringify({
|
|
3054
|
+
version: 1,
|
|
3055
|
+
externalObjectId: input.entry.externalObjectId,
|
|
3056
|
+
providerRevision: input.entry.externalVersionId,
|
|
3057
|
+
permissions: canonicalPermissions
|
|
3058
|
+
})
|
|
3059
|
+
).digest("hex");
|
|
3060
|
+
return {
|
|
3061
|
+
eligibility: "eligible",
|
|
3062
|
+
providerRevision: input.entry.externalVersionId,
|
|
3063
|
+
driveId: input.entry.driveId,
|
|
3064
|
+
aclRevision,
|
|
3065
|
+
observedAt: observedAt.toISOString(),
|
|
3066
|
+
expiresAt: new Date(observedAt.getTime() + GOOGLE_DRIVE_ACL_FRESHNESS_MS).toISOString(),
|
|
3067
|
+
providerRequests,
|
|
3068
|
+
principals: canonicalPermissions.map((permission) => ({
|
|
3069
|
+
type: permission.type,
|
|
3070
|
+
permissionId: permission.id,
|
|
3071
|
+
emailAddress: permission.emailAddress,
|
|
3072
|
+
domain: permission.domain,
|
|
3073
|
+
role: permission.role,
|
|
3074
|
+
inherited: permission.inherited,
|
|
3075
|
+
allowFileDiscovery: permission.allowFileDiscovery,
|
|
3076
|
+
expirationTime: permission.expirationTime
|
|
3077
|
+
}))
|
|
3078
|
+
};
|
|
3079
|
+
}
|
|
3080
|
+
function googleDriveDeniedAclEvidence(entry, observedAt, providerRequests) {
|
|
3081
|
+
return {
|
|
3082
|
+
eligibility: "denied",
|
|
3083
|
+
providerRevision: entry.externalVersionId,
|
|
3084
|
+
driveId: entry.driveId,
|
|
3085
|
+
aclRevision: createHash2("sha256").update(
|
|
3086
|
+
JSON.stringify({
|
|
3087
|
+
version: 1,
|
|
3088
|
+
mode: "provider_denied",
|
|
3089
|
+
externalObjectId: entry.externalObjectId,
|
|
3090
|
+
providerRevision: entry.externalVersionId
|
|
3091
|
+
})
|
|
3092
|
+
).digest("hex"),
|
|
3093
|
+
observedAt: observedAt.toISOString(),
|
|
3094
|
+
expiresAt: new Date(observedAt.getTime() + GOOGLE_DRIVE_ACL_FRESHNESS_MS).toISOString(),
|
|
3095
|
+
providerRequests,
|
|
3096
|
+
principals: []
|
|
3097
|
+
};
|
|
3098
|
+
}
|
|
3099
|
+
function parseDrivePermission(value) {
|
|
3100
|
+
const row = objectRecord(value);
|
|
3101
|
+
if (row.deleted === true) throw new SyncFailure("provider_payload_invalid", false);
|
|
3102
|
+
const type = requiredString(row.type, "permission.type");
|
|
3103
|
+
if (type !== "user" && type !== "group" && type !== "domain" && type !== "anyone") {
|
|
3104
|
+
throw new SyncFailure("provider_payload_invalid", false);
|
|
3105
|
+
}
|
|
3106
|
+
const role = requiredString(row.role, "permission.role");
|
|
3107
|
+
if (role !== "owner" && role !== "organizer" && role !== "fileOrganizer" && role !== "writer" && role !== "commenter" && role !== "reader") {
|
|
3108
|
+
throw new SyncFailure("provider_payload_invalid", false);
|
|
3109
|
+
}
|
|
3110
|
+
const permissionDetails = Array.isArray(row.permissionDetails) ? row.permissionDetails : [];
|
|
3111
|
+
return {
|
|
3112
|
+
id: requiredString(row.id, "permission.id"),
|
|
3113
|
+
type,
|
|
3114
|
+
role,
|
|
3115
|
+
emailAddress: optionalString(row.emailAddress),
|
|
3116
|
+
domain: optionalString(row.domain),
|
|
3117
|
+
allowFileDiscovery: typeof row.allowFileDiscovery === "boolean" ? row.allowFileDiscovery : null,
|
|
3118
|
+
expirationTime: optionalExactProviderTimestamp(row.expirationTime),
|
|
3119
|
+
inherited: permissionDetails.some((detail) => objectRecord(detail).inherited === true)
|
|
3120
|
+
};
|
|
3121
|
+
}
|
|
3122
|
+
function optionalExactProviderTimestamp(value) {
|
|
3123
|
+
const timestamp = optionalString(value);
|
|
3124
|
+
if (!timestamp) return null;
|
|
3125
|
+
const parsed = new Date(timestamp);
|
|
3126
|
+
if (!Number.isFinite(parsed.getTime())) throw new SyncFailure("provider_payload_invalid", false);
|
|
3127
|
+
return parsed.toISOString();
|
|
3128
|
+
}
|
|
3129
|
+
function throwGoogleDriveResponse(response) {
|
|
3130
|
+
if (response.status === 401 || response.status === 403) {
|
|
3131
|
+
throw new SyncFailure("connection_reconnect_required", true, true);
|
|
3132
|
+
}
|
|
3133
|
+
throw new SyncFailure(
|
|
3134
|
+
response.status >= 500 || response.status === 429 ? "provider_unavailable" : "provider_rejected",
|
|
3135
|
+
response.status >= 500 || response.status === 429
|
|
3136
|
+
);
|
|
3137
|
+
}
|
|
3138
|
+
async function requestGoogleDrive(url, init, operation, retry, observability) {
|
|
3139
|
+
try {
|
|
3140
|
+
return await fetchGoogleDriveProvider({
|
|
3141
|
+
fetchImpl: fetch,
|
|
3142
|
+
url,
|
|
3143
|
+
init,
|
|
3144
|
+
operation,
|
|
3145
|
+
policy: retry,
|
|
3146
|
+
observability,
|
|
3147
|
+
heartbeat: (details) => heartbeat({ provider: "google_drive", ...details })
|
|
3148
|
+
});
|
|
3149
|
+
} catch (error) {
|
|
3150
|
+
if (error instanceof GoogleDriveProviderTransportError) {
|
|
3151
|
+
throw new SyncFailure("provider_unavailable", true);
|
|
3152
|
+
}
|
|
3153
|
+
throw error;
|
|
3154
|
+
}
|
|
3155
|
+
}
|
|
3156
|
+
function selectedDriveSource(metadata, externalSourceId) {
|
|
3157
|
+
const sources = metadata.selectedSources ?? (metadata.selectedSource ? [metadata.selectedSource] : []);
|
|
3158
|
+
return sources.find((source) => source.id === externalSourceId) ?? null;
|
|
3159
|
+
}
|
|
3160
|
+
function googleDriveSyncProviderAccessAllowed(input) {
|
|
3161
|
+
return input.resolvedConnectionVersion !== void 0 && input.connectionVersion === input.resolvedConnectionVersion && input.connectionStatus === "active" && (input.lifecycleState === void 0 || input.lifecycleState === "active") && googleDriveScopesAllowCapability(input.grantedScopes, "recursive_source_sync");
|
|
3162
|
+
}
|
|
3163
|
+
async function listDriveChildren(input, authorization, retry, observability) {
|
|
3164
|
+
if (!authorization) throw new SyncFailure("connection_reconnect_required", true, true);
|
|
3165
|
+
const url = new URL(`${DRIVE_API_BASE}/files`);
|
|
3166
|
+
url.searchParams.set(
|
|
3167
|
+
"q",
|
|
3168
|
+
`'${input.folderId.replaceAll("'", "\\'")}' in parents and trashed = false`
|
|
3169
|
+
);
|
|
3170
|
+
url.searchParams.set("pageSize", String(input.pageSize));
|
|
3171
|
+
url.searchParams.set("supportsAllDrives", "true");
|
|
3172
|
+
url.searchParams.set("includeItemsFromAllDrives", "true");
|
|
3173
|
+
url.searchParams.set(
|
|
3174
|
+
"fields",
|
|
3175
|
+
"nextPageToken,incompleteSearch,files(id,name,mimeType,driveId,parents,modifiedTime,createdTime,version,md5Checksum,size,webViewLink,trashed)"
|
|
3176
|
+
);
|
|
3177
|
+
if (input.pageToken) url.searchParams.set("pageToken", input.pageToken);
|
|
3178
|
+
if (input.driveId) {
|
|
3179
|
+
url.searchParams.set("corpora", "drive");
|
|
3180
|
+
url.searchParams.set("driveId", input.driveId);
|
|
3181
|
+
}
|
|
3182
|
+
const response = await requestGoogleDrive(
|
|
3183
|
+
url,
|
|
3184
|
+
{ headers: { authorization, accept: "application/json" } },
|
|
3185
|
+
"list_children",
|
|
3186
|
+
retry,
|
|
3187
|
+
observability
|
|
3188
|
+
);
|
|
3189
|
+
if (!response.ok) {
|
|
3190
|
+
if (response.status === 401 || response.status === 403) {
|
|
3191
|
+
throw new SyncFailure("connection_reconnect_required", true, true);
|
|
3192
|
+
}
|
|
3193
|
+
throw new SyncFailure(
|
|
3194
|
+
"provider_unavailable",
|
|
3195
|
+
response.status >= 500 || response.status === 429
|
|
3196
|
+
);
|
|
2479
3197
|
}
|
|
2480
3198
|
const payload = await readResponseJsonBounded(
|
|
2481
3199
|
response,
|
|
@@ -2506,7 +3224,7 @@ function parseDriveProviderItem(value) {
|
|
|
2506
3224
|
trashed: row.trashed === true
|
|
2507
3225
|
};
|
|
2508
3226
|
}
|
|
2509
|
-
async function fetchDriveBytes(entry, authorization, maxBytes) {
|
|
3227
|
+
async function fetchDriveBytes(entry, authorization, maxBytes, retry, observability) {
|
|
2510
3228
|
if (!authorization) throw new SyncFailure("connection_reconnect_required", true, true);
|
|
2511
3229
|
const url = entry.transfer.action === "export" ? new URL(`${DRIVE_API_BASE}/files/${encodeURIComponent(entry.externalObjectId)}/export`) : new URL(`${DRIVE_API_BASE}/files/${encodeURIComponent(entry.externalObjectId)}`);
|
|
2512
3230
|
if (entry.transfer.action === "export") {
|
|
@@ -2515,11 +3233,13 @@ async function fetchDriveBytes(entry, authorization, maxBytes) {
|
|
|
2515
3233
|
url.searchParams.set("alt", "media");
|
|
2516
3234
|
url.searchParams.set("supportsAllDrives", "true");
|
|
2517
3235
|
}
|
|
2518
|
-
const response = await
|
|
2519
|
-
|
|
2520
|
-
|
|
2521
|
-
|
|
2522
|
-
|
|
3236
|
+
const response = await requestGoogleDrive(
|
|
3237
|
+
url,
|
|
3238
|
+
{ headers: { authorization } },
|
|
3239
|
+
entry.transfer.action === "export" ? "export" : "download",
|
|
3240
|
+
retry,
|
|
3241
|
+
observability
|
|
3242
|
+
);
|
|
2523
3243
|
if (!response.ok) {
|
|
2524
3244
|
if (response.status === 401 || response.status === 403) {
|
|
2525
3245
|
throw new SyncFailure("connection_reconnect_required", true, true);
|
|
@@ -2891,6 +3611,59 @@ function inventoryLimit(reason) {
|
|
|
2891
3611
|
if (reason === "known_byte_limit") return "bytes";
|
|
2892
3612
|
return null;
|
|
2893
3613
|
}
|
|
3614
|
+
function recordSyncHealthTelemetry(observability, provider, summary, outcome, failure) {
|
|
3615
|
+
observability.observeHistogram({
|
|
3616
|
+
name: "opengeni_knowledge_source_sync_terminal_batch_duration_seconds",
|
|
3617
|
+
help: "Terminal knowledge-source activity batch duration by provider and outcome.",
|
|
3618
|
+
labels: { provider, outcome },
|
|
3619
|
+
value: Math.max(0, summary.elapsedMs) / 1e3
|
|
3620
|
+
});
|
|
3621
|
+
if (summary.providerRequests > 0) {
|
|
3622
|
+
observability.incrementCounter({
|
|
3623
|
+
name: "opengeni_knowledge_source_sync_provider_requests_total",
|
|
3624
|
+
help: "Logical provider requests consumed by knowledge-source synchronization.",
|
|
3625
|
+
labels: { provider, outcome },
|
|
3626
|
+
amount: summary.providerRequests
|
|
3627
|
+
});
|
|
3628
|
+
}
|
|
3629
|
+
if (summary.limitReached) {
|
|
3630
|
+
observability.incrementCounter({
|
|
3631
|
+
name: "opengeni_knowledge_source_sync_limit_hits_total",
|
|
3632
|
+
help: "Knowledge-source synchronization runs stopped by an explicit resource budget.",
|
|
3633
|
+
labels: { provider, limit: summary.limitReached }
|
|
3634
|
+
});
|
|
3635
|
+
}
|
|
3636
|
+
if (summary.reconnectRequired) {
|
|
3637
|
+
observability.incrementCounter({
|
|
3638
|
+
name: "opengeni_knowledge_source_sync_reconnect_required_total",
|
|
3639
|
+
help: "Knowledge-source synchronization failures requiring connector reconnection.",
|
|
3640
|
+
labels: { provider }
|
|
3641
|
+
});
|
|
3642
|
+
}
|
|
3643
|
+
if (failure) {
|
|
3644
|
+
observability.incrementCounter({
|
|
3645
|
+
name: "opengeni_knowledge_source_sync_failures_total",
|
|
3646
|
+
help: "Knowledge-source synchronization terminal failures by bounded reason.",
|
|
3647
|
+
labels: {
|
|
3648
|
+
provider,
|
|
3649
|
+
reason: failure.code,
|
|
3650
|
+
retryable: failure.retryable ? "true" : "false"
|
|
3651
|
+
}
|
|
3652
|
+
});
|
|
3653
|
+
observability.warn("Knowledge source sync failed", {
|
|
3654
|
+
provider,
|
|
3655
|
+
outcome: failure.retryable ? "retryable_failed" : "failed",
|
|
3656
|
+
reason: failure.code,
|
|
3657
|
+
durationMs: summary.elapsedMs
|
|
3658
|
+
});
|
|
3659
|
+
return;
|
|
3660
|
+
}
|
|
3661
|
+
observability.info("Knowledge source sync completed", {
|
|
3662
|
+
provider,
|
|
3663
|
+
outcome: "succeeded",
|
|
3664
|
+
durationMs: summary.elapsedMs
|
|
3665
|
+
});
|
|
3666
|
+
}
|
|
2894
3667
|
var SyncFailure = class extends Error {
|
|
2895
3668
|
constructor(code, retryable, reconnectRequired = false) {
|
|
2896
3669
|
super(code);
|
|
@@ -2907,6 +3680,7 @@ import {
|
|
|
2907
3680
|
completeRetainedScreenshotMaintenance,
|
|
2908
3681
|
promoteRetainedScreenshotMaintenanceCleanup
|
|
2909
3682
|
} from "@opengeni/db";
|
|
3683
|
+
import { retryWhileMissing } from "@opengeni/storage";
|
|
2910
3684
|
var RETAINED_SCREENSHOT_PENDING_GRACE_MS = 15 * 60 * 1e3;
|
|
2911
3685
|
var RETAINED_SCREENSHOT_CLAIM_TIMEOUT_MS = 10 * 60 * 1e3;
|
|
2912
3686
|
var RETAINED_SCREENSHOT_MAINTENANCE_BATCH_SIZE = 100;
|
|
@@ -3012,8 +3786,11 @@ async function reconcileClaim(db, storage, claim, provider) {
|
|
|
3012
3786
|
}
|
|
3013
3787
|
async function observeClaimObject(storage, claim, provider) {
|
|
3014
3788
|
const file = claimFile(storage, claim);
|
|
3015
|
-
|
|
3016
|
-
|
|
3789
|
+
const head = await retryWhileMissing(async () => {
|
|
3790
|
+
if (!await provider.fileExists(storage, file)) return null;
|
|
3791
|
+
return await provider.headFile(storage, file);
|
|
3792
|
+
});
|
|
3793
|
+
if (!head) return "missing";
|
|
3017
3794
|
return head.ContentLength === claim.sizeBytes && head.ContentType === claim.mediaType && head.Metadata?.sha256 === claim.sha256 ? "matches" : "mismatch";
|
|
3018
3795
|
}
|
|
3019
3796
|
function claimFile(storage, claim) {
|
|
@@ -3361,7 +4138,7 @@ async function runWithOwnedRigVerificationSandbox(input, run, dependencies = def
|
|
|
3361
4138
|
holderId,
|
|
3362
4139
|
subjectId: null,
|
|
3363
4140
|
backend: input.settings.sandboxBackend,
|
|
3364
|
-
image: input.settings
|
|
4141
|
+
image: rigProviderImageSourceImage(input.settings, input.settings.sandboxBackend),
|
|
3365
4142
|
rigVersionId: input.rigVersionId,
|
|
3366
4143
|
leaseTtlMs: RIG_VERIFICATION_OWNER_TTL_MS,
|
|
3367
4144
|
warmingLeaseTtlMs: RIG_VERIFICATION_OWNER_TTL_MS,
|
|
@@ -4278,6 +5055,7 @@ function createRigVerificationActivities(services) {
|
|
|
4278
5055
|
// src/activities/sandbox-lease.ts
|
|
4279
5056
|
import { createHash as createHash3, randomUUID as randomUUID2 } from "crypto";
|
|
4280
5057
|
import { Context } from "@temporalio/activity";
|
|
5058
|
+
import { OpLostReason, OpState } from "@opengeni/agent-proto";
|
|
4281
5059
|
import {
|
|
4282
5060
|
accrueWarmSeconds,
|
|
4283
5061
|
adoptLegacyModalCheckpointArtifact,
|
|
@@ -4294,7 +5072,7 @@ import {
|
|
|
4294
5072
|
countSandboxLeasesByLiveness,
|
|
4295
5073
|
deferRetainedProcessReconciliation,
|
|
4296
5074
|
forceDrainOverLimitViewerOnlyBoxes,
|
|
4297
|
-
getBillingBalance as
|
|
5075
|
+
getBillingBalance as getBillingBalance3,
|
|
4298
5076
|
listCreditBalancesByAccount,
|
|
4299
5077
|
listLegacyModalCheckpointSlots,
|
|
4300
5078
|
listLiveModalSandboxLeaseAttributions,
|
|
@@ -4318,10 +5096,18 @@ import {
|
|
|
4318
5096
|
rlsContextForWorkspace as rlsContextForWorkspace2,
|
|
4319
5097
|
settleRetainedProcess
|
|
4320
5098
|
} from "@opengeni/db";
|
|
5099
|
+
import {
|
|
5100
|
+
claimConnectedMachineSessionBackgroundCommands,
|
|
5101
|
+
deferConnectedMachineBackgroundCommandReconciliation,
|
|
5102
|
+
recordConnectedMachineBackgroundCommandProof,
|
|
5103
|
+
settleClaimedConnectedMachineBackgroundCommand,
|
|
5104
|
+
settleSessionBackgroundCommandForRetainedProcess
|
|
5105
|
+
} from "@opengeni/db/session-background-commands";
|
|
4321
5106
|
import { sandboxWarmRateMicrosPerSecond } from "@opengeni/config";
|
|
4322
5107
|
import { sandboxLeaseTelemetryKey } from "@opengeni/observability";
|
|
4323
5108
|
import {
|
|
4324
5109
|
captureVerifiedWorkspaceArchive,
|
|
5110
|
+
cancelSelfhostedOp,
|
|
4325
5111
|
assertConsistentSandboxProviderIdentity,
|
|
4326
5112
|
createSandboxClientForBackend,
|
|
4327
5113
|
deleteModalCheckpointSnapshot as deleteModalCheckpointSnapshot2,
|
|
@@ -4330,18 +5116,22 @@ import {
|
|
|
4330
5116
|
inspectModalSandboxLifecycle,
|
|
4331
5117
|
isExecSessionLostBanner,
|
|
4332
5118
|
isProviderSandboxNotFoundError,
|
|
5119
|
+
NatsControlRpc,
|
|
4333
5120
|
parseExecBannerExitCode,
|
|
4334
5121
|
prepareProviderForTeardownAfterCapture,
|
|
4335
5122
|
providerWorkspaceCapturePolicy,
|
|
4336
5123
|
resolveModalCheckpointProviderBindingForLiveSandbox,
|
|
4337
5124
|
resolveModalCheckpointProviderBinding,
|
|
4338
5125
|
resolveModalCheckpointProviderBindingForSession,
|
|
5126
|
+
querySelfhostedOp,
|
|
4339
5127
|
resumeExactSandboxSession,
|
|
4340
5128
|
sandboxProviderContinuityForState,
|
|
4341
5129
|
sandboxBackendForSdkBackendId,
|
|
4342
5130
|
sandboxProviderInstanceIdFromEnvelope,
|
|
4343
5131
|
sandboxCommandExitCode as sandboxCommandExitCode2,
|
|
4344
5132
|
sandboxCommandStillRunning,
|
|
5133
|
+
SelfhostedControlError,
|
|
5134
|
+
subjectFor,
|
|
4345
5135
|
sweepModalOrphanSandboxes,
|
|
4346
5136
|
terminateManagedSandboxSession as terminateManagedSandboxSession2,
|
|
4347
5137
|
terminateModalSandboxById,
|
|
@@ -4351,6 +5141,8 @@ import { sandboxLeaseTelemetryKey as sandboxLeaseTelemetryKey2 } from "@opengeni
|
|
|
4351
5141
|
var RETAINED_PROCESS_RECONCILIATION_LIMIT = 20;
|
|
4352
5142
|
var RETAINED_PROCESS_RECONCILIATION_CLAIM_TTL_MS = 5 * 6e4;
|
|
4353
5143
|
var RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS = 5e3;
|
|
5144
|
+
var CONNECTED_COMMAND_RECONCILIATION_LIMIT = 20;
|
|
5145
|
+
var CONNECTED_COMMAND_RECONCILIATION_CLAIM_TTL_MS = 5 * 6e4;
|
|
4354
5146
|
var SandboxProviderCaptureTimeoutError = class extends Error {
|
|
4355
5147
|
constructor(sandboxGroupId, backend, timeoutMs, leaseEpoch, instanceId) {
|
|
4356
5148
|
super(
|
|
@@ -4438,6 +5230,7 @@ function createSandboxLeaseActivities(services, options = {}) {
|
|
|
4438
5230
|
capturePolicy
|
|
4439
5231
|
));
|
|
4440
5232
|
const sweepModalOrphans = options.sweepModalOrphans ?? sweepModalOrphansForConfiguredBackend;
|
|
5233
|
+
const inspectOpenSandboxInventory = options.inspectOpenSandboxKubernetesInventory ?? inspectOpenSandboxKubernetesInventory;
|
|
4441
5234
|
const probeRetainedProcess = options.probeRetainedProcess ?? probeRetainedProcessAtProvider;
|
|
4442
5235
|
const probeDrainableProvider = options.probeDrainableProvider ?? probeDrainableProviderReadiness;
|
|
4443
5236
|
async function prepareSandboxLeaseSweep() {
|
|
@@ -4446,8 +5239,15 @@ function createSandboxLeaseActivities(services, options = {}) {
|
|
|
4446
5239
|
const { db, settings, observability } = await services();
|
|
4447
5240
|
const timing = sandboxDrainTiming(settings);
|
|
4448
5241
|
if (!settings.sandboxOwnershipEnabled) {
|
|
5242
|
+
const drainableInventory2 = await reapStaleLeaseHoldersGlobal(db, {
|
|
5243
|
+
viewerHolderTtlMs: settings.sandboxViewerHolderTtlMs,
|
|
5244
|
+
turnHolderTtlMs: settings.sandboxLeaseTtlMs,
|
|
5245
|
+
interactionHolderTtlMs: settings.sandboxInteractionHolderTtlMs,
|
|
5246
|
+
idleGraceMs: settings.sandboxIdleGraceMs
|
|
5247
|
+
});
|
|
5248
|
+
const drainable2 = drainableInventory2.slice(0, SANDBOX_REAPER_CHILD_DISPATCH_LIMIT);
|
|
4449
5249
|
return {
|
|
4450
|
-
drainable:
|
|
5250
|
+
drainable: drainable2,
|
|
4451
5251
|
timeoutClass: timing.timeoutClass,
|
|
4452
5252
|
snapshotTimeoutMs: timing.snapshotTimeoutMs,
|
|
4453
5253
|
captureTimeoutMs: timing.captureTimeoutMs,
|
|
@@ -4506,8 +5306,7 @@ function createSandboxLeaseActivities(services, options = {}) {
|
|
|
4506
5306
|
instanceId: input.target.instanceId
|
|
4507
5307
|
});
|
|
4508
5308
|
try {
|
|
4509
|
-
const { db, settings, observability } = await services();
|
|
4510
|
-
if (!settings.sandboxOwnershipEnabled) return { status: "skipped" };
|
|
5309
|
+
const { db, settings, observability, objectStorage } = await services();
|
|
4511
5310
|
assertSandboxDrainInputTiming(input);
|
|
4512
5311
|
const drainSettings = settings.sandboxSnapshotTimeoutMs === input.snapshotTimeoutMs ? settings : { ...settings, sandboxSnapshotTimeoutMs: input.snapshotTimeoutMs };
|
|
4513
5312
|
const row = input.target;
|
|
@@ -4526,7 +5325,8 @@ function createSandboxLeaseActivities(services, options = {}) {
|
|
|
4526
5325
|
observability,
|
|
4527
5326
|
terminateBox,
|
|
4528
5327
|
probeDrainableProvider,
|
|
4529
|
-
captureAttempt
|
|
5328
|
+
captureAttempt,
|
|
5329
|
+
objectStorage
|
|
4530
5330
|
);
|
|
4531
5331
|
return { status: drainedCold ? "terminated" : "skipped" };
|
|
4532
5332
|
} catch (error) {
|
|
@@ -4592,8 +5392,14 @@ function createSandboxLeaseActivities(services, options = {}) {
|
|
|
4592
5392
|
let forceDrained = 0;
|
|
4593
5393
|
const rotationsRequested = input.rotationsRequested;
|
|
4594
5394
|
let modalOrphansTerminated = 0;
|
|
5395
|
+
await reconcileConnectedMachineBackgroundCommands(db, settings, observability, service.bus);
|
|
4595
5396
|
if (!settings.sandboxOwnershipEnabled) {
|
|
4596
|
-
await refreshQueueLeaseAndCreditGauges(
|
|
5397
|
+
await refreshQueueLeaseAndCreditGauges(
|
|
5398
|
+
db,
|
|
5399
|
+
settings,
|
|
5400
|
+
observability,
|
|
5401
|
+
inspectOpenSandboxInventory
|
|
5402
|
+
);
|
|
4597
5403
|
return {
|
|
4598
5404
|
metered,
|
|
4599
5405
|
forceDrained,
|
|
@@ -4647,7 +5453,12 @@ function createSandboxLeaseActivities(services, options = {}) {
|
|
|
4647
5453
|
return { claimed: 0, deleted: 0, failed: 0 };
|
|
4648
5454
|
}
|
|
4649
5455
|
);
|
|
4650
|
-
await refreshQueueLeaseAndCreditGauges(
|
|
5456
|
+
await refreshQueueLeaseAndCreditGauges(
|
|
5457
|
+
db,
|
|
5458
|
+
settings,
|
|
5459
|
+
observability,
|
|
5460
|
+
inspectOpenSandboxInventory
|
|
5461
|
+
);
|
|
4651
5462
|
if (input.examined > 0 || metered > 0 || forceDrained > 0 || modalOrphansTerminated > 0 || rotationsRequested > 0 || checkpointGc.claimed > 0) {
|
|
4652
5463
|
observability.info("sandbox reaper dispatched", {
|
|
4653
5464
|
drainable: input.examined,
|
|
@@ -4734,6 +5545,13 @@ var CHECKPOINT_GC_CLAIM_TTL_MS = 10 * 6e4;
|
|
|
4734
5545
|
var CHECKPOINT_TOMBSTONE_RETENTION_MS = 30 * 24 * 60 * 6e4;
|
|
4735
5546
|
var CHECKPOINT_TOMBSTONE_PRUNE_LIMIT = 500;
|
|
4736
5547
|
var SANDBOX_MAINTENANCE_ITEM_CONCURRENCY = 8;
|
|
5548
|
+
function recordConnectedCommandReconciliation(observability, outcome) {
|
|
5549
|
+
observability.incrementCounter({
|
|
5550
|
+
name: "opengeni_connected_background_command_reconciliation_total",
|
|
5551
|
+
help: "Exact-locator Connected Machine background-command reconciliation outcomes.",
|
|
5552
|
+
labels: { outcome }
|
|
5553
|
+
});
|
|
5554
|
+
}
|
|
4737
5555
|
async function forEachWithConcurrency(values, concurrency, visit) {
|
|
4738
5556
|
if (values.length === 0) return;
|
|
4739
5557
|
const workerCount = Math.min(values.length, Math.max(1, Math.floor(concurrency)));
|
|
@@ -4856,6 +5674,154 @@ async function adoptLegacyModalCheckpointReceipts(db, settings, observability) {
|
|
|
4856
5674
|
});
|
|
4857
5675
|
return adopted;
|
|
4858
5676
|
}
|
|
5677
|
+
async function reconcileConnectedMachineBackgroundCommands(db, settings, observability, bus) {
|
|
5678
|
+
const claimId = crypto.randomUUID();
|
|
5679
|
+
let claims;
|
|
5680
|
+
try {
|
|
5681
|
+
claims = await claimConnectedMachineSessionBackgroundCommands(db, {
|
|
5682
|
+
claimId,
|
|
5683
|
+
limit: CONNECTED_COMMAND_RECONCILIATION_LIMIT,
|
|
5684
|
+
claimTtlMs: CONNECTED_COMMAND_RECONCILIATION_CLAIM_TTL_MS
|
|
5685
|
+
});
|
|
5686
|
+
} catch (error) {
|
|
5687
|
+
recordConnectedCommandReconciliation(observability, "claim_failed");
|
|
5688
|
+
observability.warn("sandbox reaper: connected-command claim failed", {
|
|
5689
|
+
error: error instanceof Error ? error.message : String(error)
|
|
5690
|
+
});
|
|
5691
|
+
return;
|
|
5692
|
+
}
|
|
5693
|
+
const controlRpc = new NatsControlRpc(
|
|
5694
|
+
async () => bus.getRequestConnection()
|
|
5695
|
+
);
|
|
5696
|
+
await forEachWithConcurrency(claims, SANDBOX_MAINTENANCE_ITEM_CONCURRENCY, async (claim) => {
|
|
5697
|
+
let proof = claim.proof;
|
|
5698
|
+
if (!proof) {
|
|
5699
|
+
try {
|
|
5700
|
+
proof = await probeConnectedMachineBackgroundCommand(
|
|
5701
|
+
claim,
|
|
5702
|
+
controlRpc,
|
|
5703
|
+
Math.min(
|
|
5704
|
+
settings.sandboxSelfhostedControlTimeoutMs,
|
|
5705
|
+
RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS
|
|
5706
|
+
)
|
|
5707
|
+
);
|
|
5708
|
+
if (!proof) {
|
|
5709
|
+
await deferConnectedCommandClaim(db, settings, observability, claim, "provider_running");
|
|
5710
|
+
return;
|
|
5711
|
+
}
|
|
5712
|
+
} catch (error) {
|
|
5713
|
+
const offline = error instanceof SelfhostedControlError && (error.agentOffline || error.reason === "agent_reconnecting" || error.draining);
|
|
5714
|
+
if (!offline) {
|
|
5715
|
+
observability.warn("sandbox reaper: connected-command provider probe failed", {
|
|
5716
|
+
commandId: claim.commandId,
|
|
5717
|
+
enrollmentId: claim.enrollmentId,
|
|
5718
|
+
connectionInstanceId: claim.connectionInstanceId,
|
|
5719
|
+
error: error instanceof Error ? error.message : String(error)
|
|
5720
|
+
});
|
|
5721
|
+
}
|
|
5722
|
+
await deferConnectedCommandClaim(
|
|
5723
|
+
db,
|
|
5724
|
+
settings,
|
|
5725
|
+
observability,
|
|
5726
|
+
claim,
|
|
5727
|
+
offline ? "provider_offline" : "provider_error"
|
|
5728
|
+
);
|
|
5729
|
+
return;
|
|
5730
|
+
}
|
|
5731
|
+
try {
|
|
5732
|
+
await recordConnectedMachineBackgroundCommandProof(db, { claim, proof });
|
|
5733
|
+
recordConnectedCommandReconciliation(observability, `proof_${proof.outcome}`);
|
|
5734
|
+
} catch (error) {
|
|
5735
|
+
recordConnectedCommandReconciliation(observability, "proof_checkpoint_failed");
|
|
5736
|
+
observability.warn("sandbox reaper: connected-command proof checkpoint failed", {
|
|
5737
|
+
commandId: claim.commandId,
|
|
5738
|
+
error: error instanceof Error ? error.message : String(error)
|
|
5739
|
+
});
|
|
5740
|
+
return;
|
|
5741
|
+
}
|
|
5742
|
+
}
|
|
5743
|
+
try {
|
|
5744
|
+
if (!await settleClaimedConnectedMachineBackgroundCommand(db, { claim })) {
|
|
5745
|
+
throw new Error("Connected command settlement lost its exact claim or proof");
|
|
5746
|
+
}
|
|
5747
|
+
recordConnectedCommandReconciliation(observability, `settled_${proof.outcome}`);
|
|
5748
|
+
} catch (error) {
|
|
5749
|
+
recordConnectedCommandReconciliation(observability, "settlement_failed");
|
|
5750
|
+
observability.warn("sandbox reaper: connected-command settlement failed", {
|
|
5751
|
+
commandId: claim.commandId,
|
|
5752
|
+
error: error instanceof Error ? error.message : String(error)
|
|
5753
|
+
});
|
|
5754
|
+
await deferConnectedCommandClaim(db, settings, observability, claim, "settlement_failed");
|
|
5755
|
+
}
|
|
5756
|
+
});
|
|
5757
|
+
}
|
|
5758
|
+
async function probeConnectedMachineBackgroundCommand(claim, controlRpc, controlTimeoutMs, observedAt = /* @__PURE__ */ new Date()) {
|
|
5759
|
+
const input = {
|
|
5760
|
+
controlRpc,
|
|
5761
|
+
rpcSubject: subjectFor(
|
|
5762
|
+
claim.controlWorkspaceId,
|
|
5763
|
+
claim.enrollmentId,
|
|
5764
|
+
claim.connectionInstanceId
|
|
5765
|
+
),
|
|
5766
|
+
opId: claim.opId,
|
|
5767
|
+
controlTimeoutMs
|
|
5768
|
+
};
|
|
5769
|
+
const status = claim.state === "stopping" ? await cancelSelfhostedOp(input) : await querySelfhostedOp(input);
|
|
5770
|
+
return connectedCommandProofFromStatus(status, observedAt);
|
|
5771
|
+
}
|
|
5772
|
+
function connectedCommandProofFromStatus(status, observedAt) {
|
|
5773
|
+
switch (status.state) {
|
|
5774
|
+
case OpState.OP_STATE_ACCEPTED:
|
|
5775
|
+
case OpState.OP_STATE_RUNNING:
|
|
5776
|
+
return null;
|
|
5777
|
+
case OpState.OP_STATE_COMPLETE: {
|
|
5778
|
+
const exit = status.exit;
|
|
5779
|
+
if (!exit) {
|
|
5780
|
+
throw new Error("Connected command completed without an exit record");
|
|
5781
|
+
}
|
|
5782
|
+
const failureCode = exit.failureCode.replace(/[^A-Za-z0-9_-]/g, "_").slice(0, 128);
|
|
5783
|
+
const reason = exit.cancelled ? "op_cancelled" : exit.timedOut ? "op_timed_out" : failureCode ? `op_failure_${failureCode}` : "op_exit";
|
|
5784
|
+
return {
|
|
5785
|
+
outcome: "exited",
|
|
5786
|
+
exitCode: exit.exitCode,
|
|
5787
|
+
reason,
|
|
5788
|
+
observedAt
|
|
5789
|
+
};
|
|
5790
|
+
}
|
|
5791
|
+
case OpState.OP_STATE_LOST:
|
|
5792
|
+
return {
|
|
5793
|
+
outcome: "lost",
|
|
5794
|
+
exitCode: null,
|
|
5795
|
+
reason: status.lostReason === OpLostReason.OP_LOST_REASON_EVICTED ? "op_lost_evicted" : status.lostReason === OpLostReason.OP_LOST_REASON_AGENT_RESTARTED ? "op_lost_agent_restarted" : "op_lost_unspecified",
|
|
5796
|
+
observedAt
|
|
5797
|
+
};
|
|
5798
|
+
default:
|
|
5799
|
+
throw new Error(`Connected command returned invalid op state ${status.state}`);
|
|
5800
|
+
}
|
|
5801
|
+
}
|
|
5802
|
+
async function deferConnectedCommandClaim(db, settings, observability, claim, outcome) {
|
|
5803
|
+
const exponent = Math.min(4, Math.max(0, claim.reconcileAttempts - 1));
|
|
5804
|
+
const retryAfterMs = Math.min(
|
|
5805
|
+
5 * 6e4,
|
|
5806
|
+
Math.max(settings.sandboxLeaseReaperPeriodMs, 3e4) * 2 ** exponent
|
|
5807
|
+
);
|
|
5808
|
+
try {
|
|
5809
|
+
if (!await deferConnectedMachineBackgroundCommandReconciliation(db, {
|
|
5810
|
+
claim,
|
|
5811
|
+
outcome,
|
|
5812
|
+
retryAfterMs
|
|
5813
|
+
})) {
|
|
5814
|
+
throw new Error("Connected command defer lost its exact claim");
|
|
5815
|
+
}
|
|
5816
|
+
recordConnectedCommandReconciliation(observability, outcome);
|
|
5817
|
+
} catch (error) {
|
|
5818
|
+
recordConnectedCommandReconciliation(observability, "defer_failed");
|
|
5819
|
+
observability.warn("sandbox reaper: connected-command defer failed", {
|
|
5820
|
+
commandId: claim.commandId,
|
|
5821
|
+
error: error instanceof Error ? error.message : String(error)
|
|
5822
|
+
});
|
|
5823
|
+
}
|
|
5824
|
+
}
|
|
4859
5825
|
async function reconcileTerminalRetainedProcesses(db, settings, observability, probe, inspectHistoricalModalSandbox) {
|
|
4860
5826
|
const claimId = crypto.randomUUID();
|
|
4861
5827
|
let claims;
|
|
@@ -4959,7 +5925,12 @@ async function reconcileTerminalRetainedProcesses(db, settings, observability, p
|
|
|
4959
5925
|
}
|
|
4960
5926
|
} else {
|
|
4961
5927
|
try {
|
|
4962
|
-
observation = await probe(
|
|
5928
|
+
observation = await probe(
|
|
5929
|
+
settings,
|
|
5930
|
+
lease,
|
|
5931
|
+
process,
|
|
5932
|
+
claim.ownerState === "background_stopping" ? "cancel" : "observe"
|
|
5933
|
+
);
|
|
4963
5934
|
} catch (error) {
|
|
4964
5935
|
observability.warn("sandbox reaper: retained-process provider probe failed", {
|
|
4965
5936
|
processId: process.id,
|
|
@@ -5038,6 +6009,15 @@ async function reconcileTerminalRetainedProcesses(db, settings, observability, p
|
|
|
5038
6009
|
reason: proof.reason,
|
|
5039
6010
|
idleGraceMs: settings.sandboxIdleGraceMs
|
|
5040
6011
|
});
|
|
6012
|
+
await settleSessionBackgroundCommandForRetainedProcess(db, {
|
|
6013
|
+
accountId: process.accountId,
|
|
6014
|
+
workspaceId: process.workspaceId,
|
|
6015
|
+
sessionId: process.sessionId,
|
|
6016
|
+
retainedProcessId: process.id,
|
|
6017
|
+
outcome: proof.outcome,
|
|
6018
|
+
exitCode: proof.exitCode,
|
|
6019
|
+
reason: proof.reason
|
|
6020
|
+
});
|
|
5041
6021
|
recordRetainedProcessReconciliation(observability, `settled_${proof.outcome}`);
|
|
5042
6022
|
} catch (error) {
|
|
5043
6023
|
recordRetainedProcessReconciliation(observability, "settlement_failed");
|
|
@@ -5101,7 +6081,7 @@ async function withRetainedProcessProbeTimeout(promise) {
|
|
|
5101
6081
|
if (timer) clearTimeout(timer);
|
|
5102
6082
|
}
|
|
5103
6083
|
}
|
|
5104
|
-
async function probeRetainedProcessAtProvider(settings, lease, process) {
|
|
6084
|
+
async function probeRetainedProcessAtProvider(settings, lease, process, mode = "observe") {
|
|
5105
6085
|
if (lease.id !== process.leaseId || lease.sandboxGroupId !== process.sandboxGroupId || lease.leaseEpoch !== process.leaseEpoch || lease.backend !== process.providerBackend || lease.instanceId !== process.providerInstanceId || lease.resumeBackendId !== process.providerBackend || process.routeTargetId !== null) {
|
|
5106
6086
|
return { status: "deferred", reason: "identity_mismatch" };
|
|
5107
6087
|
}
|
|
@@ -5112,7 +6092,7 @@ async function probeRetainedProcessAtProvider(settings, lease, process) {
|
|
|
5112
6092
|
if (envelopeBackend !== void 0 && envelopeBackend !== process.providerBackend) {
|
|
5113
6093
|
return { status: "deferred", reason: "identity_mismatch" };
|
|
5114
6094
|
}
|
|
5115
|
-
if (
|
|
6095
|
+
if (sandboxProviderInstanceIdFromEnvelope(lease.resumeState) !== process.providerInstanceId) {
|
|
5116
6096
|
return { status: "deferred", reason: "identity_mismatch" };
|
|
5117
6097
|
}
|
|
5118
6098
|
let configuredModalBinding = null;
|
|
@@ -5215,7 +6195,7 @@ async function probeRetainedProcessAtProvider(settings, lease, process) {
|
|
|
5215
6195
|
result = await withRetainedProcessProbeTimeout(
|
|
5216
6196
|
session.writeStdin({
|
|
5217
6197
|
sessionId: process.providerSessionId,
|
|
5218
|
-
chars: "",
|
|
6198
|
+
chars: mode === "cancel" ? "" : "",
|
|
5219
6199
|
yieldTimeMs: 1e3,
|
|
5220
6200
|
maxOutputTokens: 2e3
|
|
5221
6201
|
})
|
|
@@ -5339,7 +6319,7 @@ async function accrueWarmTick(db, settings, observability) {
|
|
|
5339
6319
|
});
|
|
5340
6320
|
return { accrued, workspaceIds };
|
|
5341
6321
|
}
|
|
5342
|
-
async function refreshQueueLeaseAndCreditGauges(db, observability) {
|
|
6322
|
+
async function refreshQueueLeaseAndCreditGauges(db, settings, observability, inspectOpenSandboxInventory) {
|
|
5343
6323
|
await Promise.all([
|
|
5344
6324
|
(async () => {
|
|
5345
6325
|
try {
|
|
@@ -5389,6 +6369,21 @@ async function refreshQueueLeaseAndCreditGauges(db, observability) {
|
|
|
5389
6369
|
await countExpiredDrainingSandboxLeases(db)
|
|
5390
6370
|
);
|
|
5391
6371
|
}),
|
|
6372
|
+
...settings.sandboxBackend === "opensandbox" && settings.openSandboxKubernetesInventoryNamespace ? [
|
|
6373
|
+
refreshSandboxInventoryGauge(
|
|
6374
|
+
observability,
|
|
6375
|
+
"opensandbox_kubernetes",
|
|
6376
|
+
"OpenSandbox Kubernetes",
|
|
6377
|
+
async () => {
|
|
6378
|
+
recordOpenSandboxKubernetesInventoryGauges(
|
|
6379
|
+
observability,
|
|
6380
|
+
await inspectOpenSandboxInventory({
|
|
6381
|
+
namespace: settings.openSandboxKubernetesInventoryNamespace
|
|
6382
|
+
})
|
|
6383
|
+
);
|
|
6384
|
+
}
|
|
6385
|
+
)
|
|
6386
|
+
] : [],
|
|
5392
6387
|
(async () => {
|
|
5393
6388
|
try {
|
|
5394
6389
|
recordCreditBalanceGauges(observability, await listCreditBalancesByAccount(db));
|
|
@@ -5421,7 +6416,7 @@ async function forceDrainOverLimitWorkspaces(db, settings, workspaceIds, observa
|
|
|
5421
6416
|
async (workspaceId) => {
|
|
5422
6417
|
try {
|
|
5423
6418
|
const { accountId } = await rlsContextForWorkspace2(db, workspaceId);
|
|
5424
|
-
const balance = enforceBalance ? await
|
|
6419
|
+
const balance = enforceBalance ? await getBillingBalance3(db, accountId) : { balanceMicros: 1 };
|
|
5425
6420
|
const result = await forceDrainOverLimitViewerOnlyBoxes(db, {
|
|
5426
6421
|
workspaceId,
|
|
5427
6422
|
balanceMicros: balance.balanceMicros,
|
|
@@ -5553,7 +6548,11 @@ async function probeDrainableProviderReadiness(settings, lease) {
|
|
|
5553
6548
|
throw error;
|
|
5554
6549
|
}
|
|
5555
6550
|
}
|
|
5556
|
-
|
|
6551
|
+
function sandboxDrainReason(lease) {
|
|
6552
|
+
if (lease.rotationRequestedAt === null) return "idle";
|
|
6553
|
+
return lease.rotationReason ?? "rotation_requested";
|
|
6554
|
+
}
|
|
6555
|
+
async function terminateDrainableBox(db, settings, row, observability, terminateBox, probeDrainableProvider, attempt, objectStorage) {
|
|
5557
6556
|
const { accountId } = await rlsContextForWorkspace2(db, row.workspaceId);
|
|
5558
6557
|
const lease = await readLease(db, row.workspaceId, row.sandboxGroupId);
|
|
5559
6558
|
if (!lease) {
|
|
@@ -5716,6 +6715,7 @@ async function terminateDrainableBox(db, settings, row, observability, terminate
|
|
|
5716
6715
|
if (!lease.instanceId || !captureClaim) {
|
|
5717
6716
|
return { wrote: false, archiveRevision: null };
|
|
5718
6717
|
}
|
|
6718
|
+
const archiveMetrics = runtimeMetricsHooksForObservability(observability);
|
|
5719
6719
|
let checkpointArtifactId = null;
|
|
5720
6720
|
if (archiveBase64 && archiveMetadata?.version === 2 && (archiveMetadata.provider === "modal_snapshot_filesystem" || archiveMetadata.provider === "modal_snapshot_directory")) {
|
|
5721
6721
|
if (!providerBinding) {
|
|
@@ -5757,12 +6757,57 @@ async function terminateDrainableBox(db, settings, row, observability, terminate
|
|
|
5757
6757
|
if (!archiveMetadata) {
|
|
5758
6758
|
throw new Error("Sandbox snapshot publication requires a verified archive descriptor");
|
|
5759
6759
|
}
|
|
5760
|
-
|
|
5761
|
-
|
|
5762
|
-
|
|
5763
|
-
|
|
5764
|
-
|
|
5765
|
-
|
|
6760
|
+
const priorKeys = collectWorkspaceArchiveObjectKeys(
|
|
6761
|
+
lease.resumeState ?? null
|
|
6762
|
+
);
|
|
6763
|
+
let workspaceArchiveRef;
|
|
6764
|
+
try {
|
|
6765
|
+
const published = await putVersion1TarArchiveOrInline({
|
|
6766
|
+
backend: lease.backend,
|
|
6767
|
+
objectStorage,
|
|
6768
|
+
accountId,
|
|
6769
|
+
workspaceId: row.workspaceId,
|
|
6770
|
+
sandboxGroupId: row.sandboxGroupId,
|
|
6771
|
+
archive: {
|
|
6772
|
+
bytes: Buffer.from(archiveBase64, "base64"),
|
|
6773
|
+
descriptor: archiveMetadata,
|
|
6774
|
+
base64: archiveBase64
|
|
6775
|
+
},
|
|
6776
|
+
metrics: archiveMetrics
|
|
6777
|
+
});
|
|
6778
|
+
workspaceArchiveRef = published.workspaceArchiveRef;
|
|
6779
|
+
result = await persistDrainSnapshot(db, {
|
|
6780
|
+
...baseInput,
|
|
6781
|
+
workspaceArchiveMeta: archiveMetadata,
|
|
6782
|
+
...published,
|
|
6783
|
+
...checkpointArtifactId ? { checkpointArtifactId } : {}
|
|
6784
|
+
});
|
|
6785
|
+
if (published.workspaceArchiveRef && objectStorage) {
|
|
6786
|
+
if (!result.wrote) {
|
|
6787
|
+
await deleteUnpublishedWorkspaceArchiveObject(
|
|
6788
|
+
objectStorage,
|
|
6789
|
+
published.workspaceArchiveRef,
|
|
6790
|
+
archiveMetrics
|
|
6791
|
+
);
|
|
6792
|
+
} else {
|
|
6793
|
+
const afterLease = await readLease(db, row.workspaceId, row.sandboxGroupId);
|
|
6794
|
+
const afterKeys = collectWorkspaceArchiveObjectKeys(
|
|
6795
|
+
afterLease?.resumeState ?? null
|
|
6796
|
+
);
|
|
6797
|
+
await deleteWorkspaceArchiveObjectKeys(
|
|
6798
|
+
objectStorage,
|
|
6799
|
+
[...priorKeys].filter((key) => !afterKeys.has(key))
|
|
6800
|
+
).catch(() => void 0);
|
|
6801
|
+
}
|
|
6802
|
+
}
|
|
6803
|
+
} catch (error) {
|
|
6804
|
+
await deleteUnpublishedWorkspaceArchiveObject(
|
|
6805
|
+
objectStorage,
|
|
6806
|
+
workspaceArchiveRef,
|
|
6807
|
+
archiveMetrics
|
|
6808
|
+
);
|
|
6809
|
+
throw error;
|
|
6810
|
+
}
|
|
5766
6811
|
}
|
|
5767
6812
|
if (!result.wrote && checkpointArtifactId) {
|
|
5768
6813
|
await markSandboxCheckpointArtifactDeletePending2(db, {
|
|
@@ -5809,7 +6854,12 @@ async function terminateDrainableBox(db, settings, row, observability, terminate
|
|
|
5809
6854
|
persisted,
|
|
5810
6855
|
archiveRevision,
|
|
5811
6856
|
instanceId: lease.instanceId,
|
|
5812
|
-
providerMissingBeforeCapture: providerMissing
|
|
6857
|
+
providerMissingBeforeCapture: providerMissing,
|
|
6858
|
+
// Why the box drained: the lease's bounded rotation reason when one
|
|
6859
|
+
// was requested (provider deadline, operator/image conflict, teardown
|
|
6860
|
+
// claim), otherwise the ordinary idle drain. Additive diagnostic
|
|
6861
|
+
// fact; no identifiers beyond the ones already carried above.
|
|
6862
|
+
drainReason: sandboxDrainReason(lease)
|
|
5813
6863
|
}
|
|
5814
6864
|
});
|
|
5815
6865
|
} catch (eventError) {
|
|
@@ -6062,99 +7112,222 @@ async function terminateProviderBox(settings, lease, observability, persistArchi
|
|
|
6062
7112
|
|
|
6063
7113
|
// src/activities/scheduled-tasks.ts
|
|
6064
7114
|
import {
|
|
7115
|
+
DEFAULT_FIRST_PARTY_MCP_PERMISSIONS,
|
|
6065
7116
|
OPENGENI_SLACK_BOT_CREDENTIAL_LABEL,
|
|
6066
7117
|
OPENGENI_SLACK_BOT_CREDENTIAL_ROLE,
|
|
6067
|
-
OPENGENI_SLACK_BOT_SESSION_METADATA_KEY
|
|
7118
|
+
OPENGENI_SLACK_BOT_SESSION_METADATA_KEY,
|
|
7119
|
+
resolveWorkspaceCodexCompactionDefault,
|
|
7120
|
+
SCHEDULED_TASK_OCCURRENCE_PAYLOAD_MAX_BYTES,
|
|
7121
|
+
ScheduledTaskRunAcceptedExecution,
|
|
7122
|
+
scheduledOccurrencePayloadUtf8Bytes,
|
|
7123
|
+
stableJson
|
|
6068
7124
|
} from "@opengeni/contracts";
|
|
6069
7125
|
import {
|
|
7126
|
+
defaultSessionMcpServerIds,
|
|
6070
7127
|
openGeniSlackBotMetadata,
|
|
6071
7128
|
requireOpenGeniSlackBotConnection,
|
|
7129
|
+
resolveSessionToolPolicy,
|
|
6072
7130
|
scheduledSlackBotConnectionId
|
|
6073
7131
|
} from "@opengeni/core";
|
|
6074
7132
|
import {
|
|
6075
|
-
|
|
7133
|
+
appendSessionEvents,
|
|
6076
7134
|
addSessionSystemUpdateWithSourceMutation,
|
|
7135
|
+
bindScheduledTaskRunSessionInTransaction,
|
|
6077
7136
|
createScheduledTaskRun,
|
|
6078
|
-
|
|
6079
|
-
createSessionGoal,
|
|
7137
|
+
createSessionWithIdempotencyKeyResult,
|
|
6080
7138
|
enqueueSessionWorkflowWakeIfRunnable,
|
|
6081
7139
|
getScheduledTask as getScheduledTask2,
|
|
7140
|
+
getScheduledTaskRunAcceptedExecution,
|
|
7141
|
+
getScheduledTaskRunByProducerKey,
|
|
7142
|
+
getScheduledTargetSessionExecution,
|
|
6082
7143
|
ensureKnowledgeSourceSyncState,
|
|
6083
7144
|
getScheduledTaskPersonalConnectionDelegations,
|
|
7145
|
+
getScheduledTaskRunPersonalResourceAuthority,
|
|
7146
|
+
getScheduledTaskPersonalResourceAuthoritySubject,
|
|
7147
|
+
getScheduledTaskRevisionAuthority,
|
|
6084
7148
|
getScheduledTaskXaiProviderAccountAuthoritySnapshot,
|
|
6085
7149
|
getRig as getRig2,
|
|
7150
|
+
getScheduledScopedRigVersionMetadata,
|
|
7151
|
+
getNestedAgentDepthDeploymentPolicy,
|
|
7152
|
+
getSessionByCreateIdempotencyKey,
|
|
6086
7153
|
getVariableSet,
|
|
7154
|
+
isCodexBilledModel,
|
|
7155
|
+
initializeSessionStartAtomically,
|
|
7156
|
+
materializeScheduledTaskReusableSessionFromRun,
|
|
7157
|
+
listEnabledMcpCapabilityServerIds,
|
|
7158
|
+
listInstalledApiIntegrationServerIdsForDelegations,
|
|
6087
7159
|
markScheduledTaskRunFailedIfQueued,
|
|
7160
|
+
markScheduledTaskRunSkippedIfQueued,
|
|
6088
7161
|
recordKnowledgeSourceSyncWake,
|
|
6089
7162
|
recordUsageEvent as recordUsageEvent3,
|
|
6090
|
-
|
|
7163
|
+
requireScheduledTaskIncidentAuthorityInTransaction,
|
|
6091
7164
|
requireSession,
|
|
6092
|
-
|
|
7165
|
+
requireWorkspace,
|
|
6093
7166
|
settleScheduledTaskRunInTransaction,
|
|
6094
7167
|
SessionSpawnDeniedDbError,
|
|
6095
|
-
updateScheduledTask,
|
|
6096
7168
|
updateScheduledTaskRun as updateScheduledTaskRun2,
|
|
6097
|
-
|
|
7169
|
+
updateSessionTitle,
|
|
7170
|
+
upsertScheduledSessionGoalForRun,
|
|
7171
|
+
withSessionActivityRlsContext
|
|
6098
7172
|
} from "@opengeni/db";
|
|
6099
|
-
import {
|
|
7173
|
+
import { publishDurableSessionEvents } from "@opengeni/events";
|
|
6100
7174
|
import { resolveFirstPartyMcpToolPolicy } from "@opengeni/config";
|
|
7175
|
+
import { Context as Context2 } from "@temporalio/activity";
|
|
6101
7176
|
|
|
6102
|
-
// src/
|
|
6103
|
-
import {
|
|
6104
|
-
|
|
6105
|
-
|
|
6106
|
-
|
|
6107
|
-
|
|
6108
|
-
|
|
6109
|
-
|
|
6110
|
-
|
|
6111
|
-
|
|
6112
|
-
|
|
6113
|
-
|
|
6114
|
-
|
|
6115
|
-
|
|
6116
|
-
|
|
6117
|
-
|
|
6118
|
-
|
|
6119
|
-
|
|
6120
|
-
|
|
6121
|
-
|
|
6122
|
-
|
|
6123
|
-
|
|
6124
|
-
|
|
6125
|
-
|
|
6126
|
-
|
|
7177
|
+
// src/scheduled-alert-occurrence.ts
|
|
7178
|
+
import { createHash as createHash4 } from "crypto";
|
|
7179
|
+
var MAX_SCHEDULED_TASK_ID_BYTES = 256;
|
|
7180
|
+
var MAX_STARTS_AT_BYTES = 256;
|
|
7181
|
+
var MAX_PROVIDER_BYTES = 256;
|
|
7182
|
+
var MAX_FINGERPRINT_BYTES = 1024;
|
|
7183
|
+
var MAX_LABELS = 256;
|
|
7184
|
+
var MAX_LABEL_KEY_BYTES = 256;
|
|
7185
|
+
var MAX_LABEL_VALUE_BYTES = 4096;
|
|
7186
|
+
var OFFSET_TIMESTAMP = /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2})(?::(\d{2})(?:\.(\d+))?)?(?:Z|[+-](\d{2}):(\d{2}))$/;
|
|
7187
|
+
function scheduledAlertResponderSessionCreateIdempotencyKey(input) {
|
|
7188
|
+
const digest = createHash4("sha256").update(
|
|
7189
|
+
JSON.stringify({
|
|
7190
|
+
version: 1,
|
|
7191
|
+
occurrenceKey: input.occurrence.sessionCreateIdempotencyKey,
|
|
7192
|
+
taskAuthorityRevision: input.taskAuthorityRevision,
|
|
7193
|
+
taskExecutionDigest: input.taskExecutionDigest
|
|
7194
|
+
})
|
|
7195
|
+
).digest("hex");
|
|
7196
|
+
return `scheduled-alert-occurrence:v1:${digest}`;
|
|
7197
|
+
}
|
|
7198
|
+
function scheduledAlertOccurrenceIdentity(input) {
|
|
7199
|
+
const scheduledTaskId = boundedString(input.scheduledTaskId, MAX_SCHEDULED_TASK_ID_BYTES);
|
|
7200
|
+
if (!scheduledTaskId) return null;
|
|
7201
|
+
const alert = record(input.metadata.alert);
|
|
7202
|
+
if (!alert) return null;
|
|
7203
|
+
const status = alert.status;
|
|
7204
|
+
if (status !== "firing" && status !== "resolved") return null;
|
|
7205
|
+
const startsAt = canonicalTimestamp(alert.startsAt);
|
|
7206
|
+
const fingerprint = boundedString(alert.fingerprint, MAX_FINGERPRINT_BYTES);
|
|
7207
|
+
const labels = canonicalLabels(alert.labels);
|
|
7208
|
+
if (!startsAt || !fingerprint || !labels) return null;
|
|
7209
|
+
const provider = boundedString(
|
|
7210
|
+
alert.provider ?? input.metadata.alertProvider ?? input.metadata.provider ?? "alertmanager",
|
|
7211
|
+
MAX_PROVIDER_BYTES
|
|
7212
|
+
);
|
|
7213
|
+
if (!provider) return null;
|
|
7214
|
+
const digest = createHash4("sha256").update(
|
|
7215
|
+
JSON.stringify({
|
|
7216
|
+
version: 1,
|
|
7217
|
+
workspaceId: input.workspaceId,
|
|
7218
|
+
scheduledTaskId,
|
|
7219
|
+
provider,
|
|
7220
|
+
fingerprint,
|
|
7221
|
+
startsAt,
|
|
7222
|
+
labels
|
|
7223
|
+
})
|
|
7224
|
+
).digest("hex");
|
|
7225
|
+
return {
|
|
7226
|
+
status,
|
|
7227
|
+
sessionCreateIdempotencyKey: `scheduled-alert-occurrence:v1:${digest}`,
|
|
7228
|
+
labels: Object.fromEntries(labels)
|
|
7229
|
+
};
|
|
7230
|
+
}
|
|
7231
|
+
function record(value) {
|
|
7232
|
+
return value !== null && typeof value === "object" && !Array.isArray(value) ? value : null;
|
|
7233
|
+
}
|
|
7234
|
+
function boundedString(value, maxBytes) {
|
|
7235
|
+
if (typeof value !== "string") return null;
|
|
7236
|
+
const normalized = value.trim();
|
|
7237
|
+
if (!normalized || Buffer.byteLength(normalized, "utf8") > maxBytes) return null;
|
|
7238
|
+
return normalized;
|
|
7239
|
+
}
|
|
7240
|
+
function canonicalTimestamp(value) {
|
|
7241
|
+
if (typeof value !== "string") return null;
|
|
7242
|
+
const trimmed = value.trim();
|
|
7243
|
+
if (!trimmed || Buffer.byteLength(trimmed, "utf8") > MAX_STARTS_AT_BYTES) return null;
|
|
7244
|
+
const match = OFFSET_TIMESTAMP.exec(trimmed);
|
|
7245
|
+
if (!match) return null;
|
|
7246
|
+
const [
|
|
7247
|
+
,
|
|
7248
|
+
yearText,
|
|
7249
|
+
monthText,
|
|
7250
|
+
dayText,
|
|
7251
|
+
hourText,
|
|
7252
|
+
minuteText,
|
|
7253
|
+
secondText,
|
|
7254
|
+
,
|
|
7255
|
+
offsetHourText,
|
|
7256
|
+
offsetMinuteText
|
|
7257
|
+
] = match;
|
|
7258
|
+
const year = Number(yearText);
|
|
7259
|
+
const month = Number(monthText);
|
|
7260
|
+
const day = Number(dayText);
|
|
7261
|
+
const hour = Number(hourText);
|
|
7262
|
+
const minute = Number(minuteText);
|
|
7263
|
+
const second = secondText === void 0 ? 0 : Number(secondText);
|
|
7264
|
+
const offsetHour = offsetHourText === void 0 ? 0 : Number(offsetHourText);
|
|
7265
|
+
const offsetMinute = offsetMinuteText === void 0 ? 0 : Number(offsetMinuteText);
|
|
7266
|
+
const leapYear = year % 4 === 0 && (year % 100 !== 0 || year % 400 === 0);
|
|
7267
|
+
const daysInMonth = [31, leapYear ? 29 : 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31];
|
|
7268
|
+
if (month < 1 || month > 12 || day < 1 || day > daysInMonth[month - 1] || hour > 23 || minute > 59 || second > 59 || offsetHour > 23 || offsetMinute > 59) {
|
|
6127
7269
|
return null;
|
|
6128
7270
|
}
|
|
6129
|
-
const
|
|
6130
|
-
|
|
6131
|
-
|
|
6132
|
-
|
|
6133
|
-
|
|
6134
|
-
|
|
6135
|
-
|
|
6136
|
-
|
|
6137
|
-
|
|
7271
|
+
const parsed = new Date(trimmed);
|
|
7272
|
+
return Number.isFinite(parsed.getTime()) ? trimmed : null;
|
|
7273
|
+
}
|
|
7274
|
+
function canonicalLabels(value) {
|
|
7275
|
+
const labels = record(value);
|
|
7276
|
+
if (!labels) return null;
|
|
7277
|
+
const entries = Object.entries(labels);
|
|
7278
|
+
if (entries.length === 0 || entries.length > MAX_LABELS) return null;
|
|
7279
|
+
const canonical = [];
|
|
7280
|
+
for (const [key, rawValue] of entries) {
|
|
7281
|
+
if (!key || Buffer.byteLength(key, "utf8") > MAX_LABEL_KEY_BYTES || typeof rawValue !== "string" || Buffer.byteLength(rawValue, "utf8") > MAX_LABEL_VALUE_BYTES) {
|
|
7282
|
+
return null;
|
|
6138
7283
|
}
|
|
7284
|
+
canonical.push([key, rawValue]);
|
|
6139
7285
|
}
|
|
6140
|
-
|
|
6141
|
-
|
|
6142
|
-
|
|
6143
|
-
|
|
6144
|
-
|
|
6145
|
-
|
|
6146
|
-
|
|
6147
|
-
|
|
7286
|
+
return canonical.sort(([left], [right]) => left < right ? -1 : left > right ? 1 : 0);
|
|
7287
|
+
}
|
|
7288
|
+
|
|
7289
|
+
// src/activities/scheduled-tasks.ts
|
|
7290
|
+
function scheduledTaskRunProducerKey(input, suppliedActivityIdentity) {
|
|
7291
|
+
if (input.producerKey) return input.producerKey;
|
|
7292
|
+
if (input.triggerType !== "scheduled") return input.agentRunUsageIdempotencyKey;
|
|
7293
|
+
let info = suppliedActivityIdentity;
|
|
7294
|
+
if (!info) {
|
|
7295
|
+
try {
|
|
7296
|
+
info = Context2.current().info;
|
|
7297
|
+
} catch {
|
|
7298
|
+
throw new Error(
|
|
7299
|
+
"scheduled task delivery without a producer key requires a Temporal activity execution"
|
|
7300
|
+
);
|
|
6148
7301
|
}
|
|
6149
7302
|
}
|
|
6150
|
-
|
|
7303
|
+
if (!info.workflowExecution) {
|
|
7304
|
+
throw new Error("scheduled task Temporal activity is missing its workflow execution identity");
|
|
7305
|
+
}
|
|
7306
|
+
return `scheduled-temporal:${stableJson({
|
|
7307
|
+
namespace: info.namespace,
|
|
7308
|
+
workflowId: info.workflowExecution.workflowId
|
|
7309
|
+
})}`;
|
|
6151
7310
|
}
|
|
6152
|
-
|
|
6153
|
-
|
|
6154
|
-
|
|
7311
|
+
var SCHEDULED_SESSION_TITLE_MAX_LENGTH = 200;
|
|
7312
|
+
function scheduledTaskSessionTitle(taskName) {
|
|
7313
|
+
const name = taskName.trim() || "Scheduled run";
|
|
7314
|
+
if (name.length <= SCHEDULED_SESSION_TITLE_MAX_LENGTH) return name;
|
|
7315
|
+
return `${name.slice(0, SCHEDULED_SESSION_TITLE_MAX_LENGTH - 1).trimEnd()}\u2026`;
|
|
7316
|
+
}
|
|
7317
|
+
async function stampScheduledSessionTitle(db, input) {
|
|
7318
|
+
if (input.session.title !== null) return [];
|
|
7319
|
+
const title = scheduledTaskSessionTitle(input.taskName);
|
|
7320
|
+
const result = await updateSessionTitle(db, {
|
|
7321
|
+
workspaceId: input.workspaceId,
|
|
7322
|
+
sessionId: input.session.id,
|
|
7323
|
+
title,
|
|
7324
|
+
source: "agent"
|
|
7325
|
+
});
|
|
7326
|
+
if (!result.updated) return [];
|
|
7327
|
+
return await appendSessionEvents(db, input.workspaceId, input.session.id, [
|
|
7328
|
+
{ type: "session.title_set", payload: { title: result.title ?? title, source: "agent" } }
|
|
7329
|
+
]);
|
|
6155
7330
|
}
|
|
6156
|
-
|
|
6157
|
-
// src/activities/scheduled-tasks.ts
|
|
6158
7331
|
function createScheduledTaskActivities(services) {
|
|
6159
7332
|
return {
|
|
6160
7333
|
dispatchScheduledTaskRun: async (input) => {
|
|
@@ -6163,6 +7336,75 @@ function createScheduledTaskActivities(services) {
|
|
|
6163
7336
|
if (input.triggerType !== "scheduled" && !input.initiator) {
|
|
6164
7337
|
return { action: "blocked", reason: "malformed_manual_trigger" };
|
|
6165
7338
|
}
|
|
7339
|
+
const stableProducerKey = scheduledTaskRunProducerKey(input);
|
|
7340
|
+
const priorRun = await getScheduledTaskRunByProducerKey(db, {
|
|
7341
|
+
workspaceId: input.workspaceId,
|
|
7342
|
+
taskId: input.taskId,
|
|
7343
|
+
triggerType: input.triggerType,
|
|
7344
|
+
producerKey: stableProducerKey
|
|
7345
|
+
});
|
|
7346
|
+
if (priorRun?.actionKind === "agent_turn") {
|
|
7347
|
+
const acceptedExecution = await getScheduledTaskRunAcceptedExecution(db, {
|
|
7348
|
+
workspaceId: input.workspaceId,
|
|
7349
|
+
runId: priorRun.id
|
|
7350
|
+
});
|
|
7351
|
+
if (!acceptedExecution) {
|
|
7352
|
+
if (priorRun.status === "succeeded" || priorRun.status === "failed" || priorRun.status === "skipped") {
|
|
7353
|
+
return scheduledRunTerminalResult(priorRun.error);
|
|
7354
|
+
}
|
|
7355
|
+
throw new Error("scheduled agent run is missing accepted execution truth");
|
|
7356
|
+
}
|
|
7357
|
+
await recordScheduledTaskFiredUsage(db, acceptedExecution, priorRun);
|
|
7358
|
+
if (priorRun.status === "dispatched") {
|
|
7359
|
+
return await replayScheduledTaskDispatch({
|
|
7360
|
+
db,
|
|
7361
|
+
wakeSessionWorkflow,
|
|
7362
|
+
task: acceptedExecution.task,
|
|
7363
|
+
run: priorRun,
|
|
7364
|
+
acceptedExecution
|
|
7365
|
+
});
|
|
7366
|
+
}
|
|
7367
|
+
if (priorRun.status === "queued") {
|
|
7368
|
+
try {
|
|
7369
|
+
return await recoverBoundScheduledTaskDispatch({
|
|
7370
|
+
db,
|
|
7371
|
+
bus,
|
|
7372
|
+
settings,
|
|
7373
|
+
wakeSessionWorkflow,
|
|
7374
|
+
run: priorRun,
|
|
7375
|
+
acceptedExecution,
|
|
7376
|
+
input
|
|
7377
|
+
});
|
|
7378
|
+
} catch (error) {
|
|
7379
|
+
const terminalError = scheduledRunRecoveryTerminalError(error);
|
|
7380
|
+
if (!terminalError) throw error;
|
|
7381
|
+
await markScheduledTaskRunFailedIfQueued(
|
|
7382
|
+
db,
|
|
7383
|
+
priorRun.workspaceId,
|
|
7384
|
+
priorRun.id,
|
|
7385
|
+
terminalError.code
|
|
7386
|
+
);
|
|
7387
|
+
return scheduledRunTerminalResult(terminalError.code);
|
|
7388
|
+
}
|
|
7389
|
+
}
|
|
7390
|
+
if (priorRun.status === "succeeded") {
|
|
7391
|
+
if (!priorRun.sessionId || !priorRun.triggerEventId) {
|
|
7392
|
+
throw new Error("succeeded scheduled run is missing its delivery identity");
|
|
7393
|
+
}
|
|
7394
|
+
return {
|
|
7395
|
+
action: acceptedExecution.task.runMode === "new_session_per_run" ? "start" : "signal",
|
|
7396
|
+
accountId: acceptedExecution.task.accountId,
|
|
7397
|
+
workspaceId: acceptedExecution.task.workspaceId,
|
|
7398
|
+
sessionId: priorRun.sessionId,
|
|
7399
|
+
triggerEventId: priorRun.triggerEventId,
|
|
7400
|
+
workflowId: workflowIdForSession(priorRun.sessionId),
|
|
7401
|
+
workflowWakeRevision: null
|
|
7402
|
+
};
|
|
7403
|
+
}
|
|
7404
|
+
if (priorRun.status === "failed" || priorRun.status === "skipped") {
|
|
7405
|
+
return scheduledRunTerminalResult(priorRun.error);
|
|
7406
|
+
}
|
|
7407
|
+
}
|
|
6166
7408
|
const task = await getScheduledTask2(db, input.workspaceId, input.taskId);
|
|
6167
7409
|
if (!task) {
|
|
6168
7410
|
return { action: "deleted" };
|
|
@@ -6171,11 +7413,13 @@ function createScheduledTaskActivities(services) {
|
|
|
6171
7413
|
if (knowledgeSourceSyncEffectivelyPaused(task)) {
|
|
6172
7414
|
return { action: "blocked", reason: "knowledge_source_paused" };
|
|
6173
7415
|
}
|
|
6174
|
-
const
|
|
7416
|
+
const run = await createScheduledTaskRun(db, {
|
|
6175
7417
|
workspaceId: task.workspaceId,
|
|
6176
7418
|
taskId: task.id,
|
|
7419
|
+
taskAuthorityRevision: task.authorityRevision,
|
|
7420
|
+
taskExecutionDigest: task.executionDigest,
|
|
6177
7421
|
triggerType: input.triggerType,
|
|
6178
|
-
producerKey:
|
|
7422
|
+
producerKey: stableProducerKey,
|
|
6179
7423
|
scheduledAt: null
|
|
6180
7424
|
});
|
|
6181
7425
|
await ensureKnowledgeSourceSyncState(db, task);
|
|
@@ -6184,9 +7428,9 @@ function createScheduledTaskActivities(services) {
|
|
|
6184
7428
|
workspaceId: task.workspaceId,
|
|
6185
7429
|
sourceId: task.action.sourceId,
|
|
6186
7430
|
scheduledTaskId: task.id,
|
|
6187
|
-
scheduledTaskRunId:
|
|
7431
|
+
scheduledTaskRunId: run.id,
|
|
6188
7432
|
cause: input.triggerType,
|
|
6189
|
-
producerKey:
|
|
7433
|
+
producerKey: stableProducerKey,
|
|
6190
7434
|
sourceConfigGeneration: task.action.sourceConfigGeneration,
|
|
6191
7435
|
sourceLifecycleGeneration: task.action.sourceLifecycleGeneration
|
|
6192
7436
|
});
|
|
@@ -6197,14 +7441,20 @@ function createScheduledTaskActivities(services) {
|
|
|
6197
7441
|
quantity: 1,
|
|
6198
7442
|
unit: "run",
|
|
6199
7443
|
sourceResourceType: "scheduled_task_run",
|
|
6200
|
-
sourceResourceId:
|
|
6201
|
-
initiator: input.initiator ?? {
|
|
6202
|
-
|
|
7444
|
+
sourceResourceId: run.id,
|
|
7445
|
+
initiator: input.initiator ?? {
|
|
7446
|
+
kind: "service",
|
|
7447
|
+
subjectId: "scheduler"
|
|
7448
|
+
},
|
|
7449
|
+
initiatorContext: {
|
|
7450
|
+
scheduledTaskId: task.id,
|
|
7451
|
+
scheduledTaskRunId: run.id
|
|
7452
|
+
},
|
|
6203
7453
|
origin: "scheduled_task",
|
|
6204
|
-
idempotencyKey: `usage:knowledge_source_sync.fired:${
|
|
7454
|
+
idempotencyKey: `usage:knowledge_source_sync.fired:${run.id}`
|
|
6205
7455
|
});
|
|
6206
|
-
if (
|
|
6207
|
-
await updateScheduledTaskRun2(db, task.workspaceId,
|
|
7456
|
+
if (run.status === "queued") {
|
|
7457
|
+
await updateScheduledTaskRun2(db, task.workspaceId, run.id, {
|
|
6208
7458
|
status: "dispatched",
|
|
6209
7459
|
actionKind: "knowledge_source_sync"
|
|
6210
7460
|
});
|
|
@@ -6214,16 +7464,224 @@ function createScheduledTaskActivities(services) {
|
|
|
6214
7464
|
accountId: task.accountId,
|
|
6215
7465
|
workspaceId: task.workspaceId,
|
|
6216
7466
|
taskId: task.id,
|
|
6217
|
-
scheduledTaskRunId:
|
|
7467
|
+
scheduledTaskRunId: run.id,
|
|
6218
7468
|
sourceId: task.action.sourceId,
|
|
6219
7469
|
overlapPolicy: task.overlapPolicy
|
|
6220
7470
|
};
|
|
6221
7471
|
}
|
|
7472
|
+
if (task.status !== "active") {
|
|
7473
|
+
return { action: "blocked", reason: "scheduled_task_paused" };
|
|
7474
|
+
}
|
|
7475
|
+
const structuredAlertOccurrence = scheduledAlertOccurrenceIdentity({
|
|
7476
|
+
workspaceId: task.workspaceId,
|
|
7477
|
+
scheduledTaskId: task.id,
|
|
7478
|
+
metadata: task.metadata
|
|
7479
|
+
});
|
|
7480
|
+
const alertOccurrence = task.runMode === "new_session_per_run" ? structuredAlertOccurrence : null;
|
|
7481
|
+
const alertResponderSessionCreateIdempotencyKey = alertOccurrence ? scheduledAlertResponderSessionCreateIdempotencyKey({
|
|
7482
|
+
occurrence: alertOccurrence,
|
|
7483
|
+
taskAuthorityRevision: task.authorityRevision,
|
|
7484
|
+
taskExecutionDigest: task.executionDigest
|
|
7485
|
+
}) : null;
|
|
7486
|
+
const incidentDeclaration = incidentTelemetryPreflightDeclaration(
|
|
7487
|
+
task.agentConfig,
|
|
7488
|
+
structuredAlertOccurrence !== null
|
|
7489
|
+
);
|
|
7490
|
+
if (incidentDeclaration.action === "blocked") {
|
|
7491
|
+
return incidentDeclaration;
|
|
7492
|
+
}
|
|
6222
7493
|
const taskPersonalConnectionDelegations = await getScheduledTaskPersonalConnectionDelegations(
|
|
6223
7494
|
db,
|
|
6224
7495
|
task.workspaceId,
|
|
6225
7496
|
task.id
|
|
6226
7497
|
);
|
|
7498
|
+
const taskConnectionAuthoritySubjects = [
|
|
7499
|
+
...new Set(
|
|
7500
|
+
taskPersonalConnectionDelegations.map((delegation) => delegation.ownerSubjectId)
|
|
7501
|
+
)
|
|
7502
|
+
];
|
|
7503
|
+
if (taskConnectionAuthoritySubjects.length > 1) {
|
|
7504
|
+
throw new Error("scheduled connection authority has multiple causal humans");
|
|
7505
|
+
}
|
|
7506
|
+
const taskConnectionAuthoritySubjectId = taskConnectionAuthoritySubjects[0] ?? null;
|
|
7507
|
+
const taskPersonalResourceAuthoritySubjectId = await getScheduledTaskPersonalResourceAuthoritySubject(db, {
|
|
7508
|
+
accountId: task.accountId,
|
|
7509
|
+
workspaceId: task.workspaceId,
|
|
7510
|
+
taskId: task.id,
|
|
7511
|
+
taskAuthorityRevision: task.authorityRevision
|
|
7512
|
+
});
|
|
7513
|
+
const taskRevisionAuthority = await getScheduledTaskRevisionAuthority(db, {
|
|
7514
|
+
accountId: task.accountId,
|
|
7515
|
+
workspaceId: task.workspaceId,
|
|
7516
|
+
taskId: task.id,
|
|
7517
|
+
taskAuthorityRevision: task.authorityRevision
|
|
7518
|
+
});
|
|
7519
|
+
if (taskConnectionAuthoritySubjectId && taskPersonalResourceAuthoritySubjectId && taskConnectionAuthoritySubjectId !== taskPersonalResourceAuthoritySubjectId) {
|
|
7520
|
+
throw new Error("scheduled authority classes have different causal humans");
|
|
7521
|
+
}
|
|
7522
|
+
const taskAuthoritySubjectId = taskRevisionAuthority?.subjectId ?? taskPersonalResourceAuthoritySubjectId ?? taskConnectionAuthoritySubjectId;
|
|
7523
|
+
const model = task.agentConfig.model ?? settings.openaiModel;
|
|
7524
|
+
const reasoningEffort = task.agentConfig.reasoningEffort ?? settings.openaiReasoningEffort;
|
|
7525
|
+
const sandboxBackend = task.agentConfig.sandboxBackend ?? settings.sandboxBackend;
|
|
7526
|
+
const taskTools = withFirstPartyTools(settings, task.agentConfig.tools);
|
|
7527
|
+
const firstPartyMcpTools = resolveFirstPartyMcpToolPolicy(settings).default;
|
|
7528
|
+
const generatedTarget = task.runMode === "new_session_per_run" || task.runMode === "reusable_session" && task.reusableSessionId === null;
|
|
7529
|
+
const generatedSessionDepthPolicy = generatedTarget ? await (async () => {
|
|
7530
|
+
const workspace = await requireWorkspace(db, task.workspaceId);
|
|
7531
|
+
const deployment = await getNestedAgentDepthDeploymentPolicy(db);
|
|
7532
|
+
const taskOverride = task.agentConfig.maxNestedAgentDepth ?? null;
|
|
7533
|
+
const workspaceOverride = typeof workspace.settings.maxNestedAgentDepth === "number" ? workspace.settings.maxNestedAgentDepth : null;
|
|
7534
|
+
return taskOverride !== null ? {
|
|
7535
|
+
effectiveMaxNestedAgentDepth: taskOverride,
|
|
7536
|
+
nestedAgentDepthPolicySource: "session",
|
|
7537
|
+
codexCompactionMode: isCodexBilledModel(model) ? resolveWorkspaceCodexCompactionDefault(workspace.settings) : "portable"
|
|
7538
|
+
} : workspaceOverride !== null ? {
|
|
7539
|
+
effectiveMaxNestedAgentDepth: workspaceOverride,
|
|
7540
|
+
nestedAgentDepthPolicySource: "workspace",
|
|
7541
|
+
codexCompactionMode: isCodexBilledModel(model) ? resolveWorkspaceCodexCompactionDefault(workspace.settings) : "portable"
|
|
7542
|
+
} : {
|
|
7543
|
+
effectiveMaxNestedAgentDepth: deployment.maxNestedAgentDepth,
|
|
7544
|
+
nestedAgentDepthPolicySource: deployment.policySource,
|
|
7545
|
+
codexCompactionMode: isCodexBilledModel(model) ? resolveWorkspaceCodexCompactionDefault(workspace.settings) : "portable"
|
|
7546
|
+
};
|
|
7547
|
+
})() : null;
|
|
7548
|
+
const acceptedTargetSessionId = generatedTarget ? null : task.runMode === "existing_session" ? task.targetSessionId : task.reusableSessionId;
|
|
7549
|
+
const targetSessionExecutionBase = acceptedTargetSessionId ? await getScheduledTargetSessionExecution(
|
|
7550
|
+
db,
|
|
7551
|
+
task.workspaceId,
|
|
7552
|
+
acceptedTargetSessionId,
|
|
7553
|
+
taskAuthoritySubjectId
|
|
7554
|
+
) : null;
|
|
7555
|
+
const targetSessionExecution = targetSessionExecutionBase ? await (async () => {
|
|
7556
|
+
const [capabilityServerIds, apiIntegrationServerIds] = await Promise.all([
|
|
7557
|
+
listEnabledMcpCapabilityServerIds(db, task.workspaceId),
|
|
7558
|
+
listInstalledApiIntegrationServerIdsForDelegations(
|
|
7559
|
+
db,
|
|
7560
|
+
task.workspaceId,
|
|
7561
|
+
taskPersonalConnectionDelegations
|
|
7562
|
+
)
|
|
7563
|
+
]);
|
|
7564
|
+
const workspaceDefaultMcpServerIds = new Set(
|
|
7565
|
+
settings.mcpServers.map((server) => server.id)
|
|
7566
|
+
);
|
|
7567
|
+
for (const id of capabilityServerIds) workspaceDefaultMcpServerIds.add(id);
|
|
7568
|
+
for (const id of apiIntegrationServerIds) workspaceDefaultMcpServerIds.add(id);
|
|
7569
|
+
const availableMcpServerIds = new Set(workspaceDefaultMcpServerIds);
|
|
7570
|
+
for (const id of targetSessionExecutionBase.mcpServerIds) {
|
|
7571
|
+
availableMcpServerIds.add(id);
|
|
7572
|
+
}
|
|
7573
|
+
const effective = resolveSessionToolPolicy({
|
|
7574
|
+
toolPolicy: targetSessionExecutionBase.toolPolicy,
|
|
7575
|
+
sessionTools: targetSessionExecutionBase.tools,
|
|
7576
|
+
availableMcpServerIds,
|
|
7577
|
+
defaultMcpServerIds: defaultSessionMcpServerIds(
|
|
7578
|
+
[...workspaceDefaultMcpServerIds].map((id) => ({ id }))
|
|
7579
|
+
)
|
|
7580
|
+
});
|
|
7581
|
+
return {
|
|
7582
|
+
...targetSessionExecutionBase,
|
|
7583
|
+
effectiveMcpServerIds: [
|
|
7584
|
+
...new Set(
|
|
7585
|
+
effective.toolRefs.flatMap((tool) => tool.kind === "mcp" ? [tool.id] : [])
|
|
7586
|
+
)
|
|
7587
|
+
].sort()
|
|
7588
|
+
};
|
|
7589
|
+
})() : null;
|
|
7590
|
+
const acceptedVariableSet = generatedTarget && task.variableSetId ? await getVariableSet(
|
|
7591
|
+
db,
|
|
7592
|
+
{
|
|
7593
|
+
accountId: task.accountId,
|
|
7594
|
+
workspaceId: task.workspaceId,
|
|
7595
|
+
// Organization/workspace sets have no personal-resource causal
|
|
7596
|
+
// subject. The scoped resolver still permits those common
|
|
7597
|
+
// scopes for a service/legacy actor, while a user-owned set can
|
|
7598
|
+
// only resolve through the exact frozen 0252 subject.
|
|
7599
|
+
subjectId: taskAuthoritySubjectId ?? task.createdBy.subjectId
|
|
7600
|
+
},
|
|
7601
|
+
task.variableSetId
|
|
7602
|
+
) : null;
|
|
7603
|
+
if (generatedTarget && task.variableSetId && !acceptedVariableSet) {
|
|
7604
|
+
throw new Error(`variable set not found: ${task.variableSetId}`);
|
|
7605
|
+
}
|
|
7606
|
+
const acceptedRig = generatedTarget && task.rigId ? await getRig2(
|
|
7607
|
+
db,
|
|
7608
|
+
{
|
|
7609
|
+
accountId: task.accountId,
|
|
7610
|
+
workspaceId: task.workspaceId,
|
|
7611
|
+
subjectId: taskAuthoritySubjectId ?? task.createdBy.subjectId
|
|
7612
|
+
},
|
|
7613
|
+
task.rigId
|
|
7614
|
+
) : null;
|
|
7615
|
+
if (generatedTarget && task.rigId && (!acceptedRig || !acceptedRig.activeVersion)) {
|
|
7616
|
+
throw new Error(`rig has no active version to bind: ${task.rigId}`);
|
|
7617
|
+
}
|
|
7618
|
+
const acceptedRigDefaultVariableSets = acceptedRig?.activeVersion ? await Promise.all(
|
|
7619
|
+
acceptedRig.activeVersion.defaultVariableSetIds.map(async (variableSetId) => {
|
|
7620
|
+
const variableSet = await getVariableSet(
|
|
7621
|
+
db,
|
|
7622
|
+
{
|
|
7623
|
+
accountId: task.accountId,
|
|
7624
|
+
workspaceId: task.workspaceId,
|
|
7625
|
+
subjectId: taskAuthoritySubjectId ?? task.createdBy.subjectId
|
|
7626
|
+
},
|
|
7627
|
+
variableSetId
|
|
7628
|
+
);
|
|
7629
|
+
if (!variableSet) {
|
|
7630
|
+
throw new Error(`rig default Variable Set not found: ${variableSetId}`);
|
|
7631
|
+
}
|
|
7632
|
+
return { id: variableSet.id, generation: variableSet.generation };
|
|
7633
|
+
})
|
|
7634
|
+
) : [];
|
|
7635
|
+
let incidentPreflightRequired = incidentDeclaration.action === "required";
|
|
7636
|
+
if (incidentPreflightRequired) {
|
|
7637
|
+
const existingSessionId = task.runMode === "existing_session" ? task.targetSessionId : task.reusableSessionId;
|
|
7638
|
+
const canonicalAlertSession = task.runMode === "new_session_per_run" && alertOccurrence ? await getSessionByCreateIdempotencyKey(
|
|
7639
|
+
db,
|
|
7640
|
+
task.workspaceId,
|
|
7641
|
+
alertResponderSessionCreateIdempotencyKey
|
|
7642
|
+
) : null;
|
|
7643
|
+
const responderSession = existingSessionId ? await requireSession(db, task.workspaceId, existingSessionId) : canonicalAlertSession;
|
|
7644
|
+
const responder = await resolveIncidentTelemetryResponderMetadata({
|
|
7645
|
+
db,
|
|
7646
|
+
settings,
|
|
7647
|
+
task,
|
|
7648
|
+
session: responderSession,
|
|
7649
|
+
personalConnectionDelegations: taskPersonalConnectionDelegations,
|
|
7650
|
+
personalResourceAuthoritySubjectId: taskAuthoritySubjectId,
|
|
7651
|
+
executionPolicy: targetSessionExecution ? {
|
|
7652
|
+
tools: targetSessionExecution.tools,
|
|
7653
|
+
firstPartyMcpTools: targetSessionExecution.firstPartyMcpTools,
|
|
7654
|
+
firstPartyMcpPermissions: targetSessionExecution.firstPartyMcpPermissions,
|
|
7655
|
+
variableSetId: targetSessionExecution.variableSetId,
|
|
7656
|
+
rigId: targetSessionExecution.rigId,
|
|
7657
|
+
rigVersionId: targetSessionExecution.rigVersionId,
|
|
7658
|
+
toolPolicy: targetSessionExecution.toolPolicy,
|
|
7659
|
+
mcpServerIds: targetSessionExecution.mcpServerIds,
|
|
7660
|
+
toolPolicyVersion: targetSessionExecution.toolPolicyVersion
|
|
7661
|
+
} : {
|
|
7662
|
+
tools: taskTools,
|
|
7663
|
+
firstPartyMcpTools,
|
|
7664
|
+
firstPartyMcpPermissions: [...DEFAULT_FIRST_PARTY_MCP_PERMISSIONS],
|
|
7665
|
+
variableSetId: acceptedVariableSet?.id ?? null,
|
|
7666
|
+
rigId: acceptedRig?.id ?? null,
|
|
7667
|
+
rigVersionId: acceptedRig?.activeVersion?.id ?? null,
|
|
7668
|
+
toolPolicy: responderSession?.toolPolicy ?? {
|
|
7669
|
+
mode: "explicit",
|
|
7670
|
+
inheritedFromSessionId: null
|
|
7671
|
+
},
|
|
7672
|
+
mcpServerIds: [],
|
|
7673
|
+
toolPolicyVersion: responderSession?.toolPolicyVersion ?? null
|
|
7674
|
+
}
|
|
7675
|
+
});
|
|
7676
|
+
const preflight = evaluateIncidentTelemetryPreflight({
|
|
7677
|
+
agentConfig: task.agentConfig,
|
|
7678
|
+
incidentTriggered: structuredAlertOccurrence !== null,
|
|
7679
|
+
alertOccurrenceLabels: structuredAlertOccurrence?.labels ?? null,
|
|
7680
|
+
responder
|
|
7681
|
+
});
|
|
7682
|
+
if (preflight.action === "blocked") return preflight;
|
|
7683
|
+
incidentPreflightRequired = preflight.action === "ready";
|
|
7684
|
+
}
|
|
6227
7685
|
const taskXaiProviderAccountAuthoritySnapshot = await getScheduledTaskXaiProviderAccountAuthoritySnapshot(db, task.workspaceId, task.id);
|
|
6228
7686
|
const slackBotConnection = task.agentConfig.slackBotConnectionId ? await requireOpenGeniSlackBotConnection(
|
|
6229
7687
|
db,
|
|
@@ -6234,394 +7692,751 @@ function createScheduledTaskActivities(services) {
|
|
|
6234
7692
|
if (slackBotConnection && !slackBotMetadata) {
|
|
6235
7693
|
throw new Error("OpenGeni Slack bot connection metadata is invalid");
|
|
6236
7694
|
}
|
|
6237
|
-
const
|
|
7695
|
+
const xaiAuthoritySubjectId = taskXaiProviderAccountAuthoritySnapshot.scope === "user" && task.createdBy.kind === "subject" ? task.createdBy.subjectId : null;
|
|
7696
|
+
if (taskXaiProviderAccountAuthoritySnapshot.scope === "user" && !xaiAuthoritySubjectId) {
|
|
7697
|
+
throw new Error("scheduled user-scoped xAI authority has no causal human");
|
|
7698
|
+
}
|
|
7699
|
+
if (xaiAuthoritySubjectId && taskAuthoritySubjectId && xaiAuthoritySubjectId !== taskAuthoritySubjectId) {
|
|
7700
|
+
throw new Error("scheduled authority classes have different causal humans");
|
|
7701
|
+
}
|
|
7702
|
+
const causalHumanSubjectId = taskAuthoritySubjectId ?? xaiAuthoritySubjectId;
|
|
7703
|
+
if ((acceptedVariableSet?.scope === "user" || acceptedRig?.scope === "user") && !causalHumanSubjectId) {
|
|
7704
|
+
throw new Error("scheduled personal-resource execution has no causal human");
|
|
7705
|
+
}
|
|
6238
7706
|
const admissionDenial = await agentRunAdmissionDenial(service, {
|
|
6239
7707
|
accountId: task.accountId,
|
|
6240
7708
|
workspaceId: task.workspaceId,
|
|
6241
|
-
model,
|
|
7709
|
+
model: targetSessionExecution?.model ?? model,
|
|
6242
7710
|
requestedAgentRuns: input.agentRunUsageIdempotencyKey ? 0 : 1
|
|
6243
7711
|
});
|
|
6244
7712
|
if (admissionDenial) {
|
|
6245
7713
|
return { action: "blocked", reason: admissionDenial };
|
|
6246
7714
|
}
|
|
6247
|
-
const
|
|
6248
|
-
|
|
6249
|
-
|
|
6250
|
-
|
|
6251
|
-
|
|
6252
|
-
|
|
6253
|
-
|
|
6254
|
-
|
|
6255
|
-
|
|
6256
|
-
|
|
6257
|
-
|
|
6258
|
-
|
|
6259
|
-
|
|
6260
|
-
|
|
6261
|
-
|
|
6262
|
-
|
|
6263
|
-
|
|
6264
|
-
|
|
6265
|
-
|
|
6266
|
-
|
|
6267
|
-
|
|
6268
|
-
|
|
6269
|
-
|
|
6270
|
-
|
|
6271
|
-
|
|
6272
|
-
|
|
6273
|
-
|
|
6274
|
-
|
|
6275
|
-
|
|
6276
|
-
|
|
6277
|
-
|
|
6278
|
-
|
|
6279
|
-
|
|
6280
|
-
|
|
6281
|
-
|
|
6282
|
-
|
|
6283
|
-
|
|
6284
|
-
|
|
7715
|
+
const deferredEvents = [];
|
|
7716
|
+
const executeDispatch = async (dispatchDb, deferPublications) => {
|
|
7717
|
+
const admittedRunId = crypto.randomUUID();
|
|
7718
|
+
const generatedSessionBinding = generatedTarget ? {
|
|
7719
|
+
createIdempotencyKey: alertResponderSessionCreateIdempotencyKey ?? (task.runMode === "reusable_session" ? `scheduled-task-reusable:${task.id}:${task.authorityRevision}:${task.executionDigest}` : `scheduled-task-run:${admittedRunId}`),
|
|
7720
|
+
effectiveMaxNestedAgentDepth: generatedSessionDepthPolicy.effectiveMaxNestedAgentDepth,
|
|
7721
|
+
nestedAgentDepthPolicySource: generatedSessionDepthPolicy.nestedAgentDepthPolicySource,
|
|
7722
|
+
codexCompactionMode: generatedSessionDepthPolicy.codexCompactionMode
|
|
7723
|
+
} : null;
|
|
7724
|
+
const acceptedExecution = {
|
|
7725
|
+
version: 1,
|
|
7726
|
+
task,
|
|
7727
|
+
resolvedModel: model,
|
|
7728
|
+
resolvedReasoningEffort: reasoningEffort,
|
|
7729
|
+
resolvedLatencyMode: "standard",
|
|
7730
|
+
resolvedSandboxBackend: sandboxBackend,
|
|
7731
|
+
resolvedSandboxOs: "linux",
|
|
7732
|
+
resolvedTools: taskTools,
|
|
7733
|
+
resolvedFirstPartyMcpTools: firstPartyMcpTools,
|
|
7734
|
+
resolvedFirstPartyMcpPermissions: [...DEFAULT_FIRST_PARTY_MCP_PERMISSIONS],
|
|
7735
|
+
resolvedVariableSet: acceptedVariableSet ? { id: acceptedVariableSet.id, generation: acceptedVariableSet.generation } : null,
|
|
7736
|
+
resolvedRig: acceptedRig?.activeVersion ? {
|
|
7737
|
+
id: acceptedRig.id,
|
|
7738
|
+
versionId: acceptedRig.activeVersion.id,
|
|
7739
|
+
defaultVariableSets: acceptedRigDefaultVariableSets
|
|
7740
|
+
} : null,
|
|
7741
|
+
resolvedSlackBotConnection: slackBotConnection && slackBotMetadata && slackBotConnection.verifiedInstallVersion ? {
|
|
7742
|
+
id: slackBotConnection.id,
|
|
7743
|
+
version: slackBotConnection.version,
|
|
7744
|
+
verifiedInstallVersion: slackBotConnection.verifiedInstallVersion,
|
|
7745
|
+
metadata: slackBotMetadata
|
|
7746
|
+
} : null,
|
|
7747
|
+
targetSessionExecution,
|
|
7748
|
+
generatedSessionBinding,
|
|
7749
|
+
personalConnectionDelegations: taskPersonalConnectionDelegations,
|
|
7750
|
+
personalResourceAuthoritySubjectId: taskPersonalResourceAuthoritySubjectId,
|
|
7751
|
+
causalHumanSubjectId,
|
|
7752
|
+
causalHumanAuthority: taskRevisionAuthority,
|
|
7753
|
+
xaiProviderAccountAuthoritySnapshot: taskXaiProviderAccountAuthoritySnapshot,
|
|
7754
|
+
xaiAuthoritySubjectId,
|
|
7755
|
+
connectionAuthoritySubjectId: taskConnectionAuthoritySubjectId,
|
|
7756
|
+
triggerInitiator: input.initiator ?? { kind: "service", subjectId: "scheduler" },
|
|
7757
|
+
agentRunUsageIdempotencyKey: input.agentRunUsageIdempotencyKey ?? null,
|
|
7758
|
+
incidentPreflightRequired,
|
|
7759
|
+
alertOccurrenceLabels: structuredAlertOccurrence?.labels ?? null
|
|
7760
|
+
};
|
|
7761
|
+
const acceptedExecutionParse = ScheduledTaskRunAcceptedExecution.safeParse(acceptedExecution);
|
|
7762
|
+
if (!acceptedExecutionParse.success || scheduledOccurrencePayloadUtf8Bytes({
|
|
7763
|
+
prompt: task.agentConfig.prompt,
|
|
7764
|
+
resources: task.agentConfig.resources,
|
|
7765
|
+
tools: taskTools
|
|
7766
|
+
}) > SCHEDULED_TASK_OCCURRENCE_PAYLOAD_MAX_BYTES) {
|
|
7767
|
+
return {
|
|
7768
|
+
kind: "blocked",
|
|
7769
|
+
result: {
|
|
7770
|
+
action: "blocked",
|
|
7771
|
+
reason: "scheduled_execution_unrepresentable"
|
|
7772
|
+
}
|
|
7773
|
+
};
|
|
7774
|
+
}
|
|
7775
|
+
let run = await createScheduledTaskRun(dispatchDb, {
|
|
7776
|
+
runId: admittedRunId,
|
|
6285
7777
|
workspaceId: task.workspaceId,
|
|
6286
|
-
|
|
6287
|
-
|
|
6288
|
-
|
|
7778
|
+
taskId: task.id,
|
|
7779
|
+
taskAuthorityRevision: task.authorityRevision,
|
|
7780
|
+
taskExecutionDigest: task.executionDigest,
|
|
7781
|
+
triggerType: input.triggerType,
|
|
7782
|
+
producerKey: stableProducerKey,
|
|
7783
|
+
scheduledAt: null,
|
|
7784
|
+
acceptedExecutionSnapshot: acceptedExecution
|
|
6289
7785
|
});
|
|
6290
|
-
|
|
6291
|
-
|
|
6292
|
-
|
|
6293
|
-
|
|
6294
|
-
|
|
6295
|
-
|
|
6296
|
-
|
|
6297
|
-
|
|
6298
|
-
|
|
6299
|
-
|
|
6300
|
-
await wakeSessionWorkflow({
|
|
6301
|
-
accountId: result2.accountId,
|
|
6302
|
-
workspaceId: result2.workspaceId,
|
|
6303
|
-
sessionId: result2.sessionId,
|
|
6304
|
-
workflowId: result2.workflowId,
|
|
6305
|
-
wakeRevision: workflowWakeRevision
|
|
6306
|
-
});
|
|
7786
|
+
await recordScheduledTaskFiredUsage(dispatchDb, acceptedExecution, run);
|
|
7787
|
+
if (run.status === "failed" && run.error === "scheduled_authority_exhausted") {
|
|
7788
|
+
return {
|
|
7789
|
+
kind: "blocked",
|
|
7790
|
+
result: {
|
|
7791
|
+
action: "blocked",
|
|
7792
|
+
reason: "scheduled_authority_exhausted"
|
|
7793
|
+
},
|
|
7794
|
+
run
|
|
7795
|
+
};
|
|
6307
7796
|
}
|
|
6308
|
-
|
|
6309
|
-
|
|
6310
|
-
|
|
6311
|
-
|
|
6312
|
-
|
|
6313
|
-
|
|
6314
|
-
const goalSpec = task.agentConfig.goal ?? null;
|
|
6315
|
-
const taskTools = withFirstPartyTools(settings, task.agentConfig.tools);
|
|
6316
|
-
const existingSessionId = task.runMode === "existing_session" ? task.targetSessionId : task.reusableSessionId;
|
|
6317
|
-
if (task.runMode === "existing_session" && !existingSessionId) {
|
|
6318
|
-
throw new Error("scheduled task target session is unavailable");
|
|
7797
|
+
if (run.status === "failed" || run.status === "skipped") {
|
|
7798
|
+
return {
|
|
7799
|
+
kind: "blocked",
|
|
7800
|
+
result: scheduledRunTerminalResult(run.error),
|
|
7801
|
+
run
|
|
7802
|
+
};
|
|
6319
7803
|
}
|
|
6320
|
-
|
|
6321
|
-
|
|
6322
|
-
|
|
6323
|
-
|
|
6324
|
-
|
|
6325
|
-
|
|
6326
|
-
let frozenRigVersionId = null;
|
|
6327
|
-
if (task.rigId) {
|
|
6328
|
-
const rig = await getRig2(db, task.workspaceId, task.rigId);
|
|
6329
|
-
if (!rig || !rig.activeVersion) {
|
|
6330
|
-
throw new Error(`rig has no active version to bind: ${task.rigId}`);
|
|
6331
|
-
}
|
|
6332
|
-
frozenRigId = rig.id;
|
|
6333
|
-
frozenRigVersionId = rig.activeVersion.id;
|
|
7804
|
+
const runPersonalResourceAuthority = await getScheduledTaskRunPersonalResourceAuthority(
|
|
7805
|
+
dispatchDb,
|
|
7806
|
+
{
|
|
7807
|
+
accountId: task.accountId,
|
|
7808
|
+
workspaceId: task.workspaceId,
|
|
7809
|
+
runId: run.id
|
|
6334
7810
|
}
|
|
6335
|
-
|
|
6336
|
-
|
|
6337
|
-
|
|
6338
|
-
|
|
6339
|
-
session = await createSession(db, {
|
|
6340
|
-
accountId: task.accountId,
|
|
6341
|
-
workspaceId: task.workspaceId,
|
|
6342
|
-
initialMessage: task.agentConfig.prompt,
|
|
6343
|
-
resources: task.agentConfig.resources,
|
|
6344
|
-
tools: taskTools,
|
|
6345
|
-
firstPartyMcpTools: resolveFirstPartyMcpToolPolicy(settings).default,
|
|
6346
|
-
metadata: {
|
|
6347
|
-
...taskMetadata,
|
|
6348
|
-
model,
|
|
6349
|
-
reasoningEffort,
|
|
6350
|
-
scheduledTaskId: task.id,
|
|
6351
|
-
scheduledTaskRunId: run.id,
|
|
6352
|
-
...slackBotConnection ? {
|
|
6353
|
-
[OPENGENI_SLACK_BOT_SESSION_METADATA_KEY]: slackBotConnection.id
|
|
6354
|
-
} : {}
|
|
6355
|
-
},
|
|
6356
|
-
createdBy: {
|
|
6357
|
-
kind: "service",
|
|
6358
|
-
subjectId: "scheduler",
|
|
6359
|
-
label: "OpenGeni scheduler"
|
|
6360
|
-
},
|
|
6361
|
-
createdByContext: {
|
|
6362
|
-
scheduledTaskId: task.id,
|
|
6363
|
-
scheduledTaskRunId: run.id
|
|
6364
|
-
},
|
|
6365
|
-
model,
|
|
6366
|
-
sandboxBackend,
|
|
6367
|
-
variableSetId: task.variableSetId ?? null,
|
|
6368
|
-
rigId: frozenRigId,
|
|
6369
|
-
rigVersionId: frozenRigVersionId,
|
|
6370
|
-
personalConnectionDelegations: taskPersonalConnectionDelegations,
|
|
6371
|
-
initialXaiProviderAccountAuthoritySnapshot: taskXaiProviderAccountAuthoritySnapshot,
|
|
6372
|
-
maxNestedAgentDepthOverride: task.agentConfig.maxNestedAgentDepth ?? null,
|
|
6373
|
-
// The durable agent config was privilege-checked when the task
|
|
6374
|
-
// was created/updated. Preserve that explicit policy if a broader
|
|
6375
|
-
// workspace/deployment limit is narrowed before a later fire.
|
|
6376
|
-
allowNestedAgentDepthIncrease: true,
|
|
6377
|
-
subjectId: `scheduled_task:${task.id}`
|
|
6378
|
-
});
|
|
6379
|
-
} catch (error) {
|
|
6380
|
-
if (error instanceof SessionSpawnDeniedDbError) {
|
|
6381
|
-
throw new Error(`${error.denial.code}: denial=${error.denial.id}`, { cause: error });
|
|
6382
|
-
}
|
|
6383
|
-
throw error;
|
|
7811
|
+
);
|
|
7812
|
+
if (run?.status === "dispatched" && run.sessionId && run.triggerEventId) {
|
|
7813
|
+
if (deferPublications) {
|
|
7814
|
+
return { kind: "replay", run, acceptedExecution };
|
|
6384
7815
|
}
|
|
6385
|
-
|
|
7816
|
+
await recordUsageEvent3(dispatchDb, {
|
|
6386
7817
|
accountId: task.accountId,
|
|
6387
7818
|
workspaceId: task.workspaceId,
|
|
6388
|
-
|
|
6389
|
-
|
|
6390
|
-
|
|
6391
|
-
|
|
6392
|
-
|
|
6393
|
-
|
|
6394
|
-
|
|
6395
|
-
|
|
6396
|
-
|
|
6397
|
-
|
|
6398
|
-
reusableSessionId: session.id
|
|
6399
|
-
});
|
|
6400
|
-
}
|
|
6401
|
-
await appendAndPublishEvents(db, bus, task.workspaceId, session.id, [
|
|
6402
|
-
{
|
|
6403
|
-
type: "session.created",
|
|
6404
|
-
payload: {
|
|
6405
|
-
status: session.status,
|
|
6406
|
-
createdBy: session.createdBy,
|
|
6407
|
-
scheduledTaskId: task.id,
|
|
6408
|
-
scheduledTaskRunId: run.id,
|
|
6409
|
-
// Names/ids only; never values.
|
|
6410
|
-
...variableSet ? { variableSetId: variableSet.id, variableSetName: variableSet.name } : {},
|
|
6411
|
-
...slackBotConnection && slackBotMetadata ? {
|
|
6412
|
-
slackBotConnection: {
|
|
6413
|
-
credentialRole: OPENGENI_SLACK_BOT_CREDENTIAL_ROLE,
|
|
6414
|
-
credentialLabel: OPENGENI_SLACK_BOT_CREDENTIAL_LABEL,
|
|
6415
|
-
connectionId: slackBotConnection.id,
|
|
6416
|
-
slackTeamId: slackBotMetadata.slackTeamId
|
|
6417
|
-
}
|
|
6418
|
-
} : {}
|
|
6419
|
-
}
|
|
6420
|
-
},
|
|
6421
|
-
...goal ? [
|
|
6422
|
-
{
|
|
6423
|
-
type: "goal.set",
|
|
6424
|
-
payload: {
|
|
6425
|
-
goalId: goal.id,
|
|
6426
|
-
text: goal.text,
|
|
6427
|
-
...goal.successCriteria ? { successCriteria: goal.successCriteria } : {},
|
|
6428
|
-
version: goal.version,
|
|
6429
|
-
actor: "scheduled_task",
|
|
6430
|
-
replaced: false
|
|
6431
|
-
}
|
|
6432
|
-
}
|
|
6433
|
-
] : [],
|
|
6434
|
-
{ type: "session.status.changed", payload: { status: session.status } }
|
|
6435
|
-
]);
|
|
6436
|
-
const scheduledUpdate = await addSessionSystemUpdateWithSourceMutation(
|
|
6437
|
-
db,
|
|
6438
|
-
{
|
|
6439
|
-
accountId: task.accountId,
|
|
6440
|
-
workspaceId: task.workspaceId,
|
|
6441
|
-
sessionId: session.id,
|
|
6442
|
-
kind: "scheduled_occurrence",
|
|
6443
|
-
classification: "info",
|
|
6444
|
-
sourceId: run.id,
|
|
6445
|
-
dedupeKey: `scheduled-wake:${run.id}`,
|
|
6446
|
-
summary: task.agentConfig.prompt,
|
|
6447
|
-
payload: scheduledUserMessagePayload(
|
|
6448
|
-
task.agentConfig.prompt,
|
|
6449
|
-
task.agentConfig.resources,
|
|
6450
|
-
taskTools,
|
|
6451
|
-
task.id,
|
|
6452
|
-
run.id
|
|
6453
|
-
),
|
|
6454
|
-
lineage: {
|
|
6455
|
-
scheduledTaskId: task.id,
|
|
6456
|
-
scheduledTaskRunId: run.id,
|
|
6457
|
-
...taskXaiProviderAccountAuthoritySnapshot.scope === "user" && task.createdBy.kind === "subject" ? { xaiAuthoritySubjectId: task.createdBy.subjectId } : {}
|
|
6458
|
-
},
|
|
6459
|
-
personalConnectionDelegations: taskPersonalConnectionDelegations,
|
|
6460
|
-
xaiProviderAccountAuthoritySnapshot: taskXaiProviderAccountAuthoritySnapshot
|
|
7819
|
+
eventType: "agent_run.created",
|
|
7820
|
+
quantity: 1,
|
|
7821
|
+
unit: "run",
|
|
7822
|
+
sourceResourceType: "scheduled_task_run",
|
|
7823
|
+
sourceResourceId: run.id,
|
|
7824
|
+
sessionId: run.sessionId,
|
|
7825
|
+
initiator: acceptedExecution.triggerInitiator,
|
|
7826
|
+
initiatorContext: {
|
|
7827
|
+
scheduledTaskId: task.id,
|
|
7828
|
+
scheduledTaskRunId: run.id
|
|
6461
7829
|
},
|
|
6462
|
-
|
|
6463
|
-
|
|
6464
|
-
|
|
6465
|
-
|
|
6466
|
-
|
|
6467
|
-
|
|
6468
|
-
|
|
6469
|
-
|
|
6470
|
-
|
|
6471
|
-
|
|
6472
|
-
);
|
|
6473
|
-
|
|
6474
|
-
|
|
6475
|
-
}
|
|
6476
|
-
if (scheduledUpdate.added && scheduledUpdate.events.length > 0) {
|
|
6477
|
-
await publishDurableSessionEvents(
|
|
6478
|
-
bus,
|
|
6479
|
-
task.workspaceId,
|
|
6480
|
-
session.id,
|
|
6481
|
-
scheduledUpdate.events
|
|
6482
|
-
);
|
|
6483
|
-
}
|
|
6484
|
-
result = {
|
|
6485
|
-
action: "start",
|
|
7830
|
+
origin: "scheduled_task",
|
|
7831
|
+
idempotencyKey: acceptedExecution.agentRunUsageIdempotencyKey ?? `usage:agent_run.created:scheduled:${run.id}`
|
|
7832
|
+
});
|
|
7833
|
+
const workflowId = workflowIdForSession(run.sessionId);
|
|
7834
|
+
const workflowWakeRevision = await enqueueSessionWorkflowWakeIfRunnable(dispatchDb, {
|
|
7835
|
+
accountId: task.accountId,
|
|
7836
|
+
workspaceId: task.workspaceId,
|
|
7837
|
+
sessionId: run.sessionId,
|
|
7838
|
+
temporalWorkflowId: workflowId,
|
|
7839
|
+
reason: "scheduled_retry"
|
|
7840
|
+
});
|
|
7841
|
+
const result3 = {
|
|
7842
|
+
action: task.runMode === "new_session_per_run" ? "start" : "signal",
|
|
6486
7843
|
accountId: task.accountId,
|
|
6487
7844
|
workspaceId: task.workspaceId,
|
|
6488
|
-
sessionId:
|
|
6489
|
-
triggerEventId:
|
|
7845
|
+
sessionId: run.sessionId,
|
|
7846
|
+
triggerEventId: run.triggerEventId,
|
|
6490
7847
|
workflowId,
|
|
6491
|
-
workflowWakeRevision
|
|
7848
|
+
workflowWakeRevision
|
|
6492
7849
|
};
|
|
6493
|
-
|
|
6494
|
-
|
|
6495
|
-
|
|
6496
|
-
|
|
6497
|
-
|
|
6498
|
-
|
|
7850
|
+
return { kind: "complete", result: result3, run };
|
|
7851
|
+
}
|
|
7852
|
+
let result2;
|
|
7853
|
+
try {
|
|
7854
|
+
if (incidentPreflightRequired) {
|
|
7855
|
+
const exactAuthority = await requireScheduledTaskIncidentAuthorityInTransaction(
|
|
7856
|
+
dispatchDb,
|
|
7857
|
+
{
|
|
7858
|
+
workspaceId: task.workspaceId,
|
|
7859
|
+
taskId: task.id
|
|
7860
|
+
}
|
|
6499
7861
|
);
|
|
7862
|
+
if (stableJson({
|
|
7863
|
+
task: exactAuthority.task,
|
|
7864
|
+
personalConnectionDelegations: exactAuthority.personalConnectionDelegations
|
|
7865
|
+
}) !== stableJson({
|
|
7866
|
+
task,
|
|
7867
|
+
personalConnectionDelegations: taskPersonalConnectionDelegations
|
|
7868
|
+
})) {
|
|
7869
|
+
throw new IncidentTelemetryPreflightBlockedError(
|
|
7870
|
+
"incident_preflight_metadata_missing"
|
|
7871
|
+
);
|
|
7872
|
+
}
|
|
6500
7873
|
}
|
|
6501
|
-
|
|
6502
|
-
|
|
6503
|
-
|
|
6504
|
-
);
|
|
7874
|
+
const goalSpec = task.agentConfig.goal ?? null;
|
|
7875
|
+
const existingSessionId = task.runMode === "existing_session" ? task.targetSessionId : task.reusableSessionId;
|
|
7876
|
+
if (task.runMode === "existing_session" && !existingSessionId) {
|
|
7877
|
+
throw new Error("scheduled task target session is unavailable");
|
|
6505
7878
|
}
|
|
6506
|
-
|
|
6507
|
-
|
|
6508
|
-
|
|
6509
|
-
|
|
6510
|
-
|
|
6511
|
-
|
|
6512
|
-
|
|
6513
|
-
|
|
6514
|
-
|
|
6515
|
-
|
|
6516
|
-
|
|
6517
|
-
|
|
6518
|
-
|
|
6519
|
-
|
|
6520
|
-
|
|
6521
|
-
|
|
6522
|
-
|
|
6523
|
-
|
|
6524
|
-
|
|
6525
|
-
|
|
6526
|
-
|
|
6527
|
-
|
|
6528
|
-
|
|
6529
|
-
|
|
6530
|
-
|
|
6531
|
-
|
|
6532
|
-
|
|
7879
|
+
if (task.runMode === "new_session_per_run" || task.runMode === "reusable_session" && !existingSessionId) {
|
|
7880
|
+
const variableSet = acceptedVariableSet;
|
|
7881
|
+
if (task.variableSetId && !variableSet) {
|
|
7882
|
+
throw new Error(`variable set not found: ${task.variableSetId}`);
|
|
7883
|
+
}
|
|
7884
|
+
let frozenRigId = null;
|
|
7885
|
+
let frozenRigVersionId = null;
|
|
7886
|
+
if (task.rigId) {
|
|
7887
|
+
const rig = acceptedRig;
|
|
7888
|
+
if (!rig || !rig.activeVersion) {
|
|
7889
|
+
throw new Error(`rig has no active version to bind: ${task.rigId}`);
|
|
7890
|
+
}
|
|
7891
|
+
frozenRigId = rig.id;
|
|
7892
|
+
frozenRigVersionId = runPersonalResourceAuthority?.resources.find(
|
|
7893
|
+
(resource) => resource.resourceKind === "rig" && resource.resourceId === rig.id
|
|
7894
|
+
)?.resourceVersionId ?? rig.activeVersion.id;
|
|
7895
|
+
if (frozenRigVersionId !== rig.activeVersion.id) {
|
|
7896
|
+
throw new Error("scheduled run rig version changed during admission");
|
|
7897
|
+
}
|
|
7898
|
+
}
|
|
7899
|
+
let session;
|
|
7900
|
+
let sessionCreated = true;
|
|
7901
|
+
const taskMetadata = { ...task.agentConfig.metadata };
|
|
7902
|
+
delete taskMetadata[OPENGENI_SLACK_BOT_SESSION_METADATA_KEY];
|
|
7903
|
+
try {
|
|
7904
|
+
const sessionInput = {
|
|
7905
|
+
accountId: task.accountId,
|
|
7906
|
+
workspaceId: task.workspaceId,
|
|
7907
|
+
initialMessage: task.agentConfig.prompt,
|
|
7908
|
+
resources: task.agentConfig.resources,
|
|
7909
|
+
tools: taskTools,
|
|
7910
|
+
firstPartyMcpTools,
|
|
7911
|
+
firstPartyMcpPermissions: [...DEFAULT_FIRST_PARTY_MCP_PERMISSIONS],
|
|
7912
|
+
metadata: {
|
|
7913
|
+
...taskMetadata,
|
|
7914
|
+
model,
|
|
7915
|
+
reasoningEffort,
|
|
7916
|
+
scheduledTaskId: task.id,
|
|
7917
|
+
scheduledTaskRunMode: task.runMode,
|
|
7918
|
+
...goalSpec ? { scheduledTaskGoal: goalSpec } : {},
|
|
7919
|
+
...run ? { scheduledTaskRunId: run.id } : {},
|
|
7920
|
+
...slackBotConnection ? {
|
|
7921
|
+
[OPENGENI_SLACK_BOT_SESSION_METADATA_KEY]: slackBotConnection.id
|
|
7922
|
+
} : {}
|
|
7923
|
+
},
|
|
7924
|
+
createdBy: {
|
|
7925
|
+
kind: "service",
|
|
7926
|
+
subjectId: "scheduler",
|
|
7927
|
+
label: "OpenGeni scheduler"
|
|
7928
|
+
},
|
|
7929
|
+
createdByContext: {
|
|
7930
|
+
scheduledTaskId: task.id,
|
|
7931
|
+
...run ? { scheduledTaskRunId: run.id } : {}
|
|
7932
|
+
},
|
|
7933
|
+
model,
|
|
7934
|
+
reasoningEffort,
|
|
7935
|
+
latencyMode: "standard",
|
|
7936
|
+
sandboxBackend,
|
|
7937
|
+
sandboxOs: "linux",
|
|
7938
|
+
variableSetId: task.variableSetId ?? null,
|
|
7939
|
+
rigId: frozenRigId,
|
|
7940
|
+
rigVersionId: frozenRigVersionId,
|
|
7941
|
+
personalConnectionDelegations: [],
|
|
7942
|
+
initialXaiProviderAccountAuthoritySnapshot: taskXaiProviderAccountAuthoritySnapshot,
|
|
7943
|
+
maxNestedAgentDepthOverride: task.agentConfig.maxNestedAgentDepth ?? null,
|
|
7944
|
+
frozenNestedAgentDepthPolicy: {
|
|
7945
|
+
effectiveMaxNestedAgentDepth: generatedSessionBinding.effectiveMaxNestedAgentDepth,
|
|
7946
|
+
nestedAgentDepthPolicySource: generatedSessionBinding.nestedAgentDepthPolicySource
|
|
7947
|
+
},
|
|
7948
|
+
frozenCodexCompactionMode: generatedSessionBinding.codexCompactionMode,
|
|
7949
|
+
// The durable agent config was privilege-checked when the task
|
|
7950
|
+
// was created/updated. Preserve that explicit policy if a broader
|
|
7951
|
+
// workspace/deployment limit is narrowed before a later fire.
|
|
7952
|
+
allowNestedAgentDepthIncrease: true,
|
|
7953
|
+
subjectId: `scheduled_task:${task.id}`,
|
|
7954
|
+
beforeCreateCommit: async (tx, sessionId) => {
|
|
7955
|
+
if (incidentPreflightRequired) {
|
|
7956
|
+
const exactSession = await requireSession(tx, task.workspaceId, sessionId);
|
|
7957
|
+
const exactResponder = await resolveIncidentTelemetryResponderMetadata({
|
|
7958
|
+
db: tx,
|
|
7959
|
+
settings,
|
|
7960
|
+
task: acceptedExecution.task,
|
|
7961
|
+
session: exactSession,
|
|
7962
|
+
personalConnectionDelegations: acceptedExecution.personalConnectionDelegations,
|
|
7963
|
+
personalResourceAuthoritySubjectId: acceptedExecution.personalResourceAuthoritySubjectId,
|
|
7964
|
+
executionPolicy: {
|
|
7965
|
+
tools: exactSession.tools,
|
|
7966
|
+
firstPartyMcpTools: exactSession.firstPartyMcpTools,
|
|
7967
|
+
firstPartyMcpPermissions: exactSession.firstPartyMcpPermissions,
|
|
7968
|
+
variableSetId: exactSession.variableSetId,
|
|
7969
|
+
rigId: exactSession.rigId,
|
|
7970
|
+
rigVersionId: exactSession.rigVersionId,
|
|
7971
|
+
toolPolicy: exactSession.toolPolicy,
|
|
7972
|
+
mcpServerIds: exactSession.mcpServers.map((server) => server.id),
|
|
7973
|
+
toolPolicyVersion: exactSession.toolPolicyVersion
|
|
6533
7974
|
}
|
|
7975
|
+
});
|
|
7976
|
+
const exactPreflight = evaluateIncidentTelemetryPreflight({
|
|
7977
|
+
agentConfig: acceptedExecution.task.agentConfig,
|
|
7978
|
+
incidentTriggered: structuredAlertOccurrence !== null,
|
|
7979
|
+
alertOccurrenceLabels: structuredAlertOccurrence?.labels ?? null,
|
|
7980
|
+
responder: exactResponder
|
|
7981
|
+
});
|
|
7982
|
+
if (exactPreflight.action === "blocked") {
|
|
7983
|
+
throw new IncidentTelemetryPreflightBlockedError(exactPreflight.reason);
|
|
6534
7984
|
}
|
|
6535
|
-
|
|
6536
|
-
|
|
6537
|
-
|
|
6538
|
-
|
|
6539
|
-
|
|
6540
|
-
|
|
6541
|
-
|
|
6542
|
-
|
|
6543
|
-
|
|
6544
|
-
|
|
6545
|
-
|
|
7985
|
+
}
|
|
7986
|
+
await bindScheduledTaskRunSessionInTransaction(tx, {
|
|
7987
|
+
accountId: task.accountId,
|
|
7988
|
+
workspaceId: task.workspaceId,
|
|
7989
|
+
runId: run.id,
|
|
7990
|
+
sessionId
|
|
7991
|
+
});
|
|
7992
|
+
}
|
|
7993
|
+
};
|
|
7994
|
+
const created = await createSessionWithIdempotencyKeyResult(dispatchDb, {
|
|
7995
|
+
...sessionInput,
|
|
7996
|
+
createIdempotencyKey: generatedSessionBinding.createIdempotencyKey
|
|
7997
|
+
});
|
|
7998
|
+
if (created.denied) {
|
|
7999
|
+
throw new SessionSpawnDeniedDbError(created.denial);
|
|
8000
|
+
}
|
|
8001
|
+
session = created.session;
|
|
8002
|
+
sessionCreated = created.created;
|
|
8003
|
+
if (!sessionCreated) {
|
|
8004
|
+
await bindScheduledTaskRunSessionInTransaction(dispatchDb, {
|
|
8005
|
+
accountId: task.accountId,
|
|
8006
|
+
workspaceId: task.workspaceId,
|
|
8007
|
+
runId: run.id,
|
|
8008
|
+
sessionId: session.id
|
|
8009
|
+
});
|
|
8010
|
+
run = { ...run, sessionId: session.id };
|
|
8011
|
+
}
|
|
8012
|
+
} catch (error) {
|
|
8013
|
+
if (error instanceof IncidentTelemetryPreflightBlockedError) {
|
|
8014
|
+
throw error;
|
|
8015
|
+
}
|
|
8016
|
+
if (error instanceof SessionSpawnDeniedDbError) {
|
|
8017
|
+
throw new Error(`${error.denial.code}: denial=${error.denial.id}`, {
|
|
8018
|
+
cause: error
|
|
8019
|
+
});
|
|
8020
|
+
}
|
|
8021
|
+
throw error;
|
|
8022
|
+
}
|
|
8023
|
+
const scheduledRun = run;
|
|
8024
|
+
if (!sessionCreated) {
|
|
8025
|
+
assertReusableSessionRevivable(session.status);
|
|
8026
|
+
if ((session.variableSetId ?? null) !== (task.variableSetId ?? null)) {
|
|
8027
|
+
throw new Error(
|
|
8028
|
+
"scheduled alert occurrence variableSet attachment does not match its canonical session"
|
|
8029
|
+
);
|
|
8030
|
+
}
|
|
8031
|
+
if (scheduledSlackBotConnectionId(session.metadata) !== (task.agentConfig.slackBotConnectionId ?? null)) {
|
|
8032
|
+
throw new Error(
|
|
8033
|
+
"scheduled alert occurrence OpenGeni Slack bot binding does not match its canonical session"
|
|
8034
|
+
);
|
|
8035
|
+
}
|
|
8036
|
+
}
|
|
8037
|
+
let workflowId;
|
|
8038
|
+
if (alertOccurrence) {
|
|
8039
|
+
const started = await initializeSessionStartAtomically(dispatchDb, {
|
|
8040
|
+
accountId: task.accountId,
|
|
8041
|
+
workspaceId: task.workspaceId,
|
|
8042
|
+
sessionId: session.id,
|
|
8043
|
+
reasoningEffortFallback: reasoningEffort,
|
|
8044
|
+
createdEventPayload: {
|
|
8045
|
+
scheduledTaskId: typeof session.metadata.scheduledTaskId === "string" ? session.metadata.scheduledTaskId : task.id,
|
|
8046
|
+
scheduledTaskRunId: typeof session.metadata.scheduledTaskRunId === "string" ? session.metadata.scheduledTaskRunId : run.id,
|
|
8047
|
+
// Names/ids only; never values.
|
|
8048
|
+
...variableSet ? {
|
|
8049
|
+
variableSetId: variableSet.id,
|
|
8050
|
+
variableSetName: variableSet.name
|
|
8051
|
+
} : {},
|
|
8052
|
+
...slackBotConnection && slackBotMetadata ? {
|
|
8053
|
+
slackBotConnection: {
|
|
8054
|
+
credentialRole: OPENGENI_SLACK_BOT_CREDENTIAL_ROLE,
|
|
8055
|
+
credentialLabel: OPENGENI_SLACK_BOT_CREDENTIAL_LABEL,
|
|
8056
|
+
connectionId: slackBotConnection.id,
|
|
8057
|
+
slackTeamId: slackBotMetadata.slackTeamId
|
|
8058
|
+
}
|
|
8059
|
+
} : {}
|
|
8060
|
+
},
|
|
8061
|
+
goal: goalSpec ? {
|
|
8062
|
+
text: goalSpec.text,
|
|
8063
|
+
successCriteria: goalSpec.successCriteria ?? null,
|
|
8064
|
+
maxAutoContinuations: goalSpec.maxAutoContinuations ?? null,
|
|
8065
|
+
...goalSpec.mutationPolicy ? { mutationPolicy: goalSpec.mutationPolicy } : {},
|
|
8066
|
+
createdBy: "scheduled_task"
|
|
8067
|
+
} : null,
|
|
8068
|
+
deferInitialTurn: true,
|
|
8069
|
+
deferredStatus: "queued"
|
|
8070
|
+
});
|
|
8071
|
+
workflowId = started.temporalWorkflowId;
|
|
8072
|
+
if (started.events.length > 0) {
|
|
8073
|
+
if (deferPublications) {
|
|
8074
|
+
deferredEvents.push({
|
|
8075
|
+
sessionId: session.id,
|
|
8076
|
+
events: started.events
|
|
8077
|
+
});
|
|
8078
|
+
} else {
|
|
8079
|
+
await publishDurableSessionEvents(
|
|
8080
|
+
bus,
|
|
8081
|
+
task.workspaceId,
|
|
8082
|
+
session.id,
|
|
8083
|
+
started.events
|
|
8084
|
+
);
|
|
8085
|
+
}
|
|
8086
|
+
}
|
|
8087
|
+
} else {
|
|
8088
|
+
const started = await initializeSessionStartAtomically(dispatchDb, {
|
|
8089
|
+
accountId: task.accountId,
|
|
8090
|
+
workspaceId: task.workspaceId,
|
|
8091
|
+
sessionId: session.id,
|
|
8092
|
+
reasoningEffortFallback: reasoningEffort,
|
|
8093
|
+
createdEventPayload: {
|
|
8094
|
+
scheduledTaskId: typeof session.metadata.scheduledTaskId === "string" ? session.metadata.scheduledTaskId : task.id,
|
|
8095
|
+
scheduledTaskRunId: typeof session.metadata.scheduledTaskRunId === "string" ? session.metadata.scheduledTaskRunId : run.id,
|
|
8096
|
+
// Names/ids only; never values.
|
|
8097
|
+
...variableSet ? { variableSetId: variableSet.id, variableSetName: variableSet.name } : {},
|
|
8098
|
+
...slackBotConnection && slackBotMetadata ? {
|
|
8099
|
+
slackBotConnection: {
|
|
8100
|
+
credentialRole: OPENGENI_SLACK_BOT_CREDENTIAL_ROLE,
|
|
8101
|
+
credentialLabel: OPENGENI_SLACK_BOT_CREDENTIAL_LABEL,
|
|
8102
|
+
connectionId: slackBotConnection.id,
|
|
8103
|
+
slackTeamId: slackBotMetadata.slackTeamId
|
|
8104
|
+
}
|
|
8105
|
+
} : {}
|
|
8106
|
+
},
|
|
8107
|
+
goal: goalSpec ? {
|
|
8108
|
+
text: goalSpec.text,
|
|
8109
|
+
successCriteria: goalSpec.successCriteria ?? null,
|
|
8110
|
+
maxAutoContinuations: goalSpec.maxAutoContinuations ?? null,
|
|
8111
|
+
...goalSpec.mutationPolicy ? { mutationPolicy: goalSpec.mutationPolicy } : {},
|
|
8112
|
+
createdBy: "scheduled_task"
|
|
8113
|
+
} : null,
|
|
8114
|
+
deferInitialTurn: true,
|
|
8115
|
+
deferredStatus: "queued"
|
|
8116
|
+
});
|
|
8117
|
+
workflowId = started.temporalWorkflowId;
|
|
8118
|
+
if (started.events.length > 0) {
|
|
8119
|
+
if (deferPublications) {
|
|
8120
|
+
deferredEvents.push({ sessionId: session.id, events: started.events });
|
|
8121
|
+
} else {
|
|
8122
|
+
await publishDurableSessionEvents(
|
|
8123
|
+
bus,
|
|
8124
|
+
task.workspaceId,
|
|
8125
|
+
session.id,
|
|
8126
|
+
started.events
|
|
8127
|
+
);
|
|
8128
|
+
}
|
|
8129
|
+
}
|
|
8130
|
+
if (task.runMode === "reusable_session") {
|
|
8131
|
+
await materializeScheduledTaskReusableSessionFromRun(dispatchDb, {
|
|
8132
|
+
accountId: task.accountId,
|
|
8133
|
+
workspaceId: task.workspaceId,
|
|
8134
|
+
taskId: task.id,
|
|
8135
|
+
runId: run.id,
|
|
8136
|
+
sessionId: session.id,
|
|
8137
|
+
sourceTaskAuthorityRevision: runPersonalResourceAuthority?.taskAuthorityRevision ?? run.taskAuthorityRevision,
|
|
8138
|
+
sourceExecutionDigest: runPersonalResourceAuthority?.executionDigest ?? run.taskExecutionDigest
|
|
8139
|
+
});
|
|
8140
|
+
}
|
|
8141
|
+
}
|
|
8142
|
+
const titleEvents = await stampScheduledSessionTitle(dispatchDb, {
|
|
6546
8143
|
workspaceId: task.workspaceId,
|
|
6547
|
-
|
|
6548
|
-
|
|
6549
|
-
|
|
6550
|
-
|
|
6551
|
-
|
|
6552
|
-
|
|
6553
|
-
|
|
6554
|
-
task.
|
|
6555
|
-
|
|
6556
|
-
|
|
6557
|
-
|
|
6558
|
-
|
|
6559
|
-
|
|
6560
|
-
|
|
6561
|
-
|
|
6562
|
-
|
|
6563
|
-
|
|
8144
|
+
session,
|
|
8145
|
+
taskName: task.name
|
|
8146
|
+
});
|
|
8147
|
+
if (titleEvents.length > 0) {
|
|
8148
|
+
if (deferPublications) {
|
|
8149
|
+
deferredEvents.push({ sessionId: session.id, events: titleEvents });
|
|
8150
|
+
} else {
|
|
8151
|
+
await publishDurableSessionEvents(bus, task.workspaceId, session.id, titleEvents);
|
|
8152
|
+
}
|
|
8153
|
+
}
|
|
8154
|
+
const scheduledUpdate = await addSessionSystemUpdateWithSourceMutation(
|
|
8155
|
+
dispatchDb,
|
|
8156
|
+
{
|
|
8157
|
+
accountId: task.accountId,
|
|
8158
|
+
workspaceId: task.workspaceId,
|
|
8159
|
+
sessionId: session.id,
|
|
8160
|
+
kind: "scheduled_occurrence",
|
|
8161
|
+
classification: "info",
|
|
8162
|
+
sourceId: run.id,
|
|
8163
|
+
dedupeKey: `scheduled-task-run:${run.id}`,
|
|
8164
|
+
summary: task.agentConfig.prompt,
|
|
8165
|
+
payload: scheduledUserMessagePayload(
|
|
8166
|
+
task.agentConfig.prompt,
|
|
8167
|
+
task.agentConfig.resources,
|
|
8168
|
+
taskTools,
|
|
8169
|
+
task.id,
|
|
8170
|
+
run.id
|
|
8171
|
+
),
|
|
8172
|
+
lineage: {
|
|
8173
|
+
scheduledTaskId: task.id,
|
|
8174
|
+
scheduledTaskRunId: run.id,
|
|
8175
|
+
...causalHumanSubjectId ? { causalHumanSubjectId } : {},
|
|
8176
|
+
...taskConnectionAuthoritySubjectId ? { connectionAuthoritySubjectId: taskConnectionAuthoritySubjectId } : {},
|
|
8177
|
+
...taskXaiProviderAccountAuthoritySnapshot.scope === "user" && task.createdBy.kind === "subject" ? { xaiAuthoritySubjectId: task.createdBy.subjectId } : {}
|
|
8178
|
+
},
|
|
8179
|
+
personalConnectionDelegations: taskPersonalConnectionDelegations,
|
|
8180
|
+
xaiProviderAccountAuthoritySnapshot: taskXaiProviderAccountAuthoritySnapshot,
|
|
8181
|
+
scheduledTaskRunId: run.id
|
|
6564
8182
|
},
|
|
6565
|
-
|
|
6566
|
-
|
|
6567
|
-
|
|
6568
|
-
|
|
6569
|
-
|
|
6570
|
-
|
|
8183
|
+
async (tx, wakeEventId) => {
|
|
8184
|
+
if (!wakeEventId) {
|
|
8185
|
+
await settleScheduledTaskRunInTransaction(tx, {
|
|
8186
|
+
workspaceId: task.workspaceId,
|
|
8187
|
+
runId: scheduledRun.id,
|
|
8188
|
+
sessionId: session.id,
|
|
8189
|
+
triggerEventId: null,
|
|
8190
|
+
status: "skipped",
|
|
8191
|
+
error: "session_cancelled"
|
|
8192
|
+
});
|
|
8193
|
+
return;
|
|
8194
|
+
}
|
|
8195
|
+
await settleScheduledTaskRunInTransaction(tx, {
|
|
6571
8196
|
workspaceId: task.workspaceId,
|
|
8197
|
+
runId: scheduledRun.id,
|
|
8198
|
+
sessionId: session.id,
|
|
8199
|
+
triggerEventId: wakeEventId,
|
|
8200
|
+
status: "dispatched"
|
|
8201
|
+
});
|
|
8202
|
+
},
|
|
8203
|
+
incidentPreflightRequired ? {
|
|
8204
|
+
prepareSource: async (tx, sessionId) => await prepareIncidentTelemetrySource({
|
|
8205
|
+
tx,
|
|
8206
|
+
settings,
|
|
6572
8207
|
taskId: task.id,
|
|
6573
|
-
|
|
8208
|
+
workspaceId: task.workspaceId,
|
|
8209
|
+
sessionId,
|
|
8210
|
+
alertOccurrenceLabels: structuredAlertOccurrence?.labels ?? null,
|
|
8211
|
+
acceptedExecution
|
|
8212
|
+
})
|
|
8213
|
+
} : void 0
|
|
8214
|
+
);
|
|
8215
|
+
if (scheduledUpdate.reason === "session_cancelled") {
|
|
8216
|
+
return {
|
|
8217
|
+
kind: "blocked",
|
|
8218
|
+
result: { action: "blocked", reason: "scheduled_run_terminal" },
|
|
8219
|
+
run: { ...run, status: "skipped", error: "session_cancelled" }
|
|
8220
|
+
};
|
|
8221
|
+
}
|
|
8222
|
+
if (scheduledUpdate.added && scheduledUpdate.events.length > 0) {
|
|
8223
|
+
if (deferPublications) {
|
|
8224
|
+
deferredEvents.push({
|
|
8225
|
+
sessionId: session.id,
|
|
8226
|
+
events: scheduledUpdate.events
|
|
6574
8227
|
});
|
|
8228
|
+
} else {
|
|
8229
|
+
await publishDurableSessionEvents(
|
|
8230
|
+
bus,
|
|
8231
|
+
task.workspaceId,
|
|
8232
|
+
session.id,
|
|
8233
|
+
scheduledUpdate.events
|
|
8234
|
+
);
|
|
6575
8235
|
}
|
|
6576
|
-
|
|
6577
|
-
|
|
8236
|
+
}
|
|
8237
|
+
result2 = {
|
|
8238
|
+
action: sessionCreated ? "start" : "signal",
|
|
8239
|
+
accountId: task.accountId,
|
|
8240
|
+
workspaceId: task.workspaceId,
|
|
8241
|
+
sessionId: session.id,
|
|
8242
|
+
triggerEventId: scheduledUpdate.wakeEventId,
|
|
8243
|
+
workflowId,
|
|
8244
|
+
workflowWakeRevision: scheduledUpdate.workflowWakeRevision
|
|
8245
|
+
};
|
|
8246
|
+
} else {
|
|
8247
|
+
if (!run) throw new Error("scheduled task run was not created");
|
|
8248
|
+
const scheduledRun = run;
|
|
8249
|
+
const session = await requireSession(dispatchDb, task.workspaceId, existingSessionId);
|
|
8250
|
+
await bindScheduledTaskRunSessionInTransaction(dispatchDb, {
|
|
8251
|
+
accountId: task.accountId,
|
|
8252
|
+
workspaceId: task.workspaceId,
|
|
8253
|
+
runId: run.id,
|
|
8254
|
+
sessionId: session.id
|
|
8255
|
+
});
|
|
8256
|
+
assertReusableSessionRevivable(session.status);
|
|
8257
|
+
assertReusableSessionBindingMatches(session, task);
|
|
8258
|
+
const goalEvents = task.runMode === "reusable_session" && goalSpec && run.status === "queued" ? await upsertScheduledSessionGoalForRun(dispatchDb, {
|
|
8259
|
+
accountId: task.accountId,
|
|
8260
|
+
workspaceId: task.workspaceId,
|
|
8261
|
+
sessionId: session.id,
|
|
8262
|
+
runId: run.id,
|
|
8263
|
+
text: goalSpec.text,
|
|
8264
|
+
successCriteria: goalSpec.successCriteria ?? null,
|
|
8265
|
+
maxAutoContinuations: goalSpec.maxAutoContinuations ?? null,
|
|
8266
|
+
...goalSpec.mutationPolicy ? { mutationPolicy: goalSpec.mutationPolicy } : {}
|
|
8267
|
+
}) : [];
|
|
8268
|
+
if (goalEvents.length > 0) {
|
|
8269
|
+
if (deferPublications) {
|
|
8270
|
+
deferredEvents.push({
|
|
8271
|
+
sessionId: session.id,
|
|
8272
|
+
events: goalEvents
|
|
8273
|
+
});
|
|
8274
|
+
} else {
|
|
8275
|
+
await publishDurableSessionEvents(bus, task.workspaceId, session.id, goalEvents);
|
|
8276
|
+
}
|
|
8277
|
+
}
|
|
8278
|
+
const bundled = await addSessionSystemUpdateWithSourceMutation(
|
|
8279
|
+
dispatchDb,
|
|
8280
|
+
{
|
|
8281
|
+
accountId: task.accountId,
|
|
6578
8282
|
workspaceId: task.workspaceId,
|
|
6579
|
-
runId: run.id,
|
|
6580
8283
|
sessionId: session.id,
|
|
6581
|
-
|
|
6582
|
-
|
|
6583
|
-
|
|
8284
|
+
kind: "scheduled_occurrence",
|
|
8285
|
+
classification: "info",
|
|
8286
|
+
sourceId: run.id,
|
|
8287
|
+
dedupeKey: `scheduled-task-run:${run.id}`,
|
|
8288
|
+
summary: task.agentConfig.prompt,
|
|
8289
|
+
payload: scheduledUserMessagePayload(
|
|
8290
|
+
task.agentConfig.prompt,
|
|
8291
|
+
task.agentConfig.resources,
|
|
8292
|
+
taskTools,
|
|
8293
|
+
task.id,
|
|
8294
|
+
run.id
|
|
8295
|
+
),
|
|
8296
|
+
lineage: {
|
|
8297
|
+
scheduledTaskId: task.id,
|
|
8298
|
+
scheduledTaskRunId: run.id,
|
|
8299
|
+
...causalHumanSubjectId ? { causalHumanSubjectId } : {},
|
|
8300
|
+
...taskConnectionAuthoritySubjectId ? { connectionAuthoritySubjectId: taskConnectionAuthoritySubjectId } : {},
|
|
8301
|
+
...taskXaiProviderAccountAuthoritySnapshot.scope === "user" && task.createdBy.kind === "subject" ? { xaiAuthoritySubjectId: task.createdBy.subjectId } : {}
|
|
8302
|
+
},
|
|
8303
|
+
personalConnectionDelegations: taskPersonalConnectionDelegations,
|
|
8304
|
+
xaiProviderAccountAuthoritySnapshot: taskXaiProviderAccountAuthoritySnapshot,
|
|
8305
|
+
scheduledTaskRunId: run.id
|
|
8306
|
+
},
|
|
8307
|
+
async (tx, wakeEventId) => {
|
|
8308
|
+
if (!wakeEventId) {
|
|
8309
|
+
await settleScheduledTaskRunInTransaction(tx, {
|
|
8310
|
+
workspaceId: task.workspaceId,
|
|
8311
|
+
runId: scheduledRun.id,
|
|
8312
|
+
sessionId: session.id,
|
|
8313
|
+
triggerEventId: null,
|
|
8314
|
+
status: "skipped",
|
|
8315
|
+
error: "session_cancelled"
|
|
8316
|
+
});
|
|
8317
|
+
return;
|
|
8318
|
+
}
|
|
8319
|
+
await settleScheduledTaskRunInTransaction(tx, {
|
|
8320
|
+
workspaceId: task.workspaceId,
|
|
8321
|
+
runId: scheduledRun.id,
|
|
8322
|
+
sessionId: session.id,
|
|
8323
|
+
triggerEventId: wakeEventId,
|
|
8324
|
+
status: "dispatched"
|
|
8325
|
+
});
|
|
8326
|
+
},
|
|
8327
|
+
incidentPreflightRequired ? {
|
|
8328
|
+
prepareSource: async (tx, sessionId) => await prepareIncidentTelemetrySource({
|
|
8329
|
+
tx,
|
|
8330
|
+
settings,
|
|
8331
|
+
taskId: task.id,
|
|
8332
|
+
workspaceId: task.workspaceId,
|
|
8333
|
+
sessionId,
|
|
8334
|
+
alertOccurrenceLabels: structuredAlertOccurrence?.labels ?? null,
|
|
8335
|
+
acceptedExecution
|
|
8336
|
+
})
|
|
8337
|
+
} : void 0
|
|
8338
|
+
);
|
|
8339
|
+
if (bundled.reason === "session_cancelled") {
|
|
8340
|
+
return {
|
|
8341
|
+
kind: "blocked",
|
|
8342
|
+
result: { action: "blocked", reason: "scheduled_run_terminal" },
|
|
8343
|
+
run: { ...run, status: "skipped", error: "session_cancelled" }
|
|
8344
|
+
};
|
|
6584
8345
|
}
|
|
6585
|
-
|
|
6586
|
-
|
|
6587
|
-
|
|
6588
|
-
|
|
6589
|
-
|
|
6590
|
-
|
|
8346
|
+
if (bundled.added && bundled.events.length > 0) {
|
|
8347
|
+
if (deferPublications) {
|
|
8348
|
+
deferredEvents.push({
|
|
8349
|
+
sessionId: session.id,
|
|
8350
|
+
events: bundled.events
|
|
8351
|
+
});
|
|
8352
|
+
} else {
|
|
8353
|
+
await publishDurableSessionEvents(
|
|
8354
|
+
bus,
|
|
8355
|
+
task.workspaceId,
|
|
8356
|
+
session.id,
|
|
8357
|
+
bundled.events
|
|
8358
|
+
);
|
|
8359
|
+
}
|
|
8360
|
+
}
|
|
8361
|
+
result2 = {
|
|
8362
|
+
action: "signal",
|
|
8363
|
+
accountId: task.accountId,
|
|
8364
|
+
workspaceId: task.workspaceId,
|
|
8365
|
+
sessionId: session.id,
|
|
8366
|
+
triggerEventId: bundled.wakeEventId,
|
|
8367
|
+
workflowId: workflowIdForSession(session.id),
|
|
8368
|
+
workflowWakeRevision: bundled.workflowWakeRevision
|
|
8369
|
+
};
|
|
6591
8370
|
}
|
|
6592
|
-
|
|
6593
|
-
|
|
6594
|
-
|
|
6595
|
-
|
|
6596
|
-
|
|
6597
|
-
|
|
6598
|
-
|
|
6599
|
-
|
|
8371
|
+
} catch (error) {
|
|
8372
|
+
const terminalError = scheduledRunRecoveryTerminalError(error);
|
|
8373
|
+
if (!terminalError) throw error;
|
|
8374
|
+
await markScheduledTaskRunFailedIfQueued(
|
|
8375
|
+
dispatchDb,
|
|
8376
|
+
task.workspaceId,
|
|
8377
|
+
run.id,
|
|
8378
|
+
terminalError.code
|
|
8379
|
+
);
|
|
8380
|
+
return {
|
|
8381
|
+
kind: "blocked",
|
|
8382
|
+
result: scheduledRunTerminalResult(terminalError.code),
|
|
8383
|
+
run: { ...run, status: "failed", error: terminalError.code }
|
|
6600
8384
|
};
|
|
6601
8385
|
}
|
|
8386
|
+
await recordUsageEvent3(dispatchDb, {
|
|
8387
|
+
accountId: task.accountId,
|
|
8388
|
+
workspaceId: task.workspaceId,
|
|
8389
|
+
eventType: "agent_run.created",
|
|
8390
|
+
quantity: 1,
|
|
8391
|
+
unit: "run",
|
|
8392
|
+
sourceResourceType: "scheduled_task_run",
|
|
8393
|
+
sourceResourceId: run.id,
|
|
8394
|
+
sessionId: result2.sessionId,
|
|
8395
|
+
initiator: input.initiator ?? {
|
|
8396
|
+
kind: "service",
|
|
8397
|
+
subjectId: "scheduler"
|
|
8398
|
+
},
|
|
8399
|
+
initiatorContext: {
|
|
8400
|
+
scheduledTaskId: task.id,
|
|
8401
|
+
scheduledTaskRunId: run.id
|
|
8402
|
+
},
|
|
8403
|
+
origin: "scheduled_task",
|
|
8404
|
+
idempotencyKey: input.agentRunUsageIdempotencyKey ?? `usage:agent_run.created:scheduled:${run.id}`
|
|
8405
|
+
});
|
|
8406
|
+
return { kind: "complete", result: result2, run };
|
|
8407
|
+
};
|
|
8408
|
+
let dispatchOutcome;
|
|
8409
|
+
try {
|
|
8410
|
+
dispatchOutcome = incidentPreflightRequired ? await withSessionActivityRlsContext(
|
|
8411
|
+
db,
|
|
8412
|
+
{ accountId: task.accountId, workspaceId: task.workspaceId },
|
|
8413
|
+
async (tx) => await executeDispatch(tx, true)
|
|
8414
|
+
) : await executeDispatch(db, false);
|
|
6602
8415
|
} catch (error) {
|
|
6603
|
-
|
|
8416
|
+
if (error instanceof IncidentTelemetryPreflightBlockedError) {
|
|
8417
|
+
return { action: "blocked", reason: error.reason };
|
|
8418
|
+
}
|
|
8419
|
+
throw error;
|
|
8420
|
+
}
|
|
8421
|
+
if (dispatchOutcome.kind === "replay") {
|
|
8422
|
+
return await replayScheduledTaskDispatch({
|
|
6604
8423
|
db,
|
|
8424
|
+
wakeSessionWorkflow,
|
|
8425
|
+
task,
|
|
8426
|
+
run: dispatchOutcome.run,
|
|
8427
|
+
acceptedExecution: dispatchOutcome.acceptedExecution
|
|
8428
|
+
});
|
|
8429
|
+
}
|
|
8430
|
+
if (dispatchOutcome.kind === "blocked") return dispatchOutcome.result;
|
|
8431
|
+
for (const deferred of deferredEvents) {
|
|
8432
|
+
await publishDurableSessionEvents(
|
|
8433
|
+
bus,
|
|
6605
8434
|
task.workspaceId,
|
|
6606
|
-
|
|
6607
|
-
|
|
6608
|
-
)
|
|
6609
|
-
throw error;
|
|
8435
|
+
deferred.sessionId,
|
|
8436
|
+
deferred.events
|
|
8437
|
+
);
|
|
6610
8438
|
}
|
|
6611
|
-
|
|
6612
|
-
accountId: task.accountId,
|
|
6613
|
-
workspaceId: task.workspaceId,
|
|
6614
|
-
eventType: "agent_run.created",
|
|
6615
|
-
quantity: 1,
|
|
6616
|
-
unit: "run",
|
|
6617
|
-
sourceResourceType: "scheduled_task_run",
|
|
6618
|
-
sourceResourceId: run.id,
|
|
6619
|
-
sessionId: result.sessionId,
|
|
6620
|
-
initiator: input.initiator ?? { kind: "service", subjectId: "scheduler" },
|
|
6621
|
-
initiatorContext: { scheduledTaskId: task.id, scheduledTaskRunId: run.id },
|
|
6622
|
-
origin: "scheduled_task",
|
|
6623
|
-
idempotencyKey: input.agentRunUsageIdempotencyKey ?? `usage:agent_run.created:scheduled:${run.id}`
|
|
6624
|
-
});
|
|
8439
|
+
const result = dispatchOutcome.result;
|
|
6625
8440
|
if (wakeSessionWorkflow && result.workflowWakeRevision !== null) {
|
|
6626
8441
|
await wakeSessionWorkflow({
|
|
6627
8442
|
accountId: result.accountId,
|
|
@@ -6635,6 +8450,585 @@ function createScheduledTaskActivities(services) {
|
|
|
6635
8450
|
}
|
|
6636
8451
|
};
|
|
6637
8452
|
}
|
|
8453
|
+
var IncidentTelemetryPreflightBlockedError = class extends Error {
|
|
8454
|
+
constructor(reason) {
|
|
8455
|
+
super(reason);
|
|
8456
|
+
this.reason = reason;
|
|
8457
|
+
this.name = "IncidentTelemetryPreflightBlockedError";
|
|
8458
|
+
}
|
|
8459
|
+
};
|
|
8460
|
+
function assertReusableSessionBindingMatches(session, task) {
|
|
8461
|
+
if ((session.variableSetId ?? null) !== (task.variableSetId ?? null)) {
|
|
8462
|
+
throw new ScheduledRunTerminalAuthorityError(
|
|
8463
|
+
"scheduled_reusable_binding_changed",
|
|
8464
|
+
"scheduled task variableSet attachment does not match its reusable session"
|
|
8465
|
+
);
|
|
8466
|
+
}
|
|
8467
|
+
if (scheduledSlackBotConnectionId(session.metadata) !== (task.agentConfig.slackBotConnectionId ?? null)) {
|
|
8468
|
+
throw new ScheduledRunTerminalAuthorityError(
|
|
8469
|
+
"scheduled_reusable_binding_changed",
|
|
8470
|
+
"scheduled task OpenGeni Slack bot binding does not match its reusable session"
|
|
8471
|
+
);
|
|
8472
|
+
}
|
|
8473
|
+
}
|
|
8474
|
+
var ScheduledRunTerminalAuthorityError = class extends Error {
|
|
8475
|
+
constructor(code, message) {
|
|
8476
|
+
super(message);
|
|
8477
|
+
this.code = code;
|
|
8478
|
+
this.name = "ScheduledRunTerminalAuthorityError";
|
|
8479
|
+
}
|
|
8480
|
+
};
|
|
8481
|
+
function scheduledRunRecoveryTerminalError(error) {
|
|
8482
|
+
if (error instanceof Error && "cause" in error && error.cause !== void 0 && error.cause !== error) {
|
|
8483
|
+
const nested = scheduledRunRecoveryTerminalError(error.cause);
|
|
8484
|
+
if (nested) return nested;
|
|
8485
|
+
}
|
|
8486
|
+
if (error instanceof ScheduledRunTerminalAuthorityError) return error;
|
|
8487
|
+
if (error instanceof IncidentTelemetryPreflightBlockedError) {
|
|
8488
|
+
return new ScheduledRunTerminalAuthorityError(
|
|
8489
|
+
`scheduled_incident_${error.reason}`,
|
|
8490
|
+
error.message
|
|
8491
|
+
);
|
|
8492
|
+
}
|
|
8493
|
+
if (error instanceof SessionSpawnDeniedDbError) {
|
|
8494
|
+
return new ScheduledRunTerminalAuthorityError("scheduled_session_spawn_denied", error.message);
|
|
8495
|
+
}
|
|
8496
|
+
if (error instanceof Error && error.message.startsWith("Session not found:")) {
|
|
8497
|
+
return new ScheduledRunTerminalAuthorityError(
|
|
8498
|
+
"scheduled_target_session_unavailable",
|
|
8499
|
+
error.message
|
|
8500
|
+
);
|
|
8501
|
+
}
|
|
8502
|
+
if (typeof error === "object" && error !== null && "status" in error && error.status === 422) {
|
|
8503
|
+
return new ScheduledRunTerminalAuthorityError(
|
|
8504
|
+
"scheduled_slack_authority_unavailable",
|
|
8505
|
+
error instanceof Error ? error.message : "scheduled Slack authority is unavailable"
|
|
8506
|
+
);
|
|
8507
|
+
}
|
|
8508
|
+
if (typeof error === "object" && error !== null && "code" in error && error.code === "42501") {
|
|
8509
|
+
return new ScheduledRunTerminalAuthorityError(
|
|
8510
|
+
"scheduled_run_authority_proof_rejected",
|
|
8511
|
+
error instanceof Error ? error.message : "scheduled run authority proof was rejected"
|
|
8512
|
+
);
|
|
8513
|
+
}
|
|
8514
|
+
return null;
|
|
8515
|
+
}
|
|
8516
|
+
function scheduledRunTerminalResult(error) {
|
|
8517
|
+
if (error === "scheduled_authority_exhausted") {
|
|
8518
|
+
return { action: "blocked", reason: "scheduled_authority_exhausted" };
|
|
8519
|
+
}
|
|
8520
|
+
if (error === "scheduled_incident_incident_preflight_metadata_missing") {
|
|
8521
|
+
return { action: "blocked", reason: "incident_preflight_metadata_missing" };
|
|
8522
|
+
}
|
|
8523
|
+
if (error === "scheduled_incident_incident_responder_under_capable") {
|
|
8524
|
+
return { action: "blocked", reason: "incident_responder_under_capable" };
|
|
8525
|
+
}
|
|
8526
|
+
if (error === "scheduled_incident_incident_data_source_unsuitable") {
|
|
8527
|
+
return { action: "blocked", reason: "incident_data_source_unsuitable" };
|
|
8528
|
+
}
|
|
8529
|
+
return { action: "blocked", reason: "scheduled_run_terminal" };
|
|
8530
|
+
}
|
|
8531
|
+
async function prepareIncidentTelemetrySource(input) {
|
|
8532
|
+
const session = await requireSession(input.tx, input.workspaceId, input.sessionId);
|
|
8533
|
+
const { task, personalConnectionDelegations } = input.acceptedExecution ? {
|
|
8534
|
+
task: input.acceptedExecution.task,
|
|
8535
|
+
personalConnectionDelegations: input.acceptedExecution.personalConnectionDelegations
|
|
8536
|
+
} : await requireScheduledTaskIncidentAuthorityInTransaction(input.tx, {
|
|
8537
|
+
workspaceId: input.workspaceId,
|
|
8538
|
+
taskId: input.taskId
|
|
8539
|
+
});
|
|
8540
|
+
const responder = await resolveIncidentTelemetryResponderMetadata({
|
|
8541
|
+
db: input.tx,
|
|
8542
|
+
settings: input.settings,
|
|
8543
|
+
task,
|
|
8544
|
+
session,
|
|
8545
|
+
personalConnectionDelegations,
|
|
8546
|
+
personalResourceAuthoritySubjectId: input.acceptedExecution?.personalResourceAuthoritySubjectId ?? null,
|
|
8547
|
+
...input.acceptedExecution ? {
|
|
8548
|
+
executionPolicy: input.acceptedExecution.targetSessionExecution ? {
|
|
8549
|
+
tools: input.acceptedExecution.targetSessionExecution.tools,
|
|
8550
|
+
firstPartyMcpTools: input.acceptedExecution.targetSessionExecution.firstPartyMcpTools,
|
|
8551
|
+
firstPartyMcpPermissions: input.acceptedExecution.targetSessionExecution.firstPartyMcpPermissions,
|
|
8552
|
+
variableSetId: input.acceptedExecution.targetSessionExecution.variableSetId,
|
|
8553
|
+
rigId: input.acceptedExecution.targetSessionExecution.rigId,
|
|
8554
|
+
rigVersionId: input.acceptedExecution.targetSessionExecution.rigVersionId,
|
|
8555
|
+
toolPolicy: input.acceptedExecution.targetSessionExecution.toolPolicy,
|
|
8556
|
+
mcpServerIds: input.acceptedExecution.targetSessionExecution.mcpServerIds,
|
|
8557
|
+
toolPolicyVersion: input.acceptedExecution.targetSessionExecution.toolPolicyVersion
|
|
8558
|
+
} : {
|
|
8559
|
+
tools: input.acceptedExecution.resolvedTools,
|
|
8560
|
+
firstPartyMcpTools: input.acceptedExecution.resolvedFirstPartyMcpTools,
|
|
8561
|
+
firstPartyMcpPermissions: input.acceptedExecution.resolvedFirstPartyMcpPermissions,
|
|
8562
|
+
variableSetId: input.acceptedExecution.resolvedVariableSet?.id ?? null,
|
|
8563
|
+
rigId: input.acceptedExecution.resolvedRig?.id ?? null,
|
|
8564
|
+
rigVersionId: input.acceptedExecution.resolvedRig?.versionId ?? null,
|
|
8565
|
+
toolPolicy: session.toolPolicy,
|
|
8566
|
+
mcpServerIds: [],
|
|
8567
|
+
toolPolicyVersion: session.toolPolicyVersion
|
|
8568
|
+
}
|
|
8569
|
+
} : {}
|
|
8570
|
+
});
|
|
8571
|
+
const preflight = evaluateIncidentTelemetryPreflight({
|
|
8572
|
+
agentConfig: task.agentConfig,
|
|
8573
|
+
incidentTriggered: input.alertOccurrenceLabels !== null,
|
|
8574
|
+
alertOccurrenceLabels: input.alertOccurrenceLabels,
|
|
8575
|
+
responder
|
|
8576
|
+
});
|
|
8577
|
+
if (preflight.action === "blocked") {
|
|
8578
|
+
throw new IncidentTelemetryPreflightBlockedError(preflight.reason);
|
|
8579
|
+
}
|
|
8580
|
+
const fence = incidentTelemetryAuthorityFence({
|
|
8581
|
+
task,
|
|
8582
|
+
responder,
|
|
8583
|
+
alertOccurrenceLabels: input.alertOccurrenceLabels
|
|
8584
|
+
});
|
|
8585
|
+
if (!fence) {
|
|
8586
|
+
throw new IncidentTelemetryPreflightBlockedError("incident_preflight_metadata_missing");
|
|
8587
|
+
}
|
|
8588
|
+
return {
|
|
8589
|
+
lineage: {
|
|
8590
|
+
[INCIDENT_TELEMETRY_AUTHORITY_FENCE_LINEAGE_KEY]: fence
|
|
8591
|
+
}
|
|
8592
|
+
};
|
|
8593
|
+
}
|
|
8594
|
+
async function recordScheduledTaskFiredUsage(db, acceptedExecution, run) {
|
|
8595
|
+
const task = acceptedExecution.task;
|
|
8596
|
+
await recordUsageEvent3(db, {
|
|
8597
|
+
accountId: task.accountId,
|
|
8598
|
+
workspaceId: task.workspaceId,
|
|
8599
|
+
eventType: "scheduled_task.fired",
|
|
8600
|
+
quantity: 1,
|
|
8601
|
+
unit: "run",
|
|
8602
|
+
sourceResourceType: "scheduled_task_run",
|
|
8603
|
+
sourceResourceId: run.id,
|
|
8604
|
+
initiator: acceptedExecution.triggerInitiator,
|
|
8605
|
+
initiatorContext: { scheduledTaskId: task.id, scheduledTaskRunId: run.id },
|
|
8606
|
+
origin: "scheduled_task",
|
|
8607
|
+
idempotencyKey: `usage:scheduled_task.fired:${run.id}`
|
|
8608
|
+
});
|
|
8609
|
+
}
|
|
8610
|
+
async function recoverBoundScheduledTaskDispatch(input) {
|
|
8611
|
+
const { task } = input.acceptedExecution;
|
|
8612
|
+
if (input.run.status !== "queued") {
|
|
8613
|
+
throw new Error("scheduled run is not an exact queued recovery");
|
|
8614
|
+
}
|
|
8615
|
+
const generatedSession = task.runMode === "new_session_per_run" || task.runMode === "reusable_session" && task.reusableSessionId === null;
|
|
8616
|
+
const frozenSlack = input.acceptedExecution.resolvedSlackBotConnection;
|
|
8617
|
+
if (frozenSlack) {
|
|
8618
|
+
const currentSlack = await requireOpenGeniSlackBotConnection(
|
|
8619
|
+
input.db,
|
|
8620
|
+
task.workspaceId,
|
|
8621
|
+
frozenSlack.id
|
|
8622
|
+
);
|
|
8623
|
+
const currentMetadata = openGeniSlackBotMetadata(currentSlack.metadata);
|
|
8624
|
+
if (currentSlack.version !== frozenSlack.version || currentSlack.verifiedInstallVersion !== frozenSlack.verifiedInstallVersion || stableJson(currentMetadata) !== stableJson(frozenSlack.metadata)) {
|
|
8625
|
+
throw new ScheduledRunTerminalAuthorityError(
|
|
8626
|
+
"scheduled_slack_authority_changed",
|
|
8627
|
+
"scheduled Slack bot authority changed after run admission"
|
|
8628
|
+
);
|
|
8629
|
+
}
|
|
8630
|
+
}
|
|
8631
|
+
let session;
|
|
8632
|
+
if (input.run.sessionId) {
|
|
8633
|
+
session = await requireSession(input.db, task.workspaceId, input.run.sessionId);
|
|
8634
|
+
if (!generatedSession) assertReusableSessionBindingMatches(session, task);
|
|
8635
|
+
} else if (generatedSession) {
|
|
8636
|
+
const variableSet = input.acceptedExecution.resolvedVariableSet;
|
|
8637
|
+
if (variableSet) {
|
|
8638
|
+
const current = await getVariableSet(
|
|
8639
|
+
input.db,
|
|
8640
|
+
{
|
|
8641
|
+
accountId: task.accountId,
|
|
8642
|
+
workspaceId: task.workspaceId,
|
|
8643
|
+
subjectId: input.acceptedExecution.personalResourceAuthoritySubjectId ?? task.createdBy.subjectId
|
|
8644
|
+
},
|
|
8645
|
+
variableSet.id
|
|
8646
|
+
);
|
|
8647
|
+
if (!current || current.generation !== variableSet.generation) {
|
|
8648
|
+
throw new ScheduledRunTerminalAuthorityError(
|
|
8649
|
+
"scheduled_variable_set_authority_changed",
|
|
8650
|
+
"scheduled variable set changed after run admission"
|
|
8651
|
+
);
|
|
8652
|
+
}
|
|
8653
|
+
}
|
|
8654
|
+
const rig = input.acceptedExecution.resolvedRig;
|
|
8655
|
+
if (rig) {
|
|
8656
|
+
const retained = await getScheduledScopedRigVersionMetadata(
|
|
8657
|
+
input.db,
|
|
8658
|
+
{
|
|
8659
|
+
accountId: task.accountId,
|
|
8660
|
+
workspaceId: task.workspaceId,
|
|
8661
|
+
subjectId: input.acceptedExecution.personalResourceAuthoritySubjectId ?? task.createdBy.subjectId
|
|
8662
|
+
},
|
|
8663
|
+
rig.id,
|
|
8664
|
+
rig.versionId
|
|
8665
|
+
);
|
|
8666
|
+
if (!retained) {
|
|
8667
|
+
throw new ScheduledRunTerminalAuthorityError(
|
|
8668
|
+
"scheduled_rig_authority_changed",
|
|
8669
|
+
"scheduled rig changed after run admission"
|
|
8670
|
+
);
|
|
8671
|
+
}
|
|
8672
|
+
}
|
|
8673
|
+
const taskMetadata = { ...task.agentConfig.metadata };
|
|
8674
|
+
delete taskMetadata[OPENGENI_SLACK_BOT_SESSION_METADATA_KEY];
|
|
8675
|
+
const created = await createSessionWithIdempotencyKeyResult(input.db, {
|
|
8676
|
+
accountId: task.accountId,
|
|
8677
|
+
workspaceId: task.workspaceId,
|
|
8678
|
+
initialMessage: task.agentConfig.prompt,
|
|
8679
|
+
resources: task.agentConfig.resources,
|
|
8680
|
+
tools: input.acceptedExecution.resolvedTools,
|
|
8681
|
+
firstPartyMcpTools: input.acceptedExecution.resolvedFirstPartyMcpTools,
|
|
8682
|
+
firstPartyMcpPermissions: input.acceptedExecution.resolvedFirstPartyMcpPermissions,
|
|
8683
|
+
metadata: {
|
|
8684
|
+
...taskMetadata,
|
|
8685
|
+
model: input.acceptedExecution.resolvedModel,
|
|
8686
|
+
reasoningEffort: input.acceptedExecution.resolvedReasoningEffort,
|
|
8687
|
+
scheduledTaskId: task.id,
|
|
8688
|
+
scheduledTaskRunMode: task.runMode,
|
|
8689
|
+
...task.agentConfig.goal ? { scheduledTaskGoal: task.agentConfig.goal } : {},
|
|
8690
|
+
scheduledTaskRunId: input.run.id,
|
|
8691
|
+
...frozenSlack ? { [OPENGENI_SLACK_BOT_SESSION_METADATA_KEY]: frozenSlack.id } : {}
|
|
8692
|
+
},
|
|
8693
|
+
createdBy: {
|
|
8694
|
+
kind: "service",
|
|
8695
|
+
subjectId: "scheduler",
|
|
8696
|
+
label: "OpenGeni scheduler"
|
|
8697
|
+
},
|
|
8698
|
+
createdByContext: {
|
|
8699
|
+
scheduledTaskId: task.id,
|
|
8700
|
+
scheduledTaskRunId: input.run.id
|
|
8701
|
+
},
|
|
8702
|
+
model: input.acceptedExecution.resolvedModel,
|
|
8703
|
+
reasoningEffort: input.acceptedExecution.resolvedReasoningEffort,
|
|
8704
|
+
latencyMode: input.acceptedExecution.resolvedLatencyMode,
|
|
8705
|
+
sandboxBackend: input.acceptedExecution.resolvedSandboxBackend,
|
|
8706
|
+
sandboxOs: input.acceptedExecution.resolvedSandboxOs,
|
|
8707
|
+
variableSetId: variableSet?.id ?? null,
|
|
8708
|
+
rigId: rig?.id ?? null,
|
|
8709
|
+
rigVersionId: rig?.versionId ?? null,
|
|
8710
|
+
personalConnectionDelegations: [],
|
|
8711
|
+
initialXaiProviderAccountAuthoritySnapshot: input.acceptedExecution.xaiProviderAccountAuthoritySnapshot,
|
|
8712
|
+
maxNestedAgentDepthOverride: task.agentConfig.maxNestedAgentDepth ?? null,
|
|
8713
|
+
frozenNestedAgentDepthPolicy: {
|
|
8714
|
+
effectiveMaxNestedAgentDepth: input.acceptedExecution.generatedSessionBinding.effectiveMaxNestedAgentDepth,
|
|
8715
|
+
nestedAgentDepthPolicySource: input.acceptedExecution.generatedSessionBinding.nestedAgentDepthPolicySource
|
|
8716
|
+
},
|
|
8717
|
+
frozenCodexCompactionMode: input.acceptedExecution.generatedSessionBinding.codexCompactionMode,
|
|
8718
|
+
allowNestedAgentDepthIncrease: true,
|
|
8719
|
+
subjectId: `scheduled_task:${task.id}`,
|
|
8720
|
+
createIdempotencyKey: input.acceptedExecution.generatedSessionBinding?.createIdempotencyKey ?? (() => {
|
|
8721
|
+
throw new ScheduledRunTerminalAuthorityError(
|
|
8722
|
+
"scheduled_generated_session_binding_missing",
|
|
8723
|
+
"scheduled generated session binding is missing"
|
|
8724
|
+
);
|
|
8725
|
+
})(),
|
|
8726
|
+
beforeCreateCommit: async (tx, sessionId) => {
|
|
8727
|
+
await bindScheduledTaskRunSessionInTransaction(tx, {
|
|
8728
|
+
accountId: task.accountId,
|
|
8729
|
+
workspaceId: task.workspaceId,
|
|
8730
|
+
runId: input.run.id,
|
|
8731
|
+
sessionId
|
|
8732
|
+
});
|
|
8733
|
+
}
|
|
8734
|
+
});
|
|
8735
|
+
if (created.denied) throw new SessionSpawnDeniedDbError(created.denial);
|
|
8736
|
+
session = created.session;
|
|
8737
|
+
if (!created.created) {
|
|
8738
|
+
await bindScheduledTaskRunSessionInTransaction(input.db, {
|
|
8739
|
+
accountId: task.accountId,
|
|
8740
|
+
workspaceId: task.workspaceId,
|
|
8741
|
+
runId: input.run.id,
|
|
8742
|
+
sessionId: session.id
|
|
8743
|
+
});
|
|
8744
|
+
}
|
|
8745
|
+
} else {
|
|
8746
|
+
const targetSessionId = task.runMode === "existing_session" ? task.targetSessionId : task.reusableSessionId;
|
|
8747
|
+
if (!targetSessionId) {
|
|
8748
|
+
throw new ScheduledRunTerminalAuthorityError(
|
|
8749
|
+
"scheduled_target_session_unavailable",
|
|
8750
|
+
"scheduled accepted target session is unavailable"
|
|
8751
|
+
);
|
|
8752
|
+
}
|
|
8753
|
+
session = await requireSession(input.db, task.workspaceId, targetSessionId);
|
|
8754
|
+
assertReusableSessionBindingMatches(session, task);
|
|
8755
|
+
await bindScheduledTaskRunSessionInTransaction(input.db, {
|
|
8756
|
+
accountId: task.accountId,
|
|
8757
|
+
workspaceId: task.workspaceId,
|
|
8758
|
+
runId: input.run.id,
|
|
8759
|
+
sessionId: session.id
|
|
8760
|
+
});
|
|
8761
|
+
}
|
|
8762
|
+
if (session.status === "cancelled") {
|
|
8763
|
+
await markScheduledTaskRunSkippedIfQueued(input.db, {
|
|
8764
|
+
workspaceId: task.workspaceId,
|
|
8765
|
+
runId: input.run.id,
|
|
8766
|
+
sessionId: session.id,
|
|
8767
|
+
error: "session_cancelled"
|
|
8768
|
+
});
|
|
8769
|
+
return { action: "blocked", reason: "scheduled_run_terminal" };
|
|
8770
|
+
}
|
|
8771
|
+
if (generatedSession) {
|
|
8772
|
+
const expectedCreateKey = input.acceptedExecution.generatedSessionBinding?.createIdempotencyKey;
|
|
8773
|
+
if (!expectedCreateKey) {
|
|
8774
|
+
throw new ScheduledRunTerminalAuthorityError(
|
|
8775
|
+
"scheduled_generated_session_binding_missing",
|
|
8776
|
+
"scheduled generated session binding is missing"
|
|
8777
|
+
);
|
|
8778
|
+
}
|
|
8779
|
+
const generatedBinding = input.acceptedExecution.generatedSessionBinding;
|
|
8780
|
+
const canonicalGeneratedRunId = task.runMode === "reusable_session" && typeof session.metadata.scheduledTaskRunId === "string" ? session.metadata.scheduledTaskRunId : input.run.id;
|
|
8781
|
+
const expectedTaskMetadata = { ...task.agentConfig.metadata };
|
|
8782
|
+
delete expectedTaskMetadata[OPENGENI_SLACK_BOT_SESSION_METADATA_KEY];
|
|
8783
|
+
const expectedMetadata = {
|
|
8784
|
+
...expectedTaskMetadata,
|
|
8785
|
+
model: input.acceptedExecution.resolvedModel,
|
|
8786
|
+
reasoningEffort: input.acceptedExecution.resolvedReasoningEffort,
|
|
8787
|
+
scheduledTaskId: task.id,
|
|
8788
|
+
scheduledTaskRunMode: task.runMode,
|
|
8789
|
+
...task.agentConfig.goal ? { scheduledTaskGoal: task.agentConfig.goal } : {},
|
|
8790
|
+
scheduledTaskRunId: canonicalGeneratedRunId,
|
|
8791
|
+
...frozenSlack ? { [OPENGENI_SLACK_BOT_SESSION_METADATA_KEY]: frozenSlack.id } : {}
|
|
8792
|
+
};
|
|
8793
|
+
if (session.createIdempotencyKey !== expectedCreateKey || session.initialMessage !== task.agentConfig.prompt || session.instructions !== null || session.policyRole !== null || stableJson(session.skills) !== "[]" || stableJson(session.toolPolicy) !== stableJson({ mode: "explicit", inheritedFromSessionId: null }) || session.toolPolicyVersion !== 1 || session.mcpServers.length !== 0 || session.parentSessionId !== null || session.rootSessionId !== session.id || session.nestedAgentDepth !== 0 || session.effectiveMaxNestedAgentDepth !== generatedBinding.effectiveMaxNestedAgentDepth || session.nestedAgentDepthPolicySource !== generatedBinding.nestedAgentDepthPolicySource || session.nestedAgentDepthPolicySessionId !== (generatedBinding.nestedAgentDepthPolicySource === "session" ? session.id : null) || session.sandboxGroupId !== session.id || session.channelId !== null || session.codexCompactionMode !== generatedBinding.codexCompactionMode || stableJson(session.metadata) !== stableJson(expectedMetadata) || session.createdBy.kind !== "service" || session.createdBy.subjectId !== "scheduler" || stableJson(session.createdByContext) !== stableJson({
|
|
8794
|
+
label: "OpenGeni scheduler",
|
|
8795
|
+
scheduledTaskId: task.id,
|
|
8796
|
+
scheduledTaskRunId: canonicalGeneratedRunId
|
|
8797
|
+
}) || task.runMode === "new_session_per_run" && input.acceptedExecution.alertOccurrenceLabels === null && (session.metadata.scheduledTaskRunId !== input.run.id || session.createdByContext.scheduledTaskRunId !== input.run.id) || session.model !== input.acceptedExecution.resolvedModel || session.reasoningEffort !== input.acceptedExecution.resolvedReasoningEffort || session.latencyMode !== input.acceptedExecution.resolvedLatencyMode || session.sandboxBackend !== input.acceptedExecution.resolvedSandboxBackend || session.sandboxOs !== input.acceptedExecution.resolvedSandboxOs || stableJson(session.resources) !== stableJson(task.agentConfig.resources) || stableJson(session.tools) !== stableJson(input.acceptedExecution.resolvedTools) || stableJson(session.firstPartyMcpTools) !== stableJson(input.acceptedExecution.resolvedFirstPartyMcpTools) || stableJson(session.firstPartyMcpPermissions) !== stableJson(input.acceptedExecution.resolvedFirstPartyMcpPermissions) || session.maxNestedAgentDepthOverride !== (task.agentConfig.maxNestedAgentDepth ?? null) || (session.variableSetId ?? null) !== (input.acceptedExecution.resolvedVariableSet?.id ?? null) || (session.rigId ?? null) !== (input.acceptedExecution.resolvedRig?.id ?? null) || (session.rigVersionId ?? null) !== (input.acceptedExecution.resolvedRig?.versionId ?? null) || scheduledSlackBotConnectionId(session.metadata) !== (frozenSlack?.id ?? null)) {
|
|
8798
|
+
throw new ScheduledRunTerminalAuthorityError(
|
|
8799
|
+
"scheduled_generated_session_changed",
|
|
8800
|
+
"scheduled generated session differs from accepted execution"
|
|
8801
|
+
);
|
|
8802
|
+
}
|
|
8803
|
+
}
|
|
8804
|
+
let workflowId = workflowIdForSession(session.id);
|
|
8805
|
+
if (generatedSession) {
|
|
8806
|
+
const goalSpec = task.agentConfig.goal ?? null;
|
|
8807
|
+
const started = await initializeSessionStartAtomically(input.db, {
|
|
8808
|
+
accountId: task.accountId,
|
|
8809
|
+
workspaceId: task.workspaceId,
|
|
8810
|
+
sessionId: session.id,
|
|
8811
|
+
reasoningEffortFallback: input.acceptedExecution.resolvedReasoningEffort,
|
|
8812
|
+
createdEventPayload: {
|
|
8813
|
+
scheduledTaskId: task.id,
|
|
8814
|
+
scheduledTaskRunId: input.run.id
|
|
8815
|
+
},
|
|
8816
|
+
goal: goalSpec ? {
|
|
8817
|
+
text: goalSpec.text,
|
|
8818
|
+
successCriteria: goalSpec.successCriteria ?? null,
|
|
8819
|
+
maxAutoContinuations: goalSpec.maxAutoContinuations ?? null,
|
|
8820
|
+
...goalSpec.mutationPolicy ? { mutationPolicy: goalSpec.mutationPolicy } : {},
|
|
8821
|
+
createdBy: "scheduled_task"
|
|
8822
|
+
} : null,
|
|
8823
|
+
deferInitialTurn: true,
|
|
8824
|
+
deferredStatus: "queued"
|
|
8825
|
+
});
|
|
8826
|
+
workflowId = started.temporalWorkflowId;
|
|
8827
|
+
if (started.events.length > 0) {
|
|
8828
|
+
await publishDurableSessionEvents(input.bus, task.workspaceId, session.id, started.events);
|
|
8829
|
+
}
|
|
8830
|
+
if (task.runMode === "reusable_session") {
|
|
8831
|
+
await materializeScheduledTaskReusableSessionFromRun(input.db, {
|
|
8832
|
+
accountId: task.accountId,
|
|
8833
|
+
workspaceId: task.workspaceId,
|
|
8834
|
+
taskId: task.id,
|
|
8835
|
+
runId: input.run.id,
|
|
8836
|
+
sessionId: session.id,
|
|
8837
|
+
sourceTaskAuthorityRevision: input.run.taskAuthorityRevision,
|
|
8838
|
+
sourceExecutionDigest: input.run.taskExecutionDigest
|
|
8839
|
+
});
|
|
8840
|
+
}
|
|
8841
|
+
const titleEvents = await stampScheduledSessionTitle(input.db, {
|
|
8842
|
+
workspaceId: task.workspaceId,
|
|
8843
|
+
session,
|
|
8844
|
+
taskName: task.name
|
|
8845
|
+
});
|
|
8846
|
+
if (titleEvents.length > 0) {
|
|
8847
|
+
await publishDurableSessionEvents(input.bus, task.workspaceId, session.id, titleEvents);
|
|
8848
|
+
}
|
|
8849
|
+
}
|
|
8850
|
+
if (!generatedSession && task.runMode === "reusable_session" && task.agentConfig.goal) {
|
|
8851
|
+
const goalEvents = await upsertScheduledSessionGoalForRun(input.db, {
|
|
8852
|
+
accountId: task.accountId,
|
|
8853
|
+
workspaceId: task.workspaceId,
|
|
8854
|
+
sessionId: session.id,
|
|
8855
|
+
runId: input.run.id,
|
|
8856
|
+
text: task.agentConfig.goal.text,
|
|
8857
|
+
successCriteria: task.agentConfig.goal.successCriteria ?? null,
|
|
8858
|
+
maxAutoContinuations: task.agentConfig.goal.maxAutoContinuations ?? null,
|
|
8859
|
+
...task.agentConfig.goal.mutationPolicy ? { mutationPolicy: task.agentConfig.goal.mutationPolicy } : {}
|
|
8860
|
+
});
|
|
8861
|
+
if (goalEvents.length > 0) {
|
|
8862
|
+
await publishDurableSessionEvents(input.bus, task.workspaceId, session.id, goalEvents);
|
|
8863
|
+
}
|
|
8864
|
+
}
|
|
8865
|
+
const scheduledUpdate = await addSessionSystemUpdateWithSourceMutation(
|
|
8866
|
+
input.db,
|
|
8867
|
+
{
|
|
8868
|
+
accountId: task.accountId,
|
|
8869
|
+
workspaceId: task.workspaceId,
|
|
8870
|
+
sessionId: session.id,
|
|
8871
|
+
kind: "scheduled_occurrence",
|
|
8872
|
+
classification: "info",
|
|
8873
|
+
sourceId: input.run.id,
|
|
8874
|
+
dedupeKey: `scheduled-task-run:${input.run.id}`,
|
|
8875
|
+
summary: task.agentConfig.prompt,
|
|
8876
|
+
payload: scheduledUserMessagePayload(
|
|
8877
|
+
task.agentConfig.prompt,
|
|
8878
|
+
task.agentConfig.resources,
|
|
8879
|
+
input.acceptedExecution.resolvedTools,
|
|
8880
|
+
task.id,
|
|
8881
|
+
input.run.id
|
|
8882
|
+
),
|
|
8883
|
+
lineage: {
|
|
8884
|
+
scheduledTaskId: task.id,
|
|
8885
|
+
scheduledTaskRunId: input.run.id,
|
|
8886
|
+
...input.acceptedExecution.causalHumanSubjectId ? { causalHumanSubjectId: input.acceptedExecution.causalHumanSubjectId } : {},
|
|
8887
|
+
...input.acceptedExecution.connectionAuthoritySubjectId ? {
|
|
8888
|
+
connectionAuthoritySubjectId: input.acceptedExecution.connectionAuthoritySubjectId
|
|
8889
|
+
} : {},
|
|
8890
|
+
...input.acceptedExecution.xaiAuthoritySubjectId ? { xaiAuthoritySubjectId: input.acceptedExecution.xaiAuthoritySubjectId } : {}
|
|
8891
|
+
},
|
|
8892
|
+
personalConnectionDelegations: input.acceptedExecution.personalConnectionDelegations,
|
|
8893
|
+
xaiProviderAccountAuthoritySnapshot: input.acceptedExecution.xaiProviderAccountAuthoritySnapshot,
|
|
8894
|
+
scheduledTaskRunId: input.run.id
|
|
8895
|
+
},
|
|
8896
|
+
async (tx, wakeEventId) => {
|
|
8897
|
+
if (!wakeEventId) {
|
|
8898
|
+
await settleScheduledTaskRunInTransaction(tx, {
|
|
8899
|
+
workspaceId: task.workspaceId,
|
|
8900
|
+
runId: input.run.id,
|
|
8901
|
+
sessionId: session.id,
|
|
8902
|
+
triggerEventId: null,
|
|
8903
|
+
status: "skipped",
|
|
8904
|
+
error: "session_cancelled"
|
|
8905
|
+
});
|
|
8906
|
+
return;
|
|
8907
|
+
}
|
|
8908
|
+
await settleScheduledTaskRunInTransaction(tx, {
|
|
8909
|
+
workspaceId: task.workspaceId,
|
|
8910
|
+
runId: input.run.id,
|
|
8911
|
+
sessionId: session.id,
|
|
8912
|
+
triggerEventId: wakeEventId,
|
|
8913
|
+
status: "dispatched"
|
|
8914
|
+
});
|
|
8915
|
+
},
|
|
8916
|
+
input.acceptedExecution.incidentPreflightRequired ? {
|
|
8917
|
+
prepareSource: async (tx, sessionId) => await prepareIncidentTelemetrySource({
|
|
8918
|
+
tx,
|
|
8919
|
+
settings: input.settings,
|
|
8920
|
+
taskId: task.id,
|
|
8921
|
+
workspaceId: task.workspaceId,
|
|
8922
|
+
sessionId,
|
|
8923
|
+
alertOccurrenceLabels: input.acceptedExecution.alertOccurrenceLabels,
|
|
8924
|
+
acceptedExecution: input.acceptedExecution
|
|
8925
|
+
})
|
|
8926
|
+
} : void 0
|
|
8927
|
+
);
|
|
8928
|
+
if (scheduledUpdate.reason === "session_cancelled") {
|
|
8929
|
+
return { action: "blocked", reason: "scheduled_run_terminal" };
|
|
8930
|
+
}
|
|
8931
|
+
if (scheduledUpdate.added && scheduledUpdate.events.length > 0) {
|
|
8932
|
+
await publishDurableSessionEvents(
|
|
8933
|
+
input.bus,
|
|
8934
|
+
task.workspaceId,
|
|
8935
|
+
session.id,
|
|
8936
|
+
scheduledUpdate.events
|
|
8937
|
+
);
|
|
8938
|
+
}
|
|
8939
|
+
await recordUsageEvent3(input.db, {
|
|
8940
|
+
accountId: task.accountId,
|
|
8941
|
+
workspaceId: task.workspaceId,
|
|
8942
|
+
eventType: "agent_run.created",
|
|
8943
|
+
quantity: 1,
|
|
8944
|
+
unit: "run",
|
|
8945
|
+
sourceResourceType: "scheduled_task_run",
|
|
8946
|
+
sourceResourceId: input.run.id,
|
|
8947
|
+
sessionId: session.id,
|
|
8948
|
+
initiator: input.acceptedExecution.triggerInitiator,
|
|
8949
|
+
initiatorContext: { scheduledTaskId: task.id, scheduledTaskRunId: input.run.id },
|
|
8950
|
+
origin: "scheduled_task",
|
|
8951
|
+
idempotencyKey: input.acceptedExecution.agentRunUsageIdempotencyKey ?? `usage:agent_run.created:scheduled:${input.run.id}`
|
|
8952
|
+
});
|
|
8953
|
+
const workflowWakeRevision = await enqueueSessionWorkflowWakeIfRunnable(input.db, {
|
|
8954
|
+
accountId: task.accountId,
|
|
8955
|
+
workspaceId: task.workspaceId,
|
|
8956
|
+
sessionId: session.id,
|
|
8957
|
+
temporalWorkflowId: workflowId,
|
|
8958
|
+
reason: "scheduled_retry"
|
|
8959
|
+
});
|
|
8960
|
+
const result = {
|
|
8961
|
+
action: generatedSession ? "start" : "signal",
|
|
8962
|
+
accountId: task.accountId,
|
|
8963
|
+
workspaceId: task.workspaceId,
|
|
8964
|
+
sessionId: session.id,
|
|
8965
|
+
triggerEventId: scheduledUpdate.wakeEventId,
|
|
8966
|
+
workflowId,
|
|
8967
|
+
workflowWakeRevision
|
|
8968
|
+
};
|
|
8969
|
+
if (input.wakeSessionWorkflow && workflowWakeRevision !== null) {
|
|
8970
|
+
await input.wakeSessionWorkflow({
|
|
8971
|
+
accountId: result.accountId,
|
|
8972
|
+
workspaceId: result.workspaceId,
|
|
8973
|
+
sessionId: result.sessionId,
|
|
8974
|
+
workflowId: result.workflowId,
|
|
8975
|
+
wakeRevision: workflowWakeRevision
|
|
8976
|
+
});
|
|
8977
|
+
}
|
|
8978
|
+
return result;
|
|
8979
|
+
}
|
|
8980
|
+
async function replayScheduledTaskDispatch(input) {
|
|
8981
|
+
if (!input.run.sessionId || !input.run.triggerEventId) {
|
|
8982
|
+
throw new Error("dispatched scheduled task run is missing its delivery identity");
|
|
8983
|
+
}
|
|
8984
|
+
const sessionId = input.run.sessionId;
|
|
8985
|
+
const triggerEventId = input.run.triggerEventId;
|
|
8986
|
+
await requireSession(input.db, input.task.workspaceId, sessionId);
|
|
8987
|
+
await recordUsageEvent3(input.db, {
|
|
8988
|
+
accountId: input.task.accountId,
|
|
8989
|
+
workspaceId: input.task.workspaceId,
|
|
8990
|
+
eventType: "agent_run.created",
|
|
8991
|
+
quantity: 1,
|
|
8992
|
+
unit: "run",
|
|
8993
|
+
sourceResourceType: "scheduled_task_run",
|
|
8994
|
+
sourceResourceId: input.run.id,
|
|
8995
|
+
sessionId,
|
|
8996
|
+
initiator: input.acceptedExecution.triggerInitiator,
|
|
8997
|
+
initiatorContext: {
|
|
8998
|
+
scheduledTaskId: input.task.id,
|
|
8999
|
+
scheduledTaskRunId: input.run.id
|
|
9000
|
+
},
|
|
9001
|
+
origin: "scheduled_task",
|
|
9002
|
+
idempotencyKey: input.acceptedExecution.agentRunUsageIdempotencyKey ?? `usage:agent_run.created:scheduled:${input.run.id}`
|
|
9003
|
+
});
|
|
9004
|
+
const workflowId = workflowIdForSession(sessionId);
|
|
9005
|
+
const workflowWakeRevision = await enqueueSessionWorkflowWakeIfRunnable(input.db, {
|
|
9006
|
+
accountId: input.task.accountId,
|
|
9007
|
+
workspaceId: input.task.workspaceId,
|
|
9008
|
+
sessionId,
|
|
9009
|
+
temporalWorkflowId: workflowId,
|
|
9010
|
+
reason: "scheduled_retry"
|
|
9011
|
+
});
|
|
9012
|
+
const result = {
|
|
9013
|
+
action: input.acceptedExecution.task.runMode === "new_session_per_run" ? "start" : "signal",
|
|
9014
|
+
accountId: input.task.accountId,
|
|
9015
|
+
workspaceId: input.task.workspaceId,
|
|
9016
|
+
sessionId,
|
|
9017
|
+
triggerEventId,
|
|
9018
|
+
workflowId,
|
|
9019
|
+
workflowWakeRevision
|
|
9020
|
+
};
|
|
9021
|
+
if (input.wakeSessionWorkflow && workflowWakeRevision !== null) {
|
|
9022
|
+
await input.wakeSessionWorkflow({
|
|
9023
|
+
accountId: result.accountId,
|
|
9024
|
+
workspaceId: result.workspaceId,
|
|
9025
|
+
sessionId: result.sessionId,
|
|
9026
|
+
workflowId: result.workflowId,
|
|
9027
|
+
wakeRevision: workflowWakeRevision
|
|
9028
|
+
});
|
|
9029
|
+
}
|
|
9030
|
+
return result;
|
|
9031
|
+
}
|
|
6638
9032
|
function knowledgeSourceSyncEffectivelyPaused(task) {
|
|
6639
9033
|
if (task.status === "paused") return true;
|
|
6640
9034
|
const value = task.metadata.knowledgeSourceSync;
|
|
@@ -6644,17 +9038,17 @@ function knowledgeSourceSyncEffectivelyPaused(task) {
|
|
|
6644
9038
|
}
|
|
6645
9039
|
|
|
6646
9040
|
// src/activities/site-auth-maintenance.ts
|
|
6647
|
-
import { resolveTurnExecutionPolicyV1 } from "@opengeni/config";
|
|
9041
|
+
import { resolveTurnExecutionPolicyV1 as resolveTurnExecutionPolicyV12 } from "@opengeni/config";
|
|
6648
9042
|
import {
|
|
6649
|
-
DEFAULT_FIRST_PARTY_MCP_PERMISSIONS,
|
|
9043
|
+
DEFAULT_FIRST_PARTY_MCP_PERMISSIONS as DEFAULT_FIRST_PARTY_MCP_PERMISSIONS2,
|
|
6650
9044
|
SITE_AUTH_MAINTENANCE_CONNECTION_CONTEXT_KEY,
|
|
6651
9045
|
SITE_AUTH_MAINTENANCE_OPERATION_CONTEXT_KEY
|
|
6652
9046
|
} from "@opengeni/contracts";
|
|
6653
9047
|
import {
|
|
6654
|
-
assertWorkspaceModelPolicyAllows,
|
|
6655
|
-
canonicalConfiguredModel,
|
|
6656
|
-
createAndStartSessionWithOutcome,
|
|
6657
|
-
recordWorkspaceUsage
|
|
9048
|
+
assertWorkspaceModelPolicyAllows as assertWorkspaceModelPolicyAllows2,
|
|
9049
|
+
canonicalConfiguredModel as canonicalConfiguredModel2,
|
|
9050
|
+
createAndStartSessionWithOutcome as createAndStartSessionWithOutcome2,
|
|
9051
|
+
recordWorkspaceUsage as recordWorkspaceUsage2
|
|
6658
9052
|
} from "@opengeni/core";
|
|
6659
9053
|
import {
|
|
6660
9054
|
claimSiteAuthMaintenance,
|
|
@@ -6681,10 +9075,10 @@ function createSiteAuthMaintenanceActivities(services, options = {}) {
|
|
|
6681
9075
|
const claim = options.claim ?? claimSiteAuthMaintenance;
|
|
6682
9076
|
const confirm = options.confirm ?? confirmSiteAuthMaintenanceSessionInTransaction;
|
|
6683
9077
|
const defer = options.defer ?? deferSiteAuthMaintenance;
|
|
6684
|
-
const createSession2 = options.createSession ??
|
|
9078
|
+
const createSession2 = options.createSession ?? createAndStartSessionWithOutcome2;
|
|
6685
9079
|
const admit = options.admit ?? agentRunAdmissionDenial;
|
|
6686
|
-
const recordUsage = options.recordUsage ??
|
|
6687
|
-
const assertModelPolicy = options.assertModelPolicy ??
|
|
9080
|
+
const recordUsage = options.recordUsage ?? recordWorkspaceUsage2;
|
|
9081
|
+
const assertModelPolicy = options.assertModelPolicy ?? assertWorkspaceModelPolicyAllows2;
|
|
6688
9082
|
async function maintainSiteAuthConnections() {
|
|
6689
9083
|
const service = await services();
|
|
6690
9084
|
const claims = await claim(service.db, { claimTimeoutMs, limit: batchSize });
|
|
@@ -6730,7 +9124,7 @@ function createSiteAuthMaintenanceActivities(services, options = {}) {
|
|
|
6730
9124
|
return { maintainSiteAuthConnections };
|
|
6731
9125
|
}
|
|
6732
9126
|
async function dispatchMaintenanceSession(service, maintenance, dependencies) {
|
|
6733
|
-
const model =
|
|
9127
|
+
const model = canonicalConfiguredModel2(service.settings, service.settings.openaiModel);
|
|
6734
9128
|
if (!model) throw new Error("site auth maintenance has no configured model");
|
|
6735
9129
|
await dependencies.assertModelPolicy(
|
|
6736
9130
|
service.db,
|
|
@@ -6746,7 +9140,7 @@ async function dispatchMaintenanceSession(service, maintenance, dependencies) {
|
|
|
6746
9140
|
});
|
|
6747
9141
|
if (denial) throw new Error(`site auth maintenance admission denied: ${denial}`);
|
|
6748
9142
|
const reasoningEffort = service.settings.openaiReasoningEffort;
|
|
6749
|
-
const turnExecutionPolicy =
|
|
9143
|
+
const turnExecutionPolicy = resolveTurnExecutionPolicyV12(service.settings, {
|
|
6750
9144
|
modelId: model,
|
|
6751
9145
|
requestedModelId: null,
|
|
6752
9146
|
modelSource: "deployment",
|
|
@@ -6800,7 +9194,7 @@ async function dispatchMaintenanceSession(service, maintenance, dependencies) {
|
|
|
6800
9194
|
},
|
|
6801
9195
|
instructions: "Complete only the exact browser-auth maintenance run in the initial request. Use the canonical browser/auth tools, preserve their causal fences, expose no secrets, never publish a BrowserIdentity revision, and do not perform unrelated work.",
|
|
6802
9196
|
policyRole: "site_auth_maintenance",
|
|
6803
|
-
firstPartyMcpPermissions: [...
|
|
9197
|
+
firstPartyMcpPermissions: [...DEFAULT_FIRST_PARTY_MCP_PERMISSIONS2],
|
|
6804
9198
|
firstPartyMcpTools: [...MAINTENANCE_TOOLS],
|
|
6805
9199
|
createIdempotencyKey: `site-auth-maintenance:${maintenance.operationId}`,
|
|
6806
9200
|
subjectId: "site-auth-maintenance"
|
|
@@ -6945,14 +9339,14 @@ function createSessionStateActivities(services, overrides = {}) {
|
|
|
6945
9339
|
return { action: "failed" };
|
|
6946
9340
|
}
|
|
6947
9341
|
const requestedRetryDelayMs = input.retryDelayMs;
|
|
6948
|
-
const
|
|
9342
|
+
const retryDelayMs2 = typeof requestedRetryDelayMs === "number" && Number.isFinite(requestedRetryDelayMs) ? Math.max(1e3, Math.min(6e4, Math.trunc(requestedRetryDelayMs))) : 6e4;
|
|
6949
9343
|
await enqueueSessionWorkflowWakeFn(db, {
|
|
6950
9344
|
accountId: input.accountId,
|
|
6951
9345
|
workspaceId: input.workspaceId,
|
|
6952
9346
|
sessionId: input.sessionId,
|
|
6953
9347
|
temporalWorkflowId: workflowId,
|
|
6954
9348
|
reason: "turn_activity_failed_before_attempt_claim",
|
|
6955
|
-
notBefore: new Date(Date.now() +
|
|
9349
|
+
notBefore: new Date(Date.now() + retryDelayMs2)
|
|
6956
9350
|
});
|
|
6957
9351
|
return { action: "unclaimed" };
|
|
6958
9352
|
}
|
|
@@ -7108,7 +9502,7 @@ function createSessionStateActivities(services, overrides = {}) {
|
|
|
7108
9502
|
input.attemptId
|
|
7109
9503
|
);
|
|
7110
9504
|
if (!turn) return { action: "stale" };
|
|
7111
|
-
if (turn.id !== input.turnId || turn.triggerEventId !== input.triggerEventId || turn.executionGeneration !== input.executionGeneration || input.executionGeneration <= 1) {
|
|
9505
|
+
if (turn.id !== input.turnId || turn.triggerEventId !== input.triggerEventId || turn.executionGeneration !== input.executionGeneration || input.executionGeneration <= 1 || input.providerRecoveryCount !== providerRecoveryCountFromMetadata(turn.metadata) + 1 || input.providerRecoveryCount > MAX_AUTOMATIC_PROVIDER_RECOVERIES) {
|
|
7112
9506
|
return { action: "ineligible" };
|
|
7113
9507
|
}
|
|
7114
9508
|
const recovery = await requestSessionTurnRecoveryFn(db, input.workspaceId, {
|
|
@@ -7117,10 +9511,12 @@ function createSessionStateActivities(services, overrides = {}) {
|
|
|
7117
9511
|
triggerEventId: input.triggerEventId,
|
|
7118
9512
|
attemptId: input.attemptId,
|
|
7119
9513
|
reason: "mcp_transport_timeout",
|
|
9514
|
+
providerRecoveryCount: input.providerRecoveryCount,
|
|
7120
9515
|
detail: {
|
|
7121
9516
|
code: "mcp_transport_timeout",
|
|
7122
9517
|
retryable: true,
|
|
7123
|
-
continueDelayMs:
|
|
9518
|
+
continueDelayMs: input.continueDelayMs,
|
|
9519
|
+
providerRecoveryCount: input.providerRecoveryCount,
|
|
7124
9520
|
recoverySource: "workflow_activity_failure"
|
|
7125
9521
|
},
|
|
7126
9522
|
fromStatuses: ["running"]
|
|
@@ -7235,6 +9631,7 @@ function createControlActivityServices(dependencies) {
|
|
|
7235
9631
|
}
|
|
7236
9632
|
function createControlActivitiesFromServices(services, resolveDocumentServices) {
|
|
7237
9633
|
return {
|
|
9634
|
+
...createAutomationActivities(services),
|
|
7238
9635
|
...createDocumentActivities(services, resolveDocumentServices),
|
|
7239
9636
|
...createKnowledgeSourceSyncActivities(services, resolveDocumentServices),
|
|
7240
9637
|
...createSessionStateActivities(services),
|
|
@@ -7265,4 +9662,4 @@ export {
|
|
|
7265
9662
|
createControlActivities,
|
|
7266
9663
|
createControlActivitiesFromServices
|
|
7267
9664
|
};
|
|
7268
|
-
//# sourceMappingURL=activities-control-
|
|
9665
|
+
//# sourceMappingURL=activities-control-DKG22ONJ.js.map
|