@opengeni/worker-bundle 0.20.3 → 0.22.0-canary.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn/admission.d.ts +52 -0
- package/dist/activities/agent-turn/agent-build.d.ts +89 -0
- package/dist/activities/agent-turn/claim.d.ts +67 -0
- package/dist/activities/agent-turn/codex-capacity.d.ts +40 -0
- package/dist/activities/agent-turn/codex.d.ts +32 -0
- package/dist/activities/agent-turn/compaction-prep.d.ts +88 -0
- package/dist/activities/agent-turn/credential-leases.d.ts +56 -0
- package/dist/activities/agent-turn/errors.d.ts +253 -0
- package/dist/activities/agent-turn/failure-settlement.d.ts +33 -0
- package/dist/activities/agent-turn/file-resources.d.ts +37 -0
- package/dist/activities/agent-turn/finalization.d.ts +42 -0
- package/dist/activities/agent-turn/github-repository-bindings.d.ts +105 -0
- package/dist/activities/agent-turn/governance-model.d.ts +66 -0
- package/dist/activities/agent-turn/history-sink.d.ts +52 -0
- package/dist/activities/agent-turn/history.d.ts +63 -0
- package/dist/activities/agent-turn/media-artifacts.d.ts +145 -0
- package/dist/activities/agent-turn/model-usage.d.ts +218 -0
- package/dist/activities/agent-turn/quiescence.d.ts +135 -0
- package/dist/activities/agent-turn/run-credentials.d.ts +84 -0
- package/dist/activities/agent-turn/run.d.ts +3 -0
- package/dist/activities/agent-turn/sandbox-establish.d.ts +93 -0
- package/dist/activities/agent-turn/sandbox-provision.d.ts +89 -0
- package/dist/activities/agent-turn/sandbox-route.d.ts +151 -0
- package/dist/activities/agent-turn/sandbox-runtime.d.ts +44 -0
- package/dist/activities/agent-turn/stream-attempt.d.ts +107 -0
- package/dist/activities/agent-turn/tool-environment.d.ts +142 -0
- package/dist/activities/agent-turn/tool-policy.d.ts +127 -0
- package/dist/activities/agent-turn/tool-result-spill.d.ts +59 -0
- package/dist/activities/agent-turn/turn-context.d.ts +144 -0
- package/dist/activities/agent-turn/xai-capacity.d.ts +2 -0
- package/dist/activities/agent-turn.d.ts +12 -954
- package/dist/activities/automations.d.ts +16 -0
- package/dist/activities/common.d.ts +1 -1
- package/dist/activities/connector-attachments.d.ts +13 -0
- package/dist/activities/documents.d.ts +1 -0
- package/dist/activities/environment.d.ts +26 -15
- package/dist/activities/generated-images.d.ts +1 -1
- package/dist/activities/git-credential-renewal.d.ts +3 -1
- package/dist/activities/goals.d.ts +4 -1
- package/dist/activities/google-drive-provider.d.ts +29 -0
- package/dist/activities/google-drive-publication.d.ts +17 -4
- package/dist/activities/incident-telemetry-authority.d.ts +52 -0
- package/dist/activities/incident-telemetry-preflight.d.ts +71 -0
- package/dist/activities/knowledge-source-sync.d.ts +4 -0
- package/dist/activities/mcp-credentials.d.ts +9 -3
- package/dist/activities/open-suffix-resume.d.ts +77 -0
- package/dist/activities/packs.d.ts +3 -11
- package/dist/activities/parent-wake.d.ts +19 -0
- package/dist/activities/retained-screenshot-reaper.d.ts +1 -1
- package/dist/activities/retained-screenshots.d.ts +1 -1
- package/dist/activities/rig-verification.d.ts +1 -1
- package/dist/activities/run-input.d.ts +9 -10
- package/dist/activities/sandbox-images.d.ts +1 -14
- package/dist/activities/sandbox-lease.d.ts +21 -3
- package/dist/activities/scheduled-tasks.d.ts +47 -0
- package/dist/activities/types.d.ts +23 -2
- package/dist/activities/video-media-validation.d.ts +1 -1
- package/dist/activities/video-output-retention.d.ts +1 -2
- package/dist/activities/xai-auth.d.ts +3 -0
- package/dist/{activities-control-BJSDRSIP.js → activities-control-DKG22ONJ.js} +2646 -551
- package/dist/activities-control-DKG22ONJ.js.map +1 -0
- package/dist/activities-control.d.ts +6 -2
- package/dist/activities-turn-4IMC3MAA.js +21326 -0
- package/dist/activities-turn-4IMC3MAA.js.map +1 -0
- package/dist/activities.d.ts +7 -3
- package/dist/artifact-materializer-entry.js +41 -22
- package/dist/artifact-materializer-entry.js.map +1 -1
- package/dist/{chunk-N5TJIFCE.js → chunk-CE7TPKBS.js} +468 -1
- package/dist/chunk-CE7TPKBS.js.map +1 -0
- package/dist/chunk-CNYGVRKF.js +4698 -0
- package/dist/chunk-CNYGVRKF.js.map +1 -0
- package/dist/connection-provider-use.d.ts +18 -0
- package/dist/editable-artifact-materializer.d.ts +4 -3
- package/dist/github-rest-mcp.d.ts +26 -0
- package/dist/index.d.ts +10 -2
- package/dist/index.js +58 -14
- package/dist/index.js.map +1 -1
- package/dist/knowledge-source-sync-activity-policy.d.ts +2 -0
- package/dist/model-context-contributions.d.ts +8 -4
- package/dist/observability-metrics.d.ts +63 -4
- package/dist/opensandbox-kubernetes-inventory.d.ts +21 -0
- package/dist/personal-github-git-credentials.d.ts +9 -0
- package/dist/pr-review-credentials.d.ts +6 -0
- package/dist/sandbox-archive-storage.d.ts +45 -0
- package/dist/sandbox-resume.d.ts +65 -6
- package/dist/sandbox-routing.d.ts +38 -2
- package/dist/scheduled-alert-occurrence.d.ts +33 -0
- package/dist/workflow-bundle.js +87 -28
- package/dist/workflows/activities.d.ts +1 -0
- package/dist/workflows/automations.d.ts +6 -0
- package/dist/workflows/document-index.d.ts +1 -0
- package/dist/workflows.d.ts +1 -0
- package/package.json +17 -17
- package/src/activities/agent-turn/admission.ts +292 -0
- package/src/activities/agent-turn/agent-build.ts +802 -0
- package/src/activities/agent-turn/claim.ts +590 -0
- package/src/activities/agent-turn/codex-capacity.ts +720 -0
- package/src/activities/agent-turn/codex.ts +94 -0
- package/src/activities/agent-turn/compaction-prep.ts +722 -0
- package/src/activities/agent-turn/credential-leases.ts +240 -0
- package/src/activities/agent-turn/errors.ts +1068 -0
- package/src/activities/agent-turn/failure-settlement.ts +1436 -0
- package/src/activities/agent-turn/file-resources.ts +227 -0
- package/src/activities/agent-turn/finalization.ts +754 -0
- package/src/activities/agent-turn/github-repository-bindings.ts +300 -0
- package/src/activities/agent-turn/governance-model.ts +465 -0
- package/src/activities/agent-turn/history-sink.ts +139 -0
- package/src/activities/agent-turn/history.ts +200 -0
- package/src/activities/agent-turn/media-artifacts.ts +359 -0
- package/src/activities/agent-turn/model-usage.ts +823 -0
- package/src/activities/agent-turn/quiescence.ts +353 -0
- package/src/activities/agent-turn/run-credentials.ts +761 -0
- package/src/activities/agent-turn/run.ts +1530 -0
- package/src/activities/agent-turn/sandbox-establish.ts +1207 -0
- package/src/activities/agent-turn/sandbox-provision.ts +569 -0
- package/src/activities/agent-turn/sandbox-route.ts +358 -0
- package/src/activities/agent-turn/sandbox-runtime.ts +731 -0
- package/src/activities/agent-turn/stream-attempt.ts +1670 -0
- package/src/activities/agent-turn/tool-environment.ts +729 -0
- package/src/activities/agent-turn/tool-policy.ts +216 -0
- package/src/activities/agent-turn/tool-result-spill.ts +265 -0
- package/src/activities/agent-turn/turn-context.ts +281 -0
- package/src/activities/agent-turn/xai-capacity.ts +229 -0
- package/src/activities/agent-turn.ts +12 -12701
- package/src/activities/api-integrations.ts +3 -0
- package/src/activities/automations.ts +214 -0
- package/src/activities/codemode-dispatcher.ts +12 -1
- package/src/activities/common.ts +1 -5
- package/src/activities/connector-attachments.ts +168 -0
- package/src/activities/context-compaction.ts +18 -0
- package/src/activities/environment.ts +151 -55
- package/src/activities/generated-images.ts +26 -11
- package/src/activities/git-credential-renewal.ts +13 -3
- package/src/activities/goals.ts +85 -19
- package/src/activities/google-drive-provider.ts +178 -0
- package/src/activities/google-drive-publication.ts +147 -49
- package/src/activities/incident-telemetry-authority.ts +327 -0
- package/src/activities/incident-telemetry-preflight.ts +317 -0
- package/src/activities/knowledge-source-sync.ts +178 -37
- package/src/activities/mcp-credentials.ts +191 -17
- package/src/activities/open-suffix-resume.ts +422 -0
- package/src/activities/packs.ts +12 -60
- package/src/activities/parent-wake.ts +64 -14
- package/src/activities/retained-screenshot-reaper.ts +6 -3
- package/src/activities/retained-screenshots.ts +15 -47
- package/src/activities/rig-verification.ts +1 -1
- package/src/activities/run-input.ts +102 -142
- package/src/activities/sandbox-images.ts +7 -166
- package/src/activities/sandbox-lease.ts +406 -15
- package/src/activities/scheduled-tasks.ts +2103 -411
- package/src/activities/session-state.ts +10 -2
- package/src/activities/types.ts +32 -2
- package/src/activities/video-media-validation.ts +3 -3
- package/src/activities/video-output-retention.ts +2 -57
- package/src/activities/xai-auth.ts +8 -0
- package/src/activities-control.ts +2 -0
- package/src/activities.ts +3 -0
- package/src/activity-services.ts +13 -4
- package/src/connection-provider-use.ts +36 -0
- package/src/editable-artifact-materializer-subprocess.ts +25 -14
- package/src/editable-artifact-materializer.ts +29 -18
- package/src/github-rest-mcp.ts +713 -0
- package/src/http.ts +16 -6
- package/src/index.ts +40 -5
- package/src/knowledge-source-sync-activity-policy.ts +2 -0
- package/src/model-context-contributions.ts +33 -10
- package/src/observability-metrics.ts +357 -0
- package/src/opensandbox-kubernetes-inventory.ts +292 -0
- package/src/personal-github-git-credentials.ts +331 -0
- package/src/pr-review-credentials.ts +163 -0
- package/src/sandbox-archive-storage.ts +159 -0
- package/src/sandbox-resume.ts +449 -63
- package/src/sandbox-routing.ts +314 -16
- package/src/scheduled-alert-occurrence.ts +176 -0
- package/src/workflows/activities.ts +18 -2
- package/src/workflows/automations.ts +11 -0
- package/src/workflows/session.ts +13 -3
- package/src/workflows.ts +1 -0
- package/dist/activities-control-BJSDRSIP.js.map +0 -1
- package/dist/activities-turn-J5YO7XXX.js +0 -17243
- package/dist/activities-turn-J5YO7XXX.js.map +0 -1
- package/dist/chunk-N5TJIFCE.js.map +0 -1
- package/dist/chunk-P6Z2SLAA.js +0 -3092
- package/dist/chunk-P6Z2SLAA.js.map +0 -1
|
@@ -6,16 +6,26 @@ import {
|
|
|
6
6
|
sandboxDrainTiming
|
|
7
7
|
} from "./chunk-GRFQTTSQ.js";
|
|
8
8
|
import {
|
|
9
|
+
INCIDENT_TELEMETRY_AUTHORITY_FENCE_LINEAGE_KEY,
|
|
10
|
+
MAX_AUTOMATIC_PROVIDER_RECOVERIES,
|
|
9
11
|
assertReusableSessionRevivable,
|
|
12
|
+
collectWorkspaceArchiveObjectKeys,
|
|
10
13
|
createCodexCapacityActivities,
|
|
11
14
|
createGoalActivities,
|
|
12
15
|
createSharedActivityServices,
|
|
13
16
|
currentActivityContext,
|
|
17
|
+
deleteUnpublishedWorkspaceArchiveObject,
|
|
18
|
+
deleteWorkspaceArchiveObjectKeys,
|
|
14
19
|
deliverFailedChildTurnToParent,
|
|
20
|
+
evaluateIncidentTelemetryPreflight,
|
|
21
|
+
incidentTelemetryAuthorityFence,
|
|
22
|
+
incidentTelemetryPreflightDeclaration,
|
|
15
23
|
notifyParentOfChildIdle,
|
|
16
|
-
|
|
24
|
+
providerRecoveryCountFromMetadata,
|
|
25
|
+
putVersion1TarArchiveOrInline,
|
|
17
26
|
reconcilePendingParentSystemUpdates,
|
|
18
27
|
reconcilePendingSessionWorkflowWakes,
|
|
28
|
+
resolveIncidentTelemetryResponderMetadata,
|
|
19
29
|
resolveWorkspacePackRuntime,
|
|
20
30
|
rigProviderImageSourceImage,
|
|
21
31
|
scheduledUserMessagePayload,
|
|
@@ -23,11 +33,13 @@ import {
|
|
|
23
33
|
settingsWithRigImage,
|
|
24
34
|
withFirstPartyTools,
|
|
25
35
|
workflowIdForSession
|
|
26
|
-
} from "./chunk-
|
|
36
|
+
} from "./chunk-CNYGVRKF.js";
|
|
27
37
|
import {
|
|
38
|
+
inspectOpenSandboxKubernetesInventory,
|
|
28
39
|
recordCreditBalanceGauges,
|
|
29
40
|
recordCreditMicros,
|
|
30
41
|
recordExpiredDrainingSandboxLeaseGauges,
|
|
42
|
+
recordOpenSandboxKubernetesInventoryGauges,
|
|
31
43
|
recordRetainedProcessInventoryGauges,
|
|
32
44
|
recordRetainedProcessReconciliation,
|
|
33
45
|
recordSandboxCheckpointArtifactGauges,
|
|
@@ -38,8 +50,9 @@ import {
|
|
|
38
50
|
recordSandboxLeaseGauges,
|
|
39
51
|
recordSandboxOrphansTerminated,
|
|
40
52
|
recordSandboxRotationBacklogGauges,
|
|
41
|
-
recordTurnsQueuedGauge
|
|
42
|
-
|
|
53
|
+
recordTurnsQueuedGauge,
|
|
54
|
+
runtimeMetricsHooksForObservability
|
|
55
|
+
} from "./chunk-CE7TPKBS.js";
|
|
43
56
|
|
|
44
57
|
// src/activities/browser-state-artifact-reaper.ts
|
|
45
58
|
import {
|
|
@@ -125,14 +138,255 @@ function cleanupErrorCategory(error) {
|
|
|
125
138
|
return typeof code === "string" && SAFE_PROVIDER_ERROR_CODES.has(code) ? code : "unknown";
|
|
126
139
|
}
|
|
127
140
|
|
|
128
|
-
// src/activities/
|
|
141
|
+
// src/activities/automations.ts
|
|
142
|
+
import { resolveTurnExecutionPolicyV1 } from "@opengeni/config";
|
|
143
|
+
import {
|
|
144
|
+
assertWorkspaceModelPolicyAllows,
|
|
145
|
+
canonicalConfiguredModel,
|
|
146
|
+
createAndStartSessionWithOutcome,
|
|
147
|
+
recordWorkspaceUsage
|
|
148
|
+
} from "@opengeni/core";
|
|
149
|
+
import {
|
|
150
|
+
AutomationAuthorityRevokedError,
|
|
151
|
+
assertAutomationRunAuthorityInTransaction,
|
|
152
|
+
claimAutomationRun,
|
|
153
|
+
settleAutomationRun
|
|
154
|
+
} from "@opengeni/db";
|
|
155
|
+
|
|
156
|
+
// src/activities/agent-run-admission.ts
|
|
129
157
|
import { configuredStaticUsageLimits } from "@opengeni/config";
|
|
158
|
+
import { getBillingBalance, isCodexBilledTurn, sumUsageQuantity } from "@opengeni/db";
|
|
159
|
+
async function agentRunAdmissionDenial(services, input) {
|
|
160
|
+
const externallyBilled = await isCodexBilledTurn({
|
|
161
|
+
db: services.db,
|
|
162
|
+
settings: services.settings,
|
|
163
|
+
workspaceId: input.workspaceId,
|
|
164
|
+
model: input.model
|
|
165
|
+
});
|
|
166
|
+
if (!externallyBilled && (services.settings.billingMode === "stripe" || services.settings.usageLimitsMode === "managed")) {
|
|
167
|
+
if (services.entitlements) {
|
|
168
|
+
const decision = await services.entitlements.admitRun({
|
|
169
|
+
accountId: input.accountId,
|
|
170
|
+
workspaceId: input.workspaceId,
|
|
171
|
+
action: "agent_run:create",
|
|
172
|
+
quantity: input.requestedAgentRuns
|
|
173
|
+
});
|
|
174
|
+
if (!decision.allowed) return "insufficient_credits";
|
|
175
|
+
} else {
|
|
176
|
+
const balance = await getBillingBalance(services.db, input.accountId);
|
|
177
|
+
if (balance.balanceMicros <= 0) return "insufficient_credits";
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
if (services.settings.usageLimitsMode !== "static" && services.settings.usageLimitsMode !== "managed") {
|
|
181
|
+
return null;
|
|
182
|
+
}
|
|
183
|
+
const limits = configuredStaticUsageLimits(services.settings);
|
|
184
|
+
if (!externallyBilled && limits.maxMonthlyCostMicrosPerAccount) {
|
|
185
|
+
const used = await sumUsageQuantity(services.db, {
|
|
186
|
+
accountId: input.accountId,
|
|
187
|
+
eventType: "model.cost",
|
|
188
|
+
since: startOfUtcMonth()
|
|
189
|
+
});
|
|
190
|
+
if (used >= limits.maxMonthlyCostMicrosPerAccount) {
|
|
191
|
+
return "monthly_model_cost_limit";
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
if (limits.maxMonthlyAgentRunsPerWorkspace) {
|
|
195
|
+
const used = await sumUsageQuantity(services.db, {
|
|
196
|
+
workspaceId: input.workspaceId,
|
|
197
|
+
eventType: "agent_run.created",
|
|
198
|
+
since: startOfUtcMonth()
|
|
199
|
+
});
|
|
200
|
+
if (used + input.requestedAgentRuns > limits.maxMonthlyAgentRunsPerWorkspace) {
|
|
201
|
+
return "monthly_agent_run_limit";
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
return null;
|
|
205
|
+
}
|
|
206
|
+
function startOfUtcMonth() {
|
|
207
|
+
const now = /* @__PURE__ */ new Date();
|
|
208
|
+
return new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), 1));
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
// src/activities/automations.ts
|
|
212
|
+
function createAutomationActivities(services, options = {}) {
|
|
213
|
+
const claim = options.claim ?? claimAutomationRun;
|
|
214
|
+
const settle = options.settle ?? settleAutomationRun;
|
|
215
|
+
const assertAuthority = options.assertAuthority ?? assertAutomationRunAuthorityInTransaction;
|
|
216
|
+
const createSession2 = options.createSession ?? createAndStartSessionWithOutcome;
|
|
217
|
+
const admit = options.admit ?? agentRunAdmissionDenial;
|
|
218
|
+
const assertModelPolicy = options.assertModelPolicy ?? assertWorkspaceModelPolicyAllows;
|
|
219
|
+
const recordUsage = options.recordUsage ?? recordWorkspaceUsage;
|
|
220
|
+
return {
|
|
221
|
+
dispatchAutomationRun: async (input) => {
|
|
222
|
+
const service = await services();
|
|
223
|
+
const run = await claim(service.db, input);
|
|
224
|
+
if (!run) return { action: "not_found" };
|
|
225
|
+
if (run.status === "dispatched") {
|
|
226
|
+
if (!run.sessionId) throw new Error("dispatched automation run is missing its session");
|
|
227
|
+
return { action: "already_dispatched", sessionId: run.sessionId };
|
|
228
|
+
}
|
|
229
|
+
if (run.status === "skipped") {
|
|
230
|
+
return { action: "skipped", reason: run.errorCode ?? "authority_revoked" };
|
|
231
|
+
}
|
|
232
|
+
const accepted = run.acceptedExecution;
|
|
233
|
+
if (accepted.accountId !== input.accountId || accepted.workspaceId !== input.workspaceId || accepted.sourceId !== run.sourceId || accepted.triggerId !== run.triggerId || accepted.triggerRevision !== run.triggerRevision || accepted.eventId !== run.eventId) {
|
|
234
|
+
await settle(service.db, {
|
|
235
|
+
workspaceId: input.workspaceId,
|
|
236
|
+
runId: run.id,
|
|
237
|
+
status: "skipped",
|
|
238
|
+
errorCode: "accepted_execution_mismatch"
|
|
239
|
+
});
|
|
240
|
+
return { action: "skipped", reason: "accepted_execution_mismatch" };
|
|
241
|
+
}
|
|
242
|
+
const template = accepted.sessionTemplate;
|
|
243
|
+
const model = canonicalConfiguredModel(
|
|
244
|
+
service.settings,
|
|
245
|
+
template.model ?? service.settings.openaiModel
|
|
246
|
+
);
|
|
247
|
+
if (!model) throw new Error("automation has no configured model");
|
|
248
|
+
await assertModelPolicy(service.db, service.settings, input.workspaceId, model);
|
|
249
|
+
const denial = await admit(service, {
|
|
250
|
+
accountId: input.accountId,
|
|
251
|
+
workspaceId: input.workspaceId,
|
|
252
|
+
model,
|
|
253
|
+
requestedAgentRuns: 1
|
|
254
|
+
});
|
|
255
|
+
if (denial) {
|
|
256
|
+
await settle(service.db, {
|
|
257
|
+
workspaceId: input.workspaceId,
|
|
258
|
+
runId: run.id,
|
|
259
|
+
status: "failed",
|
|
260
|
+
errorCode: denial
|
|
261
|
+
});
|
|
262
|
+
throw new Error(`automation admission denied: ${denial}`);
|
|
263
|
+
}
|
|
264
|
+
const reasoningEffort = template.reasoningEffort ?? service.settings.openaiReasoningEffort;
|
|
265
|
+
const sandboxBackend = template.sandboxBackend ?? service.settings.sandboxBackend;
|
|
266
|
+
if (sandboxBackend === "selfhosted") {
|
|
267
|
+
await settle(service.db, {
|
|
268
|
+
workspaceId: input.workspaceId,
|
|
269
|
+
runId: run.id,
|
|
270
|
+
status: "skipped",
|
|
271
|
+
errorCode: "interactive_compute_not_allowed"
|
|
272
|
+
});
|
|
273
|
+
return { action: "skipped", reason: "interactive_compute_not_allowed" };
|
|
274
|
+
}
|
|
275
|
+
const turnExecutionPolicy = resolveTurnExecutionPolicyV1(service.settings, {
|
|
276
|
+
modelId: model,
|
|
277
|
+
requestedModelId: template.model,
|
|
278
|
+
modelSource: template.model ? "explicit" : "deployment",
|
|
279
|
+
reasoningEffort,
|
|
280
|
+
reasoningSource: template.reasoningEffort ? "explicit" : "deployment",
|
|
281
|
+
latencyMode: "standard",
|
|
282
|
+
latencyModeSource: "deployment"
|
|
283
|
+
});
|
|
284
|
+
try {
|
|
285
|
+
const created = await createSession2({
|
|
286
|
+
db: service.db,
|
|
287
|
+
bus: service.bus,
|
|
288
|
+
workflowClient: {
|
|
289
|
+
wakeSessionWorkflow: async (wake) => {
|
|
290
|
+
await service.wakeSessionWorkflow?.(wake);
|
|
291
|
+
}
|
|
292
|
+
},
|
|
293
|
+
beforeCreateCommit: async (tx, sessionId) => {
|
|
294
|
+
await assertAuthority(tx, {
|
|
295
|
+
workspaceId: input.workspaceId,
|
|
296
|
+
runId: run.id,
|
|
297
|
+
triggerId: accepted.triggerId,
|
|
298
|
+
triggerRevision: accepted.triggerRevision,
|
|
299
|
+
sourceId: accepted.sourceId,
|
|
300
|
+
sourceVersion: accepted.sourceVersion,
|
|
301
|
+
sessionId
|
|
302
|
+
});
|
|
303
|
+
},
|
|
304
|
+
accountId: input.accountId,
|
|
305
|
+
workspaceId: input.workspaceId,
|
|
306
|
+
initialMessage: accepted.initialMessage,
|
|
307
|
+
resources: template.resources,
|
|
308
|
+
skills: template.skills,
|
|
309
|
+
tools: template.tools,
|
|
310
|
+
toolPolicy: { mode: "explicit", inheritedFromSessionId: null },
|
|
311
|
+
model,
|
|
312
|
+
reasoningEffort,
|
|
313
|
+
turnExecutionPolicy,
|
|
314
|
+
sandboxBackend,
|
|
315
|
+
metadata: {
|
|
316
|
+
...template.metadata,
|
|
317
|
+
automationRunId: run.id,
|
|
318
|
+
automationSourceId: run.sourceId,
|
|
319
|
+
automationTriggerId: run.triggerId,
|
|
320
|
+
automationTriggerRevision: run.triggerRevision
|
|
321
|
+
},
|
|
322
|
+
createdBy: {
|
|
323
|
+
kind: "service",
|
|
324
|
+
subjectId: accepted.serviceSubjectId,
|
|
325
|
+
label: accepted.serviceLabel
|
|
326
|
+
},
|
|
327
|
+
createdByContext: accepted.provenance,
|
|
328
|
+
instructions: template.instructions,
|
|
329
|
+
policyRole: template.policyRole,
|
|
330
|
+
firstPartyMcpPermissions: template.firstPartyMcpPermissions,
|
|
331
|
+
firstPartyMcpTools: template.firstPartyMcpTools,
|
|
332
|
+
createIdempotencyKey: `automation-run:${run.id}`,
|
|
333
|
+
subjectId: accepted.serviceSubjectId
|
|
334
|
+
});
|
|
335
|
+
await recordUsage(
|
|
336
|
+
{ db: service.db, settings: service.settings },
|
|
337
|
+
{
|
|
338
|
+
accountId: input.accountId,
|
|
339
|
+
workspaceId: input.workspaceId,
|
|
340
|
+
subjectId: accepted.serviceSubjectId,
|
|
341
|
+
eventType: "agent_run.created",
|
|
342
|
+
quantity: 1,
|
|
343
|
+
unit: "run",
|
|
344
|
+
sourceResourceType: "automation_run",
|
|
345
|
+
sourceResourceId: run.id,
|
|
346
|
+
sessionId: created.session.id,
|
|
347
|
+
initiator: created.session.createdBy,
|
|
348
|
+
initiatorContext: created.session.createdByContext,
|
|
349
|
+
origin: "system",
|
|
350
|
+
idempotencyKey: `agent_run.created:automation:${run.id}`
|
|
351
|
+
}
|
|
352
|
+
).catch((error) => {
|
|
353
|
+
service.observability.warn("automation usage recording failed", {
|
|
354
|
+
workspaceId: input.workspaceId,
|
|
355
|
+
runId: run.id,
|
|
356
|
+
errorCategory: error instanceof Error ? error.name : "unknown"
|
|
357
|
+
});
|
|
358
|
+
});
|
|
359
|
+
return { action: "started", sessionId: created.session.id };
|
|
360
|
+
} catch (error) {
|
|
361
|
+
if (error instanceof AutomationAuthorityRevokedError) {
|
|
362
|
+
await settle(service.db, {
|
|
363
|
+
workspaceId: input.workspaceId,
|
|
364
|
+
runId: run.id,
|
|
365
|
+
status: "skipped",
|
|
366
|
+
errorCode: "authority_revoked"
|
|
367
|
+
});
|
|
368
|
+
return { action: "skipped", reason: "authority_revoked" };
|
|
369
|
+
}
|
|
370
|
+
await settle(service.db, {
|
|
371
|
+
workspaceId: input.workspaceId,
|
|
372
|
+
runId: run.id,
|
|
373
|
+
status: "failed",
|
|
374
|
+
errorCode: "dispatch_failed"
|
|
375
|
+
});
|
|
376
|
+
throw error;
|
|
377
|
+
}
|
|
378
|
+
}
|
|
379
|
+
};
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
// src/activities/documents.ts
|
|
383
|
+
import { configuredStaticUsageLimits as configuredStaticUsageLimits2 } from "@opengeni/config";
|
|
130
384
|
import {
|
|
131
|
-
getBillingBalance,
|
|
385
|
+
getBillingBalance as getBillingBalance2,
|
|
132
386
|
recordUsageEvent,
|
|
133
387
|
resolveDocumentIndexAuthority,
|
|
134
388
|
rlsContextForWorkspace,
|
|
135
|
-
sumUsageQuantity,
|
|
389
|
+
sumUsageQuantity as sumUsageQuantity2,
|
|
136
390
|
withWorkspaceUsageLock
|
|
137
391
|
} from "@opengeni/db";
|
|
138
392
|
function createDocumentActivities(services, resolveDocumentServices) {
|
|
@@ -190,7 +444,7 @@ function createDocumentActivities(services, resolveDocumentServices) {
|
|
|
190
444
|
{
|
|
191
445
|
beforeEmbed: async ({ chunkCount }) => {
|
|
192
446
|
if (settings.billingMode === "stripe" || settings.usageLimitsMode === "managed") {
|
|
193
|
-
const balance = await
|
|
447
|
+
const balance = await getBillingBalance2(lockedDb, input.accountId);
|
|
194
448
|
if (balance.balanceMicros <= 0) {
|
|
195
449
|
throw new Error("insufficient OpenGeni credits");
|
|
196
450
|
}
|
|
@@ -198,14 +452,14 @@ function createDocumentActivities(services, resolveDocumentServices) {
|
|
|
198
452
|
if (settings.usageLimitsMode !== "static" && settings.usageLimitsMode !== "managed") {
|
|
199
453
|
return;
|
|
200
454
|
}
|
|
201
|
-
const limit =
|
|
455
|
+
const limit = configuredStaticUsageLimits2(settings).maxDocumentIndexedChunksPerWorkspace;
|
|
202
456
|
if (!limit) {
|
|
203
457
|
return;
|
|
204
458
|
}
|
|
205
|
-
const used = await
|
|
459
|
+
const used = await sumUsageQuantity2(lockedDb, {
|
|
206
460
|
workspaceId: input.workspaceId,
|
|
207
461
|
eventType: "document.indexed",
|
|
208
|
-
since:
|
|
462
|
+
since: startOfUtcMonth2()
|
|
209
463
|
});
|
|
210
464
|
if (used + chunkCount > limit) {
|
|
211
465
|
throw new Error(`monthly document indexing limit reached (${limit} chunks)`);
|
|
@@ -234,7 +488,7 @@ function createDocumentActivities(services, resolveDocumentServices) {
|
|
|
234
488
|
}
|
|
235
489
|
};
|
|
236
490
|
}
|
|
237
|
-
function
|
|
491
|
+
function startOfUtcMonth2(date = /* @__PURE__ */ new Date()) {
|
|
238
492
|
return new Date(Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), 1));
|
|
239
493
|
}
|
|
240
494
|
|
|
@@ -407,6 +661,9 @@ function createFileUploadReaperActivities(services, options = {}) {
|
|
|
407
661
|
// src/activities/knowledge-source-sync.ts
|
|
408
662
|
import { createHash as createHash2 } from "crypto";
|
|
409
663
|
import { heartbeat } from "@temporalio/activity";
|
|
664
|
+
import {
|
|
665
|
+
googleDriveProviderRetryOptions
|
|
666
|
+
} from "@opengeni/config";
|
|
410
667
|
import {
|
|
411
668
|
KnowledgeSourceSyncRunSummary
|
|
412
669
|
} from "@opengeni/contracts";
|
|
@@ -471,6 +728,121 @@ import {
|
|
|
471
728
|
} from "@opengeni/db";
|
|
472
729
|
import { readResponseJsonBounded } from "@opengeni/network";
|
|
473
730
|
|
|
731
|
+
// src/activities/google-drive-provider.ts
|
|
732
|
+
var GoogleDriveProviderTransportError = class extends Error {
|
|
733
|
+
constructor() {
|
|
734
|
+
super("google_drive_provider_transport_failed");
|
|
735
|
+
this.name = "GoogleDriveProviderTransportError";
|
|
736
|
+
}
|
|
737
|
+
};
|
|
738
|
+
async function fetchGoogleDriveProvider(input) {
|
|
739
|
+
const now = input.now ?? Date.now;
|
|
740
|
+
const sleep = input.sleep ?? Bun.sleep;
|
|
741
|
+
let retryDelaySpentMs = 0;
|
|
742
|
+
for (let attempt = 1; attempt <= input.policy.attempts; attempt += 1) {
|
|
743
|
+
let response;
|
|
744
|
+
input.heartbeat?.({ operation: input.operation, attempt, phase: "request_start" });
|
|
745
|
+
try {
|
|
746
|
+
response = await input.fetchImpl(input.url, {
|
|
747
|
+
...input.init,
|
|
748
|
+
redirect: "error",
|
|
749
|
+
signal: AbortSignal.timeout(input.policy.requestTimeoutMs)
|
|
750
|
+
});
|
|
751
|
+
} catch {
|
|
752
|
+
input.heartbeat?.({ operation: input.operation, attempt, phase: "request_end" });
|
|
753
|
+
recordRequest(input.observability, input.operation, "retryable_error");
|
|
754
|
+
const delayMs2 = retryDelayMs(input.policy, attempt, null);
|
|
755
|
+
if (!canRetry(input.policy, attempt, retryDelaySpentMs, delayMs2)) {
|
|
756
|
+
throw new GoogleDriveProviderTransportError();
|
|
757
|
+
}
|
|
758
|
+
recordRetry(input.observability, input.operation, "transport", attempt, delayMs2);
|
|
759
|
+
retryDelaySpentMs += delayMs2;
|
|
760
|
+
input.heartbeat?.({ operation: input.operation, attempt, phase: "retry_delay_start" });
|
|
761
|
+
await sleep(delayMs2);
|
|
762
|
+
input.heartbeat?.({ operation: input.operation, attempt, phase: "retry_delay_end" });
|
|
763
|
+
continue;
|
|
764
|
+
}
|
|
765
|
+
input.heartbeat?.({ operation: input.operation, attempt, phase: "request_end" });
|
|
766
|
+
if (!isRetryableStatus(response.status)) {
|
|
767
|
+
recordRequest(
|
|
768
|
+
input.observability,
|
|
769
|
+
input.operation,
|
|
770
|
+
response.ok ? "succeeded" : "terminal_error"
|
|
771
|
+
);
|
|
772
|
+
return response;
|
|
773
|
+
}
|
|
774
|
+
recordRequest(input.observability, input.operation, "retryable_error");
|
|
775
|
+
const delayMs = retryDelayMs(
|
|
776
|
+
input.policy,
|
|
777
|
+
attempt,
|
|
778
|
+
retryAfterDelayMs(response.headers.get("retry-after"), now())
|
|
779
|
+
);
|
|
780
|
+
if (!canRetry(input.policy, attempt, retryDelaySpentMs, delayMs)) return response;
|
|
781
|
+
await response.body?.cancel().catch(() => void 0);
|
|
782
|
+
recordRetry(
|
|
783
|
+
input.observability,
|
|
784
|
+
input.operation,
|
|
785
|
+
response.status === 429 ? "rate_limited" : "provider_unavailable",
|
|
786
|
+
attempt,
|
|
787
|
+
delayMs
|
|
788
|
+
);
|
|
789
|
+
retryDelaySpentMs += delayMs;
|
|
790
|
+
input.heartbeat?.({ operation: input.operation, attempt, phase: "retry_delay_start" });
|
|
791
|
+
await sleep(delayMs);
|
|
792
|
+
input.heartbeat?.({ operation: input.operation, attempt, phase: "retry_delay_end" });
|
|
793
|
+
}
|
|
794
|
+
throw new GoogleDriveProviderTransportError();
|
|
795
|
+
}
|
|
796
|
+
function canRetry(policy, attempt, retryDelaySpentMs, delayMs) {
|
|
797
|
+
return attempt < policy.attempts && retryDelaySpentMs + delayMs <= policy.budgetMs;
|
|
798
|
+
}
|
|
799
|
+
function retryDelayMs(policy, attempt, retryAfterMs) {
|
|
800
|
+
const exponential = Math.min(
|
|
801
|
+
policy.maxDelayMs,
|
|
802
|
+
policy.initialDelayMs * 2 ** Math.max(0, attempt - 1)
|
|
803
|
+
);
|
|
804
|
+
return Math.min(policy.maxDelayMs, Math.max(exponential, retryAfterMs ?? 0));
|
|
805
|
+
}
|
|
806
|
+
function retryAfterDelayMs(value, nowMs) {
|
|
807
|
+
if (!value) return null;
|
|
808
|
+
const seconds = Number(value);
|
|
809
|
+
if (Number.isFinite(seconds) && seconds >= 0) return Math.round(seconds * 1e3);
|
|
810
|
+
const timestamp = Date.parse(value);
|
|
811
|
+
return Number.isFinite(timestamp) ? Math.max(0, timestamp - nowMs) : null;
|
|
812
|
+
}
|
|
813
|
+
function isRetryableStatus(status) {
|
|
814
|
+
return status === 429 || status >= 500;
|
|
815
|
+
}
|
|
816
|
+
function recordRequest(observability, operation, outcome) {
|
|
817
|
+
observability.incrementCounter({
|
|
818
|
+
name: "opengeni_google_drive_provider_requests_total",
|
|
819
|
+
help: "Google Drive provider request attempts by operation and bounded outcome.",
|
|
820
|
+
labels: { operation, outcome }
|
|
821
|
+
});
|
|
822
|
+
}
|
|
823
|
+
function recordRetry(observability, operation, reason, attempt, delayMs) {
|
|
824
|
+
observability.incrementCounter({
|
|
825
|
+
name: "opengeni_google_drive_provider_retries_total",
|
|
826
|
+
help: "Bounded Google Drive provider retries by operation and reason.",
|
|
827
|
+
labels: { operation, reason }
|
|
828
|
+
});
|
|
829
|
+
observability.observeHistogram({
|
|
830
|
+
name: "opengeni_google_drive_provider_retry_delay_seconds",
|
|
831
|
+
help: "Delay applied before a bounded Google Drive provider retry.",
|
|
832
|
+
labels: { operation, reason },
|
|
833
|
+
value: delayMs / 1e3,
|
|
834
|
+
buckets: [0.1, 0.25, 0.5, 1, 2, 5, 10, 30, 60]
|
|
835
|
+
});
|
|
836
|
+
observability.warn("Google Drive provider request retry scheduled", {
|
|
837
|
+
provider: "google_drive",
|
|
838
|
+
operation,
|
|
839
|
+
outcome: "retry_scheduled",
|
|
840
|
+
reason,
|
|
841
|
+
attempt,
|
|
842
|
+
delayMs
|
|
843
|
+
});
|
|
844
|
+
}
|
|
845
|
+
|
|
474
846
|
// src/activities/google-drive-changes.ts
|
|
475
847
|
import { createHash } from "crypto";
|
|
476
848
|
import {
|
|
@@ -805,6 +1177,10 @@ function createKnowledgeSourceSyncActivities(services, resolveDocumentServices)
|
|
|
805
1177
|
reconnectRequired: false,
|
|
806
1178
|
failures: []
|
|
807
1179
|
};
|
|
1180
|
+
observability.info("Knowledge source sync started", {
|
|
1181
|
+
provider: providerLabel,
|
|
1182
|
+
outcome: "started"
|
|
1183
|
+
});
|
|
808
1184
|
try {
|
|
809
1185
|
summary.phase = "inventory";
|
|
810
1186
|
const resolved = await getKnowledgeSourceForSyncAuthority(db, {
|
|
@@ -885,6 +1261,7 @@ function createKnowledgeSourceSyncActivities(services, resolveDocumentServices)
|
|
|
885
1261
|
const provider = await resolveKnowledgeSyncProvider({
|
|
886
1262
|
db,
|
|
887
1263
|
settings,
|
|
1264
|
+
observability,
|
|
888
1265
|
accountId: input.accountId,
|
|
889
1266
|
workspaceId: input.workspaceId,
|
|
890
1267
|
action,
|
|
@@ -1614,6 +1991,7 @@ function createKnowledgeSourceSyncActivities(services, resolveDocumentServices)
|
|
|
1614
1991
|
labels: { provider: providerLabel },
|
|
1615
1992
|
amount: summary.bytes
|
|
1616
1993
|
});
|
|
1994
|
+
recordSyncHealthTelemetry(observability, providerLabel, summary, "succeeded");
|
|
1617
1995
|
return {
|
|
1618
1996
|
action: "complete",
|
|
1619
1997
|
bufferedWake: settled.bufferedWake,
|
|
@@ -1698,6 +2076,7 @@ function createKnowledgeSourceSyncActivities(services, resolveDocumentServices)
|
|
|
1698
2076
|
labels: { provider: providerLabel },
|
|
1699
2077
|
amount: summary.bytes
|
|
1700
2078
|
});
|
|
2079
|
+
recordSyncHealthTelemetry(observability, providerLabel, summary, "failed", failure);
|
|
1701
2080
|
return {
|
|
1702
2081
|
action: "failed",
|
|
1703
2082
|
bufferedWake: settled.bufferedWake,
|
|
@@ -1878,6 +2257,8 @@ async function resolveKnowledgeSyncProvider(input) {
|
|
|
1878
2257
|
workspaceId: input.workspaceId,
|
|
1879
2258
|
initiatingSubjectId: action.initiatingSubjectId,
|
|
1880
2259
|
authorization,
|
|
2260
|
+
retry: googleDriveProviderRetryOptions(input.settings),
|
|
2261
|
+
observability: input.observability,
|
|
1881
2262
|
limits: action.limits,
|
|
1882
2263
|
connectionId: resolvedConnection.id,
|
|
1883
2264
|
fullReconciliationIntervalMs: GOOGLE_DRIVE_FULL_RECONCILIATION_INTERVAL_MS,
|
|
@@ -1963,11 +2344,18 @@ function atlassianSyncDriver(input) {
|
|
|
1963
2344
|
function googleDriveSyncDriver(input) {
|
|
1964
2345
|
const maxElapsedMs = input.limits.maxElapsedSeconds * 1e3;
|
|
1965
2346
|
const provider = input.provider ?? {
|
|
1966
|
-
getStartPageToken: async (driveId) => await getDriveStartPageToken(driveId, input.authorization),
|
|
1967
|
-
listChanges: async (pageToken, driveId, pageSize) => await listDriveChanges(
|
|
1968
|
-
|
|
1969
|
-
|
|
1970
|
-
|
|
2347
|
+
getStartPageToken: async (driveId) => await getDriveStartPageToken(driveId, input.authorization, input.retry, input.observability),
|
|
2348
|
+
listChanges: async (pageToken, driveId, pageSize) => await listDriveChanges(
|
|
2349
|
+
pageToken,
|
|
2350
|
+
driveId,
|
|
2351
|
+
pageSize,
|
|
2352
|
+
input.authorization,
|
|
2353
|
+
input.retry,
|
|
2354
|
+
input.observability
|
|
2355
|
+
),
|
|
2356
|
+
getFile: async (fileId) => await getDriveFile(fileId, input.authorization, input.retry, input.observability),
|
|
2357
|
+
listChildren: async (request) => await listDriveChildren(request, input.authorization, input.retry, input.observability),
|
|
2358
|
+
listPermissions: async (request) => await listDrivePermissions(request, input.authorization, input.retry, input.observability)
|
|
1971
2359
|
};
|
|
1972
2360
|
const clock = provider.now ?? Date.now;
|
|
1973
2361
|
const budgetLimits = {
|
|
@@ -2278,7 +2666,7 @@ function googleDriveSyncDriver(input) {
|
|
|
2278
2666
|
};
|
|
2279
2667
|
}
|
|
2280
2668
|
},
|
|
2281
|
-
fetchContent: async (entry, maxBytes) => await fetchDriveBytes(entry, input.authorization, maxBytes),
|
|
2669
|
+
fetchContent: async (entry, maxBytes) => await fetchDriveBytes(entry, input.authorization, maxBytes, input.retry, input.observability),
|
|
2282
2670
|
readAcl: async (entry, maxProviderRequests) => {
|
|
2283
2671
|
if (!provider.listPermissions) {
|
|
2284
2672
|
throw new SyncFailure("provider_rejected", false);
|
|
@@ -2466,16 +2854,18 @@ function googleDriveInventoryStopIsHard(reason, budget, limits) {
|
|
|
2466
2854
|
}
|
|
2467
2855
|
return reason === "elapsed_time_limit" && budget.elapsedMs >= limits.maxElapsedMs;
|
|
2468
2856
|
}
|
|
2469
|
-
async function getDriveStartPageToken(driveId, authorization) {
|
|
2857
|
+
async function getDriveStartPageToken(driveId, authorization, retry, observability) {
|
|
2470
2858
|
if (!authorization) throw new SyncFailure("connection_reconnect_required", true, true);
|
|
2471
2859
|
const url = new URL(`${DRIVE_API_BASE}/changes/startPageToken`);
|
|
2472
2860
|
url.searchParams.set("supportsAllDrives", "true");
|
|
2473
2861
|
if (driveId) url.searchParams.set("driveId", driveId);
|
|
2474
|
-
const response = await
|
|
2475
|
-
|
|
2476
|
-
|
|
2477
|
-
|
|
2478
|
-
|
|
2862
|
+
const response = await requestGoogleDrive(
|
|
2863
|
+
url,
|
|
2864
|
+
{ headers: { authorization, accept: "application/json" } },
|
|
2865
|
+
"start_page_token",
|
|
2866
|
+
retry,
|
|
2867
|
+
observability
|
|
2868
|
+
);
|
|
2479
2869
|
if (!response.ok) throwGoogleDriveResponse(response);
|
|
2480
2870
|
const payload = await readResponseJsonBounded(
|
|
2481
2871
|
response,
|
|
@@ -2484,7 +2874,7 @@ async function getDriveStartPageToken(driveId, authorization) {
|
|
|
2484
2874
|
);
|
|
2485
2875
|
return requiredString(payload.startPageToken, "start_page_token");
|
|
2486
2876
|
}
|
|
2487
|
-
async function listDriveChanges(pageToken, driveId, pageSize, authorization) {
|
|
2877
|
+
async function listDriveChanges(pageToken, driveId, pageSize, authorization, retry, observability) {
|
|
2488
2878
|
if (!authorization) throw new SyncFailure("connection_reconnect_required", true, true);
|
|
2489
2879
|
const url = new URL(`${DRIVE_API_BASE}/changes`);
|
|
2490
2880
|
url.searchParams.set("pageToken", pageToken);
|
|
@@ -2497,11 +2887,13 @@ async function listDriveChanges(pageToken, driveId, pageSize, authorization) {
|
|
|
2497
2887
|
"nextPageToken,newStartPageToken,changes(fileId,removed,time,driveId,file(id,name,mimeType,driveId,parents,modifiedTime,createdTime,version,md5Checksum,size,webViewLink,trashed))"
|
|
2498
2888
|
);
|
|
2499
2889
|
if (driveId) url.searchParams.set("driveId", driveId);
|
|
2500
|
-
const response = await
|
|
2501
|
-
|
|
2502
|
-
|
|
2503
|
-
|
|
2504
|
-
|
|
2890
|
+
const response = await requestGoogleDrive(
|
|
2891
|
+
url,
|
|
2892
|
+
{ headers: { authorization, accept: "application/json" } },
|
|
2893
|
+
"list_changes",
|
|
2894
|
+
retry,
|
|
2895
|
+
observability
|
|
2896
|
+
);
|
|
2505
2897
|
if (response.status === 410) throw new GoogleDriveCursorInvalidError();
|
|
2506
2898
|
if (!response.ok) throwGoogleDriveResponse(response);
|
|
2507
2899
|
const payload = await readResponseJsonBounded(
|
|
@@ -2524,7 +2916,7 @@ async function listDriveChanges(pageToken, driveId, pageSize, authorization) {
|
|
|
2524
2916
|
newStartPageToken: optionalString(payload.newStartPageToken)
|
|
2525
2917
|
};
|
|
2526
2918
|
}
|
|
2527
|
-
async function getDriveFile(fileId, authorization) {
|
|
2919
|
+
async function getDriveFile(fileId, authorization, retry, observability) {
|
|
2528
2920
|
if (!authorization) throw new SyncFailure("connection_reconnect_required", true, true);
|
|
2529
2921
|
const url = new URL(`${DRIVE_API_BASE}/files/${encodeURIComponent(fileId)}`);
|
|
2530
2922
|
url.searchParams.set("supportsAllDrives", "true");
|
|
@@ -2532,18 +2924,20 @@ async function getDriveFile(fileId, authorization) {
|
|
|
2532
2924
|
"fields",
|
|
2533
2925
|
"id,name,mimeType,driveId,parents,modifiedTime,createdTime,version,md5Checksum,size,webViewLink,trashed"
|
|
2534
2926
|
);
|
|
2535
|
-
const response = await
|
|
2536
|
-
|
|
2537
|
-
|
|
2538
|
-
|
|
2539
|
-
|
|
2927
|
+
const response = await requestGoogleDrive(
|
|
2928
|
+
url,
|
|
2929
|
+
{ headers: { authorization, accept: "application/json" } },
|
|
2930
|
+
"get_file",
|
|
2931
|
+
retry,
|
|
2932
|
+
observability
|
|
2933
|
+
);
|
|
2540
2934
|
if (response.status === 404) return null;
|
|
2541
2935
|
if (!response.ok) throwGoogleDriveResponse(response);
|
|
2542
2936
|
return parseDriveProviderItem(
|
|
2543
2937
|
await readResponseJsonBounded(response, DRIVE_JSON_MAX_BYTES, "Google Drive file metadata")
|
|
2544
2938
|
);
|
|
2545
2939
|
}
|
|
2546
|
-
async function listDrivePermissions(input, authorization) {
|
|
2940
|
+
async function listDrivePermissions(input, authorization, retry, observability) {
|
|
2547
2941
|
if (!authorization) throw new SyncFailure("connection_reconnect_required", true, true);
|
|
2548
2942
|
const url = new URL(`${DRIVE_API_BASE}/files/${encodeURIComponent(input.fileId)}/permissions`);
|
|
2549
2943
|
url.searchParams.set("supportsAllDrives", "true");
|
|
@@ -2553,11 +2947,13 @@ async function listDrivePermissions(input, authorization) {
|
|
|
2553
2947
|
"nextPageToken,permissions(id,type,emailAddress,domain,role,allowFileDiscovery,expirationTime,deleted,permissionDetails(inherited))"
|
|
2554
2948
|
);
|
|
2555
2949
|
if (input.pageToken) url.searchParams.set("pageToken", input.pageToken);
|
|
2556
|
-
const response = await
|
|
2557
|
-
|
|
2558
|
-
|
|
2559
|
-
|
|
2560
|
-
|
|
2950
|
+
const response = await requestGoogleDrive(
|
|
2951
|
+
url,
|
|
2952
|
+
{ headers: { authorization, accept: "application/json" } },
|
|
2953
|
+
"list_permissions",
|
|
2954
|
+
retry,
|
|
2955
|
+
observability
|
|
2956
|
+
);
|
|
2561
2957
|
if (response.status === 403 || response.status === 404) {
|
|
2562
2958
|
return { permissions: [], nextPageToken: null, denied: true };
|
|
2563
2959
|
}
|
|
@@ -2739,6 +3135,24 @@ function throwGoogleDriveResponse(response) {
|
|
|
2739
3135
|
response.status >= 500 || response.status === 429
|
|
2740
3136
|
);
|
|
2741
3137
|
}
|
|
3138
|
+
async function requestGoogleDrive(url, init, operation, retry, observability) {
|
|
3139
|
+
try {
|
|
3140
|
+
return await fetchGoogleDriveProvider({
|
|
3141
|
+
fetchImpl: fetch,
|
|
3142
|
+
url,
|
|
3143
|
+
init,
|
|
3144
|
+
operation,
|
|
3145
|
+
policy: retry,
|
|
3146
|
+
observability,
|
|
3147
|
+
heartbeat: (details) => heartbeat({ provider: "google_drive", ...details })
|
|
3148
|
+
});
|
|
3149
|
+
} catch (error) {
|
|
3150
|
+
if (error instanceof GoogleDriveProviderTransportError) {
|
|
3151
|
+
throw new SyncFailure("provider_unavailable", true);
|
|
3152
|
+
}
|
|
3153
|
+
throw error;
|
|
3154
|
+
}
|
|
3155
|
+
}
|
|
2742
3156
|
function selectedDriveSource(metadata, externalSourceId) {
|
|
2743
3157
|
const sources = metadata.selectedSources ?? (metadata.selectedSource ? [metadata.selectedSource] : []);
|
|
2744
3158
|
return sources.find((source) => source.id === externalSourceId) ?? null;
|
|
@@ -2746,7 +3160,7 @@ function selectedDriveSource(metadata, externalSourceId) {
|
|
|
2746
3160
|
function googleDriveSyncProviderAccessAllowed(input) {
|
|
2747
3161
|
return input.resolvedConnectionVersion !== void 0 && input.connectionVersion === input.resolvedConnectionVersion && input.connectionStatus === "active" && (input.lifecycleState === void 0 || input.lifecycleState === "active") && googleDriveScopesAllowCapability(input.grantedScopes, "recursive_source_sync");
|
|
2748
3162
|
}
|
|
2749
|
-
async function listDriveChildren(input, authorization) {
|
|
3163
|
+
async function listDriveChildren(input, authorization, retry, observability) {
|
|
2750
3164
|
if (!authorization) throw new SyncFailure("connection_reconnect_required", true, true);
|
|
2751
3165
|
const url = new URL(`${DRIVE_API_BASE}/files`);
|
|
2752
3166
|
url.searchParams.set(
|
|
@@ -2765,11 +3179,13 @@ async function listDriveChildren(input, authorization) {
|
|
|
2765
3179
|
url.searchParams.set("corpora", "drive");
|
|
2766
3180
|
url.searchParams.set("driveId", input.driveId);
|
|
2767
3181
|
}
|
|
2768
|
-
const response = await
|
|
2769
|
-
|
|
2770
|
-
|
|
2771
|
-
|
|
2772
|
-
|
|
3182
|
+
const response = await requestGoogleDrive(
|
|
3183
|
+
url,
|
|
3184
|
+
{ headers: { authorization, accept: "application/json" } },
|
|
3185
|
+
"list_children",
|
|
3186
|
+
retry,
|
|
3187
|
+
observability
|
|
3188
|
+
);
|
|
2773
3189
|
if (!response.ok) {
|
|
2774
3190
|
if (response.status === 401 || response.status === 403) {
|
|
2775
3191
|
throw new SyncFailure("connection_reconnect_required", true, true);
|
|
@@ -2808,7 +3224,7 @@ function parseDriveProviderItem(value) {
|
|
|
2808
3224
|
trashed: row.trashed === true
|
|
2809
3225
|
};
|
|
2810
3226
|
}
|
|
2811
|
-
async function fetchDriveBytes(entry, authorization, maxBytes) {
|
|
3227
|
+
async function fetchDriveBytes(entry, authorization, maxBytes, retry, observability) {
|
|
2812
3228
|
if (!authorization) throw new SyncFailure("connection_reconnect_required", true, true);
|
|
2813
3229
|
const url = entry.transfer.action === "export" ? new URL(`${DRIVE_API_BASE}/files/${encodeURIComponent(entry.externalObjectId)}/export`) : new URL(`${DRIVE_API_BASE}/files/${encodeURIComponent(entry.externalObjectId)}`);
|
|
2814
3230
|
if (entry.transfer.action === "export") {
|
|
@@ -2817,11 +3233,13 @@ async function fetchDriveBytes(entry, authorization, maxBytes) {
|
|
|
2817
3233
|
url.searchParams.set("alt", "media");
|
|
2818
3234
|
url.searchParams.set("supportsAllDrives", "true");
|
|
2819
3235
|
}
|
|
2820
|
-
const response = await
|
|
2821
|
-
|
|
2822
|
-
|
|
2823
|
-
|
|
2824
|
-
|
|
3236
|
+
const response = await requestGoogleDrive(
|
|
3237
|
+
url,
|
|
3238
|
+
{ headers: { authorization } },
|
|
3239
|
+
entry.transfer.action === "export" ? "export" : "download",
|
|
3240
|
+
retry,
|
|
3241
|
+
observability
|
|
3242
|
+
);
|
|
2825
3243
|
if (!response.ok) {
|
|
2826
3244
|
if (response.status === 401 || response.status === 403) {
|
|
2827
3245
|
throw new SyncFailure("connection_reconnect_required", true, true);
|
|
@@ -3193,6 +3611,59 @@ function inventoryLimit(reason) {
|
|
|
3193
3611
|
if (reason === "known_byte_limit") return "bytes";
|
|
3194
3612
|
return null;
|
|
3195
3613
|
}
|
|
3614
|
+
function recordSyncHealthTelemetry(observability, provider, summary, outcome, failure) {
|
|
3615
|
+
observability.observeHistogram({
|
|
3616
|
+
name: "opengeni_knowledge_source_sync_terminal_batch_duration_seconds",
|
|
3617
|
+
help: "Terminal knowledge-source activity batch duration by provider and outcome.",
|
|
3618
|
+
labels: { provider, outcome },
|
|
3619
|
+
value: Math.max(0, summary.elapsedMs) / 1e3
|
|
3620
|
+
});
|
|
3621
|
+
if (summary.providerRequests > 0) {
|
|
3622
|
+
observability.incrementCounter({
|
|
3623
|
+
name: "opengeni_knowledge_source_sync_provider_requests_total",
|
|
3624
|
+
help: "Logical provider requests consumed by knowledge-source synchronization.",
|
|
3625
|
+
labels: { provider, outcome },
|
|
3626
|
+
amount: summary.providerRequests
|
|
3627
|
+
});
|
|
3628
|
+
}
|
|
3629
|
+
if (summary.limitReached) {
|
|
3630
|
+
observability.incrementCounter({
|
|
3631
|
+
name: "opengeni_knowledge_source_sync_limit_hits_total",
|
|
3632
|
+
help: "Knowledge-source synchronization runs stopped by an explicit resource budget.",
|
|
3633
|
+
labels: { provider, limit: summary.limitReached }
|
|
3634
|
+
});
|
|
3635
|
+
}
|
|
3636
|
+
if (summary.reconnectRequired) {
|
|
3637
|
+
observability.incrementCounter({
|
|
3638
|
+
name: "opengeni_knowledge_source_sync_reconnect_required_total",
|
|
3639
|
+
help: "Knowledge-source synchronization failures requiring connector reconnection.",
|
|
3640
|
+
labels: { provider }
|
|
3641
|
+
});
|
|
3642
|
+
}
|
|
3643
|
+
if (failure) {
|
|
3644
|
+
observability.incrementCounter({
|
|
3645
|
+
name: "opengeni_knowledge_source_sync_failures_total",
|
|
3646
|
+
help: "Knowledge-source synchronization terminal failures by bounded reason.",
|
|
3647
|
+
labels: {
|
|
3648
|
+
provider,
|
|
3649
|
+
reason: failure.code,
|
|
3650
|
+
retryable: failure.retryable ? "true" : "false"
|
|
3651
|
+
}
|
|
3652
|
+
});
|
|
3653
|
+
observability.warn("Knowledge source sync failed", {
|
|
3654
|
+
provider,
|
|
3655
|
+
outcome: failure.retryable ? "retryable_failed" : "failed",
|
|
3656
|
+
reason: failure.code,
|
|
3657
|
+
durationMs: summary.elapsedMs
|
|
3658
|
+
});
|
|
3659
|
+
return;
|
|
3660
|
+
}
|
|
3661
|
+
observability.info("Knowledge source sync completed", {
|
|
3662
|
+
provider,
|
|
3663
|
+
outcome: "succeeded",
|
|
3664
|
+
durationMs: summary.elapsedMs
|
|
3665
|
+
});
|
|
3666
|
+
}
|
|
3196
3667
|
var SyncFailure = class extends Error {
|
|
3197
3668
|
constructor(code, retryable, reconnectRequired = false) {
|
|
3198
3669
|
super(code);
|
|
@@ -3209,6 +3680,7 @@ import {
|
|
|
3209
3680
|
completeRetainedScreenshotMaintenance,
|
|
3210
3681
|
promoteRetainedScreenshotMaintenanceCleanup
|
|
3211
3682
|
} from "@opengeni/db";
|
|
3683
|
+
import { retryWhileMissing } from "@opengeni/storage";
|
|
3212
3684
|
var RETAINED_SCREENSHOT_PENDING_GRACE_MS = 15 * 60 * 1e3;
|
|
3213
3685
|
var RETAINED_SCREENSHOT_CLAIM_TIMEOUT_MS = 10 * 60 * 1e3;
|
|
3214
3686
|
var RETAINED_SCREENSHOT_MAINTENANCE_BATCH_SIZE = 100;
|
|
@@ -3314,8 +3786,11 @@ async function reconcileClaim(db, storage, claim, provider) {
|
|
|
3314
3786
|
}
|
|
3315
3787
|
async function observeClaimObject(storage, claim, provider) {
|
|
3316
3788
|
const file = claimFile(storage, claim);
|
|
3317
|
-
|
|
3318
|
-
|
|
3789
|
+
const head = await retryWhileMissing(async () => {
|
|
3790
|
+
if (!await provider.fileExists(storage, file)) return null;
|
|
3791
|
+
return await provider.headFile(storage, file);
|
|
3792
|
+
});
|
|
3793
|
+
if (!head) return "missing";
|
|
3319
3794
|
return head.ContentLength === claim.sizeBytes && head.ContentType === claim.mediaType && head.Metadata?.sha256 === claim.sha256 ? "matches" : "mismatch";
|
|
3320
3795
|
}
|
|
3321
3796
|
function claimFile(storage, claim) {
|
|
@@ -3663,7 +4138,7 @@ async function runWithOwnedRigVerificationSandbox(input, run, dependencies = def
|
|
|
3663
4138
|
holderId,
|
|
3664
4139
|
subjectId: null,
|
|
3665
4140
|
backend: input.settings.sandboxBackend,
|
|
3666
|
-
image: input.settings
|
|
4141
|
+
image: rigProviderImageSourceImage(input.settings, input.settings.sandboxBackend),
|
|
3667
4142
|
rigVersionId: input.rigVersionId,
|
|
3668
4143
|
leaseTtlMs: RIG_VERIFICATION_OWNER_TTL_MS,
|
|
3669
4144
|
warmingLeaseTtlMs: RIG_VERIFICATION_OWNER_TTL_MS,
|
|
@@ -4580,6 +5055,7 @@ function createRigVerificationActivities(services) {
|
|
|
4580
5055
|
// src/activities/sandbox-lease.ts
|
|
4581
5056
|
import { createHash as createHash3, randomUUID as randomUUID2 } from "crypto";
|
|
4582
5057
|
import { Context } from "@temporalio/activity";
|
|
5058
|
+
import { OpLostReason, OpState } from "@opengeni/agent-proto";
|
|
4583
5059
|
import {
|
|
4584
5060
|
accrueWarmSeconds,
|
|
4585
5061
|
adoptLegacyModalCheckpointArtifact,
|
|
@@ -4596,7 +5072,7 @@ import {
|
|
|
4596
5072
|
countSandboxLeasesByLiveness,
|
|
4597
5073
|
deferRetainedProcessReconciliation,
|
|
4598
5074
|
forceDrainOverLimitViewerOnlyBoxes,
|
|
4599
|
-
getBillingBalance as
|
|
5075
|
+
getBillingBalance as getBillingBalance3,
|
|
4600
5076
|
listCreditBalancesByAccount,
|
|
4601
5077
|
listLegacyModalCheckpointSlots,
|
|
4602
5078
|
listLiveModalSandboxLeaseAttributions,
|
|
@@ -4620,10 +5096,18 @@ import {
|
|
|
4620
5096
|
rlsContextForWorkspace as rlsContextForWorkspace2,
|
|
4621
5097
|
settleRetainedProcess
|
|
4622
5098
|
} from "@opengeni/db";
|
|
5099
|
+
import {
|
|
5100
|
+
claimConnectedMachineSessionBackgroundCommands,
|
|
5101
|
+
deferConnectedMachineBackgroundCommandReconciliation,
|
|
5102
|
+
recordConnectedMachineBackgroundCommandProof,
|
|
5103
|
+
settleClaimedConnectedMachineBackgroundCommand,
|
|
5104
|
+
settleSessionBackgroundCommandForRetainedProcess
|
|
5105
|
+
} from "@opengeni/db/session-background-commands";
|
|
4623
5106
|
import { sandboxWarmRateMicrosPerSecond } from "@opengeni/config";
|
|
4624
5107
|
import { sandboxLeaseTelemetryKey } from "@opengeni/observability";
|
|
4625
5108
|
import {
|
|
4626
5109
|
captureVerifiedWorkspaceArchive,
|
|
5110
|
+
cancelSelfhostedOp,
|
|
4627
5111
|
assertConsistentSandboxProviderIdentity,
|
|
4628
5112
|
createSandboxClientForBackend,
|
|
4629
5113
|
deleteModalCheckpointSnapshot as deleteModalCheckpointSnapshot2,
|
|
@@ -4632,18 +5116,22 @@ import {
|
|
|
4632
5116
|
inspectModalSandboxLifecycle,
|
|
4633
5117
|
isExecSessionLostBanner,
|
|
4634
5118
|
isProviderSandboxNotFoundError,
|
|
5119
|
+
NatsControlRpc,
|
|
4635
5120
|
parseExecBannerExitCode,
|
|
4636
5121
|
prepareProviderForTeardownAfterCapture,
|
|
4637
5122
|
providerWorkspaceCapturePolicy,
|
|
4638
5123
|
resolveModalCheckpointProviderBindingForLiveSandbox,
|
|
4639
5124
|
resolveModalCheckpointProviderBinding,
|
|
4640
5125
|
resolveModalCheckpointProviderBindingForSession,
|
|
5126
|
+
querySelfhostedOp,
|
|
4641
5127
|
resumeExactSandboxSession,
|
|
4642
5128
|
sandboxProviderContinuityForState,
|
|
4643
5129
|
sandboxBackendForSdkBackendId,
|
|
4644
5130
|
sandboxProviderInstanceIdFromEnvelope,
|
|
4645
5131
|
sandboxCommandExitCode as sandboxCommandExitCode2,
|
|
4646
5132
|
sandboxCommandStillRunning,
|
|
5133
|
+
SelfhostedControlError,
|
|
5134
|
+
subjectFor,
|
|
4647
5135
|
sweepModalOrphanSandboxes,
|
|
4648
5136
|
terminateManagedSandboxSession as terminateManagedSandboxSession2,
|
|
4649
5137
|
terminateModalSandboxById,
|
|
@@ -4653,6 +5141,8 @@ import { sandboxLeaseTelemetryKey as sandboxLeaseTelemetryKey2 } from "@opengeni
|
|
|
4653
5141
|
var RETAINED_PROCESS_RECONCILIATION_LIMIT = 20;
|
|
4654
5142
|
var RETAINED_PROCESS_RECONCILIATION_CLAIM_TTL_MS = 5 * 6e4;
|
|
4655
5143
|
var RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS = 5e3;
|
|
5144
|
+
var CONNECTED_COMMAND_RECONCILIATION_LIMIT = 20;
|
|
5145
|
+
var CONNECTED_COMMAND_RECONCILIATION_CLAIM_TTL_MS = 5 * 6e4;
|
|
4656
5146
|
var SandboxProviderCaptureTimeoutError = class extends Error {
|
|
4657
5147
|
constructor(sandboxGroupId, backend, timeoutMs, leaseEpoch, instanceId) {
|
|
4658
5148
|
super(
|
|
@@ -4740,6 +5230,7 @@ function createSandboxLeaseActivities(services, options = {}) {
|
|
|
4740
5230
|
capturePolicy
|
|
4741
5231
|
));
|
|
4742
5232
|
const sweepModalOrphans = options.sweepModalOrphans ?? sweepModalOrphansForConfiguredBackend;
|
|
5233
|
+
const inspectOpenSandboxInventory = options.inspectOpenSandboxKubernetesInventory ?? inspectOpenSandboxKubernetesInventory;
|
|
4743
5234
|
const probeRetainedProcess = options.probeRetainedProcess ?? probeRetainedProcessAtProvider;
|
|
4744
5235
|
const probeDrainableProvider = options.probeDrainableProvider ?? probeDrainableProviderReadiness;
|
|
4745
5236
|
async function prepareSandboxLeaseSweep() {
|
|
@@ -4748,8 +5239,15 @@ function createSandboxLeaseActivities(services, options = {}) {
|
|
|
4748
5239
|
const { db, settings, observability } = await services();
|
|
4749
5240
|
const timing = sandboxDrainTiming(settings);
|
|
4750
5241
|
if (!settings.sandboxOwnershipEnabled) {
|
|
5242
|
+
const drainableInventory2 = await reapStaleLeaseHoldersGlobal(db, {
|
|
5243
|
+
viewerHolderTtlMs: settings.sandboxViewerHolderTtlMs,
|
|
5244
|
+
turnHolderTtlMs: settings.sandboxLeaseTtlMs,
|
|
5245
|
+
interactionHolderTtlMs: settings.sandboxInteractionHolderTtlMs,
|
|
5246
|
+
idleGraceMs: settings.sandboxIdleGraceMs
|
|
5247
|
+
});
|
|
5248
|
+
const drainable2 = drainableInventory2.slice(0, SANDBOX_REAPER_CHILD_DISPATCH_LIMIT);
|
|
4751
5249
|
return {
|
|
4752
|
-
drainable:
|
|
5250
|
+
drainable: drainable2,
|
|
4753
5251
|
timeoutClass: timing.timeoutClass,
|
|
4754
5252
|
snapshotTimeoutMs: timing.snapshotTimeoutMs,
|
|
4755
5253
|
captureTimeoutMs: timing.captureTimeoutMs,
|
|
@@ -4808,8 +5306,7 @@ function createSandboxLeaseActivities(services, options = {}) {
|
|
|
4808
5306
|
instanceId: input.target.instanceId
|
|
4809
5307
|
});
|
|
4810
5308
|
try {
|
|
4811
|
-
const { db, settings, observability } = await services();
|
|
4812
|
-
if (!settings.sandboxOwnershipEnabled) return { status: "skipped" };
|
|
5309
|
+
const { db, settings, observability, objectStorage } = await services();
|
|
4813
5310
|
assertSandboxDrainInputTiming(input);
|
|
4814
5311
|
const drainSettings = settings.sandboxSnapshotTimeoutMs === input.snapshotTimeoutMs ? settings : { ...settings, sandboxSnapshotTimeoutMs: input.snapshotTimeoutMs };
|
|
4815
5312
|
const row = input.target;
|
|
@@ -4828,7 +5325,8 @@ function createSandboxLeaseActivities(services, options = {}) {
|
|
|
4828
5325
|
observability,
|
|
4829
5326
|
terminateBox,
|
|
4830
5327
|
probeDrainableProvider,
|
|
4831
|
-
captureAttempt
|
|
5328
|
+
captureAttempt,
|
|
5329
|
+
objectStorage
|
|
4832
5330
|
);
|
|
4833
5331
|
return { status: drainedCold ? "terminated" : "skipped" };
|
|
4834
5332
|
} catch (error) {
|
|
@@ -4894,8 +5392,14 @@ function createSandboxLeaseActivities(services, options = {}) {
|
|
|
4894
5392
|
let forceDrained = 0;
|
|
4895
5393
|
const rotationsRequested = input.rotationsRequested;
|
|
4896
5394
|
let modalOrphansTerminated = 0;
|
|
5395
|
+
await reconcileConnectedMachineBackgroundCommands(db, settings, observability, service.bus);
|
|
4897
5396
|
if (!settings.sandboxOwnershipEnabled) {
|
|
4898
|
-
await refreshQueueLeaseAndCreditGauges(
|
|
5397
|
+
await refreshQueueLeaseAndCreditGauges(
|
|
5398
|
+
db,
|
|
5399
|
+
settings,
|
|
5400
|
+
observability,
|
|
5401
|
+
inspectOpenSandboxInventory
|
|
5402
|
+
);
|
|
4899
5403
|
return {
|
|
4900
5404
|
metered,
|
|
4901
5405
|
forceDrained,
|
|
@@ -4949,7 +5453,12 @@ function createSandboxLeaseActivities(services, options = {}) {
|
|
|
4949
5453
|
return { claimed: 0, deleted: 0, failed: 0 };
|
|
4950
5454
|
}
|
|
4951
5455
|
);
|
|
4952
|
-
await refreshQueueLeaseAndCreditGauges(
|
|
5456
|
+
await refreshQueueLeaseAndCreditGauges(
|
|
5457
|
+
db,
|
|
5458
|
+
settings,
|
|
5459
|
+
observability,
|
|
5460
|
+
inspectOpenSandboxInventory
|
|
5461
|
+
);
|
|
4953
5462
|
if (input.examined > 0 || metered > 0 || forceDrained > 0 || modalOrphansTerminated > 0 || rotationsRequested > 0 || checkpointGc.claimed > 0) {
|
|
4954
5463
|
observability.info("sandbox reaper dispatched", {
|
|
4955
5464
|
drainable: input.examined,
|
|
@@ -5036,6 +5545,13 @@ var CHECKPOINT_GC_CLAIM_TTL_MS = 10 * 6e4;
|
|
|
5036
5545
|
var CHECKPOINT_TOMBSTONE_RETENTION_MS = 30 * 24 * 60 * 6e4;
|
|
5037
5546
|
var CHECKPOINT_TOMBSTONE_PRUNE_LIMIT = 500;
|
|
5038
5547
|
var SANDBOX_MAINTENANCE_ITEM_CONCURRENCY = 8;
|
|
5548
|
+
function recordConnectedCommandReconciliation(observability, outcome) {
|
|
5549
|
+
observability.incrementCounter({
|
|
5550
|
+
name: "opengeni_connected_background_command_reconciliation_total",
|
|
5551
|
+
help: "Exact-locator Connected Machine background-command reconciliation outcomes.",
|
|
5552
|
+
labels: { outcome }
|
|
5553
|
+
});
|
|
5554
|
+
}
|
|
5039
5555
|
async function forEachWithConcurrency(values, concurrency, visit) {
|
|
5040
5556
|
if (values.length === 0) return;
|
|
5041
5557
|
const workerCount = Math.min(values.length, Math.max(1, Math.floor(concurrency)));
|
|
@@ -5158,55 +5674,203 @@ async function adoptLegacyModalCheckpointReceipts(db, settings, observability) {
|
|
|
5158
5674
|
});
|
|
5159
5675
|
return adopted;
|
|
5160
5676
|
}
|
|
5161
|
-
async function
|
|
5677
|
+
async function reconcileConnectedMachineBackgroundCommands(db, settings, observability, bus) {
|
|
5162
5678
|
const claimId = crypto.randomUUID();
|
|
5163
5679
|
let claims;
|
|
5164
5680
|
try {
|
|
5165
|
-
claims = await
|
|
5681
|
+
claims = await claimConnectedMachineSessionBackgroundCommands(db, {
|
|
5166
5682
|
claimId,
|
|
5167
|
-
limit:
|
|
5168
|
-
claimTtlMs:
|
|
5683
|
+
limit: CONNECTED_COMMAND_RECONCILIATION_LIMIT,
|
|
5684
|
+
claimTtlMs: CONNECTED_COMMAND_RECONCILIATION_CLAIM_TTL_MS
|
|
5169
5685
|
});
|
|
5170
5686
|
} catch (error) {
|
|
5171
|
-
|
|
5172
|
-
observability.warn("sandbox reaper:
|
|
5687
|
+
recordConnectedCommandReconciliation(observability, "claim_failed");
|
|
5688
|
+
observability.warn("sandbox reaper: connected-command claim failed", {
|
|
5173
5689
|
error: error instanceof Error ? error.message : String(error)
|
|
5174
5690
|
});
|
|
5175
5691
|
return;
|
|
5176
5692
|
}
|
|
5177
|
-
|
|
5178
|
-
|
|
5179
|
-
|
|
5180
|
-
|
|
5693
|
+
const controlRpc = new NatsControlRpc(
|
|
5694
|
+
async () => bus.getRequestConnection()
|
|
5695
|
+
);
|
|
5696
|
+
await forEachWithConcurrency(claims, SANDBOX_MAINTENANCE_ITEM_CONCURRENCY, async (claim) => {
|
|
5697
|
+
let proof = claim.proof;
|
|
5181
5698
|
if (!proof) {
|
|
5182
|
-
|
|
5183
|
-
|
|
5184
|
-
|
|
5185
|
-
|
|
5186
|
-
|
|
5187
|
-
|
|
5188
|
-
|
|
5189
|
-
|
|
5190
|
-
|
|
5191
|
-
|
|
5192
|
-
|
|
5193
|
-
|
|
5194
|
-
|
|
5195
|
-
|
|
5196
|
-
|
|
5197
|
-
|
|
5198
|
-
|
|
5199
|
-
|
|
5200
|
-
|
|
5201
|
-
|
|
5202
|
-
|
|
5203
|
-
|
|
5204
|
-
|
|
5205
|
-
|
|
5206
|
-
|
|
5207
|
-
|
|
5208
|
-
|
|
5209
|
-
|
|
5699
|
+
try {
|
|
5700
|
+
proof = await probeConnectedMachineBackgroundCommand(
|
|
5701
|
+
claim,
|
|
5702
|
+
controlRpc,
|
|
5703
|
+
Math.min(
|
|
5704
|
+
settings.sandboxSelfhostedControlTimeoutMs,
|
|
5705
|
+
RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS
|
|
5706
|
+
)
|
|
5707
|
+
);
|
|
5708
|
+
if (!proof) {
|
|
5709
|
+
await deferConnectedCommandClaim(db, settings, observability, claim, "provider_running");
|
|
5710
|
+
return;
|
|
5711
|
+
}
|
|
5712
|
+
} catch (error) {
|
|
5713
|
+
const offline = error instanceof SelfhostedControlError && (error.agentOffline || error.reason === "agent_reconnecting" || error.draining);
|
|
5714
|
+
if (!offline) {
|
|
5715
|
+
observability.warn("sandbox reaper: connected-command provider probe failed", {
|
|
5716
|
+
commandId: claim.commandId,
|
|
5717
|
+
enrollmentId: claim.enrollmentId,
|
|
5718
|
+
connectionInstanceId: claim.connectionInstanceId,
|
|
5719
|
+
error: error instanceof Error ? error.message : String(error)
|
|
5720
|
+
});
|
|
5721
|
+
}
|
|
5722
|
+
await deferConnectedCommandClaim(
|
|
5723
|
+
db,
|
|
5724
|
+
settings,
|
|
5725
|
+
observability,
|
|
5726
|
+
claim,
|
|
5727
|
+
offline ? "provider_offline" : "provider_error"
|
|
5728
|
+
);
|
|
5729
|
+
return;
|
|
5730
|
+
}
|
|
5731
|
+
try {
|
|
5732
|
+
await recordConnectedMachineBackgroundCommandProof(db, { claim, proof });
|
|
5733
|
+
recordConnectedCommandReconciliation(observability, `proof_${proof.outcome}`);
|
|
5734
|
+
} catch (error) {
|
|
5735
|
+
recordConnectedCommandReconciliation(observability, "proof_checkpoint_failed");
|
|
5736
|
+
observability.warn("sandbox reaper: connected-command proof checkpoint failed", {
|
|
5737
|
+
commandId: claim.commandId,
|
|
5738
|
+
error: error instanceof Error ? error.message : String(error)
|
|
5739
|
+
});
|
|
5740
|
+
return;
|
|
5741
|
+
}
|
|
5742
|
+
}
|
|
5743
|
+
try {
|
|
5744
|
+
if (!await settleClaimedConnectedMachineBackgroundCommand(db, { claim })) {
|
|
5745
|
+
throw new Error("Connected command settlement lost its exact claim or proof");
|
|
5746
|
+
}
|
|
5747
|
+
recordConnectedCommandReconciliation(observability, `settled_${proof.outcome}`);
|
|
5748
|
+
} catch (error) {
|
|
5749
|
+
recordConnectedCommandReconciliation(observability, "settlement_failed");
|
|
5750
|
+
observability.warn("sandbox reaper: connected-command settlement failed", {
|
|
5751
|
+
commandId: claim.commandId,
|
|
5752
|
+
error: error instanceof Error ? error.message : String(error)
|
|
5753
|
+
});
|
|
5754
|
+
await deferConnectedCommandClaim(db, settings, observability, claim, "settlement_failed");
|
|
5755
|
+
}
|
|
5756
|
+
});
|
|
5757
|
+
}
|
|
5758
|
+
async function probeConnectedMachineBackgroundCommand(claim, controlRpc, controlTimeoutMs, observedAt = /* @__PURE__ */ new Date()) {
|
|
5759
|
+
const input = {
|
|
5760
|
+
controlRpc,
|
|
5761
|
+
rpcSubject: subjectFor(
|
|
5762
|
+
claim.controlWorkspaceId,
|
|
5763
|
+
claim.enrollmentId,
|
|
5764
|
+
claim.connectionInstanceId
|
|
5765
|
+
),
|
|
5766
|
+
opId: claim.opId,
|
|
5767
|
+
controlTimeoutMs
|
|
5768
|
+
};
|
|
5769
|
+
const status = claim.state === "stopping" ? await cancelSelfhostedOp(input) : await querySelfhostedOp(input);
|
|
5770
|
+
return connectedCommandProofFromStatus(status, observedAt);
|
|
5771
|
+
}
|
|
5772
|
+
function connectedCommandProofFromStatus(status, observedAt) {
|
|
5773
|
+
switch (status.state) {
|
|
5774
|
+
case OpState.OP_STATE_ACCEPTED:
|
|
5775
|
+
case OpState.OP_STATE_RUNNING:
|
|
5776
|
+
return null;
|
|
5777
|
+
case OpState.OP_STATE_COMPLETE: {
|
|
5778
|
+
const exit = status.exit;
|
|
5779
|
+
if (!exit) {
|
|
5780
|
+
throw new Error("Connected command completed without an exit record");
|
|
5781
|
+
}
|
|
5782
|
+
const failureCode = exit.failureCode.replace(/[^A-Za-z0-9_-]/g, "_").slice(0, 128);
|
|
5783
|
+
const reason = exit.cancelled ? "op_cancelled" : exit.timedOut ? "op_timed_out" : failureCode ? `op_failure_${failureCode}` : "op_exit";
|
|
5784
|
+
return {
|
|
5785
|
+
outcome: "exited",
|
|
5786
|
+
exitCode: exit.exitCode,
|
|
5787
|
+
reason,
|
|
5788
|
+
observedAt
|
|
5789
|
+
};
|
|
5790
|
+
}
|
|
5791
|
+
case OpState.OP_STATE_LOST:
|
|
5792
|
+
return {
|
|
5793
|
+
outcome: "lost",
|
|
5794
|
+
exitCode: null,
|
|
5795
|
+
reason: status.lostReason === OpLostReason.OP_LOST_REASON_EVICTED ? "op_lost_evicted" : status.lostReason === OpLostReason.OP_LOST_REASON_AGENT_RESTARTED ? "op_lost_agent_restarted" : "op_lost_unspecified",
|
|
5796
|
+
observedAt
|
|
5797
|
+
};
|
|
5798
|
+
default:
|
|
5799
|
+
throw new Error(`Connected command returned invalid op state ${status.state}`);
|
|
5800
|
+
}
|
|
5801
|
+
}
|
|
5802
|
+
async function deferConnectedCommandClaim(db, settings, observability, claim, outcome) {
|
|
5803
|
+
const exponent = Math.min(4, Math.max(0, claim.reconcileAttempts - 1));
|
|
5804
|
+
const retryAfterMs = Math.min(
|
|
5805
|
+
5 * 6e4,
|
|
5806
|
+
Math.max(settings.sandboxLeaseReaperPeriodMs, 3e4) * 2 ** exponent
|
|
5807
|
+
);
|
|
5808
|
+
try {
|
|
5809
|
+
if (!await deferConnectedMachineBackgroundCommandReconciliation(db, {
|
|
5810
|
+
claim,
|
|
5811
|
+
outcome,
|
|
5812
|
+
retryAfterMs
|
|
5813
|
+
})) {
|
|
5814
|
+
throw new Error("Connected command defer lost its exact claim");
|
|
5815
|
+
}
|
|
5816
|
+
recordConnectedCommandReconciliation(observability, outcome);
|
|
5817
|
+
} catch (error) {
|
|
5818
|
+
recordConnectedCommandReconciliation(observability, "defer_failed");
|
|
5819
|
+
observability.warn("sandbox reaper: connected-command defer failed", {
|
|
5820
|
+
commandId: claim.commandId,
|
|
5821
|
+
error: error instanceof Error ? error.message : String(error)
|
|
5822
|
+
});
|
|
5823
|
+
}
|
|
5824
|
+
}
|
|
5825
|
+
async function reconcileTerminalRetainedProcesses(db, settings, observability, probe, inspectHistoricalModalSandbox) {
|
|
5826
|
+
const claimId = crypto.randomUUID();
|
|
5827
|
+
let claims;
|
|
5828
|
+
try {
|
|
5829
|
+
claims = await claimTerminalRetainedProcesses(db, {
|
|
5830
|
+
claimId,
|
|
5831
|
+
limit: RETAINED_PROCESS_RECONCILIATION_LIMIT,
|
|
5832
|
+
claimTtlMs: RETAINED_PROCESS_RECONCILIATION_CLAIM_TTL_MS
|
|
5833
|
+
});
|
|
5834
|
+
} catch (error) {
|
|
5835
|
+
recordRetainedProcessReconciliation(observability, "claim_failed");
|
|
5836
|
+
observability.warn("sandbox reaper: retained-process claim failed", {
|
|
5837
|
+
error: error instanceof Error ? error.message : String(error)
|
|
5838
|
+
});
|
|
5839
|
+
return;
|
|
5840
|
+
}
|
|
5841
|
+
for (const claim of claims) {
|
|
5842
|
+
let process = claim.process;
|
|
5843
|
+
const expected = retainedProcessSettlementIdentity(process);
|
|
5844
|
+
let proof = retainedProcessReconciliationProof(process);
|
|
5845
|
+
if (!proof) {
|
|
5846
|
+
let observation = null;
|
|
5847
|
+
if (process.providerBackend === "modal" && process.routeTargetId === null && !process.providerBindingKey) {
|
|
5848
|
+
try {
|
|
5849
|
+
const lifecycle = await withRetainedProcessProbeTimeout(
|
|
5850
|
+
inspectHistoricalModalSandbox(settings, process.providerInstanceId, null)
|
|
5851
|
+
);
|
|
5852
|
+
if (lifecycle.status === "not_found") {
|
|
5853
|
+
observation = {
|
|
5854
|
+
status: "deferred",
|
|
5855
|
+
reason: "provider_binding_missing"
|
|
5856
|
+
};
|
|
5857
|
+
} else {
|
|
5858
|
+
process = await bindRetainedProcessProviderIdentity(db, {
|
|
5859
|
+
accountId: process.accountId,
|
|
5860
|
+
workspaceId: process.workspaceId,
|
|
5861
|
+
sessionId: process.sessionId,
|
|
5862
|
+
processId: process.id,
|
|
5863
|
+
expected,
|
|
5864
|
+
claimId: claim.claimId,
|
|
5865
|
+
providerBindingKey: lifecycle.providerBindingKey,
|
|
5866
|
+
providerBinding: lifecycle.providerBinding
|
|
5867
|
+
});
|
|
5868
|
+
recordRetainedProcessReconciliation(observability, "provider_binding_adopted");
|
|
5869
|
+
if (lifecycle.status === "terminated") {
|
|
5870
|
+
observation = {
|
|
5871
|
+
status: "proved",
|
|
5872
|
+
proof: {
|
|
5873
|
+
outcome: "lost",
|
|
5210
5874
|
exitCode: null,
|
|
5211
5875
|
reason: "provider_instance_terminated"
|
|
5212
5876
|
}
|
|
@@ -5261,7 +5925,12 @@ async function reconcileTerminalRetainedProcesses(db, settings, observability, p
|
|
|
5261
5925
|
}
|
|
5262
5926
|
} else {
|
|
5263
5927
|
try {
|
|
5264
|
-
observation = await probe(
|
|
5928
|
+
observation = await probe(
|
|
5929
|
+
settings,
|
|
5930
|
+
lease,
|
|
5931
|
+
process,
|
|
5932
|
+
claim.ownerState === "background_stopping" ? "cancel" : "observe"
|
|
5933
|
+
);
|
|
5265
5934
|
} catch (error) {
|
|
5266
5935
|
observability.warn("sandbox reaper: retained-process provider probe failed", {
|
|
5267
5936
|
processId: process.id,
|
|
@@ -5340,6 +6009,15 @@ async function reconcileTerminalRetainedProcesses(db, settings, observability, p
|
|
|
5340
6009
|
reason: proof.reason,
|
|
5341
6010
|
idleGraceMs: settings.sandboxIdleGraceMs
|
|
5342
6011
|
});
|
|
6012
|
+
await settleSessionBackgroundCommandForRetainedProcess(db, {
|
|
6013
|
+
accountId: process.accountId,
|
|
6014
|
+
workspaceId: process.workspaceId,
|
|
6015
|
+
sessionId: process.sessionId,
|
|
6016
|
+
retainedProcessId: process.id,
|
|
6017
|
+
outcome: proof.outcome,
|
|
6018
|
+
exitCode: proof.exitCode,
|
|
6019
|
+
reason: proof.reason
|
|
6020
|
+
});
|
|
5343
6021
|
recordRetainedProcessReconciliation(observability, `settled_${proof.outcome}`);
|
|
5344
6022
|
} catch (error) {
|
|
5345
6023
|
recordRetainedProcessReconciliation(observability, "settlement_failed");
|
|
@@ -5403,7 +6081,7 @@ async function withRetainedProcessProbeTimeout(promise) {
|
|
|
5403
6081
|
if (timer) clearTimeout(timer);
|
|
5404
6082
|
}
|
|
5405
6083
|
}
|
|
5406
|
-
async function probeRetainedProcessAtProvider(settings, lease, process) {
|
|
6084
|
+
async function probeRetainedProcessAtProvider(settings, lease, process, mode = "observe") {
|
|
5407
6085
|
if (lease.id !== process.leaseId || lease.sandboxGroupId !== process.sandboxGroupId || lease.leaseEpoch !== process.leaseEpoch || lease.backend !== process.providerBackend || lease.instanceId !== process.providerInstanceId || lease.resumeBackendId !== process.providerBackend || process.routeTargetId !== null) {
|
|
5408
6086
|
return { status: "deferred", reason: "identity_mismatch" };
|
|
5409
6087
|
}
|
|
@@ -5414,7 +6092,7 @@ async function probeRetainedProcessAtProvider(settings, lease, process) {
|
|
|
5414
6092
|
if (envelopeBackend !== void 0 && envelopeBackend !== process.providerBackend) {
|
|
5415
6093
|
return { status: "deferred", reason: "identity_mismatch" };
|
|
5416
6094
|
}
|
|
5417
|
-
if (
|
|
6095
|
+
if (sandboxProviderInstanceIdFromEnvelope(lease.resumeState) !== process.providerInstanceId) {
|
|
5418
6096
|
return { status: "deferred", reason: "identity_mismatch" };
|
|
5419
6097
|
}
|
|
5420
6098
|
let configuredModalBinding = null;
|
|
@@ -5517,7 +6195,7 @@ async function probeRetainedProcessAtProvider(settings, lease, process) {
|
|
|
5517
6195
|
result = await withRetainedProcessProbeTimeout(
|
|
5518
6196
|
session.writeStdin({
|
|
5519
6197
|
sessionId: process.providerSessionId,
|
|
5520
|
-
chars: "",
|
|
6198
|
+
chars: mode === "cancel" ? "" : "",
|
|
5521
6199
|
yieldTimeMs: 1e3,
|
|
5522
6200
|
maxOutputTokens: 2e3
|
|
5523
6201
|
})
|
|
@@ -5641,7 +6319,7 @@ async function accrueWarmTick(db, settings, observability) {
|
|
|
5641
6319
|
});
|
|
5642
6320
|
return { accrued, workspaceIds };
|
|
5643
6321
|
}
|
|
5644
|
-
async function refreshQueueLeaseAndCreditGauges(db, observability) {
|
|
6322
|
+
async function refreshQueueLeaseAndCreditGauges(db, settings, observability, inspectOpenSandboxInventory) {
|
|
5645
6323
|
await Promise.all([
|
|
5646
6324
|
(async () => {
|
|
5647
6325
|
try {
|
|
@@ -5691,6 +6369,21 @@ async function refreshQueueLeaseAndCreditGauges(db, observability) {
|
|
|
5691
6369
|
await countExpiredDrainingSandboxLeases(db)
|
|
5692
6370
|
);
|
|
5693
6371
|
}),
|
|
6372
|
+
...settings.sandboxBackend === "opensandbox" && settings.openSandboxKubernetesInventoryNamespace ? [
|
|
6373
|
+
refreshSandboxInventoryGauge(
|
|
6374
|
+
observability,
|
|
6375
|
+
"opensandbox_kubernetes",
|
|
6376
|
+
"OpenSandbox Kubernetes",
|
|
6377
|
+
async () => {
|
|
6378
|
+
recordOpenSandboxKubernetesInventoryGauges(
|
|
6379
|
+
observability,
|
|
6380
|
+
await inspectOpenSandboxInventory({
|
|
6381
|
+
namespace: settings.openSandboxKubernetesInventoryNamespace
|
|
6382
|
+
})
|
|
6383
|
+
);
|
|
6384
|
+
}
|
|
6385
|
+
)
|
|
6386
|
+
] : [],
|
|
5694
6387
|
(async () => {
|
|
5695
6388
|
try {
|
|
5696
6389
|
recordCreditBalanceGauges(observability, await listCreditBalancesByAccount(db));
|
|
@@ -5723,7 +6416,7 @@ async function forceDrainOverLimitWorkspaces(db, settings, workspaceIds, observa
|
|
|
5723
6416
|
async (workspaceId) => {
|
|
5724
6417
|
try {
|
|
5725
6418
|
const { accountId } = await rlsContextForWorkspace2(db, workspaceId);
|
|
5726
|
-
const balance = enforceBalance ? await
|
|
6419
|
+
const balance = enforceBalance ? await getBillingBalance3(db, accountId) : { balanceMicros: 1 };
|
|
5727
6420
|
const result = await forceDrainOverLimitViewerOnlyBoxes(db, {
|
|
5728
6421
|
workspaceId,
|
|
5729
6422
|
balanceMicros: balance.balanceMicros,
|
|
@@ -5855,7 +6548,11 @@ async function probeDrainableProviderReadiness(settings, lease) {
|
|
|
5855
6548
|
throw error;
|
|
5856
6549
|
}
|
|
5857
6550
|
}
|
|
5858
|
-
|
|
6551
|
+
function sandboxDrainReason(lease) {
|
|
6552
|
+
if (lease.rotationRequestedAt === null) return "idle";
|
|
6553
|
+
return lease.rotationReason ?? "rotation_requested";
|
|
6554
|
+
}
|
|
6555
|
+
async function terminateDrainableBox(db, settings, row, observability, terminateBox, probeDrainableProvider, attempt, objectStorage) {
|
|
5859
6556
|
const { accountId } = await rlsContextForWorkspace2(db, row.workspaceId);
|
|
5860
6557
|
const lease = await readLease(db, row.workspaceId, row.sandboxGroupId);
|
|
5861
6558
|
if (!lease) {
|
|
@@ -6018,6 +6715,7 @@ async function terminateDrainableBox(db, settings, row, observability, terminate
|
|
|
6018
6715
|
if (!lease.instanceId || !captureClaim) {
|
|
6019
6716
|
return { wrote: false, archiveRevision: null };
|
|
6020
6717
|
}
|
|
6718
|
+
const archiveMetrics = runtimeMetricsHooksForObservability(observability);
|
|
6021
6719
|
let checkpointArtifactId = null;
|
|
6022
6720
|
if (archiveBase64 && archiveMetadata?.version === 2 && (archiveMetadata.provider === "modal_snapshot_filesystem" || archiveMetadata.provider === "modal_snapshot_directory")) {
|
|
6023
6721
|
if (!providerBinding) {
|
|
@@ -6059,12 +6757,57 @@ async function terminateDrainableBox(db, settings, row, observability, terminate
|
|
|
6059
6757
|
if (!archiveMetadata) {
|
|
6060
6758
|
throw new Error("Sandbox snapshot publication requires a verified archive descriptor");
|
|
6061
6759
|
}
|
|
6062
|
-
|
|
6063
|
-
|
|
6064
|
-
|
|
6065
|
-
|
|
6066
|
-
|
|
6067
|
-
|
|
6760
|
+
const priorKeys = collectWorkspaceArchiveObjectKeys(
|
|
6761
|
+
lease.resumeState ?? null
|
|
6762
|
+
);
|
|
6763
|
+
let workspaceArchiveRef;
|
|
6764
|
+
try {
|
|
6765
|
+
const published = await putVersion1TarArchiveOrInline({
|
|
6766
|
+
backend: lease.backend,
|
|
6767
|
+
objectStorage,
|
|
6768
|
+
accountId,
|
|
6769
|
+
workspaceId: row.workspaceId,
|
|
6770
|
+
sandboxGroupId: row.sandboxGroupId,
|
|
6771
|
+
archive: {
|
|
6772
|
+
bytes: Buffer.from(archiveBase64, "base64"),
|
|
6773
|
+
descriptor: archiveMetadata,
|
|
6774
|
+
base64: archiveBase64
|
|
6775
|
+
},
|
|
6776
|
+
metrics: archiveMetrics
|
|
6777
|
+
});
|
|
6778
|
+
workspaceArchiveRef = published.workspaceArchiveRef;
|
|
6779
|
+
result = await persistDrainSnapshot(db, {
|
|
6780
|
+
...baseInput,
|
|
6781
|
+
workspaceArchiveMeta: archiveMetadata,
|
|
6782
|
+
...published,
|
|
6783
|
+
...checkpointArtifactId ? { checkpointArtifactId } : {}
|
|
6784
|
+
});
|
|
6785
|
+
if (published.workspaceArchiveRef && objectStorage) {
|
|
6786
|
+
if (!result.wrote) {
|
|
6787
|
+
await deleteUnpublishedWorkspaceArchiveObject(
|
|
6788
|
+
objectStorage,
|
|
6789
|
+
published.workspaceArchiveRef,
|
|
6790
|
+
archiveMetrics
|
|
6791
|
+
);
|
|
6792
|
+
} else {
|
|
6793
|
+
const afterLease = await readLease(db, row.workspaceId, row.sandboxGroupId);
|
|
6794
|
+
const afterKeys = collectWorkspaceArchiveObjectKeys(
|
|
6795
|
+
afterLease?.resumeState ?? null
|
|
6796
|
+
);
|
|
6797
|
+
await deleteWorkspaceArchiveObjectKeys(
|
|
6798
|
+
objectStorage,
|
|
6799
|
+
[...priorKeys].filter((key) => !afterKeys.has(key))
|
|
6800
|
+
).catch(() => void 0);
|
|
6801
|
+
}
|
|
6802
|
+
}
|
|
6803
|
+
} catch (error) {
|
|
6804
|
+
await deleteUnpublishedWorkspaceArchiveObject(
|
|
6805
|
+
objectStorage,
|
|
6806
|
+
workspaceArchiveRef,
|
|
6807
|
+
archiveMetrics
|
|
6808
|
+
);
|
|
6809
|
+
throw error;
|
|
6810
|
+
}
|
|
6068
6811
|
}
|
|
6069
6812
|
if (!result.wrote && checkpointArtifactId) {
|
|
6070
6813
|
await markSandboxCheckpointArtifactDeletePending2(db, {
|
|
@@ -6111,7 +6854,12 @@ async function terminateDrainableBox(db, settings, row, observability, terminate
|
|
|
6111
6854
|
persisted,
|
|
6112
6855
|
archiveRevision,
|
|
6113
6856
|
instanceId: lease.instanceId,
|
|
6114
|
-
providerMissingBeforeCapture: providerMissing
|
|
6857
|
+
providerMissingBeforeCapture: providerMissing,
|
|
6858
|
+
// Why the box drained: the lease's bounded rotation reason when one
|
|
6859
|
+
// was requested (provider deadline, operator/image conflict, teardown
|
|
6860
|
+
// claim), otherwise the ordinary idle drain. Additive diagnostic
|
|
6861
|
+
// fact; no identifiers beyond the ones already carried above.
|
|
6862
|
+
drainReason: sandboxDrainReason(lease)
|
|
6115
6863
|
}
|
|
6116
6864
|
});
|
|
6117
6865
|
} catch (eventError) {
|
|
@@ -6364,99 +7112,222 @@ async function terminateProviderBox(settings, lease, observability, persistArchi
|
|
|
6364
7112
|
|
|
6365
7113
|
// src/activities/scheduled-tasks.ts
|
|
6366
7114
|
import {
|
|
7115
|
+
DEFAULT_FIRST_PARTY_MCP_PERMISSIONS,
|
|
6367
7116
|
OPENGENI_SLACK_BOT_CREDENTIAL_LABEL,
|
|
6368
7117
|
OPENGENI_SLACK_BOT_CREDENTIAL_ROLE,
|
|
6369
|
-
OPENGENI_SLACK_BOT_SESSION_METADATA_KEY
|
|
7118
|
+
OPENGENI_SLACK_BOT_SESSION_METADATA_KEY,
|
|
7119
|
+
resolveWorkspaceCodexCompactionDefault,
|
|
7120
|
+
SCHEDULED_TASK_OCCURRENCE_PAYLOAD_MAX_BYTES,
|
|
7121
|
+
ScheduledTaskRunAcceptedExecution,
|
|
7122
|
+
scheduledOccurrencePayloadUtf8Bytes,
|
|
7123
|
+
stableJson
|
|
6370
7124
|
} from "@opengeni/contracts";
|
|
6371
7125
|
import {
|
|
7126
|
+
defaultSessionMcpServerIds,
|
|
6372
7127
|
openGeniSlackBotMetadata,
|
|
6373
7128
|
requireOpenGeniSlackBotConnection,
|
|
7129
|
+
resolveSessionToolPolicy,
|
|
6374
7130
|
scheduledSlackBotConnectionId
|
|
6375
7131
|
} from "@opengeni/core";
|
|
6376
7132
|
import {
|
|
6377
|
-
|
|
7133
|
+
appendSessionEvents,
|
|
6378
7134
|
addSessionSystemUpdateWithSourceMutation,
|
|
7135
|
+
bindScheduledTaskRunSessionInTransaction,
|
|
6379
7136
|
createScheduledTaskRun,
|
|
6380
|
-
|
|
6381
|
-
createSessionGoal,
|
|
7137
|
+
createSessionWithIdempotencyKeyResult,
|
|
6382
7138
|
enqueueSessionWorkflowWakeIfRunnable,
|
|
6383
7139
|
getScheduledTask as getScheduledTask2,
|
|
7140
|
+
getScheduledTaskRunAcceptedExecution,
|
|
7141
|
+
getScheduledTaskRunByProducerKey,
|
|
7142
|
+
getScheduledTargetSessionExecution,
|
|
6384
7143
|
ensureKnowledgeSourceSyncState,
|
|
6385
7144
|
getScheduledTaskPersonalConnectionDelegations,
|
|
7145
|
+
getScheduledTaskRunPersonalResourceAuthority,
|
|
7146
|
+
getScheduledTaskPersonalResourceAuthoritySubject,
|
|
7147
|
+
getScheduledTaskRevisionAuthority,
|
|
6386
7148
|
getScheduledTaskXaiProviderAccountAuthoritySnapshot,
|
|
6387
7149
|
getRig as getRig2,
|
|
7150
|
+
getScheduledScopedRigVersionMetadata,
|
|
7151
|
+
getNestedAgentDepthDeploymentPolicy,
|
|
7152
|
+
getSessionByCreateIdempotencyKey,
|
|
6388
7153
|
getVariableSet,
|
|
7154
|
+
isCodexBilledModel,
|
|
7155
|
+
initializeSessionStartAtomically,
|
|
7156
|
+
materializeScheduledTaskReusableSessionFromRun,
|
|
7157
|
+
listEnabledMcpCapabilityServerIds,
|
|
7158
|
+
listInstalledApiIntegrationServerIdsForDelegations,
|
|
6389
7159
|
markScheduledTaskRunFailedIfQueued,
|
|
7160
|
+
markScheduledTaskRunSkippedIfQueued,
|
|
6390
7161
|
recordKnowledgeSourceSyncWake,
|
|
6391
7162
|
recordUsageEvent as recordUsageEvent3,
|
|
6392
|
-
|
|
7163
|
+
requireScheduledTaskIncidentAuthorityInTransaction,
|
|
6393
7164
|
requireSession,
|
|
6394
|
-
|
|
7165
|
+
requireWorkspace,
|
|
6395
7166
|
settleScheduledTaskRunInTransaction,
|
|
6396
7167
|
SessionSpawnDeniedDbError,
|
|
6397
|
-
updateScheduledTask,
|
|
6398
7168
|
updateScheduledTaskRun as updateScheduledTaskRun2,
|
|
6399
|
-
|
|
7169
|
+
updateSessionTitle,
|
|
7170
|
+
upsertScheduledSessionGoalForRun,
|
|
7171
|
+
withSessionActivityRlsContext
|
|
6400
7172
|
} from "@opengeni/db";
|
|
6401
|
-
import {
|
|
7173
|
+
import { publishDurableSessionEvents } from "@opengeni/events";
|
|
6402
7174
|
import { resolveFirstPartyMcpToolPolicy } from "@opengeni/config";
|
|
7175
|
+
import { Context as Context2 } from "@temporalio/activity";
|
|
6403
7176
|
|
|
6404
|
-
// src/
|
|
6405
|
-
import {
|
|
6406
|
-
|
|
6407
|
-
|
|
6408
|
-
|
|
6409
|
-
|
|
6410
|
-
|
|
6411
|
-
|
|
6412
|
-
|
|
6413
|
-
|
|
6414
|
-
|
|
6415
|
-
|
|
6416
|
-
|
|
6417
|
-
|
|
6418
|
-
|
|
6419
|
-
|
|
6420
|
-
|
|
6421
|
-
|
|
6422
|
-
|
|
6423
|
-
|
|
6424
|
-
|
|
6425
|
-
|
|
6426
|
-
|
|
6427
|
-
|
|
6428
|
-
|
|
7177
|
+
// src/scheduled-alert-occurrence.ts
|
|
7178
|
+
import { createHash as createHash4 } from "crypto";
|
|
7179
|
+
var MAX_SCHEDULED_TASK_ID_BYTES = 256;
|
|
7180
|
+
var MAX_STARTS_AT_BYTES = 256;
|
|
7181
|
+
var MAX_PROVIDER_BYTES = 256;
|
|
7182
|
+
var MAX_FINGERPRINT_BYTES = 1024;
|
|
7183
|
+
var MAX_LABELS = 256;
|
|
7184
|
+
var MAX_LABEL_KEY_BYTES = 256;
|
|
7185
|
+
var MAX_LABEL_VALUE_BYTES = 4096;
|
|
7186
|
+
var OFFSET_TIMESTAMP = /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2})(?::(\d{2})(?:\.(\d+))?)?(?:Z|[+-](\d{2}):(\d{2}))$/;
|
|
7187
|
+
function scheduledAlertResponderSessionCreateIdempotencyKey(input) {
|
|
7188
|
+
const digest = createHash4("sha256").update(
|
|
7189
|
+
JSON.stringify({
|
|
7190
|
+
version: 1,
|
|
7191
|
+
occurrenceKey: input.occurrence.sessionCreateIdempotencyKey,
|
|
7192
|
+
taskAuthorityRevision: input.taskAuthorityRevision,
|
|
7193
|
+
taskExecutionDigest: input.taskExecutionDigest
|
|
7194
|
+
})
|
|
7195
|
+
).digest("hex");
|
|
7196
|
+
return `scheduled-alert-occurrence:v1:${digest}`;
|
|
7197
|
+
}
|
|
7198
|
+
function scheduledAlertOccurrenceIdentity(input) {
|
|
7199
|
+
const scheduledTaskId = boundedString(input.scheduledTaskId, MAX_SCHEDULED_TASK_ID_BYTES);
|
|
7200
|
+
if (!scheduledTaskId) return null;
|
|
7201
|
+
const alert = record(input.metadata.alert);
|
|
7202
|
+
if (!alert) return null;
|
|
7203
|
+
const status = alert.status;
|
|
7204
|
+
if (status !== "firing" && status !== "resolved") return null;
|
|
7205
|
+
const startsAt = canonicalTimestamp(alert.startsAt);
|
|
7206
|
+
const fingerprint = boundedString(alert.fingerprint, MAX_FINGERPRINT_BYTES);
|
|
7207
|
+
const labels = canonicalLabels(alert.labels);
|
|
7208
|
+
if (!startsAt || !fingerprint || !labels) return null;
|
|
7209
|
+
const provider = boundedString(
|
|
7210
|
+
alert.provider ?? input.metadata.alertProvider ?? input.metadata.provider ?? "alertmanager",
|
|
7211
|
+
MAX_PROVIDER_BYTES
|
|
7212
|
+
);
|
|
7213
|
+
if (!provider) return null;
|
|
7214
|
+
const digest = createHash4("sha256").update(
|
|
7215
|
+
JSON.stringify({
|
|
7216
|
+
version: 1,
|
|
7217
|
+
workspaceId: input.workspaceId,
|
|
7218
|
+
scheduledTaskId,
|
|
7219
|
+
provider,
|
|
7220
|
+
fingerprint,
|
|
7221
|
+
startsAt,
|
|
7222
|
+
labels
|
|
7223
|
+
})
|
|
7224
|
+
).digest("hex");
|
|
7225
|
+
return {
|
|
7226
|
+
status,
|
|
7227
|
+
sessionCreateIdempotencyKey: `scheduled-alert-occurrence:v1:${digest}`,
|
|
7228
|
+
labels: Object.fromEntries(labels)
|
|
7229
|
+
};
|
|
7230
|
+
}
|
|
7231
|
+
function record(value) {
|
|
7232
|
+
return value !== null && typeof value === "object" && !Array.isArray(value) ? value : null;
|
|
7233
|
+
}
|
|
7234
|
+
function boundedString(value, maxBytes) {
|
|
7235
|
+
if (typeof value !== "string") return null;
|
|
7236
|
+
const normalized = value.trim();
|
|
7237
|
+
if (!normalized || Buffer.byteLength(normalized, "utf8") > maxBytes) return null;
|
|
7238
|
+
return normalized;
|
|
7239
|
+
}
|
|
7240
|
+
function canonicalTimestamp(value) {
|
|
7241
|
+
if (typeof value !== "string") return null;
|
|
7242
|
+
const trimmed = value.trim();
|
|
7243
|
+
if (!trimmed || Buffer.byteLength(trimmed, "utf8") > MAX_STARTS_AT_BYTES) return null;
|
|
7244
|
+
const match = OFFSET_TIMESTAMP.exec(trimmed);
|
|
7245
|
+
if (!match) return null;
|
|
7246
|
+
const [
|
|
7247
|
+
,
|
|
7248
|
+
yearText,
|
|
7249
|
+
monthText,
|
|
7250
|
+
dayText,
|
|
7251
|
+
hourText,
|
|
7252
|
+
minuteText,
|
|
7253
|
+
secondText,
|
|
7254
|
+
,
|
|
7255
|
+
offsetHourText,
|
|
7256
|
+
offsetMinuteText
|
|
7257
|
+
] = match;
|
|
7258
|
+
const year = Number(yearText);
|
|
7259
|
+
const month = Number(monthText);
|
|
7260
|
+
const day = Number(dayText);
|
|
7261
|
+
const hour = Number(hourText);
|
|
7262
|
+
const minute = Number(minuteText);
|
|
7263
|
+
const second = secondText === void 0 ? 0 : Number(secondText);
|
|
7264
|
+
const offsetHour = offsetHourText === void 0 ? 0 : Number(offsetHourText);
|
|
7265
|
+
const offsetMinute = offsetMinuteText === void 0 ? 0 : Number(offsetMinuteText);
|
|
7266
|
+
const leapYear = year % 4 === 0 && (year % 100 !== 0 || year % 400 === 0);
|
|
7267
|
+
const daysInMonth = [31, leapYear ? 29 : 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31];
|
|
7268
|
+
if (month < 1 || month > 12 || day < 1 || day > daysInMonth[month - 1] || hour > 23 || minute > 59 || second > 59 || offsetHour > 23 || offsetMinute > 59) {
|
|
6429
7269
|
return null;
|
|
6430
7270
|
}
|
|
6431
|
-
const
|
|
6432
|
-
|
|
6433
|
-
|
|
6434
|
-
|
|
6435
|
-
|
|
6436
|
-
|
|
6437
|
-
|
|
6438
|
-
|
|
6439
|
-
|
|
7271
|
+
const parsed = new Date(trimmed);
|
|
7272
|
+
return Number.isFinite(parsed.getTime()) ? trimmed : null;
|
|
7273
|
+
}
|
|
7274
|
+
function canonicalLabels(value) {
|
|
7275
|
+
const labels = record(value);
|
|
7276
|
+
if (!labels) return null;
|
|
7277
|
+
const entries = Object.entries(labels);
|
|
7278
|
+
if (entries.length === 0 || entries.length > MAX_LABELS) return null;
|
|
7279
|
+
const canonical = [];
|
|
7280
|
+
for (const [key, rawValue] of entries) {
|
|
7281
|
+
if (!key || Buffer.byteLength(key, "utf8") > MAX_LABEL_KEY_BYTES || typeof rawValue !== "string" || Buffer.byteLength(rawValue, "utf8") > MAX_LABEL_VALUE_BYTES) {
|
|
7282
|
+
return null;
|
|
6440
7283
|
}
|
|
7284
|
+
canonical.push([key, rawValue]);
|
|
6441
7285
|
}
|
|
6442
|
-
|
|
6443
|
-
|
|
6444
|
-
|
|
6445
|
-
|
|
6446
|
-
|
|
6447
|
-
|
|
6448
|
-
|
|
6449
|
-
|
|
7286
|
+
return canonical.sort(([left], [right]) => left < right ? -1 : left > right ? 1 : 0);
|
|
7287
|
+
}
|
|
7288
|
+
|
|
7289
|
+
// src/activities/scheduled-tasks.ts
|
|
7290
|
+
function scheduledTaskRunProducerKey(input, suppliedActivityIdentity) {
|
|
7291
|
+
if (input.producerKey) return input.producerKey;
|
|
7292
|
+
if (input.triggerType !== "scheduled") return input.agentRunUsageIdempotencyKey;
|
|
7293
|
+
let info = suppliedActivityIdentity;
|
|
7294
|
+
if (!info) {
|
|
7295
|
+
try {
|
|
7296
|
+
info = Context2.current().info;
|
|
7297
|
+
} catch {
|
|
7298
|
+
throw new Error(
|
|
7299
|
+
"scheduled task delivery without a producer key requires a Temporal activity execution"
|
|
7300
|
+
);
|
|
6450
7301
|
}
|
|
6451
7302
|
}
|
|
6452
|
-
|
|
7303
|
+
if (!info.workflowExecution) {
|
|
7304
|
+
throw new Error("scheduled task Temporal activity is missing its workflow execution identity");
|
|
7305
|
+
}
|
|
7306
|
+
return `scheduled-temporal:${stableJson({
|
|
7307
|
+
namespace: info.namespace,
|
|
7308
|
+
workflowId: info.workflowExecution.workflowId
|
|
7309
|
+
})}`;
|
|
6453
7310
|
}
|
|
6454
|
-
|
|
6455
|
-
|
|
6456
|
-
|
|
7311
|
+
var SCHEDULED_SESSION_TITLE_MAX_LENGTH = 200;
|
|
7312
|
+
function scheduledTaskSessionTitle(taskName) {
|
|
7313
|
+
const name = taskName.trim() || "Scheduled run";
|
|
7314
|
+
if (name.length <= SCHEDULED_SESSION_TITLE_MAX_LENGTH) return name;
|
|
7315
|
+
return `${name.slice(0, SCHEDULED_SESSION_TITLE_MAX_LENGTH - 1).trimEnd()}\u2026`;
|
|
7316
|
+
}
|
|
7317
|
+
async function stampScheduledSessionTitle(db, input) {
|
|
7318
|
+
if (input.session.title !== null) return [];
|
|
7319
|
+
const title = scheduledTaskSessionTitle(input.taskName);
|
|
7320
|
+
const result = await updateSessionTitle(db, {
|
|
7321
|
+
workspaceId: input.workspaceId,
|
|
7322
|
+
sessionId: input.session.id,
|
|
7323
|
+
title,
|
|
7324
|
+
source: "agent"
|
|
7325
|
+
});
|
|
7326
|
+
if (!result.updated) return [];
|
|
7327
|
+
return await appendSessionEvents(db, input.workspaceId, input.session.id, [
|
|
7328
|
+
{ type: "session.title_set", payload: { title: result.title ?? title, source: "agent" } }
|
|
7329
|
+
]);
|
|
6457
7330
|
}
|
|
6458
|
-
|
|
6459
|
-
// src/activities/scheduled-tasks.ts
|
|
6460
7331
|
function createScheduledTaskActivities(services) {
|
|
6461
7332
|
return {
|
|
6462
7333
|
dispatchScheduledTaskRun: async (input) => {
|
|
@@ -6465,6 +7336,75 @@ function createScheduledTaskActivities(services) {
|
|
|
6465
7336
|
if (input.triggerType !== "scheduled" && !input.initiator) {
|
|
6466
7337
|
return { action: "blocked", reason: "malformed_manual_trigger" };
|
|
6467
7338
|
}
|
|
7339
|
+
const stableProducerKey = scheduledTaskRunProducerKey(input);
|
|
7340
|
+
const priorRun = await getScheduledTaskRunByProducerKey(db, {
|
|
7341
|
+
workspaceId: input.workspaceId,
|
|
7342
|
+
taskId: input.taskId,
|
|
7343
|
+
triggerType: input.triggerType,
|
|
7344
|
+
producerKey: stableProducerKey
|
|
7345
|
+
});
|
|
7346
|
+
if (priorRun?.actionKind === "agent_turn") {
|
|
7347
|
+
const acceptedExecution = await getScheduledTaskRunAcceptedExecution(db, {
|
|
7348
|
+
workspaceId: input.workspaceId,
|
|
7349
|
+
runId: priorRun.id
|
|
7350
|
+
});
|
|
7351
|
+
if (!acceptedExecution) {
|
|
7352
|
+
if (priorRun.status === "succeeded" || priorRun.status === "failed" || priorRun.status === "skipped") {
|
|
7353
|
+
return scheduledRunTerminalResult(priorRun.error);
|
|
7354
|
+
}
|
|
7355
|
+
throw new Error("scheduled agent run is missing accepted execution truth");
|
|
7356
|
+
}
|
|
7357
|
+
await recordScheduledTaskFiredUsage(db, acceptedExecution, priorRun);
|
|
7358
|
+
if (priorRun.status === "dispatched") {
|
|
7359
|
+
return await replayScheduledTaskDispatch({
|
|
7360
|
+
db,
|
|
7361
|
+
wakeSessionWorkflow,
|
|
7362
|
+
task: acceptedExecution.task,
|
|
7363
|
+
run: priorRun,
|
|
7364
|
+
acceptedExecution
|
|
7365
|
+
});
|
|
7366
|
+
}
|
|
7367
|
+
if (priorRun.status === "queued") {
|
|
7368
|
+
try {
|
|
7369
|
+
return await recoverBoundScheduledTaskDispatch({
|
|
7370
|
+
db,
|
|
7371
|
+
bus,
|
|
7372
|
+
settings,
|
|
7373
|
+
wakeSessionWorkflow,
|
|
7374
|
+
run: priorRun,
|
|
7375
|
+
acceptedExecution,
|
|
7376
|
+
input
|
|
7377
|
+
});
|
|
7378
|
+
} catch (error) {
|
|
7379
|
+
const terminalError = scheduledRunRecoveryTerminalError(error);
|
|
7380
|
+
if (!terminalError) throw error;
|
|
7381
|
+
await markScheduledTaskRunFailedIfQueued(
|
|
7382
|
+
db,
|
|
7383
|
+
priorRun.workspaceId,
|
|
7384
|
+
priorRun.id,
|
|
7385
|
+
terminalError.code
|
|
7386
|
+
);
|
|
7387
|
+
return scheduledRunTerminalResult(terminalError.code);
|
|
7388
|
+
}
|
|
7389
|
+
}
|
|
7390
|
+
if (priorRun.status === "succeeded") {
|
|
7391
|
+
if (!priorRun.sessionId || !priorRun.triggerEventId) {
|
|
7392
|
+
throw new Error("succeeded scheduled run is missing its delivery identity");
|
|
7393
|
+
}
|
|
7394
|
+
return {
|
|
7395
|
+
action: acceptedExecution.task.runMode === "new_session_per_run" ? "start" : "signal",
|
|
7396
|
+
accountId: acceptedExecution.task.accountId,
|
|
7397
|
+
workspaceId: acceptedExecution.task.workspaceId,
|
|
7398
|
+
sessionId: priorRun.sessionId,
|
|
7399
|
+
triggerEventId: priorRun.triggerEventId,
|
|
7400
|
+
workflowId: workflowIdForSession(priorRun.sessionId),
|
|
7401
|
+
workflowWakeRevision: null
|
|
7402
|
+
};
|
|
7403
|
+
}
|
|
7404
|
+
if (priorRun.status === "failed" || priorRun.status === "skipped") {
|
|
7405
|
+
return scheduledRunTerminalResult(priorRun.error);
|
|
7406
|
+
}
|
|
7407
|
+
}
|
|
6468
7408
|
const task = await getScheduledTask2(db, input.workspaceId, input.taskId);
|
|
6469
7409
|
if (!task) {
|
|
6470
7410
|
return { action: "deleted" };
|
|
@@ -6473,11 +7413,13 @@ function createScheduledTaskActivities(services) {
|
|
|
6473
7413
|
if (knowledgeSourceSyncEffectivelyPaused(task)) {
|
|
6474
7414
|
return { action: "blocked", reason: "knowledge_source_paused" };
|
|
6475
7415
|
}
|
|
6476
|
-
const
|
|
7416
|
+
const run = await createScheduledTaskRun(db, {
|
|
6477
7417
|
workspaceId: task.workspaceId,
|
|
6478
7418
|
taskId: task.id,
|
|
7419
|
+
taskAuthorityRevision: task.authorityRevision,
|
|
7420
|
+
taskExecutionDigest: task.executionDigest,
|
|
6479
7421
|
triggerType: input.triggerType,
|
|
6480
|
-
producerKey:
|
|
7422
|
+
producerKey: stableProducerKey,
|
|
6481
7423
|
scheduledAt: null
|
|
6482
7424
|
});
|
|
6483
7425
|
await ensureKnowledgeSourceSyncState(db, task);
|
|
@@ -6486,9 +7428,9 @@ function createScheduledTaskActivities(services) {
|
|
|
6486
7428
|
workspaceId: task.workspaceId,
|
|
6487
7429
|
sourceId: task.action.sourceId,
|
|
6488
7430
|
scheduledTaskId: task.id,
|
|
6489
|
-
scheduledTaskRunId:
|
|
7431
|
+
scheduledTaskRunId: run.id,
|
|
6490
7432
|
cause: input.triggerType,
|
|
6491
|
-
producerKey:
|
|
7433
|
+
producerKey: stableProducerKey,
|
|
6492
7434
|
sourceConfigGeneration: task.action.sourceConfigGeneration,
|
|
6493
7435
|
sourceLifecycleGeneration: task.action.sourceLifecycleGeneration
|
|
6494
7436
|
});
|
|
@@ -6499,14 +7441,20 @@ function createScheduledTaskActivities(services) {
|
|
|
6499
7441
|
quantity: 1,
|
|
6500
7442
|
unit: "run",
|
|
6501
7443
|
sourceResourceType: "scheduled_task_run",
|
|
6502
|
-
sourceResourceId:
|
|
6503
|
-
initiator: input.initiator ?? {
|
|
6504
|
-
|
|
7444
|
+
sourceResourceId: run.id,
|
|
7445
|
+
initiator: input.initiator ?? {
|
|
7446
|
+
kind: "service",
|
|
7447
|
+
subjectId: "scheduler"
|
|
7448
|
+
},
|
|
7449
|
+
initiatorContext: {
|
|
7450
|
+
scheduledTaskId: task.id,
|
|
7451
|
+
scheduledTaskRunId: run.id
|
|
7452
|
+
},
|
|
6505
7453
|
origin: "scheduled_task",
|
|
6506
|
-
idempotencyKey: `usage:knowledge_source_sync.fired:${
|
|
7454
|
+
idempotencyKey: `usage:knowledge_source_sync.fired:${run.id}`
|
|
6507
7455
|
});
|
|
6508
|
-
if (
|
|
6509
|
-
await updateScheduledTaskRun2(db, task.workspaceId,
|
|
7456
|
+
if (run.status === "queued") {
|
|
7457
|
+
await updateScheduledTaskRun2(db, task.workspaceId, run.id, {
|
|
6510
7458
|
status: "dispatched",
|
|
6511
7459
|
actionKind: "knowledge_source_sync"
|
|
6512
7460
|
});
|
|
@@ -6516,414 +7464,979 @@ function createScheduledTaskActivities(services) {
|
|
|
6516
7464
|
accountId: task.accountId,
|
|
6517
7465
|
workspaceId: task.workspaceId,
|
|
6518
7466
|
taskId: task.id,
|
|
6519
|
-
scheduledTaskRunId:
|
|
7467
|
+
scheduledTaskRunId: run.id,
|
|
6520
7468
|
sourceId: task.action.sourceId,
|
|
6521
7469
|
overlapPolicy: task.overlapPolicy
|
|
6522
7470
|
};
|
|
6523
7471
|
}
|
|
7472
|
+
if (task.status !== "active") {
|
|
7473
|
+
return { action: "blocked", reason: "scheduled_task_paused" };
|
|
7474
|
+
}
|
|
7475
|
+
const structuredAlertOccurrence = scheduledAlertOccurrenceIdentity({
|
|
7476
|
+
workspaceId: task.workspaceId,
|
|
7477
|
+
scheduledTaskId: task.id,
|
|
7478
|
+
metadata: task.metadata
|
|
7479
|
+
});
|
|
7480
|
+
const alertOccurrence = task.runMode === "new_session_per_run" ? structuredAlertOccurrence : null;
|
|
7481
|
+
const alertResponderSessionCreateIdempotencyKey = alertOccurrence ? scheduledAlertResponderSessionCreateIdempotencyKey({
|
|
7482
|
+
occurrence: alertOccurrence,
|
|
7483
|
+
taskAuthorityRevision: task.authorityRevision,
|
|
7484
|
+
taskExecutionDigest: task.executionDigest
|
|
7485
|
+
}) : null;
|
|
7486
|
+
const incidentDeclaration = incidentTelemetryPreflightDeclaration(
|
|
7487
|
+
task.agentConfig,
|
|
7488
|
+
structuredAlertOccurrence !== null
|
|
7489
|
+
);
|
|
7490
|
+
if (incidentDeclaration.action === "blocked") {
|
|
7491
|
+
return incidentDeclaration;
|
|
7492
|
+
}
|
|
6524
7493
|
const taskPersonalConnectionDelegations = await getScheduledTaskPersonalConnectionDelegations(
|
|
6525
7494
|
db,
|
|
6526
7495
|
task.workspaceId,
|
|
6527
7496
|
task.id
|
|
6528
7497
|
);
|
|
6529
|
-
const
|
|
6530
|
-
|
|
6531
|
-
|
|
6532
|
-
|
|
6533
|
-
|
|
6534
|
-
)
|
|
6535
|
-
|
|
6536
|
-
if (slackBotConnection && !slackBotMetadata) {
|
|
6537
|
-
throw new Error("OpenGeni Slack bot connection metadata is invalid");
|
|
7498
|
+
const taskConnectionAuthoritySubjects = [
|
|
7499
|
+
...new Set(
|
|
7500
|
+
taskPersonalConnectionDelegations.map((delegation) => delegation.ownerSubjectId)
|
|
7501
|
+
)
|
|
7502
|
+
];
|
|
7503
|
+
if (taskConnectionAuthoritySubjects.length > 1) {
|
|
7504
|
+
throw new Error("scheduled connection authority has multiple causal humans");
|
|
6538
7505
|
}
|
|
6539
|
-
const
|
|
6540
|
-
const
|
|
7506
|
+
const taskConnectionAuthoritySubjectId = taskConnectionAuthoritySubjects[0] ?? null;
|
|
7507
|
+
const taskPersonalResourceAuthoritySubjectId = await getScheduledTaskPersonalResourceAuthoritySubject(db, {
|
|
6541
7508
|
accountId: task.accountId,
|
|
6542
7509
|
workspaceId: task.workspaceId,
|
|
6543
|
-
|
|
6544
|
-
|
|
7510
|
+
taskId: task.id,
|
|
7511
|
+
taskAuthorityRevision: task.authorityRevision
|
|
6545
7512
|
});
|
|
6546
|
-
|
|
6547
|
-
return { action: "blocked", reason: admissionDenial };
|
|
6548
|
-
}
|
|
6549
|
-
const run = await createScheduledTaskRun(db, {
|
|
6550
|
-
workspaceId: task.workspaceId,
|
|
6551
|
-
taskId: task.id,
|
|
6552
|
-
triggerType: input.triggerType,
|
|
6553
|
-
producerKey: input.producerKey ?? input.agentRunUsageIdempotencyKey ?? `scheduled:${crypto.randomUUID()}`,
|
|
6554
|
-
scheduledAt: null
|
|
6555
|
-
});
|
|
6556
|
-
await recordUsageEvent3(db, {
|
|
7513
|
+
const taskRevisionAuthority = await getScheduledTaskRevisionAuthority(db, {
|
|
6557
7514
|
accountId: task.accountId,
|
|
6558
7515
|
workspaceId: task.workspaceId,
|
|
6559
|
-
|
|
6560
|
-
|
|
6561
|
-
unit: "run",
|
|
6562
|
-
sourceResourceType: "scheduled_task_run",
|
|
6563
|
-
sourceResourceId: run.id,
|
|
6564
|
-
initiator: input.initiator ?? { kind: "service", subjectId: "scheduler" },
|
|
6565
|
-
initiatorContext: { scheduledTaskId: task.id, scheduledTaskRunId: run.id },
|
|
6566
|
-
origin: "scheduled_task",
|
|
6567
|
-
idempotencyKey: `usage:scheduled_task.fired:${run.id}`
|
|
7516
|
+
taskId: task.id,
|
|
7517
|
+
taskAuthorityRevision: task.authorityRevision
|
|
6568
7518
|
});
|
|
6569
|
-
if (
|
|
6570
|
-
|
|
6571
|
-
|
|
6572
|
-
|
|
6573
|
-
|
|
6574
|
-
|
|
6575
|
-
|
|
6576
|
-
|
|
6577
|
-
|
|
6578
|
-
|
|
6579
|
-
|
|
6580
|
-
|
|
6581
|
-
|
|
6582
|
-
|
|
7519
|
+
if (taskConnectionAuthoritySubjectId && taskPersonalResourceAuthoritySubjectId && taskConnectionAuthoritySubjectId !== taskPersonalResourceAuthoritySubjectId) {
|
|
7520
|
+
throw new Error("scheduled authority classes have different causal humans");
|
|
7521
|
+
}
|
|
7522
|
+
const taskAuthoritySubjectId = taskRevisionAuthority?.subjectId ?? taskPersonalResourceAuthoritySubjectId ?? taskConnectionAuthoritySubjectId;
|
|
7523
|
+
const model = task.agentConfig.model ?? settings.openaiModel;
|
|
7524
|
+
const reasoningEffort = task.agentConfig.reasoningEffort ?? settings.openaiReasoningEffort;
|
|
7525
|
+
const sandboxBackend = task.agentConfig.sandboxBackend ?? settings.sandboxBackend;
|
|
7526
|
+
const taskTools = withFirstPartyTools(settings, task.agentConfig.tools);
|
|
7527
|
+
const firstPartyMcpTools = resolveFirstPartyMcpToolPolicy(settings).default;
|
|
7528
|
+
const generatedTarget = task.runMode === "new_session_per_run" || task.runMode === "reusable_session" && task.reusableSessionId === null;
|
|
7529
|
+
const generatedSessionDepthPolicy = generatedTarget ? await (async () => {
|
|
7530
|
+
const workspace = await requireWorkspace(db, task.workspaceId);
|
|
7531
|
+
const deployment = await getNestedAgentDepthDeploymentPolicy(db);
|
|
7532
|
+
const taskOverride = task.agentConfig.maxNestedAgentDepth ?? null;
|
|
7533
|
+
const workspaceOverride = typeof workspace.settings.maxNestedAgentDepth === "number" ? workspace.settings.maxNestedAgentDepth : null;
|
|
7534
|
+
return taskOverride !== null ? {
|
|
7535
|
+
effectiveMaxNestedAgentDepth: taskOverride,
|
|
7536
|
+
nestedAgentDepthPolicySource: "session",
|
|
7537
|
+
codexCompactionMode: isCodexBilledModel(model) ? resolveWorkspaceCodexCompactionDefault(workspace.settings) : "portable"
|
|
7538
|
+
} : workspaceOverride !== null ? {
|
|
7539
|
+
effectiveMaxNestedAgentDepth: workspaceOverride,
|
|
7540
|
+
nestedAgentDepthPolicySource: "workspace",
|
|
7541
|
+
codexCompactionMode: isCodexBilledModel(model) ? resolveWorkspaceCodexCompactionDefault(workspace.settings) : "portable"
|
|
7542
|
+
} : {
|
|
7543
|
+
effectiveMaxNestedAgentDepth: deployment.maxNestedAgentDepth,
|
|
7544
|
+
nestedAgentDepthPolicySource: deployment.policySource,
|
|
7545
|
+
codexCompactionMode: isCodexBilledModel(model) ? resolveWorkspaceCodexCompactionDefault(workspace.settings) : "portable"
|
|
7546
|
+
};
|
|
7547
|
+
})() : null;
|
|
7548
|
+
const acceptedTargetSessionId = generatedTarget ? null : task.runMode === "existing_session" ? task.targetSessionId : task.reusableSessionId;
|
|
7549
|
+
const targetSessionExecutionBase = acceptedTargetSessionId ? await getScheduledTargetSessionExecution(
|
|
7550
|
+
db,
|
|
7551
|
+
task.workspaceId,
|
|
7552
|
+
acceptedTargetSessionId,
|
|
7553
|
+
taskAuthoritySubjectId
|
|
7554
|
+
) : null;
|
|
7555
|
+
const targetSessionExecution = targetSessionExecutionBase ? await (async () => {
|
|
7556
|
+
const [capabilityServerIds, apiIntegrationServerIds] = await Promise.all([
|
|
7557
|
+
listEnabledMcpCapabilityServerIds(db, task.workspaceId),
|
|
7558
|
+
listInstalledApiIntegrationServerIdsForDelegations(
|
|
7559
|
+
db,
|
|
7560
|
+
task.workspaceId,
|
|
7561
|
+
taskPersonalConnectionDelegations
|
|
7562
|
+
)
|
|
7563
|
+
]);
|
|
7564
|
+
const workspaceDefaultMcpServerIds = new Set(
|
|
7565
|
+
settings.mcpServers.map((server) => server.id)
|
|
7566
|
+
);
|
|
7567
|
+
for (const id of capabilityServerIds) workspaceDefaultMcpServerIds.add(id);
|
|
7568
|
+
for (const id of apiIntegrationServerIds) workspaceDefaultMcpServerIds.add(id);
|
|
7569
|
+
const availableMcpServerIds = new Set(workspaceDefaultMcpServerIds);
|
|
7570
|
+
for (const id of targetSessionExecutionBase.mcpServerIds) {
|
|
7571
|
+
availableMcpServerIds.add(id);
|
|
7572
|
+
}
|
|
7573
|
+
const effective = resolveSessionToolPolicy({
|
|
7574
|
+
toolPolicy: targetSessionExecutionBase.toolPolicy,
|
|
7575
|
+
sessionTools: targetSessionExecutionBase.tools,
|
|
7576
|
+
availableMcpServerIds,
|
|
7577
|
+
defaultMcpServerIds: defaultSessionMcpServerIds(
|
|
7578
|
+
[...workspaceDefaultMcpServerIds].map((id) => ({ id }))
|
|
7579
|
+
)
|
|
6583
7580
|
});
|
|
6584
|
-
|
|
6585
|
-
|
|
7581
|
+
return {
|
|
7582
|
+
...targetSessionExecutionBase,
|
|
7583
|
+
effectiveMcpServerIds: [
|
|
7584
|
+
...new Set(
|
|
7585
|
+
effective.toolRefs.flatMap((tool) => tool.kind === "mcp" ? [tool.id] : [])
|
|
7586
|
+
)
|
|
7587
|
+
].sort()
|
|
7588
|
+
};
|
|
7589
|
+
})() : null;
|
|
7590
|
+
const acceptedVariableSet = generatedTarget && task.variableSetId ? await getVariableSet(
|
|
7591
|
+
db,
|
|
7592
|
+
{
|
|
6586
7593
|
accountId: task.accountId,
|
|
6587
7594
|
workspaceId: task.workspaceId,
|
|
6588
|
-
|
|
6589
|
-
|
|
6590
|
-
|
|
6591
|
-
|
|
6592
|
-
|
|
6593
|
-
|
|
7595
|
+
// Organization/workspace sets have no personal-resource causal
|
|
7596
|
+
// subject. The scoped resolver still permits those common
|
|
7597
|
+
// scopes for a service/legacy actor, while a user-owned set can
|
|
7598
|
+
// only resolve through the exact frozen 0252 subject.
|
|
7599
|
+
subjectId: taskAuthoritySubjectId ?? task.createdBy.subjectId
|
|
7600
|
+
},
|
|
7601
|
+
task.variableSetId
|
|
7602
|
+
) : null;
|
|
7603
|
+
if (generatedTarget && task.variableSetId && !acceptedVariableSet) {
|
|
7604
|
+
throw new Error(`variable set not found: ${task.variableSetId}`);
|
|
7605
|
+
}
|
|
7606
|
+
const acceptedRig = generatedTarget && task.rigId ? await getRig2(
|
|
7607
|
+
db,
|
|
7608
|
+
{
|
|
6594
7609
|
accountId: task.accountId,
|
|
6595
7610
|
workspaceId: task.workspaceId,
|
|
6596
|
-
|
|
6597
|
-
|
|
6598
|
-
|
|
6599
|
-
|
|
6600
|
-
|
|
6601
|
-
|
|
6602
|
-
await wakeSessionWorkflow({
|
|
6603
|
-
accountId: result2.accountId,
|
|
6604
|
-
workspaceId: result2.workspaceId,
|
|
6605
|
-
sessionId: result2.sessionId,
|
|
6606
|
-
workflowId: result2.workflowId,
|
|
6607
|
-
wakeRevision: workflowWakeRevision
|
|
6608
|
-
});
|
|
6609
|
-
}
|
|
6610
|
-
return result2;
|
|
7611
|
+
subjectId: taskAuthoritySubjectId ?? task.createdBy.subjectId
|
|
7612
|
+
},
|
|
7613
|
+
task.rigId
|
|
7614
|
+
) : null;
|
|
7615
|
+
if (generatedTarget && task.rigId && (!acceptedRig || !acceptedRig.activeVersion)) {
|
|
7616
|
+
throw new Error(`rig has no active version to bind: ${task.rigId}`);
|
|
6611
7617
|
}
|
|
6612
|
-
|
|
6613
|
-
|
|
6614
|
-
|
|
6615
|
-
const sandboxBackend = task.agentConfig.sandboxBackend ?? settings.sandboxBackend;
|
|
6616
|
-
const goalSpec = task.agentConfig.goal ?? null;
|
|
6617
|
-
const taskTools = withFirstPartyTools(settings, task.agentConfig.tools);
|
|
6618
|
-
const existingSessionId = task.runMode === "existing_session" ? task.targetSessionId : task.reusableSessionId;
|
|
6619
|
-
if (task.runMode === "existing_session" && !existingSessionId) {
|
|
6620
|
-
throw new Error("scheduled task target session is unavailable");
|
|
6621
|
-
}
|
|
6622
|
-
if (task.runMode === "new_session_per_run" || task.runMode === "reusable_session" && !existingSessionId) {
|
|
6623
|
-
const variableSet = task.variableSetId ? await getVariableSet(db, task.workspaceId, task.variableSetId) : null;
|
|
6624
|
-
if (task.variableSetId && !variableSet) {
|
|
6625
|
-
throw new Error(`variable set not found: ${task.variableSetId}`);
|
|
6626
|
-
}
|
|
6627
|
-
let frozenRigId = null;
|
|
6628
|
-
let frozenRigVersionId = null;
|
|
6629
|
-
if (task.rigId) {
|
|
6630
|
-
const rig = await getRig2(db, task.workspaceId, task.rigId);
|
|
6631
|
-
if (!rig || !rig.activeVersion) {
|
|
6632
|
-
throw new Error(`rig has no active version to bind: ${task.rigId}`);
|
|
6633
|
-
}
|
|
6634
|
-
frozenRigId = rig.id;
|
|
6635
|
-
frozenRigVersionId = rig.activeVersion.id;
|
|
6636
|
-
}
|
|
6637
|
-
let session;
|
|
6638
|
-
const taskMetadata = { ...task.agentConfig.metadata };
|
|
6639
|
-
delete taskMetadata[OPENGENI_SLACK_BOT_SESSION_METADATA_KEY];
|
|
6640
|
-
try {
|
|
6641
|
-
session = await createSession(db, {
|
|
6642
|
-
accountId: task.accountId,
|
|
6643
|
-
workspaceId: task.workspaceId,
|
|
6644
|
-
initialMessage: task.agentConfig.prompt,
|
|
6645
|
-
resources: task.agentConfig.resources,
|
|
6646
|
-
tools: taskTools,
|
|
6647
|
-
firstPartyMcpTools: resolveFirstPartyMcpToolPolicy(settings).default,
|
|
6648
|
-
metadata: {
|
|
6649
|
-
...taskMetadata,
|
|
6650
|
-
model,
|
|
6651
|
-
reasoningEffort,
|
|
6652
|
-
scheduledTaskId: task.id,
|
|
6653
|
-
scheduledTaskRunId: run.id,
|
|
6654
|
-
...slackBotConnection ? {
|
|
6655
|
-
[OPENGENI_SLACK_BOT_SESSION_METADATA_KEY]: slackBotConnection.id
|
|
6656
|
-
} : {}
|
|
6657
|
-
},
|
|
6658
|
-
createdBy: {
|
|
6659
|
-
kind: "service",
|
|
6660
|
-
subjectId: "scheduler",
|
|
6661
|
-
label: "OpenGeni scheduler"
|
|
6662
|
-
},
|
|
6663
|
-
createdByContext: {
|
|
6664
|
-
scheduledTaskId: task.id,
|
|
6665
|
-
scheduledTaskRunId: run.id
|
|
6666
|
-
},
|
|
6667
|
-
model,
|
|
6668
|
-
sandboxBackend,
|
|
6669
|
-
variableSetId: task.variableSetId ?? null,
|
|
6670
|
-
rigId: frozenRigId,
|
|
6671
|
-
rigVersionId: frozenRigVersionId,
|
|
6672
|
-
personalConnectionDelegations: taskPersonalConnectionDelegations,
|
|
6673
|
-
initialXaiProviderAccountAuthoritySnapshot: taskXaiProviderAccountAuthoritySnapshot,
|
|
6674
|
-
maxNestedAgentDepthOverride: task.agentConfig.maxNestedAgentDepth ?? null,
|
|
6675
|
-
// The durable agent config was privilege-checked when the task
|
|
6676
|
-
// was created/updated. Preserve that explicit policy if a broader
|
|
6677
|
-
// workspace/deployment limit is narrowed before a later fire.
|
|
6678
|
-
allowNestedAgentDepthIncrease: true,
|
|
6679
|
-
subjectId: `scheduled_task:${task.id}`
|
|
6680
|
-
});
|
|
6681
|
-
} catch (error) {
|
|
6682
|
-
if (error instanceof SessionSpawnDeniedDbError) {
|
|
6683
|
-
throw new Error(`${error.denial.code}: denial=${error.denial.id}`, { cause: error });
|
|
6684
|
-
}
|
|
6685
|
-
throw error;
|
|
6686
|
-
}
|
|
6687
|
-
const goal = goalSpec ? await createSessionGoal(db, {
|
|
6688
|
-
accountId: task.accountId,
|
|
6689
|
-
workspaceId: task.workspaceId,
|
|
6690
|
-
sessionId: session.id,
|
|
6691
|
-
text: goalSpec.text,
|
|
6692
|
-
successCriteria: goalSpec.successCriteria ?? null,
|
|
6693
|
-
maxAutoContinuations: goalSpec.maxAutoContinuations ?? null,
|
|
6694
|
-
createdBy: "scheduled_task"
|
|
6695
|
-
}) : null;
|
|
6696
|
-
const workflowId = workflowIdForSession(session.id);
|
|
6697
|
-
await setTemporalWorkflowId(db, task.workspaceId, session.id, workflowId);
|
|
6698
|
-
if (task.runMode === "reusable_session") {
|
|
6699
|
-
await updateScheduledTask(db, task.workspaceId, task.id, {
|
|
6700
|
-
reusableSessionId: session.id
|
|
6701
|
-
});
|
|
6702
|
-
}
|
|
6703
|
-
await appendAndPublishEvents(db, bus, task.workspaceId, session.id, [
|
|
6704
|
-
{
|
|
6705
|
-
type: "session.created",
|
|
6706
|
-
payload: {
|
|
6707
|
-
status: session.status,
|
|
6708
|
-
createdBy: session.createdBy,
|
|
6709
|
-
scheduledTaskId: task.id,
|
|
6710
|
-
scheduledTaskRunId: run.id,
|
|
6711
|
-
// Names/ids only; never values.
|
|
6712
|
-
...variableSet ? { variableSetId: variableSet.id, variableSetName: variableSet.name } : {},
|
|
6713
|
-
...slackBotConnection && slackBotMetadata ? {
|
|
6714
|
-
slackBotConnection: {
|
|
6715
|
-
credentialRole: OPENGENI_SLACK_BOT_CREDENTIAL_ROLE,
|
|
6716
|
-
credentialLabel: OPENGENI_SLACK_BOT_CREDENTIAL_LABEL,
|
|
6717
|
-
connectionId: slackBotConnection.id,
|
|
6718
|
-
slackTeamId: slackBotMetadata.slackTeamId
|
|
6719
|
-
}
|
|
6720
|
-
} : {}
|
|
6721
|
-
}
|
|
6722
|
-
},
|
|
6723
|
-
...goal ? [
|
|
6724
|
-
{
|
|
6725
|
-
type: "goal.set",
|
|
6726
|
-
payload: {
|
|
6727
|
-
goalId: goal.id,
|
|
6728
|
-
text: goal.text,
|
|
6729
|
-
...goal.successCriteria ? { successCriteria: goal.successCriteria } : {},
|
|
6730
|
-
version: goal.version,
|
|
6731
|
-
actor: "scheduled_task",
|
|
6732
|
-
replaced: false
|
|
6733
|
-
}
|
|
6734
|
-
}
|
|
6735
|
-
] : [],
|
|
6736
|
-
{ type: "session.status.changed", payload: { status: session.status } }
|
|
6737
|
-
]);
|
|
6738
|
-
const scheduledUpdate = await addSessionSystemUpdateWithSourceMutation(
|
|
7618
|
+
const acceptedRigDefaultVariableSets = acceptedRig?.activeVersion ? await Promise.all(
|
|
7619
|
+
acceptedRig.activeVersion.defaultVariableSetIds.map(async (variableSetId) => {
|
|
7620
|
+
const variableSet = await getVariableSet(
|
|
6739
7621
|
db,
|
|
6740
7622
|
{
|
|
6741
7623
|
accountId: task.accountId,
|
|
6742
7624
|
workspaceId: task.workspaceId,
|
|
6743
|
-
|
|
6744
|
-
kind: "scheduled_occurrence",
|
|
6745
|
-
classification: "info",
|
|
6746
|
-
sourceId: run.id,
|
|
6747
|
-
dedupeKey: `scheduled-wake:${run.id}`,
|
|
6748
|
-
summary: task.agentConfig.prompt,
|
|
6749
|
-
payload: scheduledUserMessagePayload(
|
|
6750
|
-
task.agentConfig.prompt,
|
|
6751
|
-
task.agentConfig.resources,
|
|
6752
|
-
taskTools,
|
|
6753
|
-
task.id,
|
|
6754
|
-
run.id
|
|
6755
|
-
),
|
|
6756
|
-
lineage: {
|
|
6757
|
-
scheduledTaskId: task.id,
|
|
6758
|
-
scheduledTaskRunId: run.id,
|
|
6759
|
-
...taskXaiProviderAccountAuthoritySnapshot.scope === "user" && task.createdBy.kind === "subject" ? { xaiAuthoritySubjectId: task.createdBy.subjectId } : {}
|
|
6760
|
-
},
|
|
6761
|
-
personalConnectionDelegations: taskPersonalConnectionDelegations,
|
|
6762
|
-
xaiProviderAccountAuthoritySnapshot: taskXaiProviderAccountAuthoritySnapshot
|
|
7625
|
+
subjectId: taskAuthoritySubjectId ?? task.createdBy.subjectId
|
|
6763
7626
|
},
|
|
6764
|
-
|
|
6765
|
-
if (!wakeEventId) throw new Error("Scheduled delivery has no wake event");
|
|
6766
|
-
await settleScheduledTaskRunInTransaction(tx, {
|
|
6767
|
-
workspaceId: task.workspaceId,
|
|
6768
|
-
runId: run.id,
|
|
6769
|
-
sessionId: session.id,
|
|
6770
|
-
triggerEventId: wakeEventId,
|
|
6771
|
-
status: "dispatched"
|
|
6772
|
-
});
|
|
6773
|
-
}
|
|
7627
|
+
variableSetId
|
|
6774
7628
|
);
|
|
6775
|
-
if (
|
|
6776
|
-
throw new Error(
|
|
7629
|
+
if (!variableSet) {
|
|
7630
|
+
throw new Error(`rig default Variable Set not found: ${variableSetId}`);
|
|
6777
7631
|
}
|
|
6778
|
-
|
|
6779
|
-
|
|
6780
|
-
|
|
6781
|
-
|
|
6782
|
-
|
|
6783
|
-
|
|
6784
|
-
|
|
7632
|
+
return { id: variableSet.id, generation: variableSet.generation };
|
|
7633
|
+
})
|
|
7634
|
+
) : [];
|
|
7635
|
+
let incidentPreflightRequired = incidentDeclaration.action === "required";
|
|
7636
|
+
if (incidentPreflightRequired) {
|
|
7637
|
+
const existingSessionId = task.runMode === "existing_session" ? task.targetSessionId : task.reusableSessionId;
|
|
7638
|
+
const canonicalAlertSession = task.runMode === "new_session_per_run" && alertOccurrence ? await getSessionByCreateIdempotencyKey(
|
|
7639
|
+
db,
|
|
7640
|
+
task.workspaceId,
|
|
7641
|
+
alertResponderSessionCreateIdempotencyKey
|
|
7642
|
+
) : null;
|
|
7643
|
+
const responderSession = existingSessionId ? await requireSession(db, task.workspaceId, existingSessionId) : canonicalAlertSession;
|
|
7644
|
+
const responder = await resolveIncidentTelemetryResponderMetadata({
|
|
7645
|
+
db,
|
|
7646
|
+
settings,
|
|
7647
|
+
task,
|
|
7648
|
+
session: responderSession,
|
|
7649
|
+
personalConnectionDelegations: taskPersonalConnectionDelegations,
|
|
7650
|
+
personalResourceAuthoritySubjectId: taskAuthoritySubjectId,
|
|
7651
|
+
executionPolicy: targetSessionExecution ? {
|
|
7652
|
+
tools: targetSessionExecution.tools,
|
|
7653
|
+
firstPartyMcpTools: targetSessionExecution.firstPartyMcpTools,
|
|
7654
|
+
firstPartyMcpPermissions: targetSessionExecution.firstPartyMcpPermissions,
|
|
7655
|
+
variableSetId: targetSessionExecution.variableSetId,
|
|
7656
|
+
rigId: targetSessionExecution.rigId,
|
|
7657
|
+
rigVersionId: targetSessionExecution.rigVersionId,
|
|
7658
|
+
toolPolicy: targetSessionExecution.toolPolicy,
|
|
7659
|
+
mcpServerIds: targetSessionExecution.mcpServerIds,
|
|
7660
|
+
toolPolicyVersion: targetSessionExecution.toolPolicyVersion
|
|
7661
|
+
} : {
|
|
7662
|
+
tools: taskTools,
|
|
7663
|
+
firstPartyMcpTools,
|
|
7664
|
+
firstPartyMcpPermissions: [...DEFAULT_FIRST_PARTY_MCP_PERMISSIONS],
|
|
7665
|
+
variableSetId: acceptedVariableSet?.id ?? null,
|
|
7666
|
+
rigId: acceptedRig?.id ?? null,
|
|
7667
|
+
rigVersionId: acceptedRig?.activeVersion?.id ?? null,
|
|
7668
|
+
toolPolicy: responderSession?.toolPolicy ?? {
|
|
7669
|
+
mode: "explicit",
|
|
7670
|
+
inheritedFromSessionId: null
|
|
7671
|
+
},
|
|
7672
|
+
mcpServerIds: [],
|
|
7673
|
+
toolPolicyVersion: responderSession?.toolPolicyVersion ?? null
|
|
7674
|
+
}
|
|
7675
|
+
});
|
|
7676
|
+
const preflight = evaluateIncidentTelemetryPreflight({
|
|
7677
|
+
agentConfig: task.agentConfig,
|
|
7678
|
+
incidentTriggered: structuredAlertOccurrence !== null,
|
|
7679
|
+
alertOccurrenceLabels: structuredAlertOccurrence?.labels ?? null,
|
|
7680
|
+
responder
|
|
7681
|
+
});
|
|
7682
|
+
if (preflight.action === "blocked") return preflight;
|
|
7683
|
+
incidentPreflightRequired = preflight.action === "ready";
|
|
7684
|
+
}
|
|
7685
|
+
const taskXaiProviderAccountAuthoritySnapshot = await getScheduledTaskXaiProviderAccountAuthoritySnapshot(db, task.workspaceId, task.id);
|
|
7686
|
+
const slackBotConnection = task.agentConfig.slackBotConnectionId ? await requireOpenGeniSlackBotConnection(
|
|
7687
|
+
db,
|
|
7688
|
+
task.workspaceId,
|
|
7689
|
+
task.agentConfig.slackBotConnectionId
|
|
7690
|
+
) : null;
|
|
7691
|
+
const slackBotMetadata = slackBotConnection ? openGeniSlackBotMetadata(slackBotConnection.metadata) : null;
|
|
7692
|
+
if (slackBotConnection && !slackBotMetadata) {
|
|
7693
|
+
throw new Error("OpenGeni Slack bot connection metadata is invalid");
|
|
7694
|
+
}
|
|
7695
|
+
const xaiAuthoritySubjectId = taskXaiProviderAccountAuthoritySnapshot.scope === "user" && task.createdBy.kind === "subject" ? task.createdBy.subjectId : null;
|
|
7696
|
+
if (taskXaiProviderAccountAuthoritySnapshot.scope === "user" && !xaiAuthoritySubjectId) {
|
|
7697
|
+
throw new Error("scheduled user-scoped xAI authority has no causal human");
|
|
7698
|
+
}
|
|
7699
|
+
if (xaiAuthoritySubjectId && taskAuthoritySubjectId && xaiAuthoritySubjectId !== taskAuthoritySubjectId) {
|
|
7700
|
+
throw new Error("scheduled authority classes have different causal humans");
|
|
7701
|
+
}
|
|
7702
|
+
const causalHumanSubjectId = taskAuthoritySubjectId ?? xaiAuthoritySubjectId;
|
|
7703
|
+
if ((acceptedVariableSet?.scope === "user" || acceptedRig?.scope === "user") && !causalHumanSubjectId) {
|
|
7704
|
+
throw new Error("scheduled personal-resource execution has no causal human");
|
|
7705
|
+
}
|
|
7706
|
+
const admissionDenial = await agentRunAdmissionDenial(service, {
|
|
7707
|
+
accountId: task.accountId,
|
|
7708
|
+
workspaceId: task.workspaceId,
|
|
7709
|
+
model: targetSessionExecution?.model ?? model,
|
|
7710
|
+
requestedAgentRuns: input.agentRunUsageIdempotencyKey ? 0 : 1
|
|
7711
|
+
});
|
|
7712
|
+
if (admissionDenial) {
|
|
7713
|
+
return { action: "blocked", reason: admissionDenial };
|
|
7714
|
+
}
|
|
7715
|
+
const deferredEvents = [];
|
|
7716
|
+
const executeDispatch = async (dispatchDb, deferPublications) => {
|
|
7717
|
+
const admittedRunId = crypto.randomUUID();
|
|
7718
|
+
const generatedSessionBinding = generatedTarget ? {
|
|
7719
|
+
createIdempotencyKey: alertResponderSessionCreateIdempotencyKey ?? (task.runMode === "reusable_session" ? `scheduled-task-reusable:${task.id}:${task.authorityRevision}:${task.executionDigest}` : `scheduled-task-run:${admittedRunId}`),
|
|
7720
|
+
effectiveMaxNestedAgentDepth: generatedSessionDepthPolicy.effectiveMaxNestedAgentDepth,
|
|
7721
|
+
nestedAgentDepthPolicySource: generatedSessionDepthPolicy.nestedAgentDepthPolicySource,
|
|
7722
|
+
codexCompactionMode: generatedSessionDepthPolicy.codexCompactionMode
|
|
7723
|
+
} : null;
|
|
7724
|
+
const acceptedExecution = {
|
|
7725
|
+
version: 1,
|
|
7726
|
+
task,
|
|
7727
|
+
resolvedModel: model,
|
|
7728
|
+
resolvedReasoningEffort: reasoningEffort,
|
|
7729
|
+
resolvedLatencyMode: "standard",
|
|
7730
|
+
resolvedSandboxBackend: sandboxBackend,
|
|
7731
|
+
resolvedSandboxOs: "linux",
|
|
7732
|
+
resolvedTools: taskTools,
|
|
7733
|
+
resolvedFirstPartyMcpTools: firstPartyMcpTools,
|
|
7734
|
+
resolvedFirstPartyMcpPermissions: [...DEFAULT_FIRST_PARTY_MCP_PERMISSIONS],
|
|
7735
|
+
resolvedVariableSet: acceptedVariableSet ? { id: acceptedVariableSet.id, generation: acceptedVariableSet.generation } : null,
|
|
7736
|
+
resolvedRig: acceptedRig?.activeVersion ? {
|
|
7737
|
+
id: acceptedRig.id,
|
|
7738
|
+
versionId: acceptedRig.activeVersion.id,
|
|
7739
|
+
defaultVariableSets: acceptedRigDefaultVariableSets
|
|
7740
|
+
} : null,
|
|
7741
|
+
resolvedSlackBotConnection: slackBotConnection && slackBotMetadata && slackBotConnection.verifiedInstallVersion ? {
|
|
7742
|
+
id: slackBotConnection.id,
|
|
7743
|
+
version: slackBotConnection.version,
|
|
7744
|
+
verifiedInstallVersion: slackBotConnection.verifiedInstallVersion,
|
|
7745
|
+
metadata: slackBotMetadata
|
|
7746
|
+
} : null,
|
|
7747
|
+
targetSessionExecution,
|
|
7748
|
+
generatedSessionBinding,
|
|
7749
|
+
personalConnectionDelegations: taskPersonalConnectionDelegations,
|
|
7750
|
+
personalResourceAuthoritySubjectId: taskPersonalResourceAuthoritySubjectId,
|
|
7751
|
+
causalHumanSubjectId,
|
|
7752
|
+
causalHumanAuthority: taskRevisionAuthority,
|
|
7753
|
+
xaiProviderAccountAuthoritySnapshot: taskXaiProviderAccountAuthoritySnapshot,
|
|
7754
|
+
xaiAuthoritySubjectId,
|
|
7755
|
+
connectionAuthoritySubjectId: taskConnectionAuthoritySubjectId,
|
|
7756
|
+
triggerInitiator: input.initiator ?? { kind: "service", subjectId: "scheduler" },
|
|
7757
|
+
agentRunUsageIdempotencyKey: input.agentRunUsageIdempotencyKey ?? null,
|
|
7758
|
+
incidentPreflightRequired,
|
|
7759
|
+
alertOccurrenceLabels: structuredAlertOccurrence?.labels ?? null
|
|
7760
|
+
};
|
|
7761
|
+
const acceptedExecutionParse = ScheduledTaskRunAcceptedExecution.safeParse(acceptedExecution);
|
|
7762
|
+
if (!acceptedExecutionParse.success || scheduledOccurrencePayloadUtf8Bytes({
|
|
7763
|
+
prompt: task.agentConfig.prompt,
|
|
7764
|
+
resources: task.agentConfig.resources,
|
|
7765
|
+
tools: taskTools
|
|
7766
|
+
}) > SCHEDULED_TASK_OCCURRENCE_PAYLOAD_MAX_BYTES) {
|
|
7767
|
+
return {
|
|
7768
|
+
kind: "blocked",
|
|
7769
|
+
result: {
|
|
7770
|
+
action: "blocked",
|
|
7771
|
+
reason: "scheduled_execution_unrepresentable"
|
|
7772
|
+
}
|
|
7773
|
+
};
|
|
7774
|
+
}
|
|
7775
|
+
let run = await createScheduledTaskRun(dispatchDb, {
|
|
7776
|
+
runId: admittedRunId,
|
|
7777
|
+
workspaceId: task.workspaceId,
|
|
7778
|
+
taskId: task.id,
|
|
7779
|
+
taskAuthorityRevision: task.authorityRevision,
|
|
7780
|
+
taskExecutionDigest: task.executionDigest,
|
|
7781
|
+
triggerType: input.triggerType,
|
|
7782
|
+
producerKey: stableProducerKey,
|
|
7783
|
+
scheduledAt: null,
|
|
7784
|
+
acceptedExecutionSnapshot: acceptedExecution
|
|
7785
|
+
});
|
|
7786
|
+
await recordScheduledTaskFiredUsage(dispatchDb, acceptedExecution, run);
|
|
7787
|
+
if (run.status === "failed" && run.error === "scheduled_authority_exhausted") {
|
|
7788
|
+
return {
|
|
7789
|
+
kind: "blocked",
|
|
7790
|
+
result: {
|
|
7791
|
+
action: "blocked",
|
|
7792
|
+
reason: "scheduled_authority_exhausted"
|
|
7793
|
+
},
|
|
7794
|
+
run
|
|
7795
|
+
};
|
|
7796
|
+
}
|
|
7797
|
+
if (run.status === "failed" || run.status === "skipped") {
|
|
7798
|
+
return {
|
|
7799
|
+
kind: "blocked",
|
|
7800
|
+
result: scheduledRunTerminalResult(run.error),
|
|
7801
|
+
run
|
|
7802
|
+
};
|
|
7803
|
+
}
|
|
7804
|
+
const runPersonalResourceAuthority = await getScheduledTaskRunPersonalResourceAuthority(
|
|
7805
|
+
dispatchDb,
|
|
7806
|
+
{
|
|
7807
|
+
accountId: task.accountId,
|
|
7808
|
+
workspaceId: task.workspaceId,
|
|
7809
|
+
runId: run.id
|
|
6785
7810
|
}
|
|
6786
|
-
|
|
6787
|
-
|
|
7811
|
+
);
|
|
7812
|
+
if (run?.status === "dispatched" && run.sessionId && run.triggerEventId) {
|
|
7813
|
+
if (deferPublications) {
|
|
7814
|
+
return { kind: "replay", run, acceptedExecution };
|
|
7815
|
+
}
|
|
7816
|
+
await recordUsageEvent3(dispatchDb, {
|
|
7817
|
+
accountId: task.accountId,
|
|
7818
|
+
workspaceId: task.workspaceId,
|
|
7819
|
+
eventType: "agent_run.created",
|
|
7820
|
+
quantity: 1,
|
|
7821
|
+
unit: "run",
|
|
7822
|
+
sourceResourceType: "scheduled_task_run",
|
|
7823
|
+
sourceResourceId: run.id,
|
|
7824
|
+
sessionId: run.sessionId,
|
|
7825
|
+
initiator: acceptedExecution.triggerInitiator,
|
|
7826
|
+
initiatorContext: {
|
|
7827
|
+
scheduledTaskId: task.id,
|
|
7828
|
+
scheduledTaskRunId: run.id
|
|
7829
|
+
},
|
|
7830
|
+
origin: "scheduled_task",
|
|
7831
|
+
idempotencyKey: acceptedExecution.agentRunUsageIdempotencyKey ?? `usage:agent_run.created:scheduled:${run.id}`
|
|
7832
|
+
});
|
|
7833
|
+
const workflowId = workflowIdForSession(run.sessionId);
|
|
7834
|
+
const workflowWakeRevision = await enqueueSessionWorkflowWakeIfRunnable(dispatchDb, {
|
|
7835
|
+
accountId: task.accountId,
|
|
7836
|
+
workspaceId: task.workspaceId,
|
|
7837
|
+
sessionId: run.sessionId,
|
|
7838
|
+
temporalWorkflowId: workflowId,
|
|
7839
|
+
reason: "scheduled_retry"
|
|
7840
|
+
});
|
|
7841
|
+
const result3 = {
|
|
7842
|
+
action: task.runMode === "new_session_per_run" ? "start" : "signal",
|
|
6788
7843
|
accountId: task.accountId,
|
|
6789
7844
|
workspaceId: task.workspaceId,
|
|
6790
|
-
sessionId:
|
|
6791
|
-
triggerEventId:
|
|
7845
|
+
sessionId: run.sessionId,
|
|
7846
|
+
triggerEventId: run.triggerEventId,
|
|
6792
7847
|
workflowId,
|
|
6793
|
-
workflowWakeRevision
|
|
7848
|
+
workflowWakeRevision
|
|
6794
7849
|
};
|
|
6795
|
-
|
|
6796
|
-
|
|
6797
|
-
|
|
6798
|
-
|
|
6799
|
-
|
|
6800
|
-
|
|
7850
|
+
return { kind: "complete", result: result3, run };
|
|
7851
|
+
}
|
|
7852
|
+
let result2;
|
|
7853
|
+
try {
|
|
7854
|
+
if (incidentPreflightRequired) {
|
|
7855
|
+
const exactAuthority = await requireScheduledTaskIncidentAuthorityInTransaction(
|
|
7856
|
+
dispatchDb,
|
|
7857
|
+
{
|
|
7858
|
+
workspaceId: task.workspaceId,
|
|
7859
|
+
taskId: task.id
|
|
7860
|
+
}
|
|
6801
7861
|
);
|
|
7862
|
+
if (stableJson({
|
|
7863
|
+
task: exactAuthority.task,
|
|
7864
|
+
personalConnectionDelegations: exactAuthority.personalConnectionDelegations
|
|
7865
|
+
}) !== stableJson({
|
|
7866
|
+
task,
|
|
7867
|
+
personalConnectionDelegations: taskPersonalConnectionDelegations
|
|
7868
|
+
})) {
|
|
7869
|
+
throw new IncidentTelemetryPreflightBlockedError(
|
|
7870
|
+
"incident_preflight_metadata_missing"
|
|
7871
|
+
);
|
|
7872
|
+
}
|
|
6802
7873
|
}
|
|
6803
|
-
|
|
6804
|
-
|
|
6805
|
-
|
|
6806
|
-
);
|
|
7874
|
+
const goalSpec = task.agentConfig.goal ?? null;
|
|
7875
|
+
const existingSessionId = task.runMode === "existing_session" ? task.targetSessionId : task.reusableSessionId;
|
|
7876
|
+
if (task.runMode === "existing_session" && !existingSessionId) {
|
|
7877
|
+
throw new Error("scheduled task target session is unavailable");
|
|
6807
7878
|
}
|
|
6808
|
-
|
|
6809
|
-
|
|
6810
|
-
|
|
6811
|
-
|
|
6812
|
-
|
|
6813
|
-
|
|
6814
|
-
|
|
6815
|
-
|
|
6816
|
-
|
|
6817
|
-
|
|
6818
|
-
|
|
6819
|
-
|
|
6820
|
-
|
|
6821
|
-
|
|
6822
|
-
|
|
6823
|
-
|
|
6824
|
-
|
|
6825
|
-
|
|
6826
|
-
|
|
6827
|
-
|
|
6828
|
-
|
|
6829
|
-
|
|
6830
|
-
|
|
6831
|
-
|
|
6832
|
-
|
|
6833
|
-
|
|
6834
|
-
|
|
7879
|
+
if (task.runMode === "new_session_per_run" || task.runMode === "reusable_session" && !existingSessionId) {
|
|
7880
|
+
const variableSet = acceptedVariableSet;
|
|
7881
|
+
if (task.variableSetId && !variableSet) {
|
|
7882
|
+
throw new Error(`variable set not found: ${task.variableSetId}`);
|
|
7883
|
+
}
|
|
7884
|
+
let frozenRigId = null;
|
|
7885
|
+
let frozenRigVersionId = null;
|
|
7886
|
+
if (task.rigId) {
|
|
7887
|
+
const rig = acceptedRig;
|
|
7888
|
+
if (!rig || !rig.activeVersion) {
|
|
7889
|
+
throw new Error(`rig has no active version to bind: ${task.rigId}`);
|
|
7890
|
+
}
|
|
7891
|
+
frozenRigId = rig.id;
|
|
7892
|
+
frozenRigVersionId = runPersonalResourceAuthority?.resources.find(
|
|
7893
|
+
(resource) => resource.resourceKind === "rig" && resource.resourceId === rig.id
|
|
7894
|
+
)?.resourceVersionId ?? rig.activeVersion.id;
|
|
7895
|
+
if (frozenRigVersionId !== rig.activeVersion.id) {
|
|
7896
|
+
throw new Error("scheduled run rig version changed during admission");
|
|
7897
|
+
}
|
|
7898
|
+
}
|
|
7899
|
+
let session;
|
|
7900
|
+
let sessionCreated = true;
|
|
7901
|
+
const taskMetadata = { ...task.agentConfig.metadata };
|
|
7902
|
+
delete taskMetadata[OPENGENI_SLACK_BOT_SESSION_METADATA_KEY];
|
|
7903
|
+
try {
|
|
7904
|
+
const sessionInput = {
|
|
7905
|
+
accountId: task.accountId,
|
|
7906
|
+
workspaceId: task.workspaceId,
|
|
7907
|
+
initialMessage: task.agentConfig.prompt,
|
|
7908
|
+
resources: task.agentConfig.resources,
|
|
7909
|
+
tools: taskTools,
|
|
7910
|
+
firstPartyMcpTools,
|
|
7911
|
+
firstPartyMcpPermissions: [...DEFAULT_FIRST_PARTY_MCP_PERMISSIONS],
|
|
7912
|
+
metadata: {
|
|
7913
|
+
...taskMetadata,
|
|
7914
|
+
model,
|
|
7915
|
+
reasoningEffort,
|
|
7916
|
+
scheduledTaskId: task.id,
|
|
7917
|
+
scheduledTaskRunMode: task.runMode,
|
|
7918
|
+
...goalSpec ? { scheduledTaskGoal: goalSpec } : {},
|
|
7919
|
+
...run ? { scheduledTaskRunId: run.id } : {},
|
|
7920
|
+
...slackBotConnection ? {
|
|
7921
|
+
[OPENGENI_SLACK_BOT_SESSION_METADATA_KEY]: slackBotConnection.id
|
|
7922
|
+
} : {}
|
|
7923
|
+
},
|
|
7924
|
+
createdBy: {
|
|
7925
|
+
kind: "service",
|
|
7926
|
+
subjectId: "scheduler",
|
|
7927
|
+
label: "OpenGeni scheduler"
|
|
7928
|
+
},
|
|
7929
|
+
createdByContext: {
|
|
7930
|
+
scheduledTaskId: task.id,
|
|
7931
|
+
...run ? { scheduledTaskRunId: run.id } : {}
|
|
7932
|
+
},
|
|
7933
|
+
model,
|
|
7934
|
+
reasoningEffort,
|
|
7935
|
+
latencyMode: "standard",
|
|
7936
|
+
sandboxBackend,
|
|
7937
|
+
sandboxOs: "linux",
|
|
7938
|
+
variableSetId: task.variableSetId ?? null,
|
|
7939
|
+
rigId: frozenRigId,
|
|
7940
|
+
rigVersionId: frozenRigVersionId,
|
|
7941
|
+
personalConnectionDelegations: [],
|
|
7942
|
+
initialXaiProviderAccountAuthoritySnapshot: taskXaiProviderAccountAuthoritySnapshot,
|
|
7943
|
+
maxNestedAgentDepthOverride: task.agentConfig.maxNestedAgentDepth ?? null,
|
|
7944
|
+
frozenNestedAgentDepthPolicy: {
|
|
7945
|
+
effectiveMaxNestedAgentDepth: generatedSessionBinding.effectiveMaxNestedAgentDepth,
|
|
7946
|
+
nestedAgentDepthPolicySource: generatedSessionBinding.nestedAgentDepthPolicySource
|
|
7947
|
+
},
|
|
7948
|
+
frozenCodexCompactionMode: generatedSessionBinding.codexCompactionMode,
|
|
7949
|
+
// The durable agent config was privilege-checked when the task
|
|
7950
|
+
// was created/updated. Preserve that explicit policy if a broader
|
|
7951
|
+
// workspace/deployment limit is narrowed before a later fire.
|
|
7952
|
+
allowNestedAgentDepthIncrease: true,
|
|
7953
|
+
subjectId: `scheduled_task:${task.id}`,
|
|
7954
|
+
beforeCreateCommit: async (tx, sessionId) => {
|
|
7955
|
+
if (incidentPreflightRequired) {
|
|
7956
|
+
const exactSession = await requireSession(tx, task.workspaceId, sessionId);
|
|
7957
|
+
const exactResponder = await resolveIncidentTelemetryResponderMetadata({
|
|
7958
|
+
db: tx,
|
|
7959
|
+
settings,
|
|
7960
|
+
task: acceptedExecution.task,
|
|
7961
|
+
session: exactSession,
|
|
7962
|
+
personalConnectionDelegations: acceptedExecution.personalConnectionDelegations,
|
|
7963
|
+
personalResourceAuthoritySubjectId: acceptedExecution.personalResourceAuthoritySubjectId,
|
|
7964
|
+
executionPolicy: {
|
|
7965
|
+
tools: exactSession.tools,
|
|
7966
|
+
firstPartyMcpTools: exactSession.firstPartyMcpTools,
|
|
7967
|
+
firstPartyMcpPermissions: exactSession.firstPartyMcpPermissions,
|
|
7968
|
+
variableSetId: exactSession.variableSetId,
|
|
7969
|
+
rigId: exactSession.rigId,
|
|
7970
|
+
rigVersionId: exactSession.rigVersionId,
|
|
7971
|
+
toolPolicy: exactSession.toolPolicy,
|
|
7972
|
+
mcpServerIds: exactSession.mcpServers.map((server) => server.id),
|
|
7973
|
+
toolPolicyVersion: exactSession.toolPolicyVersion
|
|
6835
7974
|
}
|
|
7975
|
+
});
|
|
7976
|
+
const exactPreflight = evaluateIncidentTelemetryPreflight({
|
|
7977
|
+
agentConfig: acceptedExecution.task.agentConfig,
|
|
7978
|
+
incidentTriggered: structuredAlertOccurrence !== null,
|
|
7979
|
+
alertOccurrenceLabels: structuredAlertOccurrence?.labels ?? null,
|
|
7980
|
+
responder: exactResponder
|
|
7981
|
+
});
|
|
7982
|
+
if (exactPreflight.action === "blocked") {
|
|
7983
|
+
throw new IncidentTelemetryPreflightBlockedError(exactPreflight.reason);
|
|
7984
|
+
}
|
|
7985
|
+
}
|
|
7986
|
+
await bindScheduledTaskRunSessionInTransaction(tx, {
|
|
7987
|
+
accountId: task.accountId,
|
|
7988
|
+
workspaceId: task.workspaceId,
|
|
7989
|
+
runId: run.id,
|
|
7990
|
+
sessionId
|
|
7991
|
+
});
|
|
7992
|
+
}
|
|
7993
|
+
};
|
|
7994
|
+
const created = await createSessionWithIdempotencyKeyResult(dispatchDb, {
|
|
7995
|
+
...sessionInput,
|
|
7996
|
+
createIdempotencyKey: generatedSessionBinding.createIdempotencyKey
|
|
7997
|
+
});
|
|
7998
|
+
if (created.denied) {
|
|
7999
|
+
throw new SessionSpawnDeniedDbError(created.denial);
|
|
8000
|
+
}
|
|
8001
|
+
session = created.session;
|
|
8002
|
+
sessionCreated = created.created;
|
|
8003
|
+
if (!sessionCreated) {
|
|
8004
|
+
await bindScheduledTaskRunSessionInTransaction(dispatchDb, {
|
|
8005
|
+
accountId: task.accountId,
|
|
8006
|
+
workspaceId: task.workspaceId,
|
|
8007
|
+
runId: run.id,
|
|
8008
|
+
sessionId: session.id
|
|
8009
|
+
});
|
|
8010
|
+
run = { ...run, sessionId: session.id };
|
|
8011
|
+
}
|
|
8012
|
+
} catch (error) {
|
|
8013
|
+
if (error instanceof IncidentTelemetryPreflightBlockedError) {
|
|
8014
|
+
throw error;
|
|
8015
|
+
}
|
|
8016
|
+
if (error instanceof SessionSpawnDeniedDbError) {
|
|
8017
|
+
throw new Error(`${error.denial.code}: denial=${error.denial.id}`, {
|
|
8018
|
+
cause: error
|
|
8019
|
+
});
|
|
8020
|
+
}
|
|
8021
|
+
throw error;
|
|
8022
|
+
}
|
|
8023
|
+
const scheduledRun = run;
|
|
8024
|
+
if (!sessionCreated) {
|
|
8025
|
+
assertReusableSessionRevivable(session.status);
|
|
8026
|
+
if ((session.variableSetId ?? null) !== (task.variableSetId ?? null)) {
|
|
8027
|
+
throw new Error(
|
|
8028
|
+
"scheduled alert occurrence variableSet attachment does not match its canonical session"
|
|
8029
|
+
);
|
|
8030
|
+
}
|
|
8031
|
+
if (scheduledSlackBotConnectionId(session.metadata) !== (task.agentConfig.slackBotConnectionId ?? null)) {
|
|
8032
|
+
throw new Error(
|
|
8033
|
+
"scheduled alert occurrence OpenGeni Slack bot binding does not match its canonical session"
|
|
8034
|
+
);
|
|
8035
|
+
}
|
|
8036
|
+
}
|
|
8037
|
+
let workflowId;
|
|
8038
|
+
if (alertOccurrence) {
|
|
8039
|
+
const started = await initializeSessionStartAtomically(dispatchDb, {
|
|
8040
|
+
accountId: task.accountId,
|
|
8041
|
+
workspaceId: task.workspaceId,
|
|
8042
|
+
sessionId: session.id,
|
|
8043
|
+
reasoningEffortFallback: reasoningEffort,
|
|
8044
|
+
createdEventPayload: {
|
|
8045
|
+
scheduledTaskId: typeof session.metadata.scheduledTaskId === "string" ? session.metadata.scheduledTaskId : task.id,
|
|
8046
|
+
scheduledTaskRunId: typeof session.metadata.scheduledTaskRunId === "string" ? session.metadata.scheduledTaskRunId : run.id,
|
|
8047
|
+
// Names/ids only; never values.
|
|
8048
|
+
...variableSet ? {
|
|
8049
|
+
variableSetId: variableSet.id,
|
|
8050
|
+
variableSetName: variableSet.name
|
|
8051
|
+
} : {},
|
|
8052
|
+
...slackBotConnection && slackBotMetadata ? {
|
|
8053
|
+
slackBotConnection: {
|
|
8054
|
+
credentialRole: OPENGENI_SLACK_BOT_CREDENTIAL_ROLE,
|
|
8055
|
+
credentialLabel: OPENGENI_SLACK_BOT_CREDENTIAL_LABEL,
|
|
8056
|
+
connectionId: slackBotConnection.id,
|
|
8057
|
+
slackTeamId: slackBotMetadata.slackTeamId
|
|
6836
8058
|
}
|
|
6837
|
-
|
|
6838
|
-
}
|
|
8059
|
+
} : {}
|
|
8060
|
+
},
|
|
8061
|
+
goal: goalSpec ? {
|
|
8062
|
+
text: goalSpec.text,
|
|
8063
|
+
successCriteria: goalSpec.successCriteria ?? null,
|
|
8064
|
+
maxAutoContinuations: goalSpec.maxAutoContinuations ?? null,
|
|
8065
|
+
...goalSpec.mutationPolicy ? { mutationPolicy: goalSpec.mutationPolicy } : {},
|
|
8066
|
+
createdBy: "scheduled_task"
|
|
8067
|
+
} : null,
|
|
8068
|
+
deferInitialTurn: true,
|
|
8069
|
+
deferredStatus: "queued"
|
|
8070
|
+
});
|
|
8071
|
+
workflowId = started.temporalWorkflowId;
|
|
8072
|
+
if (started.events.length > 0) {
|
|
8073
|
+
if (deferPublications) {
|
|
8074
|
+
deferredEvents.push({
|
|
8075
|
+
sessionId: session.id,
|
|
8076
|
+
events: started.events
|
|
8077
|
+
});
|
|
8078
|
+
} else {
|
|
8079
|
+
await publishDurableSessionEvents(
|
|
8080
|
+
bus,
|
|
8081
|
+
task.workspaceId,
|
|
8082
|
+
session.id,
|
|
8083
|
+
started.events
|
|
8084
|
+
);
|
|
8085
|
+
}
|
|
8086
|
+
}
|
|
8087
|
+
} else {
|
|
8088
|
+
const started = await initializeSessionStartAtomically(dispatchDb, {
|
|
8089
|
+
accountId: task.accountId,
|
|
8090
|
+
workspaceId: task.workspaceId,
|
|
8091
|
+
sessionId: session.id,
|
|
8092
|
+
reasoningEffortFallback: reasoningEffort,
|
|
8093
|
+
createdEventPayload: {
|
|
8094
|
+
scheduledTaskId: typeof session.metadata.scheduledTaskId === "string" ? session.metadata.scheduledTaskId : task.id,
|
|
8095
|
+
scheduledTaskRunId: typeof session.metadata.scheduledTaskRunId === "string" ? session.metadata.scheduledTaskRunId : run.id,
|
|
8096
|
+
// Names/ids only; never values.
|
|
8097
|
+
...variableSet ? { variableSetId: variableSet.id, variableSetName: variableSet.name } : {},
|
|
8098
|
+
...slackBotConnection && slackBotMetadata ? {
|
|
8099
|
+
slackBotConnection: {
|
|
8100
|
+
credentialRole: OPENGENI_SLACK_BOT_CREDENTIAL_ROLE,
|
|
8101
|
+
credentialLabel: OPENGENI_SLACK_BOT_CREDENTIAL_LABEL,
|
|
8102
|
+
connectionId: slackBotConnection.id,
|
|
8103
|
+
slackTeamId: slackBotMetadata.slackTeamId
|
|
8104
|
+
}
|
|
8105
|
+
} : {}
|
|
8106
|
+
},
|
|
8107
|
+
goal: goalSpec ? {
|
|
8108
|
+
text: goalSpec.text,
|
|
8109
|
+
successCriteria: goalSpec.successCriteria ?? null,
|
|
8110
|
+
maxAutoContinuations: goalSpec.maxAutoContinuations ?? null,
|
|
8111
|
+
...goalSpec.mutationPolicy ? { mutationPolicy: goalSpec.mutationPolicy } : {},
|
|
8112
|
+
createdBy: "scheduled_task"
|
|
8113
|
+
} : null,
|
|
8114
|
+
deferInitialTurn: true,
|
|
8115
|
+
deferredStatus: "queued"
|
|
8116
|
+
});
|
|
8117
|
+
workflowId = started.temporalWorkflowId;
|
|
8118
|
+
if (started.events.length > 0) {
|
|
8119
|
+
if (deferPublications) {
|
|
8120
|
+
deferredEvents.push({ sessionId: session.id, events: started.events });
|
|
8121
|
+
} else {
|
|
8122
|
+
await publishDurableSessionEvents(
|
|
8123
|
+
bus,
|
|
8124
|
+
task.workspaceId,
|
|
8125
|
+
session.id,
|
|
8126
|
+
started.events
|
|
8127
|
+
);
|
|
8128
|
+
}
|
|
8129
|
+
}
|
|
8130
|
+
if (task.runMode === "reusable_session") {
|
|
8131
|
+
await materializeScheduledTaskReusableSessionFromRun(dispatchDb, {
|
|
8132
|
+
accountId: task.accountId,
|
|
8133
|
+
workspaceId: task.workspaceId,
|
|
8134
|
+
taskId: task.id,
|
|
8135
|
+
runId: run.id,
|
|
8136
|
+
sessionId: session.id,
|
|
8137
|
+
sourceTaskAuthorityRevision: runPersonalResourceAuthority?.taskAuthorityRevision ?? run.taskAuthorityRevision,
|
|
8138
|
+
sourceExecutionDigest: runPersonalResourceAuthority?.executionDigest ?? run.taskExecutionDigest
|
|
8139
|
+
});
|
|
8140
|
+
}
|
|
8141
|
+
}
|
|
8142
|
+
const titleEvents = await stampScheduledSessionTitle(dispatchDb, {
|
|
8143
|
+
workspaceId: task.workspaceId,
|
|
8144
|
+
session,
|
|
8145
|
+
taskName: task.name
|
|
8146
|
+
});
|
|
8147
|
+
if (titleEvents.length > 0) {
|
|
8148
|
+
if (deferPublications) {
|
|
8149
|
+
deferredEvents.push({ sessionId: session.id, events: titleEvents });
|
|
8150
|
+
} else {
|
|
8151
|
+
await publishDurableSessionEvents(bus, task.workspaceId, session.id, titleEvents);
|
|
8152
|
+
}
|
|
8153
|
+
}
|
|
8154
|
+
const scheduledUpdate = await addSessionSystemUpdateWithSourceMutation(
|
|
8155
|
+
dispatchDb,
|
|
8156
|
+
{
|
|
8157
|
+
accountId: task.accountId,
|
|
8158
|
+
workspaceId: task.workspaceId,
|
|
8159
|
+
sessionId: session.id,
|
|
8160
|
+
kind: "scheduled_occurrence",
|
|
8161
|
+
classification: "info",
|
|
8162
|
+
sourceId: run.id,
|
|
8163
|
+
dedupeKey: `scheduled-task-run:${run.id}`,
|
|
8164
|
+
summary: task.agentConfig.prompt,
|
|
8165
|
+
payload: scheduledUserMessagePayload(
|
|
8166
|
+
task.agentConfig.prompt,
|
|
8167
|
+
task.agentConfig.resources,
|
|
8168
|
+
taskTools,
|
|
8169
|
+
task.id,
|
|
8170
|
+
run.id
|
|
8171
|
+
),
|
|
8172
|
+
lineage: {
|
|
8173
|
+
scheduledTaskId: task.id,
|
|
8174
|
+
scheduledTaskRunId: run.id,
|
|
8175
|
+
...causalHumanSubjectId ? { causalHumanSubjectId } : {},
|
|
8176
|
+
...taskConnectionAuthoritySubjectId ? { connectionAuthoritySubjectId: taskConnectionAuthoritySubjectId } : {},
|
|
8177
|
+
...taskXaiProviderAccountAuthoritySnapshot.scope === "user" && task.createdBy.kind === "subject" ? { xaiAuthoritySubjectId: task.createdBy.subjectId } : {}
|
|
8178
|
+
},
|
|
8179
|
+
personalConnectionDelegations: taskPersonalConnectionDelegations,
|
|
8180
|
+
xaiProviderAccountAuthoritySnapshot: taskXaiProviderAccountAuthoritySnapshot,
|
|
8181
|
+
scheduledTaskRunId: run.id
|
|
8182
|
+
},
|
|
8183
|
+
async (tx, wakeEventId) => {
|
|
8184
|
+
if (!wakeEventId) {
|
|
8185
|
+
await settleScheduledTaskRunInTransaction(tx, {
|
|
8186
|
+
workspaceId: task.workspaceId,
|
|
8187
|
+
runId: scheduledRun.id,
|
|
8188
|
+
sessionId: session.id,
|
|
8189
|
+
triggerEventId: null,
|
|
8190
|
+
status: "skipped",
|
|
8191
|
+
error: "session_cancelled"
|
|
8192
|
+
});
|
|
8193
|
+
return;
|
|
8194
|
+
}
|
|
8195
|
+
await settleScheduledTaskRunInTransaction(tx, {
|
|
8196
|
+
workspaceId: task.workspaceId,
|
|
8197
|
+
runId: scheduledRun.id,
|
|
8198
|
+
sessionId: session.id,
|
|
8199
|
+
triggerEventId: wakeEventId,
|
|
8200
|
+
status: "dispatched"
|
|
8201
|
+
});
|
|
6839
8202
|
},
|
|
6840
|
-
|
|
8203
|
+
incidentPreflightRequired ? {
|
|
8204
|
+
prepareSource: async (tx, sessionId) => await prepareIncidentTelemetrySource({
|
|
8205
|
+
tx,
|
|
8206
|
+
settings,
|
|
8207
|
+
taskId: task.id,
|
|
8208
|
+
workspaceId: task.workspaceId,
|
|
8209
|
+
sessionId,
|
|
8210
|
+
alertOccurrenceLabels: structuredAlertOccurrence?.labels ?? null,
|
|
8211
|
+
acceptedExecution
|
|
8212
|
+
})
|
|
8213
|
+
} : void 0
|
|
6841
8214
|
);
|
|
6842
|
-
|
|
6843
|
-
|
|
6844
|
-
|
|
6845
|
-
|
|
6846
|
-
|
|
8215
|
+
if (scheduledUpdate.reason === "session_cancelled") {
|
|
8216
|
+
return {
|
|
8217
|
+
kind: "blocked",
|
|
8218
|
+
result: { action: "blocked", reason: "scheduled_run_terminal" },
|
|
8219
|
+
run: { ...run, status: "skipped", error: "session_cancelled" }
|
|
8220
|
+
};
|
|
8221
|
+
}
|
|
8222
|
+
if (scheduledUpdate.added && scheduledUpdate.events.length > 0) {
|
|
8223
|
+
if (deferPublications) {
|
|
8224
|
+
deferredEvents.push({
|
|
8225
|
+
sessionId: session.id,
|
|
8226
|
+
events: scheduledUpdate.events
|
|
8227
|
+
});
|
|
8228
|
+
} else {
|
|
8229
|
+
await publishDurableSessionEvents(
|
|
8230
|
+
bus,
|
|
8231
|
+
task.workspaceId,
|
|
8232
|
+
session.id,
|
|
8233
|
+
scheduledUpdate.events
|
|
8234
|
+
);
|
|
8235
|
+
}
|
|
8236
|
+
}
|
|
8237
|
+
result2 = {
|
|
8238
|
+
action: sessionCreated ? "start" : "signal",
|
|
6847
8239
|
accountId: task.accountId,
|
|
6848
8240
|
workspaceId: task.workspaceId,
|
|
6849
8241
|
sessionId: session.id,
|
|
6850
|
-
|
|
6851
|
-
|
|
6852
|
-
|
|
6853
|
-
|
|
6854
|
-
|
|
6855
|
-
|
|
6856
|
-
|
|
6857
|
-
|
|
6858
|
-
|
|
6859
|
-
|
|
6860
|
-
|
|
6861
|
-
|
|
6862
|
-
|
|
6863
|
-
|
|
6864
|
-
|
|
6865
|
-
|
|
8242
|
+
triggerEventId: scheduledUpdate.wakeEventId,
|
|
8243
|
+
workflowId,
|
|
8244
|
+
workflowWakeRevision: scheduledUpdate.workflowWakeRevision
|
|
8245
|
+
};
|
|
8246
|
+
} else {
|
|
8247
|
+
if (!run) throw new Error("scheduled task run was not created");
|
|
8248
|
+
const scheduledRun = run;
|
|
8249
|
+
const session = await requireSession(dispatchDb, task.workspaceId, existingSessionId);
|
|
8250
|
+
await bindScheduledTaskRunSessionInTransaction(dispatchDb, {
|
|
8251
|
+
accountId: task.accountId,
|
|
8252
|
+
workspaceId: task.workspaceId,
|
|
8253
|
+
runId: run.id,
|
|
8254
|
+
sessionId: session.id
|
|
8255
|
+
});
|
|
8256
|
+
assertReusableSessionRevivable(session.status);
|
|
8257
|
+
assertReusableSessionBindingMatches(session, task);
|
|
8258
|
+
const goalEvents = task.runMode === "reusable_session" && goalSpec && run.status === "queued" ? await upsertScheduledSessionGoalForRun(dispatchDb, {
|
|
8259
|
+
accountId: task.accountId,
|
|
8260
|
+
workspaceId: task.workspaceId,
|
|
8261
|
+
sessionId: session.id,
|
|
8262
|
+
runId: run.id,
|
|
8263
|
+
text: goalSpec.text,
|
|
8264
|
+
successCriteria: goalSpec.successCriteria ?? null,
|
|
8265
|
+
maxAutoContinuations: goalSpec.maxAutoContinuations ?? null,
|
|
8266
|
+
...goalSpec.mutationPolicy ? { mutationPolicy: goalSpec.mutationPolicy } : {}
|
|
8267
|
+
}) : [];
|
|
8268
|
+
if (goalEvents.length > 0) {
|
|
8269
|
+
if (deferPublications) {
|
|
8270
|
+
deferredEvents.push({
|
|
8271
|
+
sessionId: session.id,
|
|
8272
|
+
events: goalEvents
|
|
8273
|
+
});
|
|
8274
|
+
} else {
|
|
8275
|
+
await publishDurableSessionEvents(bus, task.workspaceId, session.id, goalEvents);
|
|
8276
|
+
}
|
|
8277
|
+
}
|
|
8278
|
+
const bundled = await addSessionSystemUpdateWithSourceMutation(
|
|
8279
|
+
dispatchDb,
|
|
8280
|
+
{
|
|
8281
|
+
accountId: task.accountId,
|
|
8282
|
+
workspaceId: task.workspaceId,
|
|
8283
|
+
sessionId: session.id,
|
|
8284
|
+
kind: "scheduled_occurrence",
|
|
8285
|
+
classification: "info",
|
|
8286
|
+
sourceId: run.id,
|
|
8287
|
+
dedupeKey: `scheduled-task-run:${run.id}`,
|
|
8288
|
+
summary: task.agentConfig.prompt,
|
|
8289
|
+
payload: scheduledUserMessagePayload(
|
|
8290
|
+
task.agentConfig.prompt,
|
|
8291
|
+
task.agentConfig.resources,
|
|
8292
|
+
taskTools,
|
|
8293
|
+
task.id,
|
|
8294
|
+
run.id
|
|
8295
|
+
),
|
|
8296
|
+
lineage: {
|
|
8297
|
+
scheduledTaskId: task.id,
|
|
8298
|
+
scheduledTaskRunId: run.id,
|
|
8299
|
+
...causalHumanSubjectId ? { causalHumanSubjectId } : {},
|
|
8300
|
+
...taskConnectionAuthoritySubjectId ? { connectionAuthoritySubjectId: taskConnectionAuthoritySubjectId } : {},
|
|
8301
|
+
...taskXaiProviderAccountAuthoritySnapshot.scope === "user" && task.createdBy.kind === "subject" ? { xaiAuthoritySubjectId: task.createdBy.subjectId } : {}
|
|
8302
|
+
},
|
|
8303
|
+
personalConnectionDelegations: taskPersonalConnectionDelegations,
|
|
8304
|
+
xaiProviderAccountAuthoritySnapshot: taskXaiProviderAccountAuthoritySnapshot,
|
|
8305
|
+
scheduledTaskRunId: run.id
|
|
6866
8306
|
},
|
|
6867
|
-
|
|
6868
|
-
|
|
6869
|
-
|
|
6870
|
-
|
|
6871
|
-
|
|
6872
|
-
|
|
8307
|
+
async (tx, wakeEventId) => {
|
|
8308
|
+
if (!wakeEventId) {
|
|
8309
|
+
await settleScheduledTaskRunInTransaction(tx, {
|
|
8310
|
+
workspaceId: task.workspaceId,
|
|
8311
|
+
runId: scheduledRun.id,
|
|
8312
|
+
sessionId: session.id,
|
|
8313
|
+
triggerEventId: null,
|
|
8314
|
+
status: "skipped",
|
|
8315
|
+
error: "session_cancelled"
|
|
8316
|
+
});
|
|
8317
|
+
return;
|
|
8318
|
+
}
|
|
8319
|
+
await settleScheduledTaskRunInTransaction(tx, {
|
|
6873
8320
|
workspaceId: task.workspaceId,
|
|
8321
|
+
runId: scheduledRun.id,
|
|
8322
|
+
sessionId: session.id,
|
|
8323
|
+
triggerEventId: wakeEventId,
|
|
8324
|
+
status: "dispatched"
|
|
8325
|
+
});
|
|
8326
|
+
},
|
|
8327
|
+
incidentPreflightRequired ? {
|
|
8328
|
+
prepareSource: async (tx, sessionId) => await prepareIncidentTelemetrySource({
|
|
8329
|
+
tx,
|
|
8330
|
+
settings,
|
|
6874
8331
|
taskId: task.id,
|
|
6875
|
-
|
|
8332
|
+
workspaceId: task.workspaceId,
|
|
8333
|
+
sessionId,
|
|
8334
|
+
alertOccurrenceLabels: structuredAlertOccurrence?.labels ?? null,
|
|
8335
|
+
acceptedExecution
|
|
8336
|
+
})
|
|
8337
|
+
} : void 0
|
|
8338
|
+
);
|
|
8339
|
+
if (bundled.reason === "session_cancelled") {
|
|
8340
|
+
return {
|
|
8341
|
+
kind: "blocked",
|
|
8342
|
+
result: { action: "blocked", reason: "scheduled_run_terminal" },
|
|
8343
|
+
run: { ...run, status: "skipped", error: "session_cancelled" }
|
|
8344
|
+
};
|
|
8345
|
+
}
|
|
8346
|
+
if (bundled.added && bundled.events.length > 0) {
|
|
8347
|
+
if (deferPublications) {
|
|
8348
|
+
deferredEvents.push({
|
|
8349
|
+
sessionId: session.id,
|
|
8350
|
+
events: bundled.events
|
|
6876
8351
|
});
|
|
8352
|
+
} else {
|
|
8353
|
+
await publishDurableSessionEvents(
|
|
8354
|
+
bus,
|
|
8355
|
+
task.workspaceId,
|
|
8356
|
+
session.id,
|
|
8357
|
+
bundled.events
|
|
8358
|
+
);
|
|
6877
8359
|
}
|
|
6878
|
-
if (!wakeEventId) throw new Error("Scheduled delivery has no wake event");
|
|
6879
|
-
await settleScheduledTaskRunInTransaction(tx, {
|
|
6880
|
-
workspaceId: task.workspaceId,
|
|
6881
|
-
runId: run.id,
|
|
6882
|
-
sessionId: session.id,
|
|
6883
|
-
triggerEventId: wakeEventId,
|
|
6884
|
-
status: "dispatched"
|
|
6885
|
-
});
|
|
6886
8360
|
}
|
|
6887
|
-
|
|
6888
|
-
|
|
6889
|
-
|
|
6890
|
-
|
|
6891
|
-
|
|
6892
|
-
|
|
8361
|
+
result2 = {
|
|
8362
|
+
action: "signal",
|
|
8363
|
+
accountId: task.accountId,
|
|
8364
|
+
workspaceId: task.workspaceId,
|
|
8365
|
+
sessionId: session.id,
|
|
8366
|
+
triggerEventId: bundled.wakeEventId,
|
|
8367
|
+
workflowId: workflowIdForSession(session.id),
|
|
8368
|
+
workflowWakeRevision: bundled.workflowWakeRevision
|
|
8369
|
+
};
|
|
6893
8370
|
}
|
|
6894
|
-
|
|
6895
|
-
|
|
6896
|
-
|
|
6897
|
-
|
|
6898
|
-
|
|
6899
|
-
|
|
6900
|
-
|
|
6901
|
-
|
|
8371
|
+
} catch (error) {
|
|
8372
|
+
const terminalError = scheduledRunRecoveryTerminalError(error);
|
|
8373
|
+
if (!terminalError) throw error;
|
|
8374
|
+
await markScheduledTaskRunFailedIfQueued(
|
|
8375
|
+
dispatchDb,
|
|
8376
|
+
task.workspaceId,
|
|
8377
|
+
run.id,
|
|
8378
|
+
terminalError.code
|
|
8379
|
+
);
|
|
8380
|
+
return {
|
|
8381
|
+
kind: "blocked",
|
|
8382
|
+
result: scheduledRunTerminalResult(terminalError.code),
|
|
8383
|
+
run: { ...run, status: "failed", error: terminalError.code }
|
|
6902
8384
|
};
|
|
6903
8385
|
}
|
|
8386
|
+
await recordUsageEvent3(dispatchDb, {
|
|
8387
|
+
accountId: task.accountId,
|
|
8388
|
+
workspaceId: task.workspaceId,
|
|
8389
|
+
eventType: "agent_run.created",
|
|
8390
|
+
quantity: 1,
|
|
8391
|
+
unit: "run",
|
|
8392
|
+
sourceResourceType: "scheduled_task_run",
|
|
8393
|
+
sourceResourceId: run.id,
|
|
8394
|
+
sessionId: result2.sessionId,
|
|
8395
|
+
initiator: input.initiator ?? {
|
|
8396
|
+
kind: "service",
|
|
8397
|
+
subjectId: "scheduler"
|
|
8398
|
+
},
|
|
8399
|
+
initiatorContext: {
|
|
8400
|
+
scheduledTaskId: task.id,
|
|
8401
|
+
scheduledTaskRunId: run.id
|
|
8402
|
+
},
|
|
8403
|
+
origin: "scheduled_task",
|
|
8404
|
+
idempotencyKey: input.agentRunUsageIdempotencyKey ?? `usage:agent_run.created:scheduled:${run.id}`
|
|
8405
|
+
});
|
|
8406
|
+
return { kind: "complete", result: result2, run };
|
|
8407
|
+
};
|
|
8408
|
+
let dispatchOutcome;
|
|
8409
|
+
try {
|
|
8410
|
+
dispatchOutcome = incidentPreflightRequired ? await withSessionActivityRlsContext(
|
|
8411
|
+
db,
|
|
8412
|
+
{ accountId: task.accountId, workspaceId: task.workspaceId },
|
|
8413
|
+
async (tx) => await executeDispatch(tx, true)
|
|
8414
|
+
) : await executeDispatch(db, false);
|
|
6904
8415
|
} catch (error) {
|
|
6905
|
-
|
|
8416
|
+
if (error instanceof IncidentTelemetryPreflightBlockedError) {
|
|
8417
|
+
return { action: "blocked", reason: error.reason };
|
|
8418
|
+
}
|
|
8419
|
+
throw error;
|
|
8420
|
+
}
|
|
8421
|
+
if (dispatchOutcome.kind === "replay") {
|
|
8422
|
+
return await replayScheduledTaskDispatch({
|
|
6906
8423
|
db,
|
|
8424
|
+
wakeSessionWorkflow,
|
|
8425
|
+
task,
|
|
8426
|
+
run: dispatchOutcome.run,
|
|
8427
|
+
acceptedExecution: dispatchOutcome.acceptedExecution
|
|
8428
|
+
});
|
|
8429
|
+
}
|
|
8430
|
+
if (dispatchOutcome.kind === "blocked") return dispatchOutcome.result;
|
|
8431
|
+
for (const deferred of deferredEvents) {
|
|
8432
|
+
await publishDurableSessionEvents(
|
|
8433
|
+
bus,
|
|
6907
8434
|
task.workspaceId,
|
|
6908
|
-
|
|
6909
|
-
|
|
6910
|
-
)
|
|
6911
|
-
throw error;
|
|
8435
|
+
deferred.sessionId,
|
|
8436
|
+
deferred.events
|
|
8437
|
+
);
|
|
6912
8438
|
}
|
|
6913
|
-
|
|
6914
|
-
accountId: task.accountId,
|
|
6915
|
-
workspaceId: task.workspaceId,
|
|
6916
|
-
eventType: "agent_run.created",
|
|
6917
|
-
quantity: 1,
|
|
6918
|
-
unit: "run",
|
|
6919
|
-
sourceResourceType: "scheduled_task_run",
|
|
6920
|
-
sourceResourceId: run.id,
|
|
6921
|
-
sessionId: result.sessionId,
|
|
6922
|
-
initiator: input.initiator ?? { kind: "service", subjectId: "scheduler" },
|
|
6923
|
-
initiatorContext: { scheduledTaskId: task.id, scheduledTaskRunId: run.id },
|
|
6924
|
-
origin: "scheduled_task",
|
|
6925
|
-
idempotencyKey: input.agentRunUsageIdempotencyKey ?? `usage:agent_run.created:scheduled:${run.id}`
|
|
6926
|
-
});
|
|
8439
|
+
const result = dispatchOutcome.result;
|
|
6927
8440
|
if (wakeSessionWorkflow && result.workflowWakeRevision !== null) {
|
|
6928
8441
|
await wakeSessionWorkflow({
|
|
6929
8442
|
accountId: result.accountId,
|
|
@@ -6937,6 +8450,585 @@ function createScheduledTaskActivities(services) {
|
|
|
6937
8450
|
}
|
|
6938
8451
|
};
|
|
6939
8452
|
}
|
|
8453
|
+
var IncidentTelemetryPreflightBlockedError = class extends Error {
|
|
8454
|
+
constructor(reason) {
|
|
8455
|
+
super(reason);
|
|
8456
|
+
this.reason = reason;
|
|
8457
|
+
this.name = "IncidentTelemetryPreflightBlockedError";
|
|
8458
|
+
}
|
|
8459
|
+
};
|
|
8460
|
+
function assertReusableSessionBindingMatches(session, task) {
|
|
8461
|
+
if ((session.variableSetId ?? null) !== (task.variableSetId ?? null)) {
|
|
8462
|
+
throw new ScheduledRunTerminalAuthorityError(
|
|
8463
|
+
"scheduled_reusable_binding_changed",
|
|
8464
|
+
"scheduled task variableSet attachment does not match its reusable session"
|
|
8465
|
+
);
|
|
8466
|
+
}
|
|
8467
|
+
if (scheduledSlackBotConnectionId(session.metadata) !== (task.agentConfig.slackBotConnectionId ?? null)) {
|
|
8468
|
+
throw new ScheduledRunTerminalAuthorityError(
|
|
8469
|
+
"scheduled_reusable_binding_changed",
|
|
8470
|
+
"scheduled task OpenGeni Slack bot binding does not match its reusable session"
|
|
8471
|
+
);
|
|
8472
|
+
}
|
|
8473
|
+
}
|
|
8474
|
+
var ScheduledRunTerminalAuthorityError = class extends Error {
|
|
8475
|
+
constructor(code, message) {
|
|
8476
|
+
super(message);
|
|
8477
|
+
this.code = code;
|
|
8478
|
+
this.name = "ScheduledRunTerminalAuthorityError";
|
|
8479
|
+
}
|
|
8480
|
+
};
|
|
8481
|
+
function scheduledRunRecoveryTerminalError(error) {
|
|
8482
|
+
if (error instanceof Error && "cause" in error && error.cause !== void 0 && error.cause !== error) {
|
|
8483
|
+
const nested = scheduledRunRecoveryTerminalError(error.cause);
|
|
8484
|
+
if (nested) return nested;
|
|
8485
|
+
}
|
|
8486
|
+
if (error instanceof ScheduledRunTerminalAuthorityError) return error;
|
|
8487
|
+
if (error instanceof IncidentTelemetryPreflightBlockedError) {
|
|
8488
|
+
return new ScheduledRunTerminalAuthorityError(
|
|
8489
|
+
`scheduled_incident_${error.reason}`,
|
|
8490
|
+
error.message
|
|
8491
|
+
);
|
|
8492
|
+
}
|
|
8493
|
+
if (error instanceof SessionSpawnDeniedDbError) {
|
|
8494
|
+
return new ScheduledRunTerminalAuthorityError("scheduled_session_spawn_denied", error.message);
|
|
8495
|
+
}
|
|
8496
|
+
if (error instanceof Error && error.message.startsWith("Session not found:")) {
|
|
8497
|
+
return new ScheduledRunTerminalAuthorityError(
|
|
8498
|
+
"scheduled_target_session_unavailable",
|
|
8499
|
+
error.message
|
|
8500
|
+
);
|
|
8501
|
+
}
|
|
8502
|
+
if (typeof error === "object" && error !== null && "status" in error && error.status === 422) {
|
|
8503
|
+
return new ScheduledRunTerminalAuthorityError(
|
|
8504
|
+
"scheduled_slack_authority_unavailable",
|
|
8505
|
+
error instanceof Error ? error.message : "scheduled Slack authority is unavailable"
|
|
8506
|
+
);
|
|
8507
|
+
}
|
|
8508
|
+
if (typeof error === "object" && error !== null && "code" in error && error.code === "42501") {
|
|
8509
|
+
return new ScheduledRunTerminalAuthorityError(
|
|
8510
|
+
"scheduled_run_authority_proof_rejected",
|
|
8511
|
+
error instanceof Error ? error.message : "scheduled run authority proof was rejected"
|
|
8512
|
+
);
|
|
8513
|
+
}
|
|
8514
|
+
return null;
|
|
8515
|
+
}
|
|
8516
|
+
function scheduledRunTerminalResult(error) {
|
|
8517
|
+
if (error === "scheduled_authority_exhausted") {
|
|
8518
|
+
return { action: "blocked", reason: "scheduled_authority_exhausted" };
|
|
8519
|
+
}
|
|
8520
|
+
if (error === "scheduled_incident_incident_preflight_metadata_missing") {
|
|
8521
|
+
return { action: "blocked", reason: "incident_preflight_metadata_missing" };
|
|
8522
|
+
}
|
|
8523
|
+
if (error === "scheduled_incident_incident_responder_under_capable") {
|
|
8524
|
+
return { action: "blocked", reason: "incident_responder_under_capable" };
|
|
8525
|
+
}
|
|
8526
|
+
if (error === "scheduled_incident_incident_data_source_unsuitable") {
|
|
8527
|
+
return { action: "blocked", reason: "incident_data_source_unsuitable" };
|
|
8528
|
+
}
|
|
8529
|
+
return { action: "blocked", reason: "scheduled_run_terminal" };
|
|
8530
|
+
}
|
|
8531
|
+
async function prepareIncidentTelemetrySource(input) {
|
|
8532
|
+
const session = await requireSession(input.tx, input.workspaceId, input.sessionId);
|
|
8533
|
+
const { task, personalConnectionDelegations } = input.acceptedExecution ? {
|
|
8534
|
+
task: input.acceptedExecution.task,
|
|
8535
|
+
personalConnectionDelegations: input.acceptedExecution.personalConnectionDelegations
|
|
8536
|
+
} : await requireScheduledTaskIncidentAuthorityInTransaction(input.tx, {
|
|
8537
|
+
workspaceId: input.workspaceId,
|
|
8538
|
+
taskId: input.taskId
|
|
8539
|
+
});
|
|
8540
|
+
const responder = await resolveIncidentTelemetryResponderMetadata({
|
|
8541
|
+
db: input.tx,
|
|
8542
|
+
settings: input.settings,
|
|
8543
|
+
task,
|
|
8544
|
+
session,
|
|
8545
|
+
personalConnectionDelegations,
|
|
8546
|
+
personalResourceAuthoritySubjectId: input.acceptedExecution?.personalResourceAuthoritySubjectId ?? null,
|
|
8547
|
+
...input.acceptedExecution ? {
|
|
8548
|
+
executionPolicy: input.acceptedExecution.targetSessionExecution ? {
|
|
8549
|
+
tools: input.acceptedExecution.targetSessionExecution.tools,
|
|
8550
|
+
firstPartyMcpTools: input.acceptedExecution.targetSessionExecution.firstPartyMcpTools,
|
|
8551
|
+
firstPartyMcpPermissions: input.acceptedExecution.targetSessionExecution.firstPartyMcpPermissions,
|
|
8552
|
+
variableSetId: input.acceptedExecution.targetSessionExecution.variableSetId,
|
|
8553
|
+
rigId: input.acceptedExecution.targetSessionExecution.rigId,
|
|
8554
|
+
rigVersionId: input.acceptedExecution.targetSessionExecution.rigVersionId,
|
|
8555
|
+
toolPolicy: input.acceptedExecution.targetSessionExecution.toolPolicy,
|
|
8556
|
+
mcpServerIds: input.acceptedExecution.targetSessionExecution.mcpServerIds,
|
|
8557
|
+
toolPolicyVersion: input.acceptedExecution.targetSessionExecution.toolPolicyVersion
|
|
8558
|
+
} : {
|
|
8559
|
+
tools: input.acceptedExecution.resolvedTools,
|
|
8560
|
+
firstPartyMcpTools: input.acceptedExecution.resolvedFirstPartyMcpTools,
|
|
8561
|
+
firstPartyMcpPermissions: input.acceptedExecution.resolvedFirstPartyMcpPermissions,
|
|
8562
|
+
variableSetId: input.acceptedExecution.resolvedVariableSet?.id ?? null,
|
|
8563
|
+
rigId: input.acceptedExecution.resolvedRig?.id ?? null,
|
|
8564
|
+
rigVersionId: input.acceptedExecution.resolvedRig?.versionId ?? null,
|
|
8565
|
+
toolPolicy: session.toolPolicy,
|
|
8566
|
+
mcpServerIds: [],
|
|
8567
|
+
toolPolicyVersion: session.toolPolicyVersion
|
|
8568
|
+
}
|
|
8569
|
+
} : {}
|
|
8570
|
+
});
|
|
8571
|
+
const preflight = evaluateIncidentTelemetryPreflight({
|
|
8572
|
+
agentConfig: task.agentConfig,
|
|
8573
|
+
incidentTriggered: input.alertOccurrenceLabels !== null,
|
|
8574
|
+
alertOccurrenceLabels: input.alertOccurrenceLabels,
|
|
8575
|
+
responder
|
|
8576
|
+
});
|
|
8577
|
+
if (preflight.action === "blocked") {
|
|
8578
|
+
throw new IncidentTelemetryPreflightBlockedError(preflight.reason);
|
|
8579
|
+
}
|
|
8580
|
+
const fence = incidentTelemetryAuthorityFence({
|
|
8581
|
+
task,
|
|
8582
|
+
responder,
|
|
8583
|
+
alertOccurrenceLabels: input.alertOccurrenceLabels
|
|
8584
|
+
});
|
|
8585
|
+
if (!fence) {
|
|
8586
|
+
throw new IncidentTelemetryPreflightBlockedError("incident_preflight_metadata_missing");
|
|
8587
|
+
}
|
|
8588
|
+
return {
|
|
8589
|
+
lineage: {
|
|
8590
|
+
[INCIDENT_TELEMETRY_AUTHORITY_FENCE_LINEAGE_KEY]: fence
|
|
8591
|
+
}
|
|
8592
|
+
};
|
|
8593
|
+
}
|
|
8594
|
+
async function recordScheduledTaskFiredUsage(db, acceptedExecution, run) {
|
|
8595
|
+
const task = acceptedExecution.task;
|
|
8596
|
+
await recordUsageEvent3(db, {
|
|
8597
|
+
accountId: task.accountId,
|
|
8598
|
+
workspaceId: task.workspaceId,
|
|
8599
|
+
eventType: "scheduled_task.fired",
|
|
8600
|
+
quantity: 1,
|
|
8601
|
+
unit: "run",
|
|
8602
|
+
sourceResourceType: "scheduled_task_run",
|
|
8603
|
+
sourceResourceId: run.id,
|
|
8604
|
+
initiator: acceptedExecution.triggerInitiator,
|
|
8605
|
+
initiatorContext: { scheduledTaskId: task.id, scheduledTaskRunId: run.id },
|
|
8606
|
+
origin: "scheduled_task",
|
|
8607
|
+
idempotencyKey: `usage:scheduled_task.fired:${run.id}`
|
|
8608
|
+
});
|
|
8609
|
+
}
|
|
8610
|
+
async function recoverBoundScheduledTaskDispatch(input) {
|
|
8611
|
+
const { task } = input.acceptedExecution;
|
|
8612
|
+
if (input.run.status !== "queued") {
|
|
8613
|
+
throw new Error("scheduled run is not an exact queued recovery");
|
|
8614
|
+
}
|
|
8615
|
+
const generatedSession = task.runMode === "new_session_per_run" || task.runMode === "reusable_session" && task.reusableSessionId === null;
|
|
8616
|
+
const frozenSlack = input.acceptedExecution.resolvedSlackBotConnection;
|
|
8617
|
+
if (frozenSlack) {
|
|
8618
|
+
const currentSlack = await requireOpenGeniSlackBotConnection(
|
|
8619
|
+
input.db,
|
|
8620
|
+
task.workspaceId,
|
|
8621
|
+
frozenSlack.id
|
|
8622
|
+
);
|
|
8623
|
+
const currentMetadata = openGeniSlackBotMetadata(currentSlack.metadata);
|
|
8624
|
+
if (currentSlack.version !== frozenSlack.version || currentSlack.verifiedInstallVersion !== frozenSlack.verifiedInstallVersion || stableJson(currentMetadata) !== stableJson(frozenSlack.metadata)) {
|
|
8625
|
+
throw new ScheduledRunTerminalAuthorityError(
|
|
8626
|
+
"scheduled_slack_authority_changed",
|
|
8627
|
+
"scheduled Slack bot authority changed after run admission"
|
|
8628
|
+
);
|
|
8629
|
+
}
|
|
8630
|
+
}
|
|
8631
|
+
let session;
|
|
8632
|
+
if (input.run.sessionId) {
|
|
8633
|
+
session = await requireSession(input.db, task.workspaceId, input.run.sessionId);
|
|
8634
|
+
if (!generatedSession) assertReusableSessionBindingMatches(session, task);
|
|
8635
|
+
} else if (generatedSession) {
|
|
8636
|
+
const variableSet = input.acceptedExecution.resolvedVariableSet;
|
|
8637
|
+
if (variableSet) {
|
|
8638
|
+
const current = await getVariableSet(
|
|
8639
|
+
input.db,
|
|
8640
|
+
{
|
|
8641
|
+
accountId: task.accountId,
|
|
8642
|
+
workspaceId: task.workspaceId,
|
|
8643
|
+
subjectId: input.acceptedExecution.personalResourceAuthoritySubjectId ?? task.createdBy.subjectId
|
|
8644
|
+
},
|
|
8645
|
+
variableSet.id
|
|
8646
|
+
);
|
|
8647
|
+
if (!current || current.generation !== variableSet.generation) {
|
|
8648
|
+
throw new ScheduledRunTerminalAuthorityError(
|
|
8649
|
+
"scheduled_variable_set_authority_changed",
|
|
8650
|
+
"scheduled variable set changed after run admission"
|
|
8651
|
+
);
|
|
8652
|
+
}
|
|
8653
|
+
}
|
|
8654
|
+
const rig = input.acceptedExecution.resolvedRig;
|
|
8655
|
+
if (rig) {
|
|
8656
|
+
const retained = await getScheduledScopedRigVersionMetadata(
|
|
8657
|
+
input.db,
|
|
8658
|
+
{
|
|
8659
|
+
accountId: task.accountId,
|
|
8660
|
+
workspaceId: task.workspaceId,
|
|
8661
|
+
subjectId: input.acceptedExecution.personalResourceAuthoritySubjectId ?? task.createdBy.subjectId
|
|
8662
|
+
},
|
|
8663
|
+
rig.id,
|
|
8664
|
+
rig.versionId
|
|
8665
|
+
);
|
|
8666
|
+
if (!retained) {
|
|
8667
|
+
throw new ScheduledRunTerminalAuthorityError(
|
|
8668
|
+
"scheduled_rig_authority_changed",
|
|
8669
|
+
"scheduled rig changed after run admission"
|
|
8670
|
+
);
|
|
8671
|
+
}
|
|
8672
|
+
}
|
|
8673
|
+
const taskMetadata = { ...task.agentConfig.metadata };
|
|
8674
|
+
delete taskMetadata[OPENGENI_SLACK_BOT_SESSION_METADATA_KEY];
|
|
8675
|
+
const created = await createSessionWithIdempotencyKeyResult(input.db, {
|
|
8676
|
+
accountId: task.accountId,
|
|
8677
|
+
workspaceId: task.workspaceId,
|
|
8678
|
+
initialMessage: task.agentConfig.prompt,
|
|
8679
|
+
resources: task.agentConfig.resources,
|
|
8680
|
+
tools: input.acceptedExecution.resolvedTools,
|
|
8681
|
+
firstPartyMcpTools: input.acceptedExecution.resolvedFirstPartyMcpTools,
|
|
8682
|
+
firstPartyMcpPermissions: input.acceptedExecution.resolvedFirstPartyMcpPermissions,
|
|
8683
|
+
metadata: {
|
|
8684
|
+
...taskMetadata,
|
|
8685
|
+
model: input.acceptedExecution.resolvedModel,
|
|
8686
|
+
reasoningEffort: input.acceptedExecution.resolvedReasoningEffort,
|
|
8687
|
+
scheduledTaskId: task.id,
|
|
8688
|
+
scheduledTaskRunMode: task.runMode,
|
|
8689
|
+
...task.agentConfig.goal ? { scheduledTaskGoal: task.agentConfig.goal } : {},
|
|
8690
|
+
scheduledTaskRunId: input.run.id,
|
|
8691
|
+
...frozenSlack ? { [OPENGENI_SLACK_BOT_SESSION_METADATA_KEY]: frozenSlack.id } : {}
|
|
8692
|
+
},
|
|
8693
|
+
createdBy: {
|
|
8694
|
+
kind: "service",
|
|
8695
|
+
subjectId: "scheduler",
|
|
8696
|
+
label: "OpenGeni scheduler"
|
|
8697
|
+
},
|
|
8698
|
+
createdByContext: {
|
|
8699
|
+
scheduledTaskId: task.id,
|
|
8700
|
+
scheduledTaskRunId: input.run.id
|
|
8701
|
+
},
|
|
8702
|
+
model: input.acceptedExecution.resolvedModel,
|
|
8703
|
+
reasoningEffort: input.acceptedExecution.resolvedReasoningEffort,
|
|
8704
|
+
latencyMode: input.acceptedExecution.resolvedLatencyMode,
|
|
8705
|
+
sandboxBackend: input.acceptedExecution.resolvedSandboxBackend,
|
|
8706
|
+
sandboxOs: input.acceptedExecution.resolvedSandboxOs,
|
|
8707
|
+
variableSetId: variableSet?.id ?? null,
|
|
8708
|
+
rigId: rig?.id ?? null,
|
|
8709
|
+
rigVersionId: rig?.versionId ?? null,
|
|
8710
|
+
personalConnectionDelegations: [],
|
|
8711
|
+
initialXaiProviderAccountAuthoritySnapshot: input.acceptedExecution.xaiProviderAccountAuthoritySnapshot,
|
|
8712
|
+
maxNestedAgentDepthOverride: task.agentConfig.maxNestedAgentDepth ?? null,
|
|
8713
|
+
frozenNestedAgentDepthPolicy: {
|
|
8714
|
+
effectiveMaxNestedAgentDepth: input.acceptedExecution.generatedSessionBinding.effectiveMaxNestedAgentDepth,
|
|
8715
|
+
nestedAgentDepthPolicySource: input.acceptedExecution.generatedSessionBinding.nestedAgentDepthPolicySource
|
|
8716
|
+
},
|
|
8717
|
+
frozenCodexCompactionMode: input.acceptedExecution.generatedSessionBinding.codexCompactionMode,
|
|
8718
|
+
allowNestedAgentDepthIncrease: true,
|
|
8719
|
+
subjectId: `scheduled_task:${task.id}`,
|
|
8720
|
+
createIdempotencyKey: input.acceptedExecution.generatedSessionBinding?.createIdempotencyKey ?? (() => {
|
|
8721
|
+
throw new ScheduledRunTerminalAuthorityError(
|
|
8722
|
+
"scheduled_generated_session_binding_missing",
|
|
8723
|
+
"scheduled generated session binding is missing"
|
|
8724
|
+
);
|
|
8725
|
+
})(),
|
|
8726
|
+
beforeCreateCommit: async (tx, sessionId) => {
|
|
8727
|
+
await bindScheduledTaskRunSessionInTransaction(tx, {
|
|
8728
|
+
accountId: task.accountId,
|
|
8729
|
+
workspaceId: task.workspaceId,
|
|
8730
|
+
runId: input.run.id,
|
|
8731
|
+
sessionId
|
|
8732
|
+
});
|
|
8733
|
+
}
|
|
8734
|
+
});
|
|
8735
|
+
if (created.denied) throw new SessionSpawnDeniedDbError(created.denial);
|
|
8736
|
+
session = created.session;
|
|
8737
|
+
if (!created.created) {
|
|
8738
|
+
await bindScheduledTaskRunSessionInTransaction(input.db, {
|
|
8739
|
+
accountId: task.accountId,
|
|
8740
|
+
workspaceId: task.workspaceId,
|
|
8741
|
+
runId: input.run.id,
|
|
8742
|
+
sessionId: session.id
|
|
8743
|
+
});
|
|
8744
|
+
}
|
|
8745
|
+
} else {
|
|
8746
|
+
const targetSessionId = task.runMode === "existing_session" ? task.targetSessionId : task.reusableSessionId;
|
|
8747
|
+
if (!targetSessionId) {
|
|
8748
|
+
throw new ScheduledRunTerminalAuthorityError(
|
|
8749
|
+
"scheduled_target_session_unavailable",
|
|
8750
|
+
"scheduled accepted target session is unavailable"
|
|
8751
|
+
);
|
|
8752
|
+
}
|
|
8753
|
+
session = await requireSession(input.db, task.workspaceId, targetSessionId);
|
|
8754
|
+
assertReusableSessionBindingMatches(session, task);
|
|
8755
|
+
await bindScheduledTaskRunSessionInTransaction(input.db, {
|
|
8756
|
+
accountId: task.accountId,
|
|
8757
|
+
workspaceId: task.workspaceId,
|
|
8758
|
+
runId: input.run.id,
|
|
8759
|
+
sessionId: session.id
|
|
8760
|
+
});
|
|
8761
|
+
}
|
|
8762
|
+
if (session.status === "cancelled") {
|
|
8763
|
+
await markScheduledTaskRunSkippedIfQueued(input.db, {
|
|
8764
|
+
workspaceId: task.workspaceId,
|
|
8765
|
+
runId: input.run.id,
|
|
8766
|
+
sessionId: session.id,
|
|
8767
|
+
error: "session_cancelled"
|
|
8768
|
+
});
|
|
8769
|
+
return { action: "blocked", reason: "scheduled_run_terminal" };
|
|
8770
|
+
}
|
|
8771
|
+
if (generatedSession) {
|
|
8772
|
+
const expectedCreateKey = input.acceptedExecution.generatedSessionBinding?.createIdempotencyKey;
|
|
8773
|
+
if (!expectedCreateKey) {
|
|
8774
|
+
throw new ScheduledRunTerminalAuthorityError(
|
|
8775
|
+
"scheduled_generated_session_binding_missing",
|
|
8776
|
+
"scheduled generated session binding is missing"
|
|
8777
|
+
);
|
|
8778
|
+
}
|
|
8779
|
+
const generatedBinding = input.acceptedExecution.generatedSessionBinding;
|
|
8780
|
+
const canonicalGeneratedRunId = task.runMode === "reusable_session" && typeof session.metadata.scheduledTaskRunId === "string" ? session.metadata.scheduledTaskRunId : input.run.id;
|
|
8781
|
+
const expectedTaskMetadata = { ...task.agentConfig.metadata };
|
|
8782
|
+
delete expectedTaskMetadata[OPENGENI_SLACK_BOT_SESSION_METADATA_KEY];
|
|
8783
|
+
const expectedMetadata = {
|
|
8784
|
+
...expectedTaskMetadata,
|
|
8785
|
+
model: input.acceptedExecution.resolvedModel,
|
|
8786
|
+
reasoningEffort: input.acceptedExecution.resolvedReasoningEffort,
|
|
8787
|
+
scheduledTaskId: task.id,
|
|
8788
|
+
scheduledTaskRunMode: task.runMode,
|
|
8789
|
+
...task.agentConfig.goal ? { scheduledTaskGoal: task.agentConfig.goal } : {},
|
|
8790
|
+
scheduledTaskRunId: canonicalGeneratedRunId,
|
|
8791
|
+
...frozenSlack ? { [OPENGENI_SLACK_BOT_SESSION_METADATA_KEY]: frozenSlack.id } : {}
|
|
8792
|
+
};
|
|
8793
|
+
if (session.createIdempotencyKey !== expectedCreateKey || session.initialMessage !== task.agentConfig.prompt || session.instructions !== null || session.policyRole !== null || stableJson(session.skills) !== "[]" || stableJson(session.toolPolicy) !== stableJson({ mode: "explicit", inheritedFromSessionId: null }) || session.toolPolicyVersion !== 1 || session.mcpServers.length !== 0 || session.parentSessionId !== null || session.rootSessionId !== session.id || session.nestedAgentDepth !== 0 || session.effectiveMaxNestedAgentDepth !== generatedBinding.effectiveMaxNestedAgentDepth || session.nestedAgentDepthPolicySource !== generatedBinding.nestedAgentDepthPolicySource || session.nestedAgentDepthPolicySessionId !== (generatedBinding.nestedAgentDepthPolicySource === "session" ? session.id : null) || session.sandboxGroupId !== session.id || session.channelId !== null || session.codexCompactionMode !== generatedBinding.codexCompactionMode || stableJson(session.metadata) !== stableJson(expectedMetadata) || session.createdBy.kind !== "service" || session.createdBy.subjectId !== "scheduler" || stableJson(session.createdByContext) !== stableJson({
|
|
8794
|
+
label: "OpenGeni scheduler",
|
|
8795
|
+
scheduledTaskId: task.id,
|
|
8796
|
+
scheduledTaskRunId: canonicalGeneratedRunId
|
|
8797
|
+
}) || task.runMode === "new_session_per_run" && input.acceptedExecution.alertOccurrenceLabels === null && (session.metadata.scheduledTaskRunId !== input.run.id || session.createdByContext.scheduledTaskRunId !== input.run.id) || session.model !== input.acceptedExecution.resolvedModel || session.reasoningEffort !== input.acceptedExecution.resolvedReasoningEffort || session.latencyMode !== input.acceptedExecution.resolvedLatencyMode || session.sandboxBackend !== input.acceptedExecution.resolvedSandboxBackend || session.sandboxOs !== input.acceptedExecution.resolvedSandboxOs || stableJson(session.resources) !== stableJson(task.agentConfig.resources) || stableJson(session.tools) !== stableJson(input.acceptedExecution.resolvedTools) || stableJson(session.firstPartyMcpTools) !== stableJson(input.acceptedExecution.resolvedFirstPartyMcpTools) || stableJson(session.firstPartyMcpPermissions) !== stableJson(input.acceptedExecution.resolvedFirstPartyMcpPermissions) || session.maxNestedAgentDepthOverride !== (task.agentConfig.maxNestedAgentDepth ?? null) || (session.variableSetId ?? null) !== (input.acceptedExecution.resolvedVariableSet?.id ?? null) || (session.rigId ?? null) !== (input.acceptedExecution.resolvedRig?.id ?? null) || (session.rigVersionId ?? null) !== (input.acceptedExecution.resolvedRig?.versionId ?? null) || scheduledSlackBotConnectionId(session.metadata) !== (frozenSlack?.id ?? null)) {
|
|
8798
|
+
throw new ScheduledRunTerminalAuthorityError(
|
|
8799
|
+
"scheduled_generated_session_changed",
|
|
8800
|
+
"scheduled generated session differs from accepted execution"
|
|
8801
|
+
);
|
|
8802
|
+
}
|
|
8803
|
+
}
|
|
8804
|
+
let workflowId = workflowIdForSession(session.id);
|
|
8805
|
+
if (generatedSession) {
|
|
8806
|
+
const goalSpec = task.agentConfig.goal ?? null;
|
|
8807
|
+
const started = await initializeSessionStartAtomically(input.db, {
|
|
8808
|
+
accountId: task.accountId,
|
|
8809
|
+
workspaceId: task.workspaceId,
|
|
8810
|
+
sessionId: session.id,
|
|
8811
|
+
reasoningEffortFallback: input.acceptedExecution.resolvedReasoningEffort,
|
|
8812
|
+
createdEventPayload: {
|
|
8813
|
+
scheduledTaskId: task.id,
|
|
8814
|
+
scheduledTaskRunId: input.run.id
|
|
8815
|
+
},
|
|
8816
|
+
goal: goalSpec ? {
|
|
8817
|
+
text: goalSpec.text,
|
|
8818
|
+
successCriteria: goalSpec.successCriteria ?? null,
|
|
8819
|
+
maxAutoContinuations: goalSpec.maxAutoContinuations ?? null,
|
|
8820
|
+
...goalSpec.mutationPolicy ? { mutationPolicy: goalSpec.mutationPolicy } : {},
|
|
8821
|
+
createdBy: "scheduled_task"
|
|
8822
|
+
} : null,
|
|
8823
|
+
deferInitialTurn: true,
|
|
8824
|
+
deferredStatus: "queued"
|
|
8825
|
+
});
|
|
8826
|
+
workflowId = started.temporalWorkflowId;
|
|
8827
|
+
if (started.events.length > 0) {
|
|
8828
|
+
await publishDurableSessionEvents(input.bus, task.workspaceId, session.id, started.events);
|
|
8829
|
+
}
|
|
8830
|
+
if (task.runMode === "reusable_session") {
|
|
8831
|
+
await materializeScheduledTaskReusableSessionFromRun(input.db, {
|
|
8832
|
+
accountId: task.accountId,
|
|
8833
|
+
workspaceId: task.workspaceId,
|
|
8834
|
+
taskId: task.id,
|
|
8835
|
+
runId: input.run.id,
|
|
8836
|
+
sessionId: session.id,
|
|
8837
|
+
sourceTaskAuthorityRevision: input.run.taskAuthorityRevision,
|
|
8838
|
+
sourceExecutionDigest: input.run.taskExecutionDigest
|
|
8839
|
+
});
|
|
8840
|
+
}
|
|
8841
|
+
const titleEvents = await stampScheduledSessionTitle(input.db, {
|
|
8842
|
+
workspaceId: task.workspaceId,
|
|
8843
|
+
session,
|
|
8844
|
+
taskName: task.name
|
|
8845
|
+
});
|
|
8846
|
+
if (titleEvents.length > 0) {
|
|
8847
|
+
await publishDurableSessionEvents(input.bus, task.workspaceId, session.id, titleEvents);
|
|
8848
|
+
}
|
|
8849
|
+
}
|
|
8850
|
+
if (!generatedSession && task.runMode === "reusable_session" && task.agentConfig.goal) {
|
|
8851
|
+
const goalEvents = await upsertScheduledSessionGoalForRun(input.db, {
|
|
8852
|
+
accountId: task.accountId,
|
|
8853
|
+
workspaceId: task.workspaceId,
|
|
8854
|
+
sessionId: session.id,
|
|
8855
|
+
runId: input.run.id,
|
|
8856
|
+
text: task.agentConfig.goal.text,
|
|
8857
|
+
successCriteria: task.agentConfig.goal.successCriteria ?? null,
|
|
8858
|
+
maxAutoContinuations: task.agentConfig.goal.maxAutoContinuations ?? null,
|
|
8859
|
+
...task.agentConfig.goal.mutationPolicy ? { mutationPolicy: task.agentConfig.goal.mutationPolicy } : {}
|
|
8860
|
+
});
|
|
8861
|
+
if (goalEvents.length > 0) {
|
|
8862
|
+
await publishDurableSessionEvents(input.bus, task.workspaceId, session.id, goalEvents);
|
|
8863
|
+
}
|
|
8864
|
+
}
|
|
8865
|
+
const scheduledUpdate = await addSessionSystemUpdateWithSourceMutation(
|
|
8866
|
+
input.db,
|
|
8867
|
+
{
|
|
8868
|
+
accountId: task.accountId,
|
|
8869
|
+
workspaceId: task.workspaceId,
|
|
8870
|
+
sessionId: session.id,
|
|
8871
|
+
kind: "scheduled_occurrence",
|
|
8872
|
+
classification: "info",
|
|
8873
|
+
sourceId: input.run.id,
|
|
8874
|
+
dedupeKey: `scheduled-task-run:${input.run.id}`,
|
|
8875
|
+
summary: task.agentConfig.prompt,
|
|
8876
|
+
payload: scheduledUserMessagePayload(
|
|
8877
|
+
task.agentConfig.prompt,
|
|
8878
|
+
task.agentConfig.resources,
|
|
8879
|
+
input.acceptedExecution.resolvedTools,
|
|
8880
|
+
task.id,
|
|
8881
|
+
input.run.id
|
|
8882
|
+
),
|
|
8883
|
+
lineage: {
|
|
8884
|
+
scheduledTaskId: task.id,
|
|
8885
|
+
scheduledTaskRunId: input.run.id,
|
|
8886
|
+
...input.acceptedExecution.causalHumanSubjectId ? { causalHumanSubjectId: input.acceptedExecution.causalHumanSubjectId } : {},
|
|
8887
|
+
...input.acceptedExecution.connectionAuthoritySubjectId ? {
|
|
8888
|
+
connectionAuthoritySubjectId: input.acceptedExecution.connectionAuthoritySubjectId
|
|
8889
|
+
} : {},
|
|
8890
|
+
...input.acceptedExecution.xaiAuthoritySubjectId ? { xaiAuthoritySubjectId: input.acceptedExecution.xaiAuthoritySubjectId } : {}
|
|
8891
|
+
},
|
|
8892
|
+
personalConnectionDelegations: input.acceptedExecution.personalConnectionDelegations,
|
|
8893
|
+
xaiProviderAccountAuthoritySnapshot: input.acceptedExecution.xaiProviderAccountAuthoritySnapshot,
|
|
8894
|
+
scheduledTaskRunId: input.run.id
|
|
8895
|
+
},
|
|
8896
|
+
async (tx, wakeEventId) => {
|
|
8897
|
+
if (!wakeEventId) {
|
|
8898
|
+
await settleScheduledTaskRunInTransaction(tx, {
|
|
8899
|
+
workspaceId: task.workspaceId,
|
|
8900
|
+
runId: input.run.id,
|
|
8901
|
+
sessionId: session.id,
|
|
8902
|
+
triggerEventId: null,
|
|
8903
|
+
status: "skipped",
|
|
8904
|
+
error: "session_cancelled"
|
|
8905
|
+
});
|
|
8906
|
+
return;
|
|
8907
|
+
}
|
|
8908
|
+
await settleScheduledTaskRunInTransaction(tx, {
|
|
8909
|
+
workspaceId: task.workspaceId,
|
|
8910
|
+
runId: input.run.id,
|
|
8911
|
+
sessionId: session.id,
|
|
8912
|
+
triggerEventId: wakeEventId,
|
|
8913
|
+
status: "dispatched"
|
|
8914
|
+
});
|
|
8915
|
+
},
|
|
8916
|
+
input.acceptedExecution.incidentPreflightRequired ? {
|
|
8917
|
+
prepareSource: async (tx, sessionId) => await prepareIncidentTelemetrySource({
|
|
8918
|
+
tx,
|
|
8919
|
+
settings: input.settings,
|
|
8920
|
+
taskId: task.id,
|
|
8921
|
+
workspaceId: task.workspaceId,
|
|
8922
|
+
sessionId,
|
|
8923
|
+
alertOccurrenceLabels: input.acceptedExecution.alertOccurrenceLabels,
|
|
8924
|
+
acceptedExecution: input.acceptedExecution
|
|
8925
|
+
})
|
|
8926
|
+
} : void 0
|
|
8927
|
+
);
|
|
8928
|
+
if (scheduledUpdate.reason === "session_cancelled") {
|
|
8929
|
+
return { action: "blocked", reason: "scheduled_run_terminal" };
|
|
8930
|
+
}
|
|
8931
|
+
if (scheduledUpdate.added && scheduledUpdate.events.length > 0) {
|
|
8932
|
+
await publishDurableSessionEvents(
|
|
8933
|
+
input.bus,
|
|
8934
|
+
task.workspaceId,
|
|
8935
|
+
session.id,
|
|
8936
|
+
scheduledUpdate.events
|
|
8937
|
+
);
|
|
8938
|
+
}
|
|
8939
|
+
await recordUsageEvent3(input.db, {
|
|
8940
|
+
accountId: task.accountId,
|
|
8941
|
+
workspaceId: task.workspaceId,
|
|
8942
|
+
eventType: "agent_run.created",
|
|
8943
|
+
quantity: 1,
|
|
8944
|
+
unit: "run",
|
|
8945
|
+
sourceResourceType: "scheduled_task_run",
|
|
8946
|
+
sourceResourceId: input.run.id,
|
|
8947
|
+
sessionId: session.id,
|
|
8948
|
+
initiator: input.acceptedExecution.triggerInitiator,
|
|
8949
|
+
initiatorContext: { scheduledTaskId: task.id, scheduledTaskRunId: input.run.id },
|
|
8950
|
+
origin: "scheduled_task",
|
|
8951
|
+
idempotencyKey: input.acceptedExecution.agentRunUsageIdempotencyKey ?? `usage:agent_run.created:scheduled:${input.run.id}`
|
|
8952
|
+
});
|
|
8953
|
+
const workflowWakeRevision = await enqueueSessionWorkflowWakeIfRunnable(input.db, {
|
|
8954
|
+
accountId: task.accountId,
|
|
8955
|
+
workspaceId: task.workspaceId,
|
|
8956
|
+
sessionId: session.id,
|
|
8957
|
+
temporalWorkflowId: workflowId,
|
|
8958
|
+
reason: "scheduled_retry"
|
|
8959
|
+
});
|
|
8960
|
+
const result = {
|
|
8961
|
+
action: generatedSession ? "start" : "signal",
|
|
8962
|
+
accountId: task.accountId,
|
|
8963
|
+
workspaceId: task.workspaceId,
|
|
8964
|
+
sessionId: session.id,
|
|
8965
|
+
triggerEventId: scheduledUpdate.wakeEventId,
|
|
8966
|
+
workflowId,
|
|
8967
|
+
workflowWakeRevision
|
|
8968
|
+
};
|
|
8969
|
+
if (input.wakeSessionWorkflow && workflowWakeRevision !== null) {
|
|
8970
|
+
await input.wakeSessionWorkflow({
|
|
8971
|
+
accountId: result.accountId,
|
|
8972
|
+
workspaceId: result.workspaceId,
|
|
8973
|
+
sessionId: result.sessionId,
|
|
8974
|
+
workflowId: result.workflowId,
|
|
8975
|
+
wakeRevision: workflowWakeRevision
|
|
8976
|
+
});
|
|
8977
|
+
}
|
|
8978
|
+
return result;
|
|
8979
|
+
}
|
|
8980
|
+
async function replayScheduledTaskDispatch(input) {
|
|
8981
|
+
if (!input.run.sessionId || !input.run.triggerEventId) {
|
|
8982
|
+
throw new Error("dispatched scheduled task run is missing its delivery identity");
|
|
8983
|
+
}
|
|
8984
|
+
const sessionId = input.run.sessionId;
|
|
8985
|
+
const triggerEventId = input.run.triggerEventId;
|
|
8986
|
+
await requireSession(input.db, input.task.workspaceId, sessionId);
|
|
8987
|
+
await recordUsageEvent3(input.db, {
|
|
8988
|
+
accountId: input.task.accountId,
|
|
8989
|
+
workspaceId: input.task.workspaceId,
|
|
8990
|
+
eventType: "agent_run.created",
|
|
8991
|
+
quantity: 1,
|
|
8992
|
+
unit: "run",
|
|
8993
|
+
sourceResourceType: "scheduled_task_run",
|
|
8994
|
+
sourceResourceId: input.run.id,
|
|
8995
|
+
sessionId,
|
|
8996
|
+
initiator: input.acceptedExecution.triggerInitiator,
|
|
8997
|
+
initiatorContext: {
|
|
8998
|
+
scheduledTaskId: input.task.id,
|
|
8999
|
+
scheduledTaskRunId: input.run.id
|
|
9000
|
+
},
|
|
9001
|
+
origin: "scheduled_task",
|
|
9002
|
+
idempotencyKey: input.acceptedExecution.agentRunUsageIdempotencyKey ?? `usage:agent_run.created:scheduled:${input.run.id}`
|
|
9003
|
+
});
|
|
9004
|
+
const workflowId = workflowIdForSession(sessionId);
|
|
9005
|
+
const workflowWakeRevision = await enqueueSessionWorkflowWakeIfRunnable(input.db, {
|
|
9006
|
+
accountId: input.task.accountId,
|
|
9007
|
+
workspaceId: input.task.workspaceId,
|
|
9008
|
+
sessionId,
|
|
9009
|
+
temporalWorkflowId: workflowId,
|
|
9010
|
+
reason: "scheduled_retry"
|
|
9011
|
+
});
|
|
9012
|
+
const result = {
|
|
9013
|
+
action: input.acceptedExecution.task.runMode === "new_session_per_run" ? "start" : "signal",
|
|
9014
|
+
accountId: input.task.accountId,
|
|
9015
|
+
workspaceId: input.task.workspaceId,
|
|
9016
|
+
sessionId,
|
|
9017
|
+
triggerEventId,
|
|
9018
|
+
workflowId,
|
|
9019
|
+
workflowWakeRevision
|
|
9020
|
+
};
|
|
9021
|
+
if (input.wakeSessionWorkflow && workflowWakeRevision !== null) {
|
|
9022
|
+
await input.wakeSessionWorkflow({
|
|
9023
|
+
accountId: result.accountId,
|
|
9024
|
+
workspaceId: result.workspaceId,
|
|
9025
|
+
sessionId: result.sessionId,
|
|
9026
|
+
workflowId: result.workflowId,
|
|
9027
|
+
wakeRevision: workflowWakeRevision
|
|
9028
|
+
});
|
|
9029
|
+
}
|
|
9030
|
+
return result;
|
|
9031
|
+
}
|
|
6940
9032
|
function knowledgeSourceSyncEffectivelyPaused(task) {
|
|
6941
9033
|
if (task.status === "paused") return true;
|
|
6942
9034
|
const value = task.metadata.knowledgeSourceSync;
|
|
@@ -6946,17 +9038,17 @@ function knowledgeSourceSyncEffectivelyPaused(task) {
|
|
|
6946
9038
|
}
|
|
6947
9039
|
|
|
6948
9040
|
// src/activities/site-auth-maintenance.ts
|
|
6949
|
-
import { resolveTurnExecutionPolicyV1 } from "@opengeni/config";
|
|
9041
|
+
import { resolveTurnExecutionPolicyV1 as resolveTurnExecutionPolicyV12 } from "@opengeni/config";
|
|
6950
9042
|
import {
|
|
6951
|
-
DEFAULT_FIRST_PARTY_MCP_PERMISSIONS,
|
|
9043
|
+
DEFAULT_FIRST_PARTY_MCP_PERMISSIONS as DEFAULT_FIRST_PARTY_MCP_PERMISSIONS2,
|
|
6952
9044
|
SITE_AUTH_MAINTENANCE_CONNECTION_CONTEXT_KEY,
|
|
6953
9045
|
SITE_AUTH_MAINTENANCE_OPERATION_CONTEXT_KEY
|
|
6954
9046
|
} from "@opengeni/contracts";
|
|
6955
9047
|
import {
|
|
6956
|
-
assertWorkspaceModelPolicyAllows,
|
|
6957
|
-
canonicalConfiguredModel,
|
|
6958
|
-
createAndStartSessionWithOutcome,
|
|
6959
|
-
recordWorkspaceUsage
|
|
9048
|
+
assertWorkspaceModelPolicyAllows as assertWorkspaceModelPolicyAllows2,
|
|
9049
|
+
canonicalConfiguredModel as canonicalConfiguredModel2,
|
|
9050
|
+
createAndStartSessionWithOutcome as createAndStartSessionWithOutcome2,
|
|
9051
|
+
recordWorkspaceUsage as recordWorkspaceUsage2
|
|
6960
9052
|
} from "@opengeni/core";
|
|
6961
9053
|
import {
|
|
6962
9054
|
claimSiteAuthMaintenance,
|
|
@@ -6983,10 +9075,10 @@ function createSiteAuthMaintenanceActivities(services, options = {}) {
|
|
|
6983
9075
|
const claim = options.claim ?? claimSiteAuthMaintenance;
|
|
6984
9076
|
const confirm = options.confirm ?? confirmSiteAuthMaintenanceSessionInTransaction;
|
|
6985
9077
|
const defer = options.defer ?? deferSiteAuthMaintenance;
|
|
6986
|
-
const createSession2 = options.createSession ??
|
|
9078
|
+
const createSession2 = options.createSession ?? createAndStartSessionWithOutcome2;
|
|
6987
9079
|
const admit = options.admit ?? agentRunAdmissionDenial;
|
|
6988
|
-
const recordUsage = options.recordUsage ??
|
|
6989
|
-
const assertModelPolicy = options.assertModelPolicy ??
|
|
9080
|
+
const recordUsage = options.recordUsage ?? recordWorkspaceUsage2;
|
|
9081
|
+
const assertModelPolicy = options.assertModelPolicy ?? assertWorkspaceModelPolicyAllows2;
|
|
6990
9082
|
async function maintainSiteAuthConnections() {
|
|
6991
9083
|
const service = await services();
|
|
6992
9084
|
const claims = await claim(service.db, { claimTimeoutMs, limit: batchSize });
|
|
@@ -7032,7 +9124,7 @@ function createSiteAuthMaintenanceActivities(services, options = {}) {
|
|
|
7032
9124
|
return { maintainSiteAuthConnections };
|
|
7033
9125
|
}
|
|
7034
9126
|
async function dispatchMaintenanceSession(service, maintenance, dependencies) {
|
|
7035
|
-
const model =
|
|
9127
|
+
const model = canonicalConfiguredModel2(service.settings, service.settings.openaiModel);
|
|
7036
9128
|
if (!model) throw new Error("site auth maintenance has no configured model");
|
|
7037
9129
|
await dependencies.assertModelPolicy(
|
|
7038
9130
|
service.db,
|
|
@@ -7048,7 +9140,7 @@ async function dispatchMaintenanceSession(service, maintenance, dependencies) {
|
|
|
7048
9140
|
});
|
|
7049
9141
|
if (denial) throw new Error(`site auth maintenance admission denied: ${denial}`);
|
|
7050
9142
|
const reasoningEffort = service.settings.openaiReasoningEffort;
|
|
7051
|
-
const turnExecutionPolicy =
|
|
9143
|
+
const turnExecutionPolicy = resolveTurnExecutionPolicyV12(service.settings, {
|
|
7052
9144
|
modelId: model,
|
|
7053
9145
|
requestedModelId: null,
|
|
7054
9146
|
modelSource: "deployment",
|
|
@@ -7102,7 +9194,7 @@ async function dispatchMaintenanceSession(service, maintenance, dependencies) {
|
|
|
7102
9194
|
},
|
|
7103
9195
|
instructions: "Complete only the exact browser-auth maintenance run in the initial request. Use the canonical browser/auth tools, preserve their causal fences, expose no secrets, never publish a BrowserIdentity revision, and do not perform unrelated work.",
|
|
7104
9196
|
policyRole: "site_auth_maintenance",
|
|
7105
|
-
firstPartyMcpPermissions: [...
|
|
9197
|
+
firstPartyMcpPermissions: [...DEFAULT_FIRST_PARTY_MCP_PERMISSIONS2],
|
|
7106
9198
|
firstPartyMcpTools: [...MAINTENANCE_TOOLS],
|
|
7107
9199
|
createIdempotencyKey: `site-auth-maintenance:${maintenance.operationId}`,
|
|
7108
9200
|
subjectId: "site-auth-maintenance"
|
|
@@ -7247,14 +9339,14 @@ function createSessionStateActivities(services, overrides = {}) {
|
|
|
7247
9339
|
return { action: "failed" };
|
|
7248
9340
|
}
|
|
7249
9341
|
const requestedRetryDelayMs = input.retryDelayMs;
|
|
7250
|
-
const
|
|
9342
|
+
const retryDelayMs2 = typeof requestedRetryDelayMs === "number" && Number.isFinite(requestedRetryDelayMs) ? Math.max(1e3, Math.min(6e4, Math.trunc(requestedRetryDelayMs))) : 6e4;
|
|
7251
9343
|
await enqueueSessionWorkflowWakeFn(db, {
|
|
7252
9344
|
accountId: input.accountId,
|
|
7253
9345
|
workspaceId: input.workspaceId,
|
|
7254
9346
|
sessionId: input.sessionId,
|
|
7255
9347
|
temporalWorkflowId: workflowId,
|
|
7256
9348
|
reason: "turn_activity_failed_before_attempt_claim",
|
|
7257
|
-
notBefore: new Date(Date.now() +
|
|
9349
|
+
notBefore: new Date(Date.now() + retryDelayMs2)
|
|
7258
9350
|
});
|
|
7259
9351
|
return { action: "unclaimed" };
|
|
7260
9352
|
}
|
|
@@ -7410,7 +9502,7 @@ function createSessionStateActivities(services, overrides = {}) {
|
|
|
7410
9502
|
input.attemptId
|
|
7411
9503
|
);
|
|
7412
9504
|
if (!turn) return { action: "stale" };
|
|
7413
|
-
if (turn.id !== input.turnId || turn.triggerEventId !== input.triggerEventId || turn.executionGeneration !== input.executionGeneration || input.executionGeneration <= 1) {
|
|
9505
|
+
if (turn.id !== input.turnId || turn.triggerEventId !== input.triggerEventId || turn.executionGeneration !== input.executionGeneration || input.executionGeneration <= 1 || input.providerRecoveryCount !== providerRecoveryCountFromMetadata(turn.metadata) + 1 || input.providerRecoveryCount > MAX_AUTOMATIC_PROVIDER_RECOVERIES) {
|
|
7414
9506
|
return { action: "ineligible" };
|
|
7415
9507
|
}
|
|
7416
9508
|
const recovery = await requestSessionTurnRecoveryFn(db, input.workspaceId, {
|
|
@@ -7419,10 +9511,12 @@ function createSessionStateActivities(services, overrides = {}) {
|
|
|
7419
9511
|
triggerEventId: input.triggerEventId,
|
|
7420
9512
|
attemptId: input.attemptId,
|
|
7421
9513
|
reason: "mcp_transport_timeout",
|
|
9514
|
+
providerRecoveryCount: input.providerRecoveryCount,
|
|
7422
9515
|
detail: {
|
|
7423
9516
|
code: "mcp_transport_timeout",
|
|
7424
9517
|
retryable: true,
|
|
7425
|
-
continueDelayMs:
|
|
9518
|
+
continueDelayMs: input.continueDelayMs,
|
|
9519
|
+
providerRecoveryCount: input.providerRecoveryCount,
|
|
7426
9520
|
recoverySource: "workflow_activity_failure"
|
|
7427
9521
|
},
|
|
7428
9522
|
fromStatuses: ["running"]
|
|
@@ -7537,6 +9631,7 @@ function createControlActivityServices(dependencies) {
|
|
|
7537
9631
|
}
|
|
7538
9632
|
function createControlActivitiesFromServices(services, resolveDocumentServices) {
|
|
7539
9633
|
return {
|
|
9634
|
+
...createAutomationActivities(services),
|
|
7540
9635
|
...createDocumentActivities(services, resolveDocumentServices),
|
|
7541
9636
|
...createKnowledgeSourceSyncActivities(services, resolveDocumentServices),
|
|
7542
9637
|
...createSessionStateActivities(services),
|
|
@@ -7567,4 +9662,4 @@ export {
|
|
|
7567
9662
|
createControlActivities,
|
|
7568
9663
|
createControlActivitiesFromServices
|
|
7569
9664
|
};
|
|
7570
|
-
//# sourceMappingURL=activities-control-
|
|
9665
|
+
//# sourceMappingURL=activities-control-DKG22ONJ.js.map
|