@opengeni/worker-bundle 0.20.3 → 0.22.1-canary.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (184) hide show
  1. package/dist/activities/agent-turn/admission.d.ts +52 -0
  2. package/dist/activities/agent-turn/agent-build.d.ts +89 -0
  3. package/dist/activities/agent-turn/claim.d.ts +67 -0
  4. package/dist/activities/agent-turn/codex-capacity.d.ts +40 -0
  5. package/dist/activities/agent-turn/codex.d.ts +32 -0
  6. package/dist/activities/agent-turn/compaction-prep.d.ts +88 -0
  7. package/dist/activities/agent-turn/credential-leases.d.ts +56 -0
  8. package/dist/activities/agent-turn/errors.d.ts +253 -0
  9. package/dist/activities/agent-turn/failure-settlement.d.ts +33 -0
  10. package/dist/activities/agent-turn/file-resources.d.ts +37 -0
  11. package/dist/activities/agent-turn/finalization.d.ts +42 -0
  12. package/dist/activities/agent-turn/github-repository-bindings.d.ts +105 -0
  13. package/dist/activities/agent-turn/governance-model.d.ts +66 -0
  14. package/dist/activities/agent-turn/history-sink.d.ts +52 -0
  15. package/dist/activities/agent-turn/history.d.ts +63 -0
  16. package/dist/activities/agent-turn/media-artifacts.d.ts +145 -0
  17. package/dist/activities/agent-turn/model-usage.d.ts +218 -0
  18. package/dist/activities/agent-turn/quiescence.d.ts +135 -0
  19. package/dist/activities/agent-turn/run-credentials.d.ts +84 -0
  20. package/dist/activities/agent-turn/run.d.ts +3 -0
  21. package/dist/activities/agent-turn/sandbox-establish.d.ts +93 -0
  22. package/dist/activities/agent-turn/sandbox-provision.d.ts +89 -0
  23. package/dist/activities/agent-turn/sandbox-route.d.ts +151 -0
  24. package/dist/activities/agent-turn/sandbox-runtime.d.ts +44 -0
  25. package/dist/activities/agent-turn/stream-attempt.d.ts +107 -0
  26. package/dist/activities/agent-turn/tool-environment.d.ts +142 -0
  27. package/dist/activities/agent-turn/tool-policy.d.ts +127 -0
  28. package/dist/activities/agent-turn/tool-result-spill.d.ts +59 -0
  29. package/dist/activities/agent-turn/turn-context.d.ts +144 -0
  30. package/dist/activities/agent-turn/xai-capacity.d.ts +2 -0
  31. package/dist/activities/agent-turn.d.ts +12 -954
  32. package/dist/activities/automations.d.ts +16 -0
  33. package/dist/activities/common.d.ts +1 -1
  34. package/dist/activities/connector-attachments.d.ts +13 -0
  35. package/dist/activities/documents.d.ts +1 -0
  36. package/dist/activities/environment.d.ts +26 -15
  37. package/dist/activities/generated-images.d.ts +1 -1
  38. package/dist/activities/git-credential-renewal.d.ts +3 -1
  39. package/dist/activities/goals.d.ts +4 -1
  40. package/dist/activities/google-drive-provider.d.ts +29 -0
  41. package/dist/activities/google-drive-publication.d.ts +17 -4
  42. package/dist/activities/incident-telemetry-authority.d.ts +52 -0
  43. package/dist/activities/incident-telemetry-preflight.d.ts +71 -0
  44. package/dist/activities/knowledge-source-sync.d.ts +4 -0
  45. package/dist/activities/mcp-credentials.d.ts +9 -3
  46. package/dist/activities/open-suffix-resume.d.ts +77 -0
  47. package/dist/activities/packs.d.ts +3 -11
  48. package/dist/activities/parent-wake.d.ts +19 -0
  49. package/dist/activities/retained-screenshot-reaper.d.ts +1 -1
  50. package/dist/activities/retained-screenshots.d.ts +1 -1
  51. package/dist/activities/rig-verification.d.ts +1 -1
  52. package/dist/activities/run-input.d.ts +9 -10
  53. package/dist/activities/sandbox-images.d.ts +1 -14
  54. package/dist/activities/sandbox-lease.d.ts +21 -3
  55. package/dist/activities/scheduled-tasks.d.ts +47 -0
  56. package/dist/activities/types.d.ts +23 -2
  57. package/dist/activities/video-media-validation.d.ts +1 -1
  58. package/dist/activities/video-output-retention.d.ts +1 -2
  59. package/dist/activities/xai-auth.d.ts +3 -0
  60. package/dist/{activities-control-BJSDRSIP.js → activities-control-DKG22ONJ.js} +2646 -551
  61. package/dist/activities-control-DKG22ONJ.js.map +1 -0
  62. package/dist/activities-control.d.ts +6 -2
  63. package/dist/activities-turn-4IMC3MAA.js +21326 -0
  64. package/dist/activities-turn-4IMC3MAA.js.map +1 -0
  65. package/dist/activities.d.ts +7 -3
  66. package/dist/artifact-materializer-entry.js +41 -22
  67. package/dist/artifact-materializer-entry.js.map +1 -1
  68. package/dist/{chunk-N5TJIFCE.js → chunk-CE7TPKBS.js} +468 -1
  69. package/dist/chunk-CE7TPKBS.js.map +1 -0
  70. package/dist/chunk-CNYGVRKF.js +4698 -0
  71. package/dist/chunk-CNYGVRKF.js.map +1 -0
  72. package/dist/connection-provider-use.d.ts +18 -0
  73. package/dist/editable-artifact-materializer.d.ts +4 -3
  74. package/dist/github-rest-mcp.d.ts +26 -0
  75. package/dist/index.d.ts +10 -2
  76. package/dist/index.js +58 -14
  77. package/dist/index.js.map +1 -1
  78. package/dist/knowledge-source-sync-activity-policy.d.ts +2 -0
  79. package/dist/model-context-contributions.d.ts +8 -4
  80. package/dist/observability-metrics.d.ts +63 -4
  81. package/dist/opensandbox-kubernetes-inventory.d.ts +21 -0
  82. package/dist/personal-github-git-credentials.d.ts +9 -0
  83. package/dist/pr-review-credentials.d.ts +6 -0
  84. package/dist/sandbox-archive-storage.d.ts +45 -0
  85. package/dist/sandbox-resume.d.ts +65 -6
  86. package/dist/sandbox-routing.d.ts +38 -2
  87. package/dist/scheduled-alert-occurrence.d.ts +33 -0
  88. package/dist/workflow-bundle.js +87 -28
  89. package/dist/workflows/activities.d.ts +1 -0
  90. package/dist/workflows/automations.d.ts +6 -0
  91. package/dist/workflows/document-index.d.ts +1 -0
  92. package/dist/workflows.d.ts +1 -0
  93. package/package.json +17 -17
  94. package/src/activities/agent-turn/admission.ts +292 -0
  95. package/src/activities/agent-turn/agent-build.ts +802 -0
  96. package/src/activities/agent-turn/claim.ts +590 -0
  97. package/src/activities/agent-turn/codex-capacity.ts +720 -0
  98. package/src/activities/agent-turn/codex.ts +94 -0
  99. package/src/activities/agent-turn/compaction-prep.ts +722 -0
  100. package/src/activities/agent-turn/credential-leases.ts +240 -0
  101. package/src/activities/agent-turn/errors.ts +1068 -0
  102. package/src/activities/agent-turn/failure-settlement.ts +1436 -0
  103. package/src/activities/agent-turn/file-resources.ts +227 -0
  104. package/src/activities/agent-turn/finalization.ts +754 -0
  105. package/src/activities/agent-turn/github-repository-bindings.ts +300 -0
  106. package/src/activities/agent-turn/governance-model.ts +465 -0
  107. package/src/activities/agent-turn/history-sink.ts +139 -0
  108. package/src/activities/agent-turn/history.ts +200 -0
  109. package/src/activities/agent-turn/media-artifacts.ts +359 -0
  110. package/src/activities/agent-turn/model-usage.ts +823 -0
  111. package/src/activities/agent-turn/quiescence.ts +353 -0
  112. package/src/activities/agent-turn/run-credentials.ts +761 -0
  113. package/src/activities/agent-turn/run.ts +1530 -0
  114. package/src/activities/agent-turn/sandbox-establish.ts +1207 -0
  115. package/src/activities/agent-turn/sandbox-provision.ts +569 -0
  116. package/src/activities/agent-turn/sandbox-route.ts +358 -0
  117. package/src/activities/agent-turn/sandbox-runtime.ts +731 -0
  118. package/src/activities/agent-turn/stream-attempt.ts +1670 -0
  119. package/src/activities/agent-turn/tool-environment.ts +729 -0
  120. package/src/activities/agent-turn/tool-policy.ts +216 -0
  121. package/src/activities/agent-turn/tool-result-spill.ts +265 -0
  122. package/src/activities/agent-turn/turn-context.ts +281 -0
  123. package/src/activities/agent-turn/xai-capacity.ts +229 -0
  124. package/src/activities/agent-turn.ts +12 -12701
  125. package/src/activities/api-integrations.ts +3 -0
  126. package/src/activities/automations.ts +214 -0
  127. package/src/activities/codemode-dispatcher.ts +12 -1
  128. package/src/activities/common.ts +1 -5
  129. package/src/activities/connector-attachments.ts +168 -0
  130. package/src/activities/context-compaction.ts +18 -0
  131. package/src/activities/environment.ts +151 -55
  132. package/src/activities/generated-images.ts +26 -11
  133. package/src/activities/git-credential-renewal.ts +13 -3
  134. package/src/activities/goals.ts +85 -19
  135. package/src/activities/google-drive-provider.ts +178 -0
  136. package/src/activities/google-drive-publication.ts +147 -49
  137. package/src/activities/incident-telemetry-authority.ts +327 -0
  138. package/src/activities/incident-telemetry-preflight.ts +317 -0
  139. package/src/activities/knowledge-source-sync.ts +178 -37
  140. package/src/activities/mcp-credentials.ts +191 -17
  141. package/src/activities/open-suffix-resume.ts +422 -0
  142. package/src/activities/packs.ts +12 -60
  143. package/src/activities/parent-wake.ts +64 -14
  144. package/src/activities/retained-screenshot-reaper.ts +6 -3
  145. package/src/activities/retained-screenshots.ts +15 -47
  146. package/src/activities/rig-verification.ts +1 -1
  147. package/src/activities/run-input.ts +102 -142
  148. package/src/activities/sandbox-images.ts +7 -166
  149. package/src/activities/sandbox-lease.ts +406 -15
  150. package/src/activities/scheduled-tasks.ts +2103 -411
  151. package/src/activities/session-state.ts +10 -2
  152. package/src/activities/types.ts +32 -2
  153. package/src/activities/video-media-validation.ts +3 -3
  154. package/src/activities/video-output-retention.ts +2 -57
  155. package/src/activities/xai-auth.ts +8 -0
  156. package/src/activities-control.ts +2 -0
  157. package/src/activities.ts +3 -0
  158. package/src/activity-services.ts +13 -4
  159. package/src/connection-provider-use.ts +36 -0
  160. package/src/editable-artifact-materializer-subprocess.ts +25 -14
  161. package/src/editable-artifact-materializer.ts +29 -18
  162. package/src/github-rest-mcp.ts +713 -0
  163. package/src/http.ts +16 -6
  164. package/src/index.ts +40 -5
  165. package/src/knowledge-source-sync-activity-policy.ts +2 -0
  166. package/src/model-context-contributions.ts +33 -10
  167. package/src/observability-metrics.ts +357 -0
  168. package/src/opensandbox-kubernetes-inventory.ts +292 -0
  169. package/src/personal-github-git-credentials.ts +331 -0
  170. package/src/pr-review-credentials.ts +163 -0
  171. package/src/sandbox-archive-storage.ts +159 -0
  172. package/src/sandbox-resume.ts +449 -63
  173. package/src/sandbox-routing.ts +314 -16
  174. package/src/scheduled-alert-occurrence.ts +176 -0
  175. package/src/workflows/activities.ts +18 -2
  176. package/src/workflows/automations.ts +11 -0
  177. package/src/workflows/session.ts +13 -3
  178. package/src/workflows.ts +1 -0
  179. package/dist/activities-control-BJSDRSIP.js.map +0 -1
  180. package/dist/activities-turn-J5YO7XXX.js +0 -17243
  181. package/dist/activities-turn-J5YO7XXX.js.map +0 -1
  182. package/dist/chunk-N5TJIFCE.js.map +0 -1
  183. package/dist/chunk-P6Z2SLAA.js +0 -3092
  184. package/dist/chunk-P6Z2SLAA.js.map +0 -1
@@ -6,16 +6,26 @@ import {
6
6
  sandboxDrainTiming
7
7
  } from "./chunk-GRFQTTSQ.js";
8
8
  import {
9
+ INCIDENT_TELEMETRY_AUTHORITY_FENCE_LINEAGE_KEY,
10
+ MAX_AUTOMATIC_PROVIDER_RECOVERIES,
9
11
  assertReusableSessionRevivable,
12
+ collectWorkspaceArchiveObjectKeys,
10
13
  createCodexCapacityActivities,
11
14
  createGoalActivities,
12
15
  createSharedActivityServices,
13
16
  currentActivityContext,
17
+ deleteUnpublishedWorkspaceArchiveObject,
18
+ deleteWorkspaceArchiveObjectKeys,
14
19
  deliverFailedChildTurnToParent,
20
+ evaluateIncidentTelemetryPreflight,
21
+ incidentTelemetryAuthorityFence,
22
+ incidentTelemetryPreflightDeclaration,
15
23
  notifyParentOfChildIdle,
16
- providerIdentityFromResumeState,
24
+ providerRecoveryCountFromMetadata,
25
+ putVersion1TarArchiveOrInline,
17
26
  reconcilePendingParentSystemUpdates,
18
27
  reconcilePendingSessionWorkflowWakes,
28
+ resolveIncidentTelemetryResponderMetadata,
19
29
  resolveWorkspacePackRuntime,
20
30
  rigProviderImageSourceImage,
21
31
  scheduledUserMessagePayload,
@@ -23,11 +33,13 @@ import {
23
33
  settingsWithRigImage,
24
34
  withFirstPartyTools,
25
35
  workflowIdForSession
26
- } from "./chunk-P6Z2SLAA.js";
36
+ } from "./chunk-CNYGVRKF.js";
27
37
  import {
38
+ inspectOpenSandboxKubernetesInventory,
28
39
  recordCreditBalanceGauges,
29
40
  recordCreditMicros,
30
41
  recordExpiredDrainingSandboxLeaseGauges,
42
+ recordOpenSandboxKubernetesInventoryGauges,
31
43
  recordRetainedProcessInventoryGauges,
32
44
  recordRetainedProcessReconciliation,
33
45
  recordSandboxCheckpointArtifactGauges,
@@ -38,8 +50,9 @@ import {
38
50
  recordSandboxLeaseGauges,
39
51
  recordSandboxOrphansTerminated,
40
52
  recordSandboxRotationBacklogGauges,
41
- recordTurnsQueuedGauge
42
- } from "./chunk-N5TJIFCE.js";
53
+ recordTurnsQueuedGauge,
54
+ runtimeMetricsHooksForObservability
55
+ } from "./chunk-CE7TPKBS.js";
43
56
 
44
57
  // src/activities/browser-state-artifact-reaper.ts
45
58
  import {
@@ -125,14 +138,255 @@ function cleanupErrorCategory(error) {
125
138
  return typeof code === "string" && SAFE_PROVIDER_ERROR_CODES.has(code) ? code : "unknown";
126
139
  }
127
140
 
128
- // src/activities/documents.ts
141
+ // src/activities/automations.ts
142
+ import { resolveTurnExecutionPolicyV1 } from "@opengeni/config";
143
+ import {
144
+ assertWorkspaceModelPolicyAllows,
145
+ canonicalConfiguredModel,
146
+ createAndStartSessionWithOutcome,
147
+ recordWorkspaceUsage
148
+ } from "@opengeni/core";
149
+ import {
150
+ AutomationAuthorityRevokedError,
151
+ assertAutomationRunAuthorityInTransaction,
152
+ claimAutomationRun,
153
+ settleAutomationRun
154
+ } from "@opengeni/db";
155
+
156
+ // src/activities/agent-run-admission.ts
129
157
  import { configuredStaticUsageLimits } from "@opengeni/config";
158
+ import { getBillingBalance, isCodexBilledTurn, sumUsageQuantity } from "@opengeni/db";
159
+ async function agentRunAdmissionDenial(services, input) {
160
+ const externallyBilled = await isCodexBilledTurn({
161
+ db: services.db,
162
+ settings: services.settings,
163
+ workspaceId: input.workspaceId,
164
+ model: input.model
165
+ });
166
+ if (!externallyBilled && (services.settings.billingMode === "stripe" || services.settings.usageLimitsMode === "managed")) {
167
+ if (services.entitlements) {
168
+ const decision = await services.entitlements.admitRun({
169
+ accountId: input.accountId,
170
+ workspaceId: input.workspaceId,
171
+ action: "agent_run:create",
172
+ quantity: input.requestedAgentRuns
173
+ });
174
+ if (!decision.allowed) return "insufficient_credits";
175
+ } else {
176
+ const balance = await getBillingBalance(services.db, input.accountId);
177
+ if (balance.balanceMicros <= 0) return "insufficient_credits";
178
+ }
179
+ }
180
+ if (services.settings.usageLimitsMode !== "static" && services.settings.usageLimitsMode !== "managed") {
181
+ return null;
182
+ }
183
+ const limits = configuredStaticUsageLimits(services.settings);
184
+ if (!externallyBilled && limits.maxMonthlyCostMicrosPerAccount) {
185
+ const used = await sumUsageQuantity(services.db, {
186
+ accountId: input.accountId,
187
+ eventType: "model.cost",
188
+ since: startOfUtcMonth()
189
+ });
190
+ if (used >= limits.maxMonthlyCostMicrosPerAccount) {
191
+ return "monthly_model_cost_limit";
192
+ }
193
+ }
194
+ if (limits.maxMonthlyAgentRunsPerWorkspace) {
195
+ const used = await sumUsageQuantity(services.db, {
196
+ workspaceId: input.workspaceId,
197
+ eventType: "agent_run.created",
198
+ since: startOfUtcMonth()
199
+ });
200
+ if (used + input.requestedAgentRuns > limits.maxMonthlyAgentRunsPerWorkspace) {
201
+ return "monthly_agent_run_limit";
202
+ }
203
+ }
204
+ return null;
205
+ }
206
+ function startOfUtcMonth() {
207
+ const now = /* @__PURE__ */ new Date();
208
+ return new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), 1));
209
+ }
210
+
211
+ // src/activities/automations.ts
212
+ function createAutomationActivities(services, options = {}) {
213
+ const claim = options.claim ?? claimAutomationRun;
214
+ const settle = options.settle ?? settleAutomationRun;
215
+ const assertAuthority = options.assertAuthority ?? assertAutomationRunAuthorityInTransaction;
216
+ const createSession2 = options.createSession ?? createAndStartSessionWithOutcome;
217
+ const admit = options.admit ?? agentRunAdmissionDenial;
218
+ const assertModelPolicy = options.assertModelPolicy ?? assertWorkspaceModelPolicyAllows;
219
+ const recordUsage = options.recordUsage ?? recordWorkspaceUsage;
220
+ return {
221
+ dispatchAutomationRun: async (input) => {
222
+ const service = await services();
223
+ const run = await claim(service.db, input);
224
+ if (!run) return { action: "not_found" };
225
+ if (run.status === "dispatched") {
226
+ if (!run.sessionId) throw new Error("dispatched automation run is missing its session");
227
+ return { action: "already_dispatched", sessionId: run.sessionId };
228
+ }
229
+ if (run.status === "skipped") {
230
+ return { action: "skipped", reason: run.errorCode ?? "authority_revoked" };
231
+ }
232
+ const accepted = run.acceptedExecution;
233
+ if (accepted.accountId !== input.accountId || accepted.workspaceId !== input.workspaceId || accepted.sourceId !== run.sourceId || accepted.triggerId !== run.triggerId || accepted.triggerRevision !== run.triggerRevision || accepted.eventId !== run.eventId) {
234
+ await settle(service.db, {
235
+ workspaceId: input.workspaceId,
236
+ runId: run.id,
237
+ status: "skipped",
238
+ errorCode: "accepted_execution_mismatch"
239
+ });
240
+ return { action: "skipped", reason: "accepted_execution_mismatch" };
241
+ }
242
+ const template = accepted.sessionTemplate;
243
+ const model = canonicalConfiguredModel(
244
+ service.settings,
245
+ template.model ?? service.settings.openaiModel
246
+ );
247
+ if (!model) throw new Error("automation has no configured model");
248
+ await assertModelPolicy(service.db, service.settings, input.workspaceId, model);
249
+ const denial = await admit(service, {
250
+ accountId: input.accountId,
251
+ workspaceId: input.workspaceId,
252
+ model,
253
+ requestedAgentRuns: 1
254
+ });
255
+ if (denial) {
256
+ await settle(service.db, {
257
+ workspaceId: input.workspaceId,
258
+ runId: run.id,
259
+ status: "failed",
260
+ errorCode: denial
261
+ });
262
+ throw new Error(`automation admission denied: ${denial}`);
263
+ }
264
+ const reasoningEffort = template.reasoningEffort ?? service.settings.openaiReasoningEffort;
265
+ const sandboxBackend = template.sandboxBackend ?? service.settings.sandboxBackend;
266
+ if (sandboxBackend === "selfhosted") {
267
+ await settle(service.db, {
268
+ workspaceId: input.workspaceId,
269
+ runId: run.id,
270
+ status: "skipped",
271
+ errorCode: "interactive_compute_not_allowed"
272
+ });
273
+ return { action: "skipped", reason: "interactive_compute_not_allowed" };
274
+ }
275
+ const turnExecutionPolicy = resolveTurnExecutionPolicyV1(service.settings, {
276
+ modelId: model,
277
+ requestedModelId: template.model,
278
+ modelSource: template.model ? "explicit" : "deployment",
279
+ reasoningEffort,
280
+ reasoningSource: template.reasoningEffort ? "explicit" : "deployment",
281
+ latencyMode: "standard",
282
+ latencyModeSource: "deployment"
283
+ });
284
+ try {
285
+ const created = await createSession2({
286
+ db: service.db,
287
+ bus: service.bus,
288
+ workflowClient: {
289
+ wakeSessionWorkflow: async (wake) => {
290
+ await service.wakeSessionWorkflow?.(wake);
291
+ }
292
+ },
293
+ beforeCreateCommit: async (tx, sessionId) => {
294
+ await assertAuthority(tx, {
295
+ workspaceId: input.workspaceId,
296
+ runId: run.id,
297
+ triggerId: accepted.triggerId,
298
+ triggerRevision: accepted.triggerRevision,
299
+ sourceId: accepted.sourceId,
300
+ sourceVersion: accepted.sourceVersion,
301
+ sessionId
302
+ });
303
+ },
304
+ accountId: input.accountId,
305
+ workspaceId: input.workspaceId,
306
+ initialMessage: accepted.initialMessage,
307
+ resources: template.resources,
308
+ skills: template.skills,
309
+ tools: template.tools,
310
+ toolPolicy: { mode: "explicit", inheritedFromSessionId: null },
311
+ model,
312
+ reasoningEffort,
313
+ turnExecutionPolicy,
314
+ sandboxBackend,
315
+ metadata: {
316
+ ...template.metadata,
317
+ automationRunId: run.id,
318
+ automationSourceId: run.sourceId,
319
+ automationTriggerId: run.triggerId,
320
+ automationTriggerRevision: run.triggerRevision
321
+ },
322
+ createdBy: {
323
+ kind: "service",
324
+ subjectId: accepted.serviceSubjectId,
325
+ label: accepted.serviceLabel
326
+ },
327
+ createdByContext: accepted.provenance,
328
+ instructions: template.instructions,
329
+ policyRole: template.policyRole,
330
+ firstPartyMcpPermissions: template.firstPartyMcpPermissions,
331
+ firstPartyMcpTools: template.firstPartyMcpTools,
332
+ createIdempotencyKey: `automation-run:${run.id}`,
333
+ subjectId: accepted.serviceSubjectId
334
+ });
335
+ await recordUsage(
336
+ { db: service.db, settings: service.settings },
337
+ {
338
+ accountId: input.accountId,
339
+ workspaceId: input.workspaceId,
340
+ subjectId: accepted.serviceSubjectId,
341
+ eventType: "agent_run.created",
342
+ quantity: 1,
343
+ unit: "run",
344
+ sourceResourceType: "automation_run",
345
+ sourceResourceId: run.id,
346
+ sessionId: created.session.id,
347
+ initiator: created.session.createdBy,
348
+ initiatorContext: created.session.createdByContext,
349
+ origin: "system",
350
+ idempotencyKey: `agent_run.created:automation:${run.id}`
351
+ }
352
+ ).catch((error) => {
353
+ service.observability.warn("automation usage recording failed", {
354
+ workspaceId: input.workspaceId,
355
+ runId: run.id,
356
+ errorCategory: error instanceof Error ? error.name : "unknown"
357
+ });
358
+ });
359
+ return { action: "started", sessionId: created.session.id };
360
+ } catch (error) {
361
+ if (error instanceof AutomationAuthorityRevokedError) {
362
+ await settle(service.db, {
363
+ workspaceId: input.workspaceId,
364
+ runId: run.id,
365
+ status: "skipped",
366
+ errorCode: "authority_revoked"
367
+ });
368
+ return { action: "skipped", reason: "authority_revoked" };
369
+ }
370
+ await settle(service.db, {
371
+ workspaceId: input.workspaceId,
372
+ runId: run.id,
373
+ status: "failed",
374
+ errorCode: "dispatch_failed"
375
+ });
376
+ throw error;
377
+ }
378
+ }
379
+ };
380
+ }
381
+
382
+ // src/activities/documents.ts
383
+ import { configuredStaticUsageLimits as configuredStaticUsageLimits2 } from "@opengeni/config";
130
384
  import {
131
- getBillingBalance,
385
+ getBillingBalance as getBillingBalance2,
132
386
  recordUsageEvent,
133
387
  resolveDocumentIndexAuthority,
134
388
  rlsContextForWorkspace,
135
- sumUsageQuantity,
389
+ sumUsageQuantity as sumUsageQuantity2,
136
390
  withWorkspaceUsageLock
137
391
  } from "@opengeni/db";
138
392
  function createDocumentActivities(services, resolveDocumentServices) {
@@ -190,7 +444,7 @@ function createDocumentActivities(services, resolveDocumentServices) {
190
444
  {
191
445
  beforeEmbed: async ({ chunkCount }) => {
192
446
  if (settings.billingMode === "stripe" || settings.usageLimitsMode === "managed") {
193
- const balance = await getBillingBalance(lockedDb, input.accountId);
447
+ const balance = await getBillingBalance2(lockedDb, input.accountId);
194
448
  if (balance.balanceMicros <= 0) {
195
449
  throw new Error("insufficient OpenGeni credits");
196
450
  }
@@ -198,14 +452,14 @@ function createDocumentActivities(services, resolveDocumentServices) {
198
452
  if (settings.usageLimitsMode !== "static" && settings.usageLimitsMode !== "managed") {
199
453
  return;
200
454
  }
201
- const limit = configuredStaticUsageLimits(settings).maxDocumentIndexedChunksPerWorkspace;
455
+ const limit = configuredStaticUsageLimits2(settings).maxDocumentIndexedChunksPerWorkspace;
202
456
  if (!limit) {
203
457
  return;
204
458
  }
205
- const used = await sumUsageQuantity(lockedDb, {
459
+ const used = await sumUsageQuantity2(lockedDb, {
206
460
  workspaceId: input.workspaceId,
207
461
  eventType: "document.indexed",
208
- since: startOfUtcMonth()
462
+ since: startOfUtcMonth2()
209
463
  });
210
464
  if (used + chunkCount > limit) {
211
465
  throw new Error(`monthly document indexing limit reached (${limit} chunks)`);
@@ -234,7 +488,7 @@ function createDocumentActivities(services, resolveDocumentServices) {
234
488
  }
235
489
  };
236
490
  }
237
- function startOfUtcMonth(date = /* @__PURE__ */ new Date()) {
491
+ function startOfUtcMonth2(date = /* @__PURE__ */ new Date()) {
238
492
  return new Date(Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), 1));
239
493
  }
240
494
 
@@ -407,6 +661,9 @@ function createFileUploadReaperActivities(services, options = {}) {
407
661
  // src/activities/knowledge-source-sync.ts
408
662
  import { createHash as createHash2 } from "crypto";
409
663
  import { heartbeat } from "@temporalio/activity";
664
+ import {
665
+ googleDriveProviderRetryOptions
666
+ } from "@opengeni/config";
410
667
  import {
411
668
  KnowledgeSourceSyncRunSummary
412
669
  } from "@opengeni/contracts";
@@ -471,6 +728,121 @@ import {
471
728
  } from "@opengeni/db";
472
729
  import { readResponseJsonBounded } from "@opengeni/network";
473
730
 
731
+ // src/activities/google-drive-provider.ts
732
+ var GoogleDriveProviderTransportError = class extends Error {
733
+ constructor() {
734
+ super("google_drive_provider_transport_failed");
735
+ this.name = "GoogleDriveProviderTransportError";
736
+ }
737
+ };
738
+ async function fetchGoogleDriveProvider(input) {
739
+ const now = input.now ?? Date.now;
740
+ const sleep = input.sleep ?? Bun.sleep;
741
+ let retryDelaySpentMs = 0;
742
+ for (let attempt = 1; attempt <= input.policy.attempts; attempt += 1) {
743
+ let response;
744
+ input.heartbeat?.({ operation: input.operation, attempt, phase: "request_start" });
745
+ try {
746
+ response = await input.fetchImpl(input.url, {
747
+ ...input.init,
748
+ redirect: "error",
749
+ signal: AbortSignal.timeout(input.policy.requestTimeoutMs)
750
+ });
751
+ } catch {
752
+ input.heartbeat?.({ operation: input.operation, attempt, phase: "request_end" });
753
+ recordRequest(input.observability, input.operation, "retryable_error");
754
+ const delayMs2 = retryDelayMs(input.policy, attempt, null);
755
+ if (!canRetry(input.policy, attempt, retryDelaySpentMs, delayMs2)) {
756
+ throw new GoogleDriveProviderTransportError();
757
+ }
758
+ recordRetry(input.observability, input.operation, "transport", attempt, delayMs2);
759
+ retryDelaySpentMs += delayMs2;
760
+ input.heartbeat?.({ operation: input.operation, attempt, phase: "retry_delay_start" });
761
+ await sleep(delayMs2);
762
+ input.heartbeat?.({ operation: input.operation, attempt, phase: "retry_delay_end" });
763
+ continue;
764
+ }
765
+ input.heartbeat?.({ operation: input.operation, attempt, phase: "request_end" });
766
+ if (!isRetryableStatus(response.status)) {
767
+ recordRequest(
768
+ input.observability,
769
+ input.operation,
770
+ response.ok ? "succeeded" : "terminal_error"
771
+ );
772
+ return response;
773
+ }
774
+ recordRequest(input.observability, input.operation, "retryable_error");
775
+ const delayMs = retryDelayMs(
776
+ input.policy,
777
+ attempt,
778
+ retryAfterDelayMs(response.headers.get("retry-after"), now())
779
+ );
780
+ if (!canRetry(input.policy, attempt, retryDelaySpentMs, delayMs)) return response;
781
+ await response.body?.cancel().catch(() => void 0);
782
+ recordRetry(
783
+ input.observability,
784
+ input.operation,
785
+ response.status === 429 ? "rate_limited" : "provider_unavailable",
786
+ attempt,
787
+ delayMs
788
+ );
789
+ retryDelaySpentMs += delayMs;
790
+ input.heartbeat?.({ operation: input.operation, attempt, phase: "retry_delay_start" });
791
+ await sleep(delayMs);
792
+ input.heartbeat?.({ operation: input.operation, attempt, phase: "retry_delay_end" });
793
+ }
794
+ throw new GoogleDriveProviderTransportError();
795
+ }
796
+ function canRetry(policy, attempt, retryDelaySpentMs, delayMs) {
797
+ return attempt < policy.attempts && retryDelaySpentMs + delayMs <= policy.budgetMs;
798
+ }
799
+ function retryDelayMs(policy, attempt, retryAfterMs) {
800
+ const exponential = Math.min(
801
+ policy.maxDelayMs,
802
+ policy.initialDelayMs * 2 ** Math.max(0, attempt - 1)
803
+ );
804
+ return Math.min(policy.maxDelayMs, Math.max(exponential, retryAfterMs ?? 0));
805
+ }
806
+ function retryAfterDelayMs(value, nowMs) {
807
+ if (!value) return null;
808
+ const seconds = Number(value);
809
+ if (Number.isFinite(seconds) && seconds >= 0) return Math.round(seconds * 1e3);
810
+ const timestamp = Date.parse(value);
811
+ return Number.isFinite(timestamp) ? Math.max(0, timestamp - nowMs) : null;
812
+ }
813
+ function isRetryableStatus(status) {
814
+ return status === 429 || status >= 500;
815
+ }
816
+ function recordRequest(observability, operation, outcome) {
817
+ observability.incrementCounter({
818
+ name: "opengeni_google_drive_provider_requests_total",
819
+ help: "Google Drive provider request attempts by operation and bounded outcome.",
820
+ labels: { operation, outcome }
821
+ });
822
+ }
823
+ function recordRetry(observability, operation, reason, attempt, delayMs) {
824
+ observability.incrementCounter({
825
+ name: "opengeni_google_drive_provider_retries_total",
826
+ help: "Bounded Google Drive provider retries by operation and reason.",
827
+ labels: { operation, reason }
828
+ });
829
+ observability.observeHistogram({
830
+ name: "opengeni_google_drive_provider_retry_delay_seconds",
831
+ help: "Delay applied before a bounded Google Drive provider retry.",
832
+ labels: { operation, reason },
833
+ value: delayMs / 1e3,
834
+ buckets: [0.1, 0.25, 0.5, 1, 2, 5, 10, 30, 60]
835
+ });
836
+ observability.warn("Google Drive provider request retry scheduled", {
837
+ provider: "google_drive",
838
+ operation,
839
+ outcome: "retry_scheduled",
840
+ reason,
841
+ attempt,
842
+ delayMs
843
+ });
844
+ }
845
+
474
846
  // src/activities/google-drive-changes.ts
475
847
  import { createHash } from "crypto";
476
848
  import {
@@ -805,6 +1177,10 @@ function createKnowledgeSourceSyncActivities(services, resolveDocumentServices)
805
1177
  reconnectRequired: false,
806
1178
  failures: []
807
1179
  };
1180
+ observability.info("Knowledge source sync started", {
1181
+ provider: providerLabel,
1182
+ outcome: "started"
1183
+ });
808
1184
  try {
809
1185
  summary.phase = "inventory";
810
1186
  const resolved = await getKnowledgeSourceForSyncAuthority(db, {
@@ -885,6 +1261,7 @@ function createKnowledgeSourceSyncActivities(services, resolveDocumentServices)
885
1261
  const provider = await resolveKnowledgeSyncProvider({
886
1262
  db,
887
1263
  settings,
1264
+ observability,
888
1265
  accountId: input.accountId,
889
1266
  workspaceId: input.workspaceId,
890
1267
  action,
@@ -1614,6 +1991,7 @@ function createKnowledgeSourceSyncActivities(services, resolveDocumentServices)
1614
1991
  labels: { provider: providerLabel },
1615
1992
  amount: summary.bytes
1616
1993
  });
1994
+ recordSyncHealthTelemetry(observability, providerLabel, summary, "succeeded");
1617
1995
  return {
1618
1996
  action: "complete",
1619
1997
  bufferedWake: settled.bufferedWake,
@@ -1698,6 +2076,7 @@ function createKnowledgeSourceSyncActivities(services, resolveDocumentServices)
1698
2076
  labels: { provider: providerLabel },
1699
2077
  amount: summary.bytes
1700
2078
  });
2079
+ recordSyncHealthTelemetry(observability, providerLabel, summary, "failed", failure);
1701
2080
  return {
1702
2081
  action: "failed",
1703
2082
  bufferedWake: settled.bufferedWake,
@@ -1878,6 +2257,8 @@ async function resolveKnowledgeSyncProvider(input) {
1878
2257
  workspaceId: input.workspaceId,
1879
2258
  initiatingSubjectId: action.initiatingSubjectId,
1880
2259
  authorization,
2260
+ retry: googleDriveProviderRetryOptions(input.settings),
2261
+ observability: input.observability,
1881
2262
  limits: action.limits,
1882
2263
  connectionId: resolvedConnection.id,
1883
2264
  fullReconciliationIntervalMs: GOOGLE_DRIVE_FULL_RECONCILIATION_INTERVAL_MS,
@@ -1963,11 +2344,18 @@ function atlassianSyncDriver(input) {
1963
2344
  function googleDriveSyncDriver(input) {
1964
2345
  const maxElapsedMs = input.limits.maxElapsedSeconds * 1e3;
1965
2346
  const provider = input.provider ?? {
1966
- getStartPageToken: async (driveId) => await getDriveStartPageToken(driveId, input.authorization),
1967
- listChanges: async (pageToken, driveId, pageSize) => await listDriveChanges(pageToken, driveId, pageSize, input.authorization),
1968
- getFile: async (fileId) => await getDriveFile(fileId, input.authorization),
1969
- listChildren: async (request) => await listDriveChildren(request, input.authorization),
1970
- listPermissions: async (request) => await listDrivePermissions(request, input.authorization)
2347
+ getStartPageToken: async (driveId) => await getDriveStartPageToken(driveId, input.authorization, input.retry, input.observability),
2348
+ listChanges: async (pageToken, driveId, pageSize) => await listDriveChanges(
2349
+ pageToken,
2350
+ driveId,
2351
+ pageSize,
2352
+ input.authorization,
2353
+ input.retry,
2354
+ input.observability
2355
+ ),
2356
+ getFile: async (fileId) => await getDriveFile(fileId, input.authorization, input.retry, input.observability),
2357
+ listChildren: async (request) => await listDriveChildren(request, input.authorization, input.retry, input.observability),
2358
+ listPermissions: async (request) => await listDrivePermissions(request, input.authorization, input.retry, input.observability)
1971
2359
  };
1972
2360
  const clock = provider.now ?? Date.now;
1973
2361
  const budgetLimits = {
@@ -2278,7 +2666,7 @@ function googleDriveSyncDriver(input) {
2278
2666
  };
2279
2667
  }
2280
2668
  },
2281
- fetchContent: async (entry, maxBytes) => await fetchDriveBytes(entry, input.authorization, maxBytes),
2669
+ fetchContent: async (entry, maxBytes) => await fetchDriveBytes(entry, input.authorization, maxBytes, input.retry, input.observability),
2282
2670
  readAcl: async (entry, maxProviderRequests) => {
2283
2671
  if (!provider.listPermissions) {
2284
2672
  throw new SyncFailure("provider_rejected", false);
@@ -2466,16 +2854,18 @@ function googleDriveInventoryStopIsHard(reason, budget, limits) {
2466
2854
  }
2467
2855
  return reason === "elapsed_time_limit" && budget.elapsedMs >= limits.maxElapsedMs;
2468
2856
  }
2469
- async function getDriveStartPageToken(driveId, authorization) {
2857
+ async function getDriveStartPageToken(driveId, authorization, retry, observability) {
2470
2858
  if (!authorization) throw new SyncFailure("connection_reconnect_required", true, true);
2471
2859
  const url = new URL(`${DRIVE_API_BASE}/changes/startPageToken`);
2472
2860
  url.searchParams.set("supportsAllDrives", "true");
2473
2861
  if (driveId) url.searchParams.set("driveId", driveId);
2474
- const response = await fetch(url, {
2475
- headers: { authorization, accept: "application/json" },
2476
- redirect: "error",
2477
- signal: AbortSignal.timeout(15e3)
2478
- });
2862
+ const response = await requestGoogleDrive(
2863
+ url,
2864
+ { headers: { authorization, accept: "application/json" } },
2865
+ "start_page_token",
2866
+ retry,
2867
+ observability
2868
+ );
2479
2869
  if (!response.ok) throwGoogleDriveResponse(response);
2480
2870
  const payload = await readResponseJsonBounded(
2481
2871
  response,
@@ -2484,7 +2874,7 @@ async function getDriveStartPageToken(driveId, authorization) {
2484
2874
  );
2485
2875
  return requiredString(payload.startPageToken, "start_page_token");
2486
2876
  }
2487
- async function listDriveChanges(pageToken, driveId, pageSize, authorization) {
2877
+ async function listDriveChanges(pageToken, driveId, pageSize, authorization, retry, observability) {
2488
2878
  if (!authorization) throw new SyncFailure("connection_reconnect_required", true, true);
2489
2879
  const url = new URL(`${DRIVE_API_BASE}/changes`);
2490
2880
  url.searchParams.set("pageToken", pageToken);
@@ -2497,11 +2887,13 @@ async function listDriveChanges(pageToken, driveId, pageSize, authorization) {
2497
2887
  "nextPageToken,newStartPageToken,changes(fileId,removed,time,driveId,file(id,name,mimeType,driveId,parents,modifiedTime,createdTime,version,md5Checksum,size,webViewLink,trashed))"
2498
2888
  );
2499
2889
  if (driveId) url.searchParams.set("driveId", driveId);
2500
- const response = await fetch(url, {
2501
- headers: { authorization, accept: "application/json" },
2502
- redirect: "error",
2503
- signal: AbortSignal.timeout(15e3)
2504
- });
2890
+ const response = await requestGoogleDrive(
2891
+ url,
2892
+ { headers: { authorization, accept: "application/json" } },
2893
+ "list_changes",
2894
+ retry,
2895
+ observability
2896
+ );
2505
2897
  if (response.status === 410) throw new GoogleDriveCursorInvalidError();
2506
2898
  if (!response.ok) throwGoogleDriveResponse(response);
2507
2899
  const payload = await readResponseJsonBounded(
@@ -2524,7 +2916,7 @@ async function listDriveChanges(pageToken, driveId, pageSize, authorization) {
2524
2916
  newStartPageToken: optionalString(payload.newStartPageToken)
2525
2917
  };
2526
2918
  }
2527
- async function getDriveFile(fileId, authorization) {
2919
+ async function getDriveFile(fileId, authorization, retry, observability) {
2528
2920
  if (!authorization) throw new SyncFailure("connection_reconnect_required", true, true);
2529
2921
  const url = new URL(`${DRIVE_API_BASE}/files/${encodeURIComponent(fileId)}`);
2530
2922
  url.searchParams.set("supportsAllDrives", "true");
@@ -2532,18 +2924,20 @@ async function getDriveFile(fileId, authorization) {
2532
2924
  "fields",
2533
2925
  "id,name,mimeType,driveId,parents,modifiedTime,createdTime,version,md5Checksum,size,webViewLink,trashed"
2534
2926
  );
2535
- const response = await fetch(url, {
2536
- headers: { authorization, accept: "application/json" },
2537
- redirect: "error",
2538
- signal: AbortSignal.timeout(15e3)
2539
- });
2927
+ const response = await requestGoogleDrive(
2928
+ url,
2929
+ { headers: { authorization, accept: "application/json" } },
2930
+ "get_file",
2931
+ retry,
2932
+ observability
2933
+ );
2540
2934
  if (response.status === 404) return null;
2541
2935
  if (!response.ok) throwGoogleDriveResponse(response);
2542
2936
  return parseDriveProviderItem(
2543
2937
  await readResponseJsonBounded(response, DRIVE_JSON_MAX_BYTES, "Google Drive file metadata")
2544
2938
  );
2545
2939
  }
2546
- async function listDrivePermissions(input, authorization) {
2940
+ async function listDrivePermissions(input, authorization, retry, observability) {
2547
2941
  if (!authorization) throw new SyncFailure("connection_reconnect_required", true, true);
2548
2942
  const url = new URL(`${DRIVE_API_BASE}/files/${encodeURIComponent(input.fileId)}/permissions`);
2549
2943
  url.searchParams.set("supportsAllDrives", "true");
@@ -2553,11 +2947,13 @@ async function listDrivePermissions(input, authorization) {
2553
2947
  "nextPageToken,permissions(id,type,emailAddress,domain,role,allowFileDiscovery,expirationTime,deleted,permissionDetails(inherited))"
2554
2948
  );
2555
2949
  if (input.pageToken) url.searchParams.set("pageToken", input.pageToken);
2556
- const response = await fetch(url, {
2557
- headers: { authorization, accept: "application/json" },
2558
- redirect: "error",
2559
- signal: AbortSignal.timeout(15e3)
2560
- });
2950
+ const response = await requestGoogleDrive(
2951
+ url,
2952
+ { headers: { authorization, accept: "application/json" } },
2953
+ "list_permissions",
2954
+ retry,
2955
+ observability
2956
+ );
2561
2957
  if (response.status === 403 || response.status === 404) {
2562
2958
  return { permissions: [], nextPageToken: null, denied: true };
2563
2959
  }
@@ -2739,6 +3135,24 @@ function throwGoogleDriveResponse(response) {
2739
3135
  response.status >= 500 || response.status === 429
2740
3136
  );
2741
3137
  }
3138
+ async function requestGoogleDrive(url, init, operation, retry, observability) {
3139
+ try {
3140
+ return await fetchGoogleDriveProvider({
3141
+ fetchImpl: fetch,
3142
+ url,
3143
+ init,
3144
+ operation,
3145
+ policy: retry,
3146
+ observability,
3147
+ heartbeat: (details) => heartbeat({ provider: "google_drive", ...details })
3148
+ });
3149
+ } catch (error) {
3150
+ if (error instanceof GoogleDriveProviderTransportError) {
3151
+ throw new SyncFailure("provider_unavailable", true);
3152
+ }
3153
+ throw error;
3154
+ }
3155
+ }
2742
3156
  function selectedDriveSource(metadata, externalSourceId) {
2743
3157
  const sources = metadata.selectedSources ?? (metadata.selectedSource ? [metadata.selectedSource] : []);
2744
3158
  return sources.find((source) => source.id === externalSourceId) ?? null;
@@ -2746,7 +3160,7 @@ function selectedDriveSource(metadata, externalSourceId) {
2746
3160
  function googleDriveSyncProviderAccessAllowed(input) {
2747
3161
  return input.resolvedConnectionVersion !== void 0 && input.connectionVersion === input.resolvedConnectionVersion && input.connectionStatus === "active" && (input.lifecycleState === void 0 || input.lifecycleState === "active") && googleDriveScopesAllowCapability(input.grantedScopes, "recursive_source_sync");
2748
3162
  }
2749
- async function listDriveChildren(input, authorization) {
3163
+ async function listDriveChildren(input, authorization, retry, observability) {
2750
3164
  if (!authorization) throw new SyncFailure("connection_reconnect_required", true, true);
2751
3165
  const url = new URL(`${DRIVE_API_BASE}/files`);
2752
3166
  url.searchParams.set(
@@ -2765,11 +3179,13 @@ async function listDriveChildren(input, authorization) {
2765
3179
  url.searchParams.set("corpora", "drive");
2766
3180
  url.searchParams.set("driveId", input.driveId);
2767
3181
  }
2768
- const response = await fetch(url, {
2769
- headers: { authorization, accept: "application/json" },
2770
- redirect: "error",
2771
- signal: AbortSignal.timeout(15e3)
2772
- });
3182
+ const response = await requestGoogleDrive(
3183
+ url,
3184
+ { headers: { authorization, accept: "application/json" } },
3185
+ "list_children",
3186
+ retry,
3187
+ observability
3188
+ );
2773
3189
  if (!response.ok) {
2774
3190
  if (response.status === 401 || response.status === 403) {
2775
3191
  throw new SyncFailure("connection_reconnect_required", true, true);
@@ -2808,7 +3224,7 @@ function parseDriveProviderItem(value) {
2808
3224
  trashed: row.trashed === true
2809
3225
  };
2810
3226
  }
2811
- async function fetchDriveBytes(entry, authorization, maxBytes) {
3227
+ async function fetchDriveBytes(entry, authorization, maxBytes, retry, observability) {
2812
3228
  if (!authorization) throw new SyncFailure("connection_reconnect_required", true, true);
2813
3229
  const url = entry.transfer.action === "export" ? new URL(`${DRIVE_API_BASE}/files/${encodeURIComponent(entry.externalObjectId)}/export`) : new URL(`${DRIVE_API_BASE}/files/${encodeURIComponent(entry.externalObjectId)}`);
2814
3230
  if (entry.transfer.action === "export") {
@@ -2817,11 +3233,13 @@ async function fetchDriveBytes(entry, authorization, maxBytes) {
2817
3233
  url.searchParams.set("alt", "media");
2818
3234
  url.searchParams.set("supportsAllDrives", "true");
2819
3235
  }
2820
- const response = await fetch(url, {
2821
- headers: { authorization },
2822
- redirect: "error",
2823
- signal: AbortSignal.timeout(3e4)
2824
- });
3236
+ const response = await requestGoogleDrive(
3237
+ url,
3238
+ { headers: { authorization } },
3239
+ entry.transfer.action === "export" ? "export" : "download",
3240
+ retry,
3241
+ observability
3242
+ );
2825
3243
  if (!response.ok) {
2826
3244
  if (response.status === 401 || response.status === 403) {
2827
3245
  throw new SyncFailure("connection_reconnect_required", true, true);
@@ -3193,6 +3611,59 @@ function inventoryLimit(reason) {
3193
3611
  if (reason === "known_byte_limit") return "bytes";
3194
3612
  return null;
3195
3613
  }
3614
+ function recordSyncHealthTelemetry(observability, provider, summary, outcome, failure) {
3615
+ observability.observeHistogram({
3616
+ name: "opengeni_knowledge_source_sync_terminal_batch_duration_seconds",
3617
+ help: "Terminal knowledge-source activity batch duration by provider and outcome.",
3618
+ labels: { provider, outcome },
3619
+ value: Math.max(0, summary.elapsedMs) / 1e3
3620
+ });
3621
+ if (summary.providerRequests > 0) {
3622
+ observability.incrementCounter({
3623
+ name: "opengeni_knowledge_source_sync_provider_requests_total",
3624
+ help: "Logical provider requests consumed by knowledge-source synchronization.",
3625
+ labels: { provider, outcome },
3626
+ amount: summary.providerRequests
3627
+ });
3628
+ }
3629
+ if (summary.limitReached) {
3630
+ observability.incrementCounter({
3631
+ name: "opengeni_knowledge_source_sync_limit_hits_total",
3632
+ help: "Knowledge-source synchronization runs stopped by an explicit resource budget.",
3633
+ labels: { provider, limit: summary.limitReached }
3634
+ });
3635
+ }
3636
+ if (summary.reconnectRequired) {
3637
+ observability.incrementCounter({
3638
+ name: "opengeni_knowledge_source_sync_reconnect_required_total",
3639
+ help: "Knowledge-source synchronization failures requiring connector reconnection.",
3640
+ labels: { provider }
3641
+ });
3642
+ }
3643
+ if (failure) {
3644
+ observability.incrementCounter({
3645
+ name: "opengeni_knowledge_source_sync_failures_total",
3646
+ help: "Knowledge-source synchronization terminal failures by bounded reason.",
3647
+ labels: {
3648
+ provider,
3649
+ reason: failure.code,
3650
+ retryable: failure.retryable ? "true" : "false"
3651
+ }
3652
+ });
3653
+ observability.warn("Knowledge source sync failed", {
3654
+ provider,
3655
+ outcome: failure.retryable ? "retryable_failed" : "failed",
3656
+ reason: failure.code,
3657
+ durationMs: summary.elapsedMs
3658
+ });
3659
+ return;
3660
+ }
3661
+ observability.info("Knowledge source sync completed", {
3662
+ provider,
3663
+ outcome: "succeeded",
3664
+ durationMs: summary.elapsedMs
3665
+ });
3666
+ }
3196
3667
  var SyncFailure = class extends Error {
3197
3668
  constructor(code, retryable, reconnectRequired = false) {
3198
3669
  super(code);
@@ -3209,6 +3680,7 @@ import {
3209
3680
  completeRetainedScreenshotMaintenance,
3210
3681
  promoteRetainedScreenshotMaintenanceCleanup
3211
3682
  } from "@opengeni/db";
3683
+ import { retryWhileMissing } from "@opengeni/storage";
3212
3684
  var RETAINED_SCREENSHOT_PENDING_GRACE_MS = 15 * 60 * 1e3;
3213
3685
  var RETAINED_SCREENSHOT_CLAIM_TIMEOUT_MS = 10 * 60 * 1e3;
3214
3686
  var RETAINED_SCREENSHOT_MAINTENANCE_BATCH_SIZE = 100;
@@ -3314,8 +3786,11 @@ async function reconcileClaim(db, storage, claim, provider) {
3314
3786
  }
3315
3787
  async function observeClaimObject(storage, claim, provider) {
3316
3788
  const file = claimFile(storage, claim);
3317
- if (!await provider.fileExists(storage, file)) return "missing";
3318
- const head = await provider.headFile(storage, file);
3789
+ const head = await retryWhileMissing(async () => {
3790
+ if (!await provider.fileExists(storage, file)) return null;
3791
+ return await provider.headFile(storage, file);
3792
+ });
3793
+ if (!head) return "missing";
3319
3794
  return head.ContentLength === claim.sizeBytes && head.ContentType === claim.mediaType && head.Metadata?.sha256 === claim.sha256 ? "matches" : "mismatch";
3320
3795
  }
3321
3796
  function claimFile(storage, claim) {
@@ -3663,7 +4138,7 @@ async function runWithOwnedRigVerificationSandbox(input, run, dependencies = def
3663
4138
  holderId,
3664
4139
  subjectId: null,
3665
4140
  backend: input.settings.sandboxBackend,
3666
- image: input.settings.modalImageRef ?? input.settings.dockerImage,
4141
+ image: rigProviderImageSourceImage(input.settings, input.settings.sandboxBackend),
3667
4142
  rigVersionId: input.rigVersionId,
3668
4143
  leaseTtlMs: RIG_VERIFICATION_OWNER_TTL_MS,
3669
4144
  warmingLeaseTtlMs: RIG_VERIFICATION_OWNER_TTL_MS,
@@ -4580,6 +5055,7 @@ function createRigVerificationActivities(services) {
4580
5055
  // src/activities/sandbox-lease.ts
4581
5056
  import { createHash as createHash3, randomUUID as randomUUID2 } from "crypto";
4582
5057
  import { Context } from "@temporalio/activity";
5058
+ import { OpLostReason, OpState } from "@opengeni/agent-proto";
4583
5059
  import {
4584
5060
  accrueWarmSeconds,
4585
5061
  adoptLegacyModalCheckpointArtifact,
@@ -4596,7 +5072,7 @@ import {
4596
5072
  countSandboxLeasesByLiveness,
4597
5073
  deferRetainedProcessReconciliation,
4598
5074
  forceDrainOverLimitViewerOnlyBoxes,
4599
- getBillingBalance as getBillingBalance2,
5075
+ getBillingBalance as getBillingBalance3,
4600
5076
  listCreditBalancesByAccount,
4601
5077
  listLegacyModalCheckpointSlots,
4602
5078
  listLiveModalSandboxLeaseAttributions,
@@ -4620,10 +5096,18 @@ import {
4620
5096
  rlsContextForWorkspace as rlsContextForWorkspace2,
4621
5097
  settleRetainedProcess
4622
5098
  } from "@opengeni/db";
5099
+ import {
5100
+ claimConnectedMachineSessionBackgroundCommands,
5101
+ deferConnectedMachineBackgroundCommandReconciliation,
5102
+ recordConnectedMachineBackgroundCommandProof,
5103
+ settleClaimedConnectedMachineBackgroundCommand,
5104
+ settleSessionBackgroundCommandForRetainedProcess
5105
+ } from "@opengeni/db/session-background-commands";
4623
5106
  import { sandboxWarmRateMicrosPerSecond } from "@opengeni/config";
4624
5107
  import { sandboxLeaseTelemetryKey } from "@opengeni/observability";
4625
5108
  import {
4626
5109
  captureVerifiedWorkspaceArchive,
5110
+ cancelSelfhostedOp,
4627
5111
  assertConsistentSandboxProviderIdentity,
4628
5112
  createSandboxClientForBackend,
4629
5113
  deleteModalCheckpointSnapshot as deleteModalCheckpointSnapshot2,
@@ -4632,18 +5116,22 @@ import {
4632
5116
  inspectModalSandboxLifecycle,
4633
5117
  isExecSessionLostBanner,
4634
5118
  isProviderSandboxNotFoundError,
5119
+ NatsControlRpc,
4635
5120
  parseExecBannerExitCode,
4636
5121
  prepareProviderForTeardownAfterCapture,
4637
5122
  providerWorkspaceCapturePolicy,
4638
5123
  resolveModalCheckpointProviderBindingForLiveSandbox,
4639
5124
  resolveModalCheckpointProviderBinding,
4640
5125
  resolveModalCheckpointProviderBindingForSession,
5126
+ querySelfhostedOp,
4641
5127
  resumeExactSandboxSession,
4642
5128
  sandboxProviderContinuityForState,
4643
5129
  sandboxBackendForSdkBackendId,
4644
5130
  sandboxProviderInstanceIdFromEnvelope,
4645
5131
  sandboxCommandExitCode as sandboxCommandExitCode2,
4646
5132
  sandboxCommandStillRunning,
5133
+ SelfhostedControlError,
5134
+ subjectFor,
4647
5135
  sweepModalOrphanSandboxes,
4648
5136
  terminateManagedSandboxSession as terminateManagedSandboxSession2,
4649
5137
  terminateModalSandboxById,
@@ -4653,6 +5141,8 @@ import { sandboxLeaseTelemetryKey as sandboxLeaseTelemetryKey2 } from "@opengeni
4653
5141
  var RETAINED_PROCESS_RECONCILIATION_LIMIT = 20;
4654
5142
  var RETAINED_PROCESS_RECONCILIATION_CLAIM_TTL_MS = 5 * 6e4;
4655
5143
  var RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS = 5e3;
5144
+ var CONNECTED_COMMAND_RECONCILIATION_LIMIT = 20;
5145
+ var CONNECTED_COMMAND_RECONCILIATION_CLAIM_TTL_MS = 5 * 6e4;
4656
5146
  var SandboxProviderCaptureTimeoutError = class extends Error {
4657
5147
  constructor(sandboxGroupId, backend, timeoutMs, leaseEpoch, instanceId) {
4658
5148
  super(
@@ -4740,6 +5230,7 @@ function createSandboxLeaseActivities(services, options = {}) {
4740
5230
  capturePolicy
4741
5231
  ));
4742
5232
  const sweepModalOrphans = options.sweepModalOrphans ?? sweepModalOrphansForConfiguredBackend;
5233
+ const inspectOpenSandboxInventory = options.inspectOpenSandboxKubernetesInventory ?? inspectOpenSandboxKubernetesInventory;
4743
5234
  const probeRetainedProcess = options.probeRetainedProcess ?? probeRetainedProcessAtProvider;
4744
5235
  const probeDrainableProvider = options.probeDrainableProvider ?? probeDrainableProviderReadiness;
4745
5236
  async function prepareSandboxLeaseSweep() {
@@ -4748,8 +5239,15 @@ function createSandboxLeaseActivities(services, options = {}) {
4748
5239
  const { db, settings, observability } = await services();
4749
5240
  const timing = sandboxDrainTiming(settings);
4750
5241
  if (!settings.sandboxOwnershipEnabled) {
5242
+ const drainableInventory2 = await reapStaleLeaseHoldersGlobal(db, {
5243
+ viewerHolderTtlMs: settings.sandboxViewerHolderTtlMs,
5244
+ turnHolderTtlMs: settings.sandboxLeaseTtlMs,
5245
+ interactionHolderTtlMs: settings.sandboxInteractionHolderTtlMs,
5246
+ idleGraceMs: settings.sandboxIdleGraceMs
5247
+ });
5248
+ const drainable2 = drainableInventory2.slice(0, SANDBOX_REAPER_CHILD_DISPATCH_LIMIT);
4751
5249
  return {
4752
- drainable: [],
5250
+ drainable: drainable2,
4753
5251
  timeoutClass: timing.timeoutClass,
4754
5252
  snapshotTimeoutMs: timing.snapshotTimeoutMs,
4755
5253
  captureTimeoutMs: timing.captureTimeoutMs,
@@ -4808,8 +5306,7 @@ function createSandboxLeaseActivities(services, options = {}) {
4808
5306
  instanceId: input.target.instanceId
4809
5307
  });
4810
5308
  try {
4811
- const { db, settings, observability } = await services();
4812
- if (!settings.sandboxOwnershipEnabled) return { status: "skipped" };
5309
+ const { db, settings, observability, objectStorage } = await services();
4813
5310
  assertSandboxDrainInputTiming(input);
4814
5311
  const drainSettings = settings.sandboxSnapshotTimeoutMs === input.snapshotTimeoutMs ? settings : { ...settings, sandboxSnapshotTimeoutMs: input.snapshotTimeoutMs };
4815
5312
  const row = input.target;
@@ -4828,7 +5325,8 @@ function createSandboxLeaseActivities(services, options = {}) {
4828
5325
  observability,
4829
5326
  terminateBox,
4830
5327
  probeDrainableProvider,
4831
- captureAttempt
5328
+ captureAttempt,
5329
+ objectStorage
4832
5330
  );
4833
5331
  return { status: drainedCold ? "terminated" : "skipped" };
4834
5332
  } catch (error) {
@@ -4894,8 +5392,14 @@ function createSandboxLeaseActivities(services, options = {}) {
4894
5392
  let forceDrained = 0;
4895
5393
  const rotationsRequested = input.rotationsRequested;
4896
5394
  let modalOrphansTerminated = 0;
5395
+ await reconcileConnectedMachineBackgroundCommands(db, settings, observability, service.bus);
4897
5396
  if (!settings.sandboxOwnershipEnabled) {
4898
- await refreshQueueLeaseAndCreditGauges(db, observability);
5397
+ await refreshQueueLeaseAndCreditGauges(
5398
+ db,
5399
+ settings,
5400
+ observability,
5401
+ inspectOpenSandboxInventory
5402
+ );
4899
5403
  return {
4900
5404
  metered,
4901
5405
  forceDrained,
@@ -4949,7 +5453,12 @@ function createSandboxLeaseActivities(services, options = {}) {
4949
5453
  return { claimed: 0, deleted: 0, failed: 0 };
4950
5454
  }
4951
5455
  );
4952
- await refreshQueueLeaseAndCreditGauges(db, observability);
5456
+ await refreshQueueLeaseAndCreditGauges(
5457
+ db,
5458
+ settings,
5459
+ observability,
5460
+ inspectOpenSandboxInventory
5461
+ );
4953
5462
  if (input.examined > 0 || metered > 0 || forceDrained > 0 || modalOrphansTerminated > 0 || rotationsRequested > 0 || checkpointGc.claimed > 0) {
4954
5463
  observability.info("sandbox reaper dispatched", {
4955
5464
  drainable: input.examined,
@@ -5036,6 +5545,13 @@ var CHECKPOINT_GC_CLAIM_TTL_MS = 10 * 6e4;
5036
5545
  var CHECKPOINT_TOMBSTONE_RETENTION_MS = 30 * 24 * 60 * 6e4;
5037
5546
  var CHECKPOINT_TOMBSTONE_PRUNE_LIMIT = 500;
5038
5547
  var SANDBOX_MAINTENANCE_ITEM_CONCURRENCY = 8;
5548
+ function recordConnectedCommandReconciliation(observability, outcome) {
5549
+ observability.incrementCounter({
5550
+ name: "opengeni_connected_background_command_reconciliation_total",
5551
+ help: "Exact-locator Connected Machine background-command reconciliation outcomes.",
5552
+ labels: { outcome }
5553
+ });
5554
+ }
5039
5555
  async function forEachWithConcurrency(values, concurrency, visit) {
5040
5556
  if (values.length === 0) return;
5041
5557
  const workerCount = Math.min(values.length, Math.max(1, Math.floor(concurrency)));
@@ -5158,55 +5674,203 @@ async function adoptLegacyModalCheckpointReceipts(db, settings, observability) {
5158
5674
  });
5159
5675
  return adopted;
5160
5676
  }
5161
- async function reconcileTerminalRetainedProcesses(db, settings, observability, probe, inspectHistoricalModalSandbox) {
5677
+ async function reconcileConnectedMachineBackgroundCommands(db, settings, observability, bus) {
5162
5678
  const claimId = crypto.randomUUID();
5163
5679
  let claims;
5164
5680
  try {
5165
- claims = await claimTerminalRetainedProcesses(db, {
5681
+ claims = await claimConnectedMachineSessionBackgroundCommands(db, {
5166
5682
  claimId,
5167
- limit: RETAINED_PROCESS_RECONCILIATION_LIMIT,
5168
- claimTtlMs: RETAINED_PROCESS_RECONCILIATION_CLAIM_TTL_MS
5683
+ limit: CONNECTED_COMMAND_RECONCILIATION_LIMIT,
5684
+ claimTtlMs: CONNECTED_COMMAND_RECONCILIATION_CLAIM_TTL_MS
5169
5685
  });
5170
5686
  } catch (error) {
5171
- recordRetainedProcessReconciliation(observability, "claim_failed");
5172
- observability.warn("sandbox reaper: retained-process claim failed", {
5687
+ recordConnectedCommandReconciliation(observability, "claim_failed");
5688
+ observability.warn("sandbox reaper: connected-command claim failed", {
5173
5689
  error: error instanceof Error ? error.message : String(error)
5174
5690
  });
5175
5691
  return;
5176
5692
  }
5177
- for (const claim of claims) {
5178
- let process = claim.process;
5179
- const expected = retainedProcessSettlementIdentity(process);
5180
- let proof = retainedProcessReconciliationProof(process);
5693
+ const controlRpc = new NatsControlRpc(
5694
+ async () => bus.getRequestConnection()
5695
+ );
5696
+ await forEachWithConcurrency(claims, SANDBOX_MAINTENANCE_ITEM_CONCURRENCY, async (claim) => {
5697
+ let proof = claim.proof;
5181
5698
  if (!proof) {
5182
- let observation = null;
5183
- if (process.providerBackend === "modal" && process.routeTargetId === null && !process.providerBindingKey) {
5184
- try {
5185
- const lifecycle = await withRetainedProcessProbeTimeout(
5186
- inspectHistoricalModalSandbox(settings, process.providerInstanceId, null)
5187
- );
5188
- if (lifecycle.status === "not_found") {
5189
- observation = {
5190
- status: "deferred",
5191
- reason: "provider_binding_missing"
5192
- };
5193
- } else {
5194
- process = await bindRetainedProcessProviderIdentity(db, {
5195
- accountId: process.accountId,
5196
- workspaceId: process.workspaceId,
5197
- sessionId: process.sessionId,
5198
- processId: process.id,
5199
- expected,
5200
- claimId: claim.claimId,
5201
- providerBindingKey: lifecycle.providerBindingKey,
5202
- providerBinding: lifecycle.providerBinding
5203
- });
5204
- recordRetainedProcessReconciliation(observability, "provider_binding_adopted");
5205
- if (lifecycle.status === "terminated") {
5206
- observation = {
5207
- status: "proved",
5208
- proof: {
5209
- outcome: "lost",
5699
+ try {
5700
+ proof = await probeConnectedMachineBackgroundCommand(
5701
+ claim,
5702
+ controlRpc,
5703
+ Math.min(
5704
+ settings.sandboxSelfhostedControlTimeoutMs,
5705
+ RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS
5706
+ )
5707
+ );
5708
+ if (!proof) {
5709
+ await deferConnectedCommandClaim(db, settings, observability, claim, "provider_running");
5710
+ return;
5711
+ }
5712
+ } catch (error) {
5713
+ const offline = error instanceof SelfhostedControlError && (error.agentOffline || error.reason === "agent_reconnecting" || error.draining);
5714
+ if (!offline) {
5715
+ observability.warn("sandbox reaper: connected-command provider probe failed", {
5716
+ commandId: claim.commandId,
5717
+ enrollmentId: claim.enrollmentId,
5718
+ connectionInstanceId: claim.connectionInstanceId,
5719
+ error: error instanceof Error ? error.message : String(error)
5720
+ });
5721
+ }
5722
+ await deferConnectedCommandClaim(
5723
+ db,
5724
+ settings,
5725
+ observability,
5726
+ claim,
5727
+ offline ? "provider_offline" : "provider_error"
5728
+ );
5729
+ return;
5730
+ }
5731
+ try {
5732
+ await recordConnectedMachineBackgroundCommandProof(db, { claim, proof });
5733
+ recordConnectedCommandReconciliation(observability, `proof_${proof.outcome}`);
5734
+ } catch (error) {
5735
+ recordConnectedCommandReconciliation(observability, "proof_checkpoint_failed");
5736
+ observability.warn("sandbox reaper: connected-command proof checkpoint failed", {
5737
+ commandId: claim.commandId,
5738
+ error: error instanceof Error ? error.message : String(error)
5739
+ });
5740
+ return;
5741
+ }
5742
+ }
5743
+ try {
5744
+ if (!await settleClaimedConnectedMachineBackgroundCommand(db, { claim })) {
5745
+ throw new Error("Connected command settlement lost its exact claim or proof");
5746
+ }
5747
+ recordConnectedCommandReconciliation(observability, `settled_${proof.outcome}`);
5748
+ } catch (error) {
5749
+ recordConnectedCommandReconciliation(observability, "settlement_failed");
5750
+ observability.warn("sandbox reaper: connected-command settlement failed", {
5751
+ commandId: claim.commandId,
5752
+ error: error instanceof Error ? error.message : String(error)
5753
+ });
5754
+ await deferConnectedCommandClaim(db, settings, observability, claim, "settlement_failed");
5755
+ }
5756
+ });
5757
+ }
5758
+ async function probeConnectedMachineBackgroundCommand(claim, controlRpc, controlTimeoutMs, observedAt = /* @__PURE__ */ new Date()) {
5759
+ const input = {
5760
+ controlRpc,
5761
+ rpcSubject: subjectFor(
5762
+ claim.controlWorkspaceId,
5763
+ claim.enrollmentId,
5764
+ claim.connectionInstanceId
5765
+ ),
5766
+ opId: claim.opId,
5767
+ controlTimeoutMs
5768
+ };
5769
+ const status = claim.state === "stopping" ? await cancelSelfhostedOp(input) : await querySelfhostedOp(input);
5770
+ return connectedCommandProofFromStatus(status, observedAt);
5771
+ }
5772
+ function connectedCommandProofFromStatus(status, observedAt) {
5773
+ switch (status.state) {
5774
+ case OpState.OP_STATE_ACCEPTED:
5775
+ case OpState.OP_STATE_RUNNING:
5776
+ return null;
5777
+ case OpState.OP_STATE_COMPLETE: {
5778
+ const exit = status.exit;
5779
+ if (!exit) {
5780
+ throw new Error("Connected command completed without an exit record");
5781
+ }
5782
+ const failureCode = exit.failureCode.replace(/[^A-Za-z0-9_-]/g, "_").slice(0, 128);
5783
+ const reason = exit.cancelled ? "op_cancelled" : exit.timedOut ? "op_timed_out" : failureCode ? `op_failure_${failureCode}` : "op_exit";
5784
+ return {
5785
+ outcome: "exited",
5786
+ exitCode: exit.exitCode,
5787
+ reason,
5788
+ observedAt
5789
+ };
5790
+ }
5791
+ case OpState.OP_STATE_LOST:
5792
+ return {
5793
+ outcome: "lost",
5794
+ exitCode: null,
5795
+ reason: status.lostReason === OpLostReason.OP_LOST_REASON_EVICTED ? "op_lost_evicted" : status.lostReason === OpLostReason.OP_LOST_REASON_AGENT_RESTARTED ? "op_lost_agent_restarted" : "op_lost_unspecified",
5796
+ observedAt
5797
+ };
5798
+ default:
5799
+ throw new Error(`Connected command returned invalid op state ${status.state}`);
5800
+ }
5801
+ }
5802
+ async function deferConnectedCommandClaim(db, settings, observability, claim, outcome) {
5803
+ const exponent = Math.min(4, Math.max(0, claim.reconcileAttempts - 1));
5804
+ const retryAfterMs = Math.min(
5805
+ 5 * 6e4,
5806
+ Math.max(settings.sandboxLeaseReaperPeriodMs, 3e4) * 2 ** exponent
5807
+ );
5808
+ try {
5809
+ if (!await deferConnectedMachineBackgroundCommandReconciliation(db, {
5810
+ claim,
5811
+ outcome,
5812
+ retryAfterMs
5813
+ })) {
5814
+ throw new Error("Connected command defer lost its exact claim");
5815
+ }
5816
+ recordConnectedCommandReconciliation(observability, outcome);
5817
+ } catch (error) {
5818
+ recordConnectedCommandReconciliation(observability, "defer_failed");
5819
+ observability.warn("sandbox reaper: connected-command defer failed", {
5820
+ commandId: claim.commandId,
5821
+ error: error instanceof Error ? error.message : String(error)
5822
+ });
5823
+ }
5824
+ }
5825
+ async function reconcileTerminalRetainedProcesses(db, settings, observability, probe, inspectHistoricalModalSandbox) {
5826
+ const claimId = crypto.randomUUID();
5827
+ let claims;
5828
+ try {
5829
+ claims = await claimTerminalRetainedProcesses(db, {
5830
+ claimId,
5831
+ limit: RETAINED_PROCESS_RECONCILIATION_LIMIT,
5832
+ claimTtlMs: RETAINED_PROCESS_RECONCILIATION_CLAIM_TTL_MS
5833
+ });
5834
+ } catch (error) {
5835
+ recordRetainedProcessReconciliation(observability, "claim_failed");
5836
+ observability.warn("sandbox reaper: retained-process claim failed", {
5837
+ error: error instanceof Error ? error.message : String(error)
5838
+ });
5839
+ return;
5840
+ }
5841
+ for (const claim of claims) {
5842
+ let process = claim.process;
5843
+ const expected = retainedProcessSettlementIdentity(process);
5844
+ let proof = retainedProcessReconciliationProof(process);
5845
+ if (!proof) {
5846
+ let observation = null;
5847
+ if (process.providerBackend === "modal" && process.routeTargetId === null && !process.providerBindingKey) {
5848
+ try {
5849
+ const lifecycle = await withRetainedProcessProbeTimeout(
5850
+ inspectHistoricalModalSandbox(settings, process.providerInstanceId, null)
5851
+ );
5852
+ if (lifecycle.status === "not_found") {
5853
+ observation = {
5854
+ status: "deferred",
5855
+ reason: "provider_binding_missing"
5856
+ };
5857
+ } else {
5858
+ process = await bindRetainedProcessProviderIdentity(db, {
5859
+ accountId: process.accountId,
5860
+ workspaceId: process.workspaceId,
5861
+ sessionId: process.sessionId,
5862
+ processId: process.id,
5863
+ expected,
5864
+ claimId: claim.claimId,
5865
+ providerBindingKey: lifecycle.providerBindingKey,
5866
+ providerBinding: lifecycle.providerBinding
5867
+ });
5868
+ recordRetainedProcessReconciliation(observability, "provider_binding_adopted");
5869
+ if (lifecycle.status === "terminated") {
5870
+ observation = {
5871
+ status: "proved",
5872
+ proof: {
5873
+ outcome: "lost",
5210
5874
  exitCode: null,
5211
5875
  reason: "provider_instance_terminated"
5212
5876
  }
@@ -5261,7 +5925,12 @@ async function reconcileTerminalRetainedProcesses(db, settings, observability, p
5261
5925
  }
5262
5926
  } else {
5263
5927
  try {
5264
- observation = await probe(settings, lease, process);
5928
+ observation = await probe(
5929
+ settings,
5930
+ lease,
5931
+ process,
5932
+ claim.ownerState === "background_stopping" ? "cancel" : "observe"
5933
+ );
5265
5934
  } catch (error) {
5266
5935
  observability.warn("sandbox reaper: retained-process provider probe failed", {
5267
5936
  processId: process.id,
@@ -5340,6 +6009,15 @@ async function reconcileTerminalRetainedProcesses(db, settings, observability, p
5340
6009
  reason: proof.reason,
5341
6010
  idleGraceMs: settings.sandboxIdleGraceMs
5342
6011
  });
6012
+ await settleSessionBackgroundCommandForRetainedProcess(db, {
6013
+ accountId: process.accountId,
6014
+ workspaceId: process.workspaceId,
6015
+ sessionId: process.sessionId,
6016
+ retainedProcessId: process.id,
6017
+ outcome: proof.outcome,
6018
+ exitCode: proof.exitCode,
6019
+ reason: proof.reason
6020
+ });
5343
6021
  recordRetainedProcessReconciliation(observability, `settled_${proof.outcome}`);
5344
6022
  } catch (error) {
5345
6023
  recordRetainedProcessReconciliation(observability, "settlement_failed");
@@ -5403,7 +6081,7 @@ async function withRetainedProcessProbeTimeout(promise) {
5403
6081
  if (timer) clearTimeout(timer);
5404
6082
  }
5405
6083
  }
5406
- async function probeRetainedProcessAtProvider(settings, lease, process) {
6084
+ async function probeRetainedProcessAtProvider(settings, lease, process, mode = "observe") {
5407
6085
  if (lease.id !== process.leaseId || lease.sandboxGroupId !== process.sandboxGroupId || lease.leaseEpoch !== process.leaseEpoch || lease.backend !== process.providerBackend || lease.instanceId !== process.providerInstanceId || lease.resumeBackendId !== process.providerBackend || process.routeTargetId !== null) {
5408
6086
  return { status: "deferred", reason: "identity_mismatch" };
5409
6087
  }
@@ -5414,7 +6092,7 @@ async function probeRetainedProcessAtProvider(settings, lease, process) {
5414
6092
  if (envelopeBackend !== void 0 && envelopeBackend !== process.providerBackend) {
5415
6093
  return { status: "deferred", reason: "identity_mismatch" };
5416
6094
  }
5417
- if (providerIdentityFromResumeState(lease.resumeState) !== process.providerInstanceId) {
6095
+ if (sandboxProviderInstanceIdFromEnvelope(lease.resumeState) !== process.providerInstanceId) {
5418
6096
  return { status: "deferred", reason: "identity_mismatch" };
5419
6097
  }
5420
6098
  let configuredModalBinding = null;
@@ -5517,7 +6195,7 @@ async function probeRetainedProcessAtProvider(settings, lease, process) {
5517
6195
  result = await withRetainedProcessProbeTimeout(
5518
6196
  session.writeStdin({
5519
6197
  sessionId: process.providerSessionId,
5520
- chars: "",
6198
+ chars: mode === "cancel" ? "" : "",
5521
6199
  yieldTimeMs: 1e3,
5522
6200
  maxOutputTokens: 2e3
5523
6201
  })
@@ -5641,7 +6319,7 @@ async function accrueWarmTick(db, settings, observability) {
5641
6319
  });
5642
6320
  return { accrued, workspaceIds };
5643
6321
  }
5644
- async function refreshQueueLeaseAndCreditGauges(db, observability) {
6322
+ async function refreshQueueLeaseAndCreditGauges(db, settings, observability, inspectOpenSandboxInventory) {
5645
6323
  await Promise.all([
5646
6324
  (async () => {
5647
6325
  try {
@@ -5691,6 +6369,21 @@ async function refreshQueueLeaseAndCreditGauges(db, observability) {
5691
6369
  await countExpiredDrainingSandboxLeases(db)
5692
6370
  );
5693
6371
  }),
6372
+ ...settings.sandboxBackend === "opensandbox" && settings.openSandboxKubernetesInventoryNamespace ? [
6373
+ refreshSandboxInventoryGauge(
6374
+ observability,
6375
+ "opensandbox_kubernetes",
6376
+ "OpenSandbox Kubernetes",
6377
+ async () => {
6378
+ recordOpenSandboxKubernetesInventoryGauges(
6379
+ observability,
6380
+ await inspectOpenSandboxInventory({
6381
+ namespace: settings.openSandboxKubernetesInventoryNamespace
6382
+ })
6383
+ );
6384
+ }
6385
+ )
6386
+ ] : [],
5694
6387
  (async () => {
5695
6388
  try {
5696
6389
  recordCreditBalanceGauges(observability, await listCreditBalancesByAccount(db));
@@ -5723,7 +6416,7 @@ async function forceDrainOverLimitWorkspaces(db, settings, workspaceIds, observa
5723
6416
  async (workspaceId) => {
5724
6417
  try {
5725
6418
  const { accountId } = await rlsContextForWorkspace2(db, workspaceId);
5726
- const balance = enforceBalance ? await getBillingBalance2(db, accountId) : { balanceMicros: 1 };
6419
+ const balance = enforceBalance ? await getBillingBalance3(db, accountId) : { balanceMicros: 1 };
5727
6420
  const result = await forceDrainOverLimitViewerOnlyBoxes(db, {
5728
6421
  workspaceId,
5729
6422
  balanceMicros: balance.balanceMicros,
@@ -5855,7 +6548,11 @@ async function probeDrainableProviderReadiness(settings, lease) {
5855
6548
  throw error;
5856
6549
  }
5857
6550
  }
5858
- async function terminateDrainableBox(db, settings, row, observability, terminateBox, probeDrainableProvider, attempt) {
6551
+ function sandboxDrainReason(lease) {
6552
+ if (lease.rotationRequestedAt === null) return "idle";
6553
+ return lease.rotationReason ?? "rotation_requested";
6554
+ }
6555
+ async function terminateDrainableBox(db, settings, row, observability, terminateBox, probeDrainableProvider, attempt, objectStorage) {
5859
6556
  const { accountId } = await rlsContextForWorkspace2(db, row.workspaceId);
5860
6557
  const lease = await readLease(db, row.workspaceId, row.sandboxGroupId);
5861
6558
  if (!lease) {
@@ -6018,6 +6715,7 @@ async function terminateDrainableBox(db, settings, row, observability, terminate
6018
6715
  if (!lease.instanceId || !captureClaim) {
6019
6716
  return { wrote: false, archiveRevision: null };
6020
6717
  }
6718
+ const archiveMetrics = runtimeMetricsHooksForObservability(observability);
6021
6719
  let checkpointArtifactId = null;
6022
6720
  if (archiveBase64 && archiveMetadata?.version === 2 && (archiveMetadata.provider === "modal_snapshot_filesystem" || archiveMetadata.provider === "modal_snapshot_directory")) {
6023
6721
  if (!providerBinding) {
@@ -6059,12 +6757,57 @@ async function terminateDrainableBox(db, settings, row, observability, terminate
6059
6757
  if (!archiveMetadata) {
6060
6758
  throw new Error("Sandbox snapshot publication requires a verified archive descriptor");
6061
6759
  }
6062
- result = await persistDrainSnapshot(db, {
6063
- ...baseInput,
6064
- workspaceArchive: archiveBase64,
6065
- workspaceArchiveMeta: archiveMetadata,
6066
- ...checkpointArtifactId ? { checkpointArtifactId } : {}
6067
- });
6760
+ const priorKeys = collectWorkspaceArchiveObjectKeys(
6761
+ lease.resumeState ?? null
6762
+ );
6763
+ let workspaceArchiveRef;
6764
+ try {
6765
+ const published = await putVersion1TarArchiveOrInline({
6766
+ backend: lease.backend,
6767
+ objectStorage,
6768
+ accountId,
6769
+ workspaceId: row.workspaceId,
6770
+ sandboxGroupId: row.sandboxGroupId,
6771
+ archive: {
6772
+ bytes: Buffer.from(archiveBase64, "base64"),
6773
+ descriptor: archiveMetadata,
6774
+ base64: archiveBase64
6775
+ },
6776
+ metrics: archiveMetrics
6777
+ });
6778
+ workspaceArchiveRef = published.workspaceArchiveRef;
6779
+ result = await persistDrainSnapshot(db, {
6780
+ ...baseInput,
6781
+ workspaceArchiveMeta: archiveMetadata,
6782
+ ...published,
6783
+ ...checkpointArtifactId ? { checkpointArtifactId } : {}
6784
+ });
6785
+ if (published.workspaceArchiveRef && objectStorage) {
6786
+ if (!result.wrote) {
6787
+ await deleteUnpublishedWorkspaceArchiveObject(
6788
+ objectStorage,
6789
+ published.workspaceArchiveRef,
6790
+ archiveMetrics
6791
+ );
6792
+ } else {
6793
+ const afterLease = await readLease(db, row.workspaceId, row.sandboxGroupId);
6794
+ const afterKeys = collectWorkspaceArchiveObjectKeys(
6795
+ afterLease?.resumeState ?? null
6796
+ );
6797
+ await deleteWorkspaceArchiveObjectKeys(
6798
+ objectStorage,
6799
+ [...priorKeys].filter((key) => !afterKeys.has(key))
6800
+ ).catch(() => void 0);
6801
+ }
6802
+ }
6803
+ } catch (error) {
6804
+ await deleteUnpublishedWorkspaceArchiveObject(
6805
+ objectStorage,
6806
+ workspaceArchiveRef,
6807
+ archiveMetrics
6808
+ );
6809
+ throw error;
6810
+ }
6068
6811
  }
6069
6812
  if (!result.wrote && checkpointArtifactId) {
6070
6813
  await markSandboxCheckpointArtifactDeletePending2(db, {
@@ -6111,7 +6854,12 @@ async function terminateDrainableBox(db, settings, row, observability, terminate
6111
6854
  persisted,
6112
6855
  archiveRevision,
6113
6856
  instanceId: lease.instanceId,
6114
- providerMissingBeforeCapture: providerMissing
6857
+ providerMissingBeforeCapture: providerMissing,
6858
+ // Why the box drained: the lease's bounded rotation reason when one
6859
+ // was requested (provider deadline, operator/image conflict, teardown
6860
+ // claim), otherwise the ordinary idle drain. Additive diagnostic
6861
+ // fact; no identifiers beyond the ones already carried above.
6862
+ drainReason: sandboxDrainReason(lease)
6115
6863
  }
6116
6864
  });
6117
6865
  } catch (eventError) {
@@ -6364,99 +7112,222 @@ async function terminateProviderBox(settings, lease, observability, persistArchi
6364
7112
 
6365
7113
  // src/activities/scheduled-tasks.ts
6366
7114
  import {
7115
+ DEFAULT_FIRST_PARTY_MCP_PERMISSIONS,
6367
7116
  OPENGENI_SLACK_BOT_CREDENTIAL_LABEL,
6368
7117
  OPENGENI_SLACK_BOT_CREDENTIAL_ROLE,
6369
- OPENGENI_SLACK_BOT_SESSION_METADATA_KEY
7118
+ OPENGENI_SLACK_BOT_SESSION_METADATA_KEY,
7119
+ resolveWorkspaceCodexCompactionDefault,
7120
+ SCHEDULED_TASK_OCCURRENCE_PAYLOAD_MAX_BYTES,
7121
+ ScheduledTaskRunAcceptedExecution,
7122
+ scheduledOccurrencePayloadUtf8Bytes,
7123
+ stableJson
6370
7124
  } from "@opengeni/contracts";
6371
7125
  import {
7126
+ defaultSessionMcpServerIds,
6372
7127
  openGeniSlackBotMetadata,
6373
7128
  requireOpenGeniSlackBotConnection,
7129
+ resolveSessionToolPolicy,
6374
7130
  scheduledSlackBotConnectionId
6375
7131
  } from "@opengeni/core";
6376
7132
  import {
6377
- appendSessionEventsWithLockedSessionUpdate,
7133
+ appendSessionEvents,
6378
7134
  addSessionSystemUpdateWithSourceMutation,
7135
+ bindScheduledTaskRunSessionInTransaction,
6379
7136
  createScheduledTaskRun,
6380
- createSession,
6381
- createSessionGoal,
7137
+ createSessionWithIdempotencyKeyResult,
6382
7138
  enqueueSessionWorkflowWakeIfRunnable,
6383
7139
  getScheduledTask as getScheduledTask2,
7140
+ getScheduledTaskRunAcceptedExecution,
7141
+ getScheduledTaskRunByProducerKey,
7142
+ getScheduledTargetSessionExecution,
6384
7143
  ensureKnowledgeSourceSyncState,
6385
7144
  getScheduledTaskPersonalConnectionDelegations,
7145
+ getScheduledTaskRunPersonalResourceAuthority,
7146
+ getScheduledTaskPersonalResourceAuthoritySubject,
7147
+ getScheduledTaskRevisionAuthority,
6386
7148
  getScheduledTaskXaiProviderAccountAuthoritySnapshot,
6387
7149
  getRig as getRig2,
7150
+ getScheduledScopedRigVersionMetadata,
7151
+ getNestedAgentDepthDeploymentPolicy,
7152
+ getSessionByCreateIdempotencyKey,
6388
7153
  getVariableSet,
7154
+ isCodexBilledModel,
7155
+ initializeSessionStartAtomically,
7156
+ materializeScheduledTaskReusableSessionFromRun,
7157
+ listEnabledMcpCapabilityServerIds,
7158
+ listInstalledApiIntegrationServerIdsForDelegations,
6389
7159
  markScheduledTaskRunFailedIfQueued,
7160
+ markScheduledTaskRunSkippedIfQueued,
6390
7161
  recordKnowledgeSourceSyncWake,
6391
7162
  recordUsageEvent as recordUsageEvent3,
6392
- requireScheduledTaskTargetInTransaction,
7163
+ requireScheduledTaskIncidentAuthorityInTransaction,
6393
7164
  requireSession,
6394
- setTemporalWorkflowId,
7165
+ requireWorkspace,
6395
7166
  settleScheduledTaskRunInTransaction,
6396
7167
  SessionSpawnDeniedDbError,
6397
- updateScheduledTask,
6398
7168
  updateScheduledTaskRun as updateScheduledTaskRun2,
6399
- upsertSessionGoal
7169
+ updateSessionTitle,
7170
+ upsertScheduledSessionGoalForRun,
7171
+ withSessionActivityRlsContext
6400
7172
  } from "@opengeni/db";
6401
- import { appendAndPublishEvents, publishDurableSessionEvents } from "@opengeni/events";
7173
+ import { publishDurableSessionEvents } from "@opengeni/events";
6402
7174
  import { resolveFirstPartyMcpToolPolicy } from "@opengeni/config";
7175
+ import { Context as Context2 } from "@temporalio/activity";
6403
7176
 
6404
- // src/activities/agent-run-admission.ts
6405
- import { configuredStaticUsageLimits as configuredStaticUsageLimits2 } from "@opengeni/config";
6406
- import { getBillingBalance as getBillingBalance3, isCodexBilledTurn, sumUsageQuantity as sumUsageQuantity2 } from "@opengeni/db";
6407
- async function agentRunAdmissionDenial(services, input) {
6408
- const externallyBilled = await isCodexBilledTurn({
6409
- db: services.db,
6410
- settings: services.settings,
6411
- workspaceId: input.workspaceId,
6412
- model: input.model
6413
- });
6414
- if (!externallyBilled && (services.settings.billingMode === "stripe" || services.settings.usageLimitsMode === "managed")) {
6415
- if (services.entitlements) {
6416
- const decision = await services.entitlements.admitRun({
6417
- accountId: input.accountId,
6418
- workspaceId: input.workspaceId,
6419
- action: "agent_run:create",
6420
- quantity: input.requestedAgentRuns
6421
- });
6422
- if (!decision.allowed) return "insufficient_credits";
6423
- } else {
6424
- const balance = await getBillingBalance3(services.db, input.accountId);
6425
- if (balance.balanceMicros <= 0) return "insufficient_credits";
6426
- }
6427
- }
6428
- if (services.settings.usageLimitsMode !== "static" && services.settings.usageLimitsMode !== "managed") {
7177
+ // src/scheduled-alert-occurrence.ts
7178
+ import { createHash as createHash4 } from "crypto";
7179
+ var MAX_SCHEDULED_TASK_ID_BYTES = 256;
7180
+ var MAX_STARTS_AT_BYTES = 256;
7181
+ var MAX_PROVIDER_BYTES = 256;
7182
+ var MAX_FINGERPRINT_BYTES = 1024;
7183
+ var MAX_LABELS = 256;
7184
+ var MAX_LABEL_KEY_BYTES = 256;
7185
+ var MAX_LABEL_VALUE_BYTES = 4096;
7186
+ var OFFSET_TIMESTAMP = /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2})(?::(\d{2})(?:\.(\d+))?)?(?:Z|[+-](\d{2}):(\d{2}))$/;
7187
+ function scheduledAlertResponderSessionCreateIdempotencyKey(input) {
7188
+ const digest = createHash4("sha256").update(
7189
+ JSON.stringify({
7190
+ version: 1,
7191
+ occurrenceKey: input.occurrence.sessionCreateIdempotencyKey,
7192
+ taskAuthorityRevision: input.taskAuthorityRevision,
7193
+ taskExecutionDigest: input.taskExecutionDigest
7194
+ })
7195
+ ).digest("hex");
7196
+ return `scheduled-alert-occurrence:v1:${digest}`;
7197
+ }
7198
+ function scheduledAlertOccurrenceIdentity(input) {
7199
+ const scheduledTaskId = boundedString(input.scheduledTaskId, MAX_SCHEDULED_TASK_ID_BYTES);
7200
+ if (!scheduledTaskId) return null;
7201
+ const alert = record(input.metadata.alert);
7202
+ if (!alert) return null;
7203
+ const status = alert.status;
7204
+ if (status !== "firing" && status !== "resolved") return null;
7205
+ const startsAt = canonicalTimestamp(alert.startsAt);
7206
+ const fingerprint = boundedString(alert.fingerprint, MAX_FINGERPRINT_BYTES);
7207
+ const labels = canonicalLabels(alert.labels);
7208
+ if (!startsAt || !fingerprint || !labels) return null;
7209
+ const provider = boundedString(
7210
+ alert.provider ?? input.metadata.alertProvider ?? input.metadata.provider ?? "alertmanager",
7211
+ MAX_PROVIDER_BYTES
7212
+ );
7213
+ if (!provider) return null;
7214
+ const digest = createHash4("sha256").update(
7215
+ JSON.stringify({
7216
+ version: 1,
7217
+ workspaceId: input.workspaceId,
7218
+ scheduledTaskId,
7219
+ provider,
7220
+ fingerprint,
7221
+ startsAt,
7222
+ labels
7223
+ })
7224
+ ).digest("hex");
7225
+ return {
7226
+ status,
7227
+ sessionCreateIdempotencyKey: `scheduled-alert-occurrence:v1:${digest}`,
7228
+ labels: Object.fromEntries(labels)
7229
+ };
7230
+ }
7231
+ function record(value) {
7232
+ return value !== null && typeof value === "object" && !Array.isArray(value) ? value : null;
7233
+ }
7234
+ function boundedString(value, maxBytes) {
7235
+ if (typeof value !== "string") return null;
7236
+ const normalized = value.trim();
7237
+ if (!normalized || Buffer.byteLength(normalized, "utf8") > maxBytes) return null;
7238
+ return normalized;
7239
+ }
7240
+ function canonicalTimestamp(value) {
7241
+ if (typeof value !== "string") return null;
7242
+ const trimmed = value.trim();
7243
+ if (!trimmed || Buffer.byteLength(trimmed, "utf8") > MAX_STARTS_AT_BYTES) return null;
7244
+ const match = OFFSET_TIMESTAMP.exec(trimmed);
7245
+ if (!match) return null;
7246
+ const [
7247
+ ,
7248
+ yearText,
7249
+ monthText,
7250
+ dayText,
7251
+ hourText,
7252
+ minuteText,
7253
+ secondText,
7254
+ ,
7255
+ offsetHourText,
7256
+ offsetMinuteText
7257
+ ] = match;
7258
+ const year = Number(yearText);
7259
+ const month = Number(monthText);
7260
+ const day = Number(dayText);
7261
+ const hour = Number(hourText);
7262
+ const minute = Number(minuteText);
7263
+ const second = secondText === void 0 ? 0 : Number(secondText);
7264
+ const offsetHour = offsetHourText === void 0 ? 0 : Number(offsetHourText);
7265
+ const offsetMinute = offsetMinuteText === void 0 ? 0 : Number(offsetMinuteText);
7266
+ const leapYear = year % 4 === 0 && (year % 100 !== 0 || year % 400 === 0);
7267
+ const daysInMonth = [31, leapYear ? 29 : 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31];
7268
+ if (month < 1 || month > 12 || day < 1 || day > daysInMonth[month - 1] || hour > 23 || minute > 59 || second > 59 || offsetHour > 23 || offsetMinute > 59) {
6429
7269
  return null;
6430
7270
  }
6431
- const limits = configuredStaticUsageLimits2(services.settings);
6432
- if (!externallyBilled && limits.maxMonthlyCostMicrosPerAccount) {
6433
- const used = await sumUsageQuantity2(services.db, {
6434
- accountId: input.accountId,
6435
- eventType: "model.cost",
6436
- since: startOfUtcMonth2()
6437
- });
6438
- if (used >= limits.maxMonthlyCostMicrosPerAccount) {
6439
- return "monthly_model_cost_limit";
7271
+ const parsed = new Date(trimmed);
7272
+ return Number.isFinite(parsed.getTime()) ? trimmed : null;
7273
+ }
7274
+ function canonicalLabels(value) {
7275
+ const labels = record(value);
7276
+ if (!labels) return null;
7277
+ const entries = Object.entries(labels);
7278
+ if (entries.length === 0 || entries.length > MAX_LABELS) return null;
7279
+ const canonical = [];
7280
+ for (const [key, rawValue] of entries) {
7281
+ if (!key || Buffer.byteLength(key, "utf8") > MAX_LABEL_KEY_BYTES || typeof rawValue !== "string" || Buffer.byteLength(rawValue, "utf8") > MAX_LABEL_VALUE_BYTES) {
7282
+ return null;
6440
7283
  }
7284
+ canonical.push([key, rawValue]);
6441
7285
  }
6442
- if (limits.maxMonthlyAgentRunsPerWorkspace) {
6443
- const used = await sumUsageQuantity2(services.db, {
6444
- workspaceId: input.workspaceId,
6445
- eventType: "agent_run.created",
6446
- since: startOfUtcMonth2()
6447
- });
6448
- if (used + input.requestedAgentRuns > limits.maxMonthlyAgentRunsPerWorkspace) {
6449
- return "monthly_agent_run_limit";
7286
+ return canonical.sort(([left], [right]) => left < right ? -1 : left > right ? 1 : 0);
7287
+ }
7288
+
7289
+ // src/activities/scheduled-tasks.ts
7290
+ function scheduledTaskRunProducerKey(input, suppliedActivityIdentity) {
7291
+ if (input.producerKey) return input.producerKey;
7292
+ if (input.triggerType !== "scheduled") return input.agentRunUsageIdempotencyKey;
7293
+ let info = suppliedActivityIdentity;
7294
+ if (!info) {
7295
+ try {
7296
+ info = Context2.current().info;
7297
+ } catch {
7298
+ throw new Error(
7299
+ "scheduled task delivery without a producer key requires a Temporal activity execution"
7300
+ );
6450
7301
  }
6451
7302
  }
6452
- return null;
7303
+ if (!info.workflowExecution) {
7304
+ throw new Error("scheduled task Temporal activity is missing its workflow execution identity");
7305
+ }
7306
+ return `scheduled-temporal:${stableJson({
7307
+ namespace: info.namespace,
7308
+ workflowId: info.workflowExecution.workflowId
7309
+ })}`;
6453
7310
  }
6454
- function startOfUtcMonth2() {
6455
- const now = /* @__PURE__ */ new Date();
6456
- return new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), 1));
7311
+ var SCHEDULED_SESSION_TITLE_MAX_LENGTH = 200;
7312
+ function scheduledTaskSessionTitle(taskName) {
7313
+ const name = taskName.trim() || "Scheduled run";
7314
+ if (name.length <= SCHEDULED_SESSION_TITLE_MAX_LENGTH) return name;
7315
+ return `${name.slice(0, SCHEDULED_SESSION_TITLE_MAX_LENGTH - 1).trimEnd()}\u2026`;
7316
+ }
7317
+ async function stampScheduledSessionTitle(db, input) {
7318
+ if (input.session.title !== null) return [];
7319
+ const title = scheduledTaskSessionTitle(input.taskName);
7320
+ const result = await updateSessionTitle(db, {
7321
+ workspaceId: input.workspaceId,
7322
+ sessionId: input.session.id,
7323
+ title,
7324
+ source: "agent"
7325
+ });
7326
+ if (!result.updated) return [];
7327
+ return await appendSessionEvents(db, input.workspaceId, input.session.id, [
7328
+ { type: "session.title_set", payload: { title: result.title ?? title, source: "agent" } }
7329
+ ]);
6457
7330
  }
6458
-
6459
- // src/activities/scheduled-tasks.ts
6460
7331
  function createScheduledTaskActivities(services) {
6461
7332
  return {
6462
7333
  dispatchScheduledTaskRun: async (input) => {
@@ -6465,6 +7336,75 @@ function createScheduledTaskActivities(services) {
6465
7336
  if (input.triggerType !== "scheduled" && !input.initiator) {
6466
7337
  return { action: "blocked", reason: "malformed_manual_trigger" };
6467
7338
  }
7339
+ const stableProducerKey = scheduledTaskRunProducerKey(input);
7340
+ const priorRun = await getScheduledTaskRunByProducerKey(db, {
7341
+ workspaceId: input.workspaceId,
7342
+ taskId: input.taskId,
7343
+ triggerType: input.triggerType,
7344
+ producerKey: stableProducerKey
7345
+ });
7346
+ if (priorRun?.actionKind === "agent_turn") {
7347
+ const acceptedExecution = await getScheduledTaskRunAcceptedExecution(db, {
7348
+ workspaceId: input.workspaceId,
7349
+ runId: priorRun.id
7350
+ });
7351
+ if (!acceptedExecution) {
7352
+ if (priorRun.status === "succeeded" || priorRun.status === "failed" || priorRun.status === "skipped") {
7353
+ return scheduledRunTerminalResult(priorRun.error);
7354
+ }
7355
+ throw new Error("scheduled agent run is missing accepted execution truth");
7356
+ }
7357
+ await recordScheduledTaskFiredUsage(db, acceptedExecution, priorRun);
7358
+ if (priorRun.status === "dispatched") {
7359
+ return await replayScheduledTaskDispatch({
7360
+ db,
7361
+ wakeSessionWorkflow,
7362
+ task: acceptedExecution.task,
7363
+ run: priorRun,
7364
+ acceptedExecution
7365
+ });
7366
+ }
7367
+ if (priorRun.status === "queued") {
7368
+ try {
7369
+ return await recoverBoundScheduledTaskDispatch({
7370
+ db,
7371
+ bus,
7372
+ settings,
7373
+ wakeSessionWorkflow,
7374
+ run: priorRun,
7375
+ acceptedExecution,
7376
+ input
7377
+ });
7378
+ } catch (error) {
7379
+ const terminalError = scheduledRunRecoveryTerminalError(error);
7380
+ if (!terminalError) throw error;
7381
+ await markScheduledTaskRunFailedIfQueued(
7382
+ db,
7383
+ priorRun.workspaceId,
7384
+ priorRun.id,
7385
+ terminalError.code
7386
+ );
7387
+ return scheduledRunTerminalResult(terminalError.code);
7388
+ }
7389
+ }
7390
+ if (priorRun.status === "succeeded") {
7391
+ if (!priorRun.sessionId || !priorRun.triggerEventId) {
7392
+ throw new Error("succeeded scheduled run is missing its delivery identity");
7393
+ }
7394
+ return {
7395
+ action: acceptedExecution.task.runMode === "new_session_per_run" ? "start" : "signal",
7396
+ accountId: acceptedExecution.task.accountId,
7397
+ workspaceId: acceptedExecution.task.workspaceId,
7398
+ sessionId: priorRun.sessionId,
7399
+ triggerEventId: priorRun.triggerEventId,
7400
+ workflowId: workflowIdForSession(priorRun.sessionId),
7401
+ workflowWakeRevision: null
7402
+ };
7403
+ }
7404
+ if (priorRun.status === "failed" || priorRun.status === "skipped") {
7405
+ return scheduledRunTerminalResult(priorRun.error);
7406
+ }
7407
+ }
6468
7408
  const task = await getScheduledTask2(db, input.workspaceId, input.taskId);
6469
7409
  if (!task) {
6470
7410
  return { action: "deleted" };
@@ -6473,11 +7413,13 @@ function createScheduledTaskActivities(services) {
6473
7413
  if (knowledgeSourceSyncEffectivelyPaused(task)) {
6474
7414
  return { action: "blocked", reason: "knowledge_source_paused" };
6475
7415
  }
6476
- const run2 = await createScheduledTaskRun(db, {
7416
+ const run = await createScheduledTaskRun(db, {
6477
7417
  workspaceId: task.workspaceId,
6478
7418
  taskId: task.id,
7419
+ taskAuthorityRevision: task.authorityRevision,
7420
+ taskExecutionDigest: task.executionDigest,
6479
7421
  triggerType: input.triggerType,
6480
- producerKey: input.producerKey ?? input.agentRunUsageIdempotencyKey ?? `knowledge-source-sync:${crypto.randomUUID()}`,
7422
+ producerKey: stableProducerKey,
6481
7423
  scheduledAt: null
6482
7424
  });
6483
7425
  await ensureKnowledgeSourceSyncState(db, task);
@@ -6486,9 +7428,9 @@ function createScheduledTaskActivities(services) {
6486
7428
  workspaceId: task.workspaceId,
6487
7429
  sourceId: task.action.sourceId,
6488
7430
  scheduledTaskId: task.id,
6489
- scheduledTaskRunId: run2.id,
7431
+ scheduledTaskRunId: run.id,
6490
7432
  cause: input.triggerType,
6491
- producerKey: input.producerKey ?? input.agentRunUsageIdempotencyKey ?? `knowledge-source-sync:${run2.id}`,
7433
+ producerKey: stableProducerKey,
6492
7434
  sourceConfigGeneration: task.action.sourceConfigGeneration,
6493
7435
  sourceLifecycleGeneration: task.action.sourceLifecycleGeneration
6494
7436
  });
@@ -6499,14 +7441,20 @@ function createScheduledTaskActivities(services) {
6499
7441
  quantity: 1,
6500
7442
  unit: "run",
6501
7443
  sourceResourceType: "scheduled_task_run",
6502
- sourceResourceId: run2.id,
6503
- initiator: input.initiator ?? { kind: "service", subjectId: "scheduler" },
6504
- initiatorContext: { scheduledTaskId: task.id, scheduledTaskRunId: run2.id },
7444
+ sourceResourceId: run.id,
7445
+ initiator: input.initiator ?? {
7446
+ kind: "service",
7447
+ subjectId: "scheduler"
7448
+ },
7449
+ initiatorContext: {
7450
+ scheduledTaskId: task.id,
7451
+ scheduledTaskRunId: run.id
7452
+ },
6505
7453
  origin: "scheduled_task",
6506
- idempotencyKey: `usage:knowledge_source_sync.fired:${run2.id}`
7454
+ idempotencyKey: `usage:knowledge_source_sync.fired:${run.id}`
6507
7455
  });
6508
- if (run2.status === "queued") {
6509
- await updateScheduledTaskRun2(db, task.workspaceId, run2.id, {
7456
+ if (run.status === "queued") {
7457
+ await updateScheduledTaskRun2(db, task.workspaceId, run.id, {
6510
7458
  status: "dispatched",
6511
7459
  actionKind: "knowledge_source_sync"
6512
7460
  });
@@ -6516,414 +7464,979 @@ function createScheduledTaskActivities(services) {
6516
7464
  accountId: task.accountId,
6517
7465
  workspaceId: task.workspaceId,
6518
7466
  taskId: task.id,
6519
- scheduledTaskRunId: run2.id,
7467
+ scheduledTaskRunId: run.id,
6520
7468
  sourceId: task.action.sourceId,
6521
7469
  overlapPolicy: task.overlapPolicy
6522
7470
  };
6523
7471
  }
7472
+ if (task.status !== "active") {
7473
+ return { action: "blocked", reason: "scheduled_task_paused" };
7474
+ }
7475
+ const structuredAlertOccurrence = scheduledAlertOccurrenceIdentity({
7476
+ workspaceId: task.workspaceId,
7477
+ scheduledTaskId: task.id,
7478
+ metadata: task.metadata
7479
+ });
7480
+ const alertOccurrence = task.runMode === "new_session_per_run" ? structuredAlertOccurrence : null;
7481
+ const alertResponderSessionCreateIdempotencyKey = alertOccurrence ? scheduledAlertResponderSessionCreateIdempotencyKey({
7482
+ occurrence: alertOccurrence,
7483
+ taskAuthorityRevision: task.authorityRevision,
7484
+ taskExecutionDigest: task.executionDigest
7485
+ }) : null;
7486
+ const incidentDeclaration = incidentTelemetryPreflightDeclaration(
7487
+ task.agentConfig,
7488
+ structuredAlertOccurrence !== null
7489
+ );
7490
+ if (incidentDeclaration.action === "blocked") {
7491
+ return incidentDeclaration;
7492
+ }
6524
7493
  const taskPersonalConnectionDelegations = await getScheduledTaskPersonalConnectionDelegations(
6525
7494
  db,
6526
7495
  task.workspaceId,
6527
7496
  task.id
6528
7497
  );
6529
- const taskXaiProviderAccountAuthoritySnapshot = await getScheduledTaskXaiProviderAccountAuthoritySnapshot(db, task.workspaceId, task.id);
6530
- const slackBotConnection = task.agentConfig.slackBotConnectionId ? await requireOpenGeniSlackBotConnection(
6531
- db,
6532
- task.workspaceId,
6533
- task.agentConfig.slackBotConnectionId
6534
- ) : null;
6535
- const slackBotMetadata = slackBotConnection ? openGeniSlackBotMetadata(slackBotConnection.metadata) : null;
6536
- if (slackBotConnection && !slackBotMetadata) {
6537
- throw new Error("OpenGeni Slack bot connection metadata is invalid");
7498
+ const taskConnectionAuthoritySubjects = [
7499
+ ...new Set(
7500
+ taskPersonalConnectionDelegations.map((delegation) => delegation.ownerSubjectId)
7501
+ )
7502
+ ];
7503
+ if (taskConnectionAuthoritySubjects.length > 1) {
7504
+ throw new Error("scheduled connection authority has multiple causal humans");
6538
7505
  }
6539
- const model = task.agentConfig.model ?? settings.openaiModel;
6540
- const admissionDenial = await agentRunAdmissionDenial(service, {
7506
+ const taskConnectionAuthoritySubjectId = taskConnectionAuthoritySubjects[0] ?? null;
7507
+ const taskPersonalResourceAuthoritySubjectId = await getScheduledTaskPersonalResourceAuthoritySubject(db, {
6541
7508
  accountId: task.accountId,
6542
7509
  workspaceId: task.workspaceId,
6543
- model,
6544
- requestedAgentRuns: input.agentRunUsageIdempotencyKey ? 0 : 1
7510
+ taskId: task.id,
7511
+ taskAuthorityRevision: task.authorityRevision
6545
7512
  });
6546
- if (admissionDenial) {
6547
- return { action: "blocked", reason: admissionDenial };
6548
- }
6549
- const run = await createScheduledTaskRun(db, {
6550
- workspaceId: task.workspaceId,
6551
- taskId: task.id,
6552
- triggerType: input.triggerType,
6553
- producerKey: input.producerKey ?? input.agentRunUsageIdempotencyKey ?? `scheduled:${crypto.randomUUID()}`,
6554
- scheduledAt: null
6555
- });
6556
- await recordUsageEvent3(db, {
7513
+ const taskRevisionAuthority = await getScheduledTaskRevisionAuthority(db, {
6557
7514
  accountId: task.accountId,
6558
7515
  workspaceId: task.workspaceId,
6559
- eventType: "scheduled_task.fired",
6560
- quantity: 1,
6561
- unit: "run",
6562
- sourceResourceType: "scheduled_task_run",
6563
- sourceResourceId: run.id,
6564
- initiator: input.initiator ?? { kind: "service", subjectId: "scheduler" },
6565
- initiatorContext: { scheduledTaskId: task.id, scheduledTaskRunId: run.id },
6566
- origin: "scheduled_task",
6567
- idempotencyKey: `usage:scheduled_task.fired:${run.id}`
7516
+ taskId: task.id,
7517
+ taskAuthorityRevision: task.authorityRevision
6568
7518
  });
6569
- if (run.status === "dispatched" && run.sessionId && run.triggerEventId) {
6570
- await recordUsageEvent3(db, {
6571
- accountId: task.accountId,
6572
- workspaceId: task.workspaceId,
6573
- eventType: "agent_run.created",
6574
- quantity: 1,
6575
- unit: "run",
6576
- sourceResourceType: "scheduled_task_run",
6577
- sourceResourceId: run.id,
6578
- sessionId: run.sessionId,
6579
- initiator: input.initiator ?? { kind: "service", subjectId: "scheduler" },
6580
- initiatorContext: { scheduledTaskId: task.id, scheduledTaskRunId: run.id },
6581
- origin: "scheduled_task",
6582
- idempotencyKey: input.agentRunUsageIdempotencyKey ?? `usage:agent_run.created:scheduled:${run.id}`
7519
+ if (taskConnectionAuthoritySubjectId && taskPersonalResourceAuthoritySubjectId && taskConnectionAuthoritySubjectId !== taskPersonalResourceAuthoritySubjectId) {
7520
+ throw new Error("scheduled authority classes have different causal humans");
7521
+ }
7522
+ const taskAuthoritySubjectId = taskRevisionAuthority?.subjectId ?? taskPersonalResourceAuthoritySubjectId ?? taskConnectionAuthoritySubjectId;
7523
+ const model = task.agentConfig.model ?? settings.openaiModel;
7524
+ const reasoningEffort = task.agentConfig.reasoningEffort ?? settings.openaiReasoningEffort;
7525
+ const sandboxBackend = task.agentConfig.sandboxBackend ?? settings.sandboxBackend;
7526
+ const taskTools = withFirstPartyTools(settings, task.agentConfig.tools);
7527
+ const firstPartyMcpTools = resolveFirstPartyMcpToolPolicy(settings).default;
7528
+ const generatedTarget = task.runMode === "new_session_per_run" || task.runMode === "reusable_session" && task.reusableSessionId === null;
7529
+ const generatedSessionDepthPolicy = generatedTarget ? await (async () => {
7530
+ const workspace = await requireWorkspace(db, task.workspaceId);
7531
+ const deployment = await getNestedAgentDepthDeploymentPolicy(db);
7532
+ const taskOverride = task.agentConfig.maxNestedAgentDepth ?? null;
7533
+ const workspaceOverride = typeof workspace.settings.maxNestedAgentDepth === "number" ? workspace.settings.maxNestedAgentDepth : null;
7534
+ return taskOverride !== null ? {
7535
+ effectiveMaxNestedAgentDepth: taskOverride,
7536
+ nestedAgentDepthPolicySource: "session",
7537
+ codexCompactionMode: isCodexBilledModel(model) ? resolveWorkspaceCodexCompactionDefault(workspace.settings) : "portable"
7538
+ } : workspaceOverride !== null ? {
7539
+ effectiveMaxNestedAgentDepth: workspaceOverride,
7540
+ nestedAgentDepthPolicySource: "workspace",
7541
+ codexCompactionMode: isCodexBilledModel(model) ? resolveWorkspaceCodexCompactionDefault(workspace.settings) : "portable"
7542
+ } : {
7543
+ effectiveMaxNestedAgentDepth: deployment.maxNestedAgentDepth,
7544
+ nestedAgentDepthPolicySource: deployment.policySource,
7545
+ codexCompactionMode: isCodexBilledModel(model) ? resolveWorkspaceCodexCompactionDefault(workspace.settings) : "portable"
7546
+ };
7547
+ })() : null;
7548
+ const acceptedTargetSessionId = generatedTarget ? null : task.runMode === "existing_session" ? task.targetSessionId : task.reusableSessionId;
7549
+ const targetSessionExecutionBase = acceptedTargetSessionId ? await getScheduledTargetSessionExecution(
7550
+ db,
7551
+ task.workspaceId,
7552
+ acceptedTargetSessionId,
7553
+ taskAuthoritySubjectId
7554
+ ) : null;
7555
+ const targetSessionExecution = targetSessionExecutionBase ? await (async () => {
7556
+ const [capabilityServerIds, apiIntegrationServerIds] = await Promise.all([
7557
+ listEnabledMcpCapabilityServerIds(db, task.workspaceId),
7558
+ listInstalledApiIntegrationServerIdsForDelegations(
7559
+ db,
7560
+ task.workspaceId,
7561
+ taskPersonalConnectionDelegations
7562
+ )
7563
+ ]);
7564
+ const workspaceDefaultMcpServerIds = new Set(
7565
+ settings.mcpServers.map((server) => server.id)
7566
+ );
7567
+ for (const id of capabilityServerIds) workspaceDefaultMcpServerIds.add(id);
7568
+ for (const id of apiIntegrationServerIds) workspaceDefaultMcpServerIds.add(id);
7569
+ const availableMcpServerIds = new Set(workspaceDefaultMcpServerIds);
7570
+ for (const id of targetSessionExecutionBase.mcpServerIds) {
7571
+ availableMcpServerIds.add(id);
7572
+ }
7573
+ const effective = resolveSessionToolPolicy({
7574
+ toolPolicy: targetSessionExecutionBase.toolPolicy,
7575
+ sessionTools: targetSessionExecutionBase.tools,
7576
+ availableMcpServerIds,
7577
+ defaultMcpServerIds: defaultSessionMcpServerIds(
7578
+ [...workspaceDefaultMcpServerIds].map((id) => ({ id }))
7579
+ )
6583
7580
  });
6584
- const workflowId = workflowIdForSession(run.sessionId);
6585
- const workflowWakeRevision = await enqueueSessionWorkflowWakeIfRunnable(db, {
7581
+ return {
7582
+ ...targetSessionExecutionBase,
7583
+ effectiveMcpServerIds: [
7584
+ ...new Set(
7585
+ effective.toolRefs.flatMap((tool) => tool.kind === "mcp" ? [tool.id] : [])
7586
+ )
7587
+ ].sort()
7588
+ };
7589
+ })() : null;
7590
+ const acceptedVariableSet = generatedTarget && task.variableSetId ? await getVariableSet(
7591
+ db,
7592
+ {
6586
7593
  accountId: task.accountId,
6587
7594
  workspaceId: task.workspaceId,
6588
- sessionId: run.sessionId,
6589
- temporalWorkflowId: workflowId,
6590
- reason: "scheduled_retry"
6591
- });
6592
- const result2 = {
6593
- action: task.runMode === "new_session_per_run" ? "start" : "signal",
7595
+ // Organization/workspace sets have no personal-resource causal
7596
+ // subject. The scoped resolver still permits those common
7597
+ // scopes for a service/legacy actor, while a user-owned set can
7598
+ // only resolve through the exact frozen 0252 subject.
7599
+ subjectId: taskAuthoritySubjectId ?? task.createdBy.subjectId
7600
+ },
7601
+ task.variableSetId
7602
+ ) : null;
7603
+ if (generatedTarget && task.variableSetId && !acceptedVariableSet) {
7604
+ throw new Error(`variable set not found: ${task.variableSetId}`);
7605
+ }
7606
+ const acceptedRig = generatedTarget && task.rigId ? await getRig2(
7607
+ db,
7608
+ {
6594
7609
  accountId: task.accountId,
6595
7610
  workspaceId: task.workspaceId,
6596
- sessionId: run.sessionId,
6597
- triggerEventId: run.triggerEventId,
6598
- workflowId,
6599
- workflowWakeRevision
6600
- };
6601
- if (wakeSessionWorkflow && workflowWakeRevision !== null) {
6602
- await wakeSessionWorkflow({
6603
- accountId: result2.accountId,
6604
- workspaceId: result2.workspaceId,
6605
- sessionId: result2.sessionId,
6606
- workflowId: result2.workflowId,
6607
- wakeRevision: workflowWakeRevision
6608
- });
6609
- }
6610
- return result2;
7611
+ subjectId: taskAuthoritySubjectId ?? task.createdBy.subjectId
7612
+ },
7613
+ task.rigId
7614
+ ) : null;
7615
+ if (generatedTarget && task.rigId && (!acceptedRig || !acceptedRig.activeVersion)) {
7616
+ throw new Error(`rig has no active version to bind: ${task.rigId}`);
6611
7617
  }
6612
- let result;
6613
- try {
6614
- const reasoningEffort = task.agentConfig.reasoningEffort ?? settings.openaiReasoningEffort;
6615
- const sandboxBackend = task.agentConfig.sandboxBackend ?? settings.sandboxBackend;
6616
- const goalSpec = task.agentConfig.goal ?? null;
6617
- const taskTools = withFirstPartyTools(settings, task.agentConfig.tools);
6618
- const existingSessionId = task.runMode === "existing_session" ? task.targetSessionId : task.reusableSessionId;
6619
- if (task.runMode === "existing_session" && !existingSessionId) {
6620
- throw new Error("scheduled task target session is unavailable");
6621
- }
6622
- if (task.runMode === "new_session_per_run" || task.runMode === "reusable_session" && !existingSessionId) {
6623
- const variableSet = task.variableSetId ? await getVariableSet(db, task.workspaceId, task.variableSetId) : null;
6624
- if (task.variableSetId && !variableSet) {
6625
- throw new Error(`variable set not found: ${task.variableSetId}`);
6626
- }
6627
- let frozenRigId = null;
6628
- let frozenRigVersionId = null;
6629
- if (task.rigId) {
6630
- const rig = await getRig2(db, task.workspaceId, task.rigId);
6631
- if (!rig || !rig.activeVersion) {
6632
- throw new Error(`rig has no active version to bind: ${task.rigId}`);
6633
- }
6634
- frozenRigId = rig.id;
6635
- frozenRigVersionId = rig.activeVersion.id;
6636
- }
6637
- let session;
6638
- const taskMetadata = { ...task.agentConfig.metadata };
6639
- delete taskMetadata[OPENGENI_SLACK_BOT_SESSION_METADATA_KEY];
6640
- try {
6641
- session = await createSession(db, {
6642
- accountId: task.accountId,
6643
- workspaceId: task.workspaceId,
6644
- initialMessage: task.agentConfig.prompt,
6645
- resources: task.agentConfig.resources,
6646
- tools: taskTools,
6647
- firstPartyMcpTools: resolveFirstPartyMcpToolPolicy(settings).default,
6648
- metadata: {
6649
- ...taskMetadata,
6650
- model,
6651
- reasoningEffort,
6652
- scheduledTaskId: task.id,
6653
- scheduledTaskRunId: run.id,
6654
- ...slackBotConnection ? {
6655
- [OPENGENI_SLACK_BOT_SESSION_METADATA_KEY]: slackBotConnection.id
6656
- } : {}
6657
- },
6658
- createdBy: {
6659
- kind: "service",
6660
- subjectId: "scheduler",
6661
- label: "OpenGeni scheduler"
6662
- },
6663
- createdByContext: {
6664
- scheduledTaskId: task.id,
6665
- scheduledTaskRunId: run.id
6666
- },
6667
- model,
6668
- sandboxBackend,
6669
- variableSetId: task.variableSetId ?? null,
6670
- rigId: frozenRigId,
6671
- rigVersionId: frozenRigVersionId,
6672
- personalConnectionDelegations: taskPersonalConnectionDelegations,
6673
- initialXaiProviderAccountAuthoritySnapshot: taskXaiProviderAccountAuthoritySnapshot,
6674
- maxNestedAgentDepthOverride: task.agentConfig.maxNestedAgentDepth ?? null,
6675
- // The durable agent config was privilege-checked when the task
6676
- // was created/updated. Preserve that explicit policy if a broader
6677
- // workspace/deployment limit is narrowed before a later fire.
6678
- allowNestedAgentDepthIncrease: true,
6679
- subjectId: `scheduled_task:${task.id}`
6680
- });
6681
- } catch (error) {
6682
- if (error instanceof SessionSpawnDeniedDbError) {
6683
- throw new Error(`${error.denial.code}: denial=${error.denial.id}`, { cause: error });
6684
- }
6685
- throw error;
6686
- }
6687
- const goal = goalSpec ? await createSessionGoal(db, {
6688
- accountId: task.accountId,
6689
- workspaceId: task.workspaceId,
6690
- sessionId: session.id,
6691
- text: goalSpec.text,
6692
- successCriteria: goalSpec.successCriteria ?? null,
6693
- maxAutoContinuations: goalSpec.maxAutoContinuations ?? null,
6694
- createdBy: "scheduled_task"
6695
- }) : null;
6696
- const workflowId = workflowIdForSession(session.id);
6697
- await setTemporalWorkflowId(db, task.workspaceId, session.id, workflowId);
6698
- if (task.runMode === "reusable_session") {
6699
- await updateScheduledTask(db, task.workspaceId, task.id, {
6700
- reusableSessionId: session.id
6701
- });
6702
- }
6703
- await appendAndPublishEvents(db, bus, task.workspaceId, session.id, [
6704
- {
6705
- type: "session.created",
6706
- payload: {
6707
- status: session.status,
6708
- createdBy: session.createdBy,
6709
- scheduledTaskId: task.id,
6710
- scheduledTaskRunId: run.id,
6711
- // Names/ids only; never values.
6712
- ...variableSet ? { variableSetId: variableSet.id, variableSetName: variableSet.name } : {},
6713
- ...slackBotConnection && slackBotMetadata ? {
6714
- slackBotConnection: {
6715
- credentialRole: OPENGENI_SLACK_BOT_CREDENTIAL_ROLE,
6716
- credentialLabel: OPENGENI_SLACK_BOT_CREDENTIAL_LABEL,
6717
- connectionId: slackBotConnection.id,
6718
- slackTeamId: slackBotMetadata.slackTeamId
6719
- }
6720
- } : {}
6721
- }
6722
- },
6723
- ...goal ? [
6724
- {
6725
- type: "goal.set",
6726
- payload: {
6727
- goalId: goal.id,
6728
- text: goal.text,
6729
- ...goal.successCriteria ? { successCriteria: goal.successCriteria } : {},
6730
- version: goal.version,
6731
- actor: "scheduled_task",
6732
- replaced: false
6733
- }
6734
- }
6735
- ] : [],
6736
- { type: "session.status.changed", payload: { status: session.status } }
6737
- ]);
6738
- const scheduledUpdate = await addSessionSystemUpdateWithSourceMutation(
7618
+ const acceptedRigDefaultVariableSets = acceptedRig?.activeVersion ? await Promise.all(
7619
+ acceptedRig.activeVersion.defaultVariableSetIds.map(async (variableSetId) => {
7620
+ const variableSet = await getVariableSet(
6739
7621
  db,
6740
7622
  {
6741
7623
  accountId: task.accountId,
6742
7624
  workspaceId: task.workspaceId,
6743
- sessionId: session.id,
6744
- kind: "scheduled_occurrence",
6745
- classification: "info",
6746
- sourceId: run.id,
6747
- dedupeKey: `scheduled-wake:${run.id}`,
6748
- summary: task.agentConfig.prompt,
6749
- payload: scheduledUserMessagePayload(
6750
- task.agentConfig.prompt,
6751
- task.agentConfig.resources,
6752
- taskTools,
6753
- task.id,
6754
- run.id
6755
- ),
6756
- lineage: {
6757
- scheduledTaskId: task.id,
6758
- scheduledTaskRunId: run.id,
6759
- ...taskXaiProviderAccountAuthoritySnapshot.scope === "user" && task.createdBy.kind === "subject" ? { xaiAuthoritySubjectId: task.createdBy.subjectId } : {}
6760
- },
6761
- personalConnectionDelegations: taskPersonalConnectionDelegations,
6762
- xaiProviderAccountAuthoritySnapshot: taskXaiProviderAccountAuthoritySnapshot
7625
+ subjectId: taskAuthoritySubjectId ?? task.createdBy.subjectId
6763
7626
  },
6764
- async (tx, wakeEventId) => {
6765
- if (!wakeEventId) throw new Error("Scheduled delivery has no wake event");
6766
- await settleScheduledTaskRunInTransaction(tx, {
6767
- workspaceId: task.workspaceId,
6768
- runId: run.id,
6769
- sessionId: session.id,
6770
- triggerEventId: wakeEventId,
6771
- status: "dispatched"
6772
- });
6773
- }
7627
+ variableSetId
6774
7628
  );
6775
- if (scheduledUpdate.reason === "session_cancelled") {
6776
- throw new Error("new scheduled session was cancelled during dispatch");
7629
+ if (!variableSet) {
7630
+ throw new Error(`rig default Variable Set not found: ${variableSetId}`);
6777
7631
  }
6778
- if (scheduledUpdate.added && scheduledUpdate.events.length > 0) {
6779
- await publishDurableSessionEvents(
6780
- bus,
6781
- task.workspaceId,
6782
- session.id,
6783
- scheduledUpdate.events
6784
- );
7632
+ return { id: variableSet.id, generation: variableSet.generation };
7633
+ })
7634
+ ) : [];
7635
+ let incidentPreflightRequired = incidentDeclaration.action === "required";
7636
+ if (incidentPreflightRequired) {
7637
+ const existingSessionId = task.runMode === "existing_session" ? task.targetSessionId : task.reusableSessionId;
7638
+ const canonicalAlertSession = task.runMode === "new_session_per_run" && alertOccurrence ? await getSessionByCreateIdempotencyKey(
7639
+ db,
7640
+ task.workspaceId,
7641
+ alertResponderSessionCreateIdempotencyKey
7642
+ ) : null;
7643
+ const responderSession = existingSessionId ? await requireSession(db, task.workspaceId, existingSessionId) : canonicalAlertSession;
7644
+ const responder = await resolveIncidentTelemetryResponderMetadata({
7645
+ db,
7646
+ settings,
7647
+ task,
7648
+ session: responderSession,
7649
+ personalConnectionDelegations: taskPersonalConnectionDelegations,
7650
+ personalResourceAuthoritySubjectId: taskAuthoritySubjectId,
7651
+ executionPolicy: targetSessionExecution ? {
7652
+ tools: targetSessionExecution.tools,
7653
+ firstPartyMcpTools: targetSessionExecution.firstPartyMcpTools,
7654
+ firstPartyMcpPermissions: targetSessionExecution.firstPartyMcpPermissions,
7655
+ variableSetId: targetSessionExecution.variableSetId,
7656
+ rigId: targetSessionExecution.rigId,
7657
+ rigVersionId: targetSessionExecution.rigVersionId,
7658
+ toolPolicy: targetSessionExecution.toolPolicy,
7659
+ mcpServerIds: targetSessionExecution.mcpServerIds,
7660
+ toolPolicyVersion: targetSessionExecution.toolPolicyVersion
7661
+ } : {
7662
+ tools: taskTools,
7663
+ firstPartyMcpTools,
7664
+ firstPartyMcpPermissions: [...DEFAULT_FIRST_PARTY_MCP_PERMISSIONS],
7665
+ variableSetId: acceptedVariableSet?.id ?? null,
7666
+ rigId: acceptedRig?.id ?? null,
7667
+ rigVersionId: acceptedRig?.activeVersion?.id ?? null,
7668
+ toolPolicy: responderSession?.toolPolicy ?? {
7669
+ mode: "explicit",
7670
+ inheritedFromSessionId: null
7671
+ },
7672
+ mcpServerIds: [],
7673
+ toolPolicyVersion: responderSession?.toolPolicyVersion ?? null
7674
+ }
7675
+ });
7676
+ const preflight = evaluateIncidentTelemetryPreflight({
7677
+ agentConfig: task.agentConfig,
7678
+ incidentTriggered: structuredAlertOccurrence !== null,
7679
+ alertOccurrenceLabels: structuredAlertOccurrence?.labels ?? null,
7680
+ responder
7681
+ });
7682
+ if (preflight.action === "blocked") return preflight;
7683
+ incidentPreflightRequired = preflight.action === "ready";
7684
+ }
7685
+ const taskXaiProviderAccountAuthoritySnapshot = await getScheduledTaskXaiProviderAccountAuthoritySnapshot(db, task.workspaceId, task.id);
7686
+ const slackBotConnection = task.agentConfig.slackBotConnectionId ? await requireOpenGeniSlackBotConnection(
7687
+ db,
7688
+ task.workspaceId,
7689
+ task.agentConfig.slackBotConnectionId
7690
+ ) : null;
7691
+ const slackBotMetadata = slackBotConnection ? openGeniSlackBotMetadata(slackBotConnection.metadata) : null;
7692
+ if (slackBotConnection && !slackBotMetadata) {
7693
+ throw new Error("OpenGeni Slack bot connection metadata is invalid");
7694
+ }
7695
+ const xaiAuthoritySubjectId = taskXaiProviderAccountAuthoritySnapshot.scope === "user" && task.createdBy.kind === "subject" ? task.createdBy.subjectId : null;
7696
+ if (taskXaiProviderAccountAuthoritySnapshot.scope === "user" && !xaiAuthoritySubjectId) {
7697
+ throw new Error("scheduled user-scoped xAI authority has no causal human");
7698
+ }
7699
+ if (xaiAuthoritySubjectId && taskAuthoritySubjectId && xaiAuthoritySubjectId !== taskAuthoritySubjectId) {
7700
+ throw new Error("scheduled authority classes have different causal humans");
7701
+ }
7702
+ const causalHumanSubjectId = taskAuthoritySubjectId ?? xaiAuthoritySubjectId;
7703
+ if ((acceptedVariableSet?.scope === "user" || acceptedRig?.scope === "user") && !causalHumanSubjectId) {
7704
+ throw new Error("scheduled personal-resource execution has no causal human");
7705
+ }
7706
+ const admissionDenial = await agentRunAdmissionDenial(service, {
7707
+ accountId: task.accountId,
7708
+ workspaceId: task.workspaceId,
7709
+ model: targetSessionExecution?.model ?? model,
7710
+ requestedAgentRuns: input.agentRunUsageIdempotencyKey ? 0 : 1
7711
+ });
7712
+ if (admissionDenial) {
7713
+ return { action: "blocked", reason: admissionDenial };
7714
+ }
7715
+ const deferredEvents = [];
7716
+ const executeDispatch = async (dispatchDb, deferPublications) => {
7717
+ const admittedRunId = crypto.randomUUID();
7718
+ const generatedSessionBinding = generatedTarget ? {
7719
+ createIdempotencyKey: alertResponderSessionCreateIdempotencyKey ?? (task.runMode === "reusable_session" ? `scheduled-task-reusable:${task.id}:${task.authorityRevision}:${task.executionDigest}` : `scheduled-task-run:${admittedRunId}`),
7720
+ effectiveMaxNestedAgentDepth: generatedSessionDepthPolicy.effectiveMaxNestedAgentDepth,
7721
+ nestedAgentDepthPolicySource: generatedSessionDepthPolicy.nestedAgentDepthPolicySource,
7722
+ codexCompactionMode: generatedSessionDepthPolicy.codexCompactionMode
7723
+ } : null;
7724
+ const acceptedExecution = {
7725
+ version: 1,
7726
+ task,
7727
+ resolvedModel: model,
7728
+ resolvedReasoningEffort: reasoningEffort,
7729
+ resolvedLatencyMode: "standard",
7730
+ resolvedSandboxBackend: sandboxBackend,
7731
+ resolvedSandboxOs: "linux",
7732
+ resolvedTools: taskTools,
7733
+ resolvedFirstPartyMcpTools: firstPartyMcpTools,
7734
+ resolvedFirstPartyMcpPermissions: [...DEFAULT_FIRST_PARTY_MCP_PERMISSIONS],
7735
+ resolvedVariableSet: acceptedVariableSet ? { id: acceptedVariableSet.id, generation: acceptedVariableSet.generation } : null,
7736
+ resolvedRig: acceptedRig?.activeVersion ? {
7737
+ id: acceptedRig.id,
7738
+ versionId: acceptedRig.activeVersion.id,
7739
+ defaultVariableSets: acceptedRigDefaultVariableSets
7740
+ } : null,
7741
+ resolvedSlackBotConnection: slackBotConnection && slackBotMetadata && slackBotConnection.verifiedInstallVersion ? {
7742
+ id: slackBotConnection.id,
7743
+ version: slackBotConnection.version,
7744
+ verifiedInstallVersion: slackBotConnection.verifiedInstallVersion,
7745
+ metadata: slackBotMetadata
7746
+ } : null,
7747
+ targetSessionExecution,
7748
+ generatedSessionBinding,
7749
+ personalConnectionDelegations: taskPersonalConnectionDelegations,
7750
+ personalResourceAuthoritySubjectId: taskPersonalResourceAuthoritySubjectId,
7751
+ causalHumanSubjectId,
7752
+ causalHumanAuthority: taskRevisionAuthority,
7753
+ xaiProviderAccountAuthoritySnapshot: taskXaiProviderAccountAuthoritySnapshot,
7754
+ xaiAuthoritySubjectId,
7755
+ connectionAuthoritySubjectId: taskConnectionAuthoritySubjectId,
7756
+ triggerInitiator: input.initiator ?? { kind: "service", subjectId: "scheduler" },
7757
+ agentRunUsageIdempotencyKey: input.agentRunUsageIdempotencyKey ?? null,
7758
+ incidentPreflightRequired,
7759
+ alertOccurrenceLabels: structuredAlertOccurrence?.labels ?? null
7760
+ };
7761
+ const acceptedExecutionParse = ScheduledTaskRunAcceptedExecution.safeParse(acceptedExecution);
7762
+ if (!acceptedExecutionParse.success || scheduledOccurrencePayloadUtf8Bytes({
7763
+ prompt: task.agentConfig.prompt,
7764
+ resources: task.agentConfig.resources,
7765
+ tools: taskTools
7766
+ }) > SCHEDULED_TASK_OCCURRENCE_PAYLOAD_MAX_BYTES) {
7767
+ return {
7768
+ kind: "blocked",
7769
+ result: {
7770
+ action: "blocked",
7771
+ reason: "scheduled_execution_unrepresentable"
7772
+ }
7773
+ };
7774
+ }
7775
+ let run = await createScheduledTaskRun(dispatchDb, {
7776
+ runId: admittedRunId,
7777
+ workspaceId: task.workspaceId,
7778
+ taskId: task.id,
7779
+ taskAuthorityRevision: task.authorityRevision,
7780
+ taskExecutionDigest: task.executionDigest,
7781
+ triggerType: input.triggerType,
7782
+ producerKey: stableProducerKey,
7783
+ scheduledAt: null,
7784
+ acceptedExecutionSnapshot: acceptedExecution
7785
+ });
7786
+ await recordScheduledTaskFiredUsage(dispatchDb, acceptedExecution, run);
7787
+ if (run.status === "failed" && run.error === "scheduled_authority_exhausted") {
7788
+ return {
7789
+ kind: "blocked",
7790
+ result: {
7791
+ action: "blocked",
7792
+ reason: "scheduled_authority_exhausted"
7793
+ },
7794
+ run
7795
+ };
7796
+ }
7797
+ if (run.status === "failed" || run.status === "skipped") {
7798
+ return {
7799
+ kind: "blocked",
7800
+ result: scheduledRunTerminalResult(run.error),
7801
+ run
7802
+ };
7803
+ }
7804
+ const runPersonalResourceAuthority = await getScheduledTaskRunPersonalResourceAuthority(
7805
+ dispatchDb,
7806
+ {
7807
+ accountId: task.accountId,
7808
+ workspaceId: task.workspaceId,
7809
+ runId: run.id
6785
7810
  }
6786
- result = {
6787
- action: "start",
7811
+ );
7812
+ if (run?.status === "dispatched" && run.sessionId && run.triggerEventId) {
7813
+ if (deferPublications) {
7814
+ return { kind: "replay", run, acceptedExecution };
7815
+ }
7816
+ await recordUsageEvent3(dispatchDb, {
7817
+ accountId: task.accountId,
7818
+ workspaceId: task.workspaceId,
7819
+ eventType: "agent_run.created",
7820
+ quantity: 1,
7821
+ unit: "run",
7822
+ sourceResourceType: "scheduled_task_run",
7823
+ sourceResourceId: run.id,
7824
+ sessionId: run.sessionId,
7825
+ initiator: acceptedExecution.triggerInitiator,
7826
+ initiatorContext: {
7827
+ scheduledTaskId: task.id,
7828
+ scheduledTaskRunId: run.id
7829
+ },
7830
+ origin: "scheduled_task",
7831
+ idempotencyKey: acceptedExecution.agentRunUsageIdempotencyKey ?? `usage:agent_run.created:scheduled:${run.id}`
7832
+ });
7833
+ const workflowId = workflowIdForSession(run.sessionId);
7834
+ const workflowWakeRevision = await enqueueSessionWorkflowWakeIfRunnable(dispatchDb, {
7835
+ accountId: task.accountId,
7836
+ workspaceId: task.workspaceId,
7837
+ sessionId: run.sessionId,
7838
+ temporalWorkflowId: workflowId,
7839
+ reason: "scheduled_retry"
7840
+ });
7841
+ const result3 = {
7842
+ action: task.runMode === "new_session_per_run" ? "start" : "signal",
6788
7843
  accountId: task.accountId,
6789
7844
  workspaceId: task.workspaceId,
6790
- sessionId: session.id,
6791
- triggerEventId: scheduledUpdate.wakeEventId,
7845
+ sessionId: run.sessionId,
7846
+ triggerEventId: run.triggerEventId,
6792
7847
  workflowId,
6793
- workflowWakeRevision: scheduledUpdate.workflowWakeRevision
7848
+ workflowWakeRevision
6794
7849
  };
6795
- } else {
6796
- const session = await requireSession(db, task.workspaceId, existingSessionId);
6797
- assertReusableSessionRevivable(session.status);
6798
- if ((session.variableSetId ?? null) !== (task.variableSetId ?? null)) {
6799
- throw new Error(
6800
- "scheduled task variableSet attachment does not match its reusable session"
7850
+ return { kind: "complete", result: result3, run };
7851
+ }
7852
+ let result2;
7853
+ try {
7854
+ if (incidentPreflightRequired) {
7855
+ const exactAuthority = await requireScheduledTaskIncidentAuthorityInTransaction(
7856
+ dispatchDb,
7857
+ {
7858
+ workspaceId: task.workspaceId,
7859
+ taskId: task.id
7860
+ }
6801
7861
  );
7862
+ if (stableJson({
7863
+ task: exactAuthority.task,
7864
+ personalConnectionDelegations: exactAuthority.personalConnectionDelegations
7865
+ }) !== stableJson({
7866
+ task,
7867
+ personalConnectionDelegations: taskPersonalConnectionDelegations
7868
+ })) {
7869
+ throw new IncidentTelemetryPreflightBlockedError(
7870
+ "incident_preflight_metadata_missing"
7871
+ );
7872
+ }
6802
7873
  }
6803
- if (scheduledSlackBotConnectionId(session.metadata) !== (task.agentConfig.slackBotConnectionId ?? null)) {
6804
- throw new Error(
6805
- "scheduled task OpenGeni Slack bot binding does not match its reusable session"
6806
- );
7874
+ const goalSpec = task.agentConfig.goal ?? null;
7875
+ const existingSessionId = task.runMode === "existing_session" ? task.targetSessionId : task.reusableSessionId;
7876
+ if (task.runMode === "existing_session" && !existingSessionId) {
7877
+ throw new Error("scheduled task target session is unavailable");
6807
7878
  }
6808
- const reusableGoal = task.runMode === "reusable_session" && goalSpec && run.status === "queued" ? await upsertSessionGoal(db, {
6809
- accountId: task.accountId,
6810
- workspaceId: task.workspaceId,
6811
- sessionId: session.id,
6812
- text: goalSpec.text,
6813
- successCriteria: goalSpec.successCriteria ?? null,
6814
- maxAutoContinuations: goalSpec.maxAutoContinuations ?? null,
6815
- createdBy: "scheduled_task"
6816
- }) : null;
6817
- if (reusableGoal) {
6818
- const goalEvents = await appendSessionEventsWithLockedSessionUpdate(
6819
- db,
6820
- task.workspaceId,
6821
- session.id,
6822
- (locked) => {
6823
- assertReusableSessionRevivable(locked.status);
6824
- return {
6825
- events: [
6826
- {
6827
- type: "goal.set",
6828
- payload: {
6829
- goalId: reusableGoal.goal.id,
6830
- text: reusableGoal.goal.text,
6831
- ...reusableGoal.goal.successCriteria ? { successCriteria: reusableGoal.goal.successCriteria } : {},
6832
- version: reusableGoal.goal.version,
6833
- actor: "scheduled_task",
6834
- replaced: reusableGoal.replaced
7879
+ if (task.runMode === "new_session_per_run" || task.runMode === "reusable_session" && !existingSessionId) {
7880
+ const variableSet = acceptedVariableSet;
7881
+ if (task.variableSetId && !variableSet) {
7882
+ throw new Error(`variable set not found: ${task.variableSetId}`);
7883
+ }
7884
+ let frozenRigId = null;
7885
+ let frozenRigVersionId = null;
7886
+ if (task.rigId) {
7887
+ const rig = acceptedRig;
7888
+ if (!rig || !rig.activeVersion) {
7889
+ throw new Error(`rig has no active version to bind: ${task.rigId}`);
7890
+ }
7891
+ frozenRigId = rig.id;
7892
+ frozenRigVersionId = runPersonalResourceAuthority?.resources.find(
7893
+ (resource) => resource.resourceKind === "rig" && resource.resourceId === rig.id
7894
+ )?.resourceVersionId ?? rig.activeVersion.id;
7895
+ if (frozenRigVersionId !== rig.activeVersion.id) {
7896
+ throw new Error("scheduled run rig version changed during admission");
7897
+ }
7898
+ }
7899
+ let session;
7900
+ let sessionCreated = true;
7901
+ const taskMetadata = { ...task.agentConfig.metadata };
7902
+ delete taskMetadata[OPENGENI_SLACK_BOT_SESSION_METADATA_KEY];
7903
+ try {
7904
+ const sessionInput = {
7905
+ accountId: task.accountId,
7906
+ workspaceId: task.workspaceId,
7907
+ initialMessage: task.agentConfig.prompt,
7908
+ resources: task.agentConfig.resources,
7909
+ tools: taskTools,
7910
+ firstPartyMcpTools,
7911
+ firstPartyMcpPermissions: [...DEFAULT_FIRST_PARTY_MCP_PERMISSIONS],
7912
+ metadata: {
7913
+ ...taskMetadata,
7914
+ model,
7915
+ reasoningEffort,
7916
+ scheduledTaskId: task.id,
7917
+ scheduledTaskRunMode: task.runMode,
7918
+ ...goalSpec ? { scheduledTaskGoal: goalSpec } : {},
7919
+ ...run ? { scheduledTaskRunId: run.id } : {},
7920
+ ...slackBotConnection ? {
7921
+ [OPENGENI_SLACK_BOT_SESSION_METADATA_KEY]: slackBotConnection.id
7922
+ } : {}
7923
+ },
7924
+ createdBy: {
7925
+ kind: "service",
7926
+ subjectId: "scheduler",
7927
+ label: "OpenGeni scheduler"
7928
+ },
7929
+ createdByContext: {
7930
+ scheduledTaskId: task.id,
7931
+ ...run ? { scheduledTaskRunId: run.id } : {}
7932
+ },
7933
+ model,
7934
+ reasoningEffort,
7935
+ latencyMode: "standard",
7936
+ sandboxBackend,
7937
+ sandboxOs: "linux",
7938
+ variableSetId: task.variableSetId ?? null,
7939
+ rigId: frozenRigId,
7940
+ rigVersionId: frozenRigVersionId,
7941
+ personalConnectionDelegations: [],
7942
+ initialXaiProviderAccountAuthoritySnapshot: taskXaiProviderAccountAuthoritySnapshot,
7943
+ maxNestedAgentDepthOverride: task.agentConfig.maxNestedAgentDepth ?? null,
7944
+ frozenNestedAgentDepthPolicy: {
7945
+ effectiveMaxNestedAgentDepth: generatedSessionBinding.effectiveMaxNestedAgentDepth,
7946
+ nestedAgentDepthPolicySource: generatedSessionBinding.nestedAgentDepthPolicySource
7947
+ },
7948
+ frozenCodexCompactionMode: generatedSessionBinding.codexCompactionMode,
7949
+ // The durable agent config was privilege-checked when the task
7950
+ // was created/updated. Preserve that explicit policy if a broader
7951
+ // workspace/deployment limit is narrowed before a later fire.
7952
+ allowNestedAgentDepthIncrease: true,
7953
+ subjectId: `scheduled_task:${task.id}`,
7954
+ beforeCreateCommit: async (tx, sessionId) => {
7955
+ if (incidentPreflightRequired) {
7956
+ const exactSession = await requireSession(tx, task.workspaceId, sessionId);
7957
+ const exactResponder = await resolveIncidentTelemetryResponderMetadata({
7958
+ db: tx,
7959
+ settings,
7960
+ task: acceptedExecution.task,
7961
+ session: exactSession,
7962
+ personalConnectionDelegations: acceptedExecution.personalConnectionDelegations,
7963
+ personalResourceAuthoritySubjectId: acceptedExecution.personalResourceAuthoritySubjectId,
7964
+ executionPolicy: {
7965
+ tools: exactSession.tools,
7966
+ firstPartyMcpTools: exactSession.firstPartyMcpTools,
7967
+ firstPartyMcpPermissions: exactSession.firstPartyMcpPermissions,
7968
+ variableSetId: exactSession.variableSetId,
7969
+ rigId: exactSession.rigId,
7970
+ rigVersionId: exactSession.rigVersionId,
7971
+ toolPolicy: exactSession.toolPolicy,
7972
+ mcpServerIds: exactSession.mcpServers.map((server) => server.id),
7973
+ toolPolicyVersion: exactSession.toolPolicyVersion
6835
7974
  }
7975
+ });
7976
+ const exactPreflight = evaluateIncidentTelemetryPreflight({
7977
+ agentConfig: acceptedExecution.task.agentConfig,
7978
+ incidentTriggered: structuredAlertOccurrence !== null,
7979
+ alertOccurrenceLabels: structuredAlertOccurrence?.labels ?? null,
7980
+ responder: exactResponder
7981
+ });
7982
+ if (exactPreflight.action === "blocked") {
7983
+ throw new IncidentTelemetryPreflightBlockedError(exactPreflight.reason);
7984
+ }
7985
+ }
7986
+ await bindScheduledTaskRunSessionInTransaction(tx, {
7987
+ accountId: task.accountId,
7988
+ workspaceId: task.workspaceId,
7989
+ runId: run.id,
7990
+ sessionId
7991
+ });
7992
+ }
7993
+ };
7994
+ const created = await createSessionWithIdempotencyKeyResult(dispatchDb, {
7995
+ ...sessionInput,
7996
+ createIdempotencyKey: generatedSessionBinding.createIdempotencyKey
7997
+ });
7998
+ if (created.denied) {
7999
+ throw new SessionSpawnDeniedDbError(created.denial);
8000
+ }
8001
+ session = created.session;
8002
+ sessionCreated = created.created;
8003
+ if (!sessionCreated) {
8004
+ await bindScheduledTaskRunSessionInTransaction(dispatchDb, {
8005
+ accountId: task.accountId,
8006
+ workspaceId: task.workspaceId,
8007
+ runId: run.id,
8008
+ sessionId: session.id
8009
+ });
8010
+ run = { ...run, sessionId: session.id };
8011
+ }
8012
+ } catch (error) {
8013
+ if (error instanceof IncidentTelemetryPreflightBlockedError) {
8014
+ throw error;
8015
+ }
8016
+ if (error instanceof SessionSpawnDeniedDbError) {
8017
+ throw new Error(`${error.denial.code}: denial=${error.denial.id}`, {
8018
+ cause: error
8019
+ });
8020
+ }
8021
+ throw error;
8022
+ }
8023
+ const scheduledRun = run;
8024
+ if (!sessionCreated) {
8025
+ assertReusableSessionRevivable(session.status);
8026
+ if ((session.variableSetId ?? null) !== (task.variableSetId ?? null)) {
8027
+ throw new Error(
8028
+ "scheduled alert occurrence variableSet attachment does not match its canonical session"
8029
+ );
8030
+ }
8031
+ if (scheduledSlackBotConnectionId(session.metadata) !== (task.agentConfig.slackBotConnectionId ?? null)) {
8032
+ throw new Error(
8033
+ "scheduled alert occurrence OpenGeni Slack bot binding does not match its canonical session"
8034
+ );
8035
+ }
8036
+ }
8037
+ let workflowId;
8038
+ if (alertOccurrence) {
8039
+ const started = await initializeSessionStartAtomically(dispatchDb, {
8040
+ accountId: task.accountId,
8041
+ workspaceId: task.workspaceId,
8042
+ sessionId: session.id,
8043
+ reasoningEffortFallback: reasoningEffort,
8044
+ createdEventPayload: {
8045
+ scheduledTaskId: typeof session.metadata.scheduledTaskId === "string" ? session.metadata.scheduledTaskId : task.id,
8046
+ scheduledTaskRunId: typeof session.metadata.scheduledTaskRunId === "string" ? session.metadata.scheduledTaskRunId : run.id,
8047
+ // Names/ids only; never values.
8048
+ ...variableSet ? {
8049
+ variableSetId: variableSet.id,
8050
+ variableSetName: variableSet.name
8051
+ } : {},
8052
+ ...slackBotConnection && slackBotMetadata ? {
8053
+ slackBotConnection: {
8054
+ credentialRole: OPENGENI_SLACK_BOT_CREDENTIAL_ROLE,
8055
+ credentialLabel: OPENGENI_SLACK_BOT_CREDENTIAL_LABEL,
8056
+ connectionId: slackBotConnection.id,
8057
+ slackTeamId: slackBotMetadata.slackTeamId
6836
8058
  }
6837
- ]
6838
- };
8059
+ } : {}
8060
+ },
8061
+ goal: goalSpec ? {
8062
+ text: goalSpec.text,
8063
+ successCriteria: goalSpec.successCriteria ?? null,
8064
+ maxAutoContinuations: goalSpec.maxAutoContinuations ?? null,
8065
+ ...goalSpec.mutationPolicy ? { mutationPolicy: goalSpec.mutationPolicy } : {},
8066
+ createdBy: "scheduled_task"
8067
+ } : null,
8068
+ deferInitialTurn: true,
8069
+ deferredStatus: "queued"
8070
+ });
8071
+ workflowId = started.temporalWorkflowId;
8072
+ if (started.events.length > 0) {
8073
+ if (deferPublications) {
8074
+ deferredEvents.push({
8075
+ sessionId: session.id,
8076
+ events: started.events
8077
+ });
8078
+ } else {
8079
+ await publishDurableSessionEvents(
8080
+ bus,
8081
+ task.workspaceId,
8082
+ session.id,
8083
+ started.events
8084
+ );
8085
+ }
8086
+ }
8087
+ } else {
8088
+ const started = await initializeSessionStartAtomically(dispatchDb, {
8089
+ accountId: task.accountId,
8090
+ workspaceId: task.workspaceId,
8091
+ sessionId: session.id,
8092
+ reasoningEffortFallback: reasoningEffort,
8093
+ createdEventPayload: {
8094
+ scheduledTaskId: typeof session.metadata.scheduledTaskId === "string" ? session.metadata.scheduledTaskId : task.id,
8095
+ scheduledTaskRunId: typeof session.metadata.scheduledTaskRunId === "string" ? session.metadata.scheduledTaskRunId : run.id,
8096
+ // Names/ids only; never values.
8097
+ ...variableSet ? { variableSetId: variableSet.id, variableSetName: variableSet.name } : {},
8098
+ ...slackBotConnection && slackBotMetadata ? {
8099
+ slackBotConnection: {
8100
+ credentialRole: OPENGENI_SLACK_BOT_CREDENTIAL_ROLE,
8101
+ credentialLabel: OPENGENI_SLACK_BOT_CREDENTIAL_LABEL,
8102
+ connectionId: slackBotConnection.id,
8103
+ slackTeamId: slackBotMetadata.slackTeamId
8104
+ }
8105
+ } : {}
8106
+ },
8107
+ goal: goalSpec ? {
8108
+ text: goalSpec.text,
8109
+ successCriteria: goalSpec.successCriteria ?? null,
8110
+ maxAutoContinuations: goalSpec.maxAutoContinuations ?? null,
8111
+ ...goalSpec.mutationPolicy ? { mutationPolicy: goalSpec.mutationPolicy } : {},
8112
+ createdBy: "scheduled_task"
8113
+ } : null,
8114
+ deferInitialTurn: true,
8115
+ deferredStatus: "queued"
8116
+ });
8117
+ workflowId = started.temporalWorkflowId;
8118
+ if (started.events.length > 0) {
8119
+ if (deferPublications) {
8120
+ deferredEvents.push({ sessionId: session.id, events: started.events });
8121
+ } else {
8122
+ await publishDurableSessionEvents(
8123
+ bus,
8124
+ task.workspaceId,
8125
+ session.id,
8126
+ started.events
8127
+ );
8128
+ }
8129
+ }
8130
+ if (task.runMode === "reusable_session") {
8131
+ await materializeScheduledTaskReusableSessionFromRun(dispatchDb, {
8132
+ accountId: task.accountId,
8133
+ workspaceId: task.workspaceId,
8134
+ taskId: task.id,
8135
+ runId: run.id,
8136
+ sessionId: session.id,
8137
+ sourceTaskAuthorityRevision: runPersonalResourceAuthority?.taskAuthorityRevision ?? run.taskAuthorityRevision,
8138
+ sourceExecutionDigest: runPersonalResourceAuthority?.executionDigest ?? run.taskExecutionDigest
8139
+ });
8140
+ }
8141
+ }
8142
+ const titleEvents = await stampScheduledSessionTitle(dispatchDb, {
8143
+ workspaceId: task.workspaceId,
8144
+ session,
8145
+ taskName: task.name
8146
+ });
8147
+ if (titleEvents.length > 0) {
8148
+ if (deferPublications) {
8149
+ deferredEvents.push({ sessionId: session.id, events: titleEvents });
8150
+ } else {
8151
+ await publishDurableSessionEvents(bus, task.workspaceId, session.id, titleEvents);
8152
+ }
8153
+ }
8154
+ const scheduledUpdate = await addSessionSystemUpdateWithSourceMutation(
8155
+ dispatchDb,
8156
+ {
8157
+ accountId: task.accountId,
8158
+ workspaceId: task.workspaceId,
8159
+ sessionId: session.id,
8160
+ kind: "scheduled_occurrence",
8161
+ classification: "info",
8162
+ sourceId: run.id,
8163
+ dedupeKey: `scheduled-task-run:${run.id}`,
8164
+ summary: task.agentConfig.prompt,
8165
+ payload: scheduledUserMessagePayload(
8166
+ task.agentConfig.prompt,
8167
+ task.agentConfig.resources,
8168
+ taskTools,
8169
+ task.id,
8170
+ run.id
8171
+ ),
8172
+ lineage: {
8173
+ scheduledTaskId: task.id,
8174
+ scheduledTaskRunId: run.id,
8175
+ ...causalHumanSubjectId ? { causalHumanSubjectId } : {},
8176
+ ...taskConnectionAuthoritySubjectId ? { connectionAuthoritySubjectId: taskConnectionAuthoritySubjectId } : {},
8177
+ ...taskXaiProviderAccountAuthoritySnapshot.scope === "user" && task.createdBy.kind === "subject" ? { xaiAuthoritySubjectId: task.createdBy.subjectId } : {}
8178
+ },
8179
+ personalConnectionDelegations: taskPersonalConnectionDelegations,
8180
+ xaiProviderAccountAuthoritySnapshot: taskXaiProviderAccountAuthoritySnapshot,
8181
+ scheduledTaskRunId: run.id
8182
+ },
8183
+ async (tx, wakeEventId) => {
8184
+ if (!wakeEventId) {
8185
+ await settleScheduledTaskRunInTransaction(tx, {
8186
+ workspaceId: task.workspaceId,
8187
+ runId: scheduledRun.id,
8188
+ sessionId: session.id,
8189
+ triggerEventId: null,
8190
+ status: "skipped",
8191
+ error: "session_cancelled"
8192
+ });
8193
+ return;
8194
+ }
8195
+ await settleScheduledTaskRunInTransaction(tx, {
8196
+ workspaceId: task.workspaceId,
8197
+ runId: scheduledRun.id,
8198
+ sessionId: session.id,
8199
+ triggerEventId: wakeEventId,
8200
+ status: "dispatched"
8201
+ });
6839
8202
  },
6840
- { activity: "semantic" }
8203
+ incidentPreflightRequired ? {
8204
+ prepareSource: async (tx, sessionId) => await prepareIncidentTelemetrySource({
8205
+ tx,
8206
+ settings,
8207
+ taskId: task.id,
8208
+ workspaceId: task.workspaceId,
8209
+ sessionId,
8210
+ alertOccurrenceLabels: structuredAlertOccurrence?.labels ?? null,
8211
+ acceptedExecution
8212
+ })
8213
+ } : void 0
6841
8214
  );
6842
- await publishDurableSessionEvents(bus, task.workspaceId, session.id, goalEvents);
6843
- }
6844
- const bundled = await addSessionSystemUpdateWithSourceMutation(
6845
- db,
6846
- {
8215
+ if (scheduledUpdate.reason === "session_cancelled") {
8216
+ return {
8217
+ kind: "blocked",
8218
+ result: { action: "blocked", reason: "scheduled_run_terminal" },
8219
+ run: { ...run, status: "skipped", error: "session_cancelled" }
8220
+ };
8221
+ }
8222
+ if (scheduledUpdate.added && scheduledUpdate.events.length > 0) {
8223
+ if (deferPublications) {
8224
+ deferredEvents.push({
8225
+ sessionId: session.id,
8226
+ events: scheduledUpdate.events
8227
+ });
8228
+ } else {
8229
+ await publishDurableSessionEvents(
8230
+ bus,
8231
+ task.workspaceId,
8232
+ session.id,
8233
+ scheduledUpdate.events
8234
+ );
8235
+ }
8236
+ }
8237
+ result2 = {
8238
+ action: sessionCreated ? "start" : "signal",
6847
8239
  accountId: task.accountId,
6848
8240
  workspaceId: task.workspaceId,
6849
8241
  sessionId: session.id,
6850
- kind: "scheduled_occurrence",
6851
- classification: "info",
6852
- sourceId: run.id,
6853
- dedupeKey: `scheduled-wake:${run.id}`,
6854
- summary: task.agentConfig.prompt,
6855
- payload: scheduledUserMessagePayload(
6856
- task.agentConfig.prompt,
6857
- task.agentConfig.resources,
6858
- taskTools,
6859
- task.id,
6860
- run.id
6861
- ),
6862
- lineage: {
6863
- scheduledTaskId: task.id,
6864
- scheduledTaskRunId: run.id,
6865
- ...taskXaiProviderAccountAuthoritySnapshot.scope === "user" && task.createdBy.kind === "subject" ? { xaiAuthoritySubjectId: task.createdBy.subjectId } : {}
8242
+ triggerEventId: scheduledUpdate.wakeEventId,
8243
+ workflowId,
8244
+ workflowWakeRevision: scheduledUpdate.workflowWakeRevision
8245
+ };
8246
+ } else {
8247
+ if (!run) throw new Error("scheduled task run was not created");
8248
+ const scheduledRun = run;
8249
+ const session = await requireSession(dispatchDb, task.workspaceId, existingSessionId);
8250
+ await bindScheduledTaskRunSessionInTransaction(dispatchDb, {
8251
+ accountId: task.accountId,
8252
+ workspaceId: task.workspaceId,
8253
+ runId: run.id,
8254
+ sessionId: session.id
8255
+ });
8256
+ assertReusableSessionRevivable(session.status);
8257
+ assertReusableSessionBindingMatches(session, task);
8258
+ const goalEvents = task.runMode === "reusable_session" && goalSpec && run.status === "queued" ? await upsertScheduledSessionGoalForRun(dispatchDb, {
8259
+ accountId: task.accountId,
8260
+ workspaceId: task.workspaceId,
8261
+ sessionId: session.id,
8262
+ runId: run.id,
8263
+ text: goalSpec.text,
8264
+ successCriteria: goalSpec.successCriteria ?? null,
8265
+ maxAutoContinuations: goalSpec.maxAutoContinuations ?? null,
8266
+ ...goalSpec.mutationPolicy ? { mutationPolicy: goalSpec.mutationPolicy } : {}
8267
+ }) : [];
8268
+ if (goalEvents.length > 0) {
8269
+ if (deferPublications) {
8270
+ deferredEvents.push({
8271
+ sessionId: session.id,
8272
+ events: goalEvents
8273
+ });
8274
+ } else {
8275
+ await publishDurableSessionEvents(bus, task.workspaceId, session.id, goalEvents);
8276
+ }
8277
+ }
8278
+ const bundled = await addSessionSystemUpdateWithSourceMutation(
8279
+ dispatchDb,
8280
+ {
8281
+ accountId: task.accountId,
8282
+ workspaceId: task.workspaceId,
8283
+ sessionId: session.id,
8284
+ kind: "scheduled_occurrence",
8285
+ classification: "info",
8286
+ sourceId: run.id,
8287
+ dedupeKey: `scheduled-task-run:${run.id}`,
8288
+ summary: task.agentConfig.prompt,
8289
+ payload: scheduledUserMessagePayload(
8290
+ task.agentConfig.prompt,
8291
+ task.agentConfig.resources,
8292
+ taskTools,
8293
+ task.id,
8294
+ run.id
8295
+ ),
8296
+ lineage: {
8297
+ scheduledTaskId: task.id,
8298
+ scheduledTaskRunId: run.id,
8299
+ ...causalHumanSubjectId ? { causalHumanSubjectId } : {},
8300
+ ...taskConnectionAuthoritySubjectId ? { connectionAuthoritySubjectId: taskConnectionAuthoritySubjectId } : {},
8301
+ ...taskXaiProviderAccountAuthoritySnapshot.scope === "user" && task.createdBy.kind === "subject" ? { xaiAuthoritySubjectId: task.createdBy.subjectId } : {}
8302
+ },
8303
+ personalConnectionDelegations: taskPersonalConnectionDelegations,
8304
+ xaiProviderAccountAuthoritySnapshot: taskXaiProviderAccountAuthoritySnapshot,
8305
+ scheduledTaskRunId: run.id
6866
8306
  },
6867
- personalConnectionDelegations: taskPersonalConnectionDelegations,
6868
- xaiProviderAccountAuthoritySnapshot: taskXaiProviderAccountAuthoritySnapshot
6869
- },
6870
- async (tx, wakeEventId) => {
6871
- if (task.runMode === "existing_session") {
6872
- await requireScheduledTaskTargetInTransaction(tx, {
8307
+ async (tx, wakeEventId) => {
8308
+ if (!wakeEventId) {
8309
+ await settleScheduledTaskRunInTransaction(tx, {
8310
+ workspaceId: task.workspaceId,
8311
+ runId: scheduledRun.id,
8312
+ sessionId: session.id,
8313
+ triggerEventId: null,
8314
+ status: "skipped",
8315
+ error: "session_cancelled"
8316
+ });
8317
+ return;
8318
+ }
8319
+ await settleScheduledTaskRunInTransaction(tx, {
6873
8320
  workspaceId: task.workspaceId,
8321
+ runId: scheduledRun.id,
8322
+ sessionId: session.id,
8323
+ triggerEventId: wakeEventId,
8324
+ status: "dispatched"
8325
+ });
8326
+ },
8327
+ incidentPreflightRequired ? {
8328
+ prepareSource: async (tx, sessionId) => await prepareIncidentTelemetrySource({
8329
+ tx,
8330
+ settings,
6874
8331
  taskId: task.id,
6875
- targetSessionId: session.id
8332
+ workspaceId: task.workspaceId,
8333
+ sessionId,
8334
+ alertOccurrenceLabels: structuredAlertOccurrence?.labels ?? null,
8335
+ acceptedExecution
8336
+ })
8337
+ } : void 0
8338
+ );
8339
+ if (bundled.reason === "session_cancelled") {
8340
+ return {
8341
+ kind: "blocked",
8342
+ result: { action: "blocked", reason: "scheduled_run_terminal" },
8343
+ run: { ...run, status: "skipped", error: "session_cancelled" }
8344
+ };
8345
+ }
8346
+ if (bundled.added && bundled.events.length > 0) {
8347
+ if (deferPublications) {
8348
+ deferredEvents.push({
8349
+ sessionId: session.id,
8350
+ events: bundled.events
6876
8351
  });
8352
+ } else {
8353
+ await publishDurableSessionEvents(
8354
+ bus,
8355
+ task.workspaceId,
8356
+ session.id,
8357
+ bundled.events
8358
+ );
6877
8359
  }
6878
- if (!wakeEventId) throw new Error("Scheduled delivery has no wake event");
6879
- await settleScheduledTaskRunInTransaction(tx, {
6880
- workspaceId: task.workspaceId,
6881
- runId: run.id,
6882
- sessionId: session.id,
6883
- triggerEventId: wakeEventId,
6884
- status: "dispatched"
6885
- });
6886
8360
  }
6887
- );
6888
- if (bundled.reason === "session_cancelled") {
6889
- throw new Error(`scheduled wake was not added: ${bundled.reason}`);
6890
- }
6891
- if (bundled.added && bundled.events.length > 0) {
6892
- await publishDurableSessionEvents(bus, task.workspaceId, session.id, bundled.events);
8361
+ result2 = {
8362
+ action: "signal",
8363
+ accountId: task.accountId,
8364
+ workspaceId: task.workspaceId,
8365
+ sessionId: session.id,
8366
+ triggerEventId: bundled.wakeEventId,
8367
+ workflowId: workflowIdForSession(session.id),
8368
+ workflowWakeRevision: bundled.workflowWakeRevision
8369
+ };
6893
8370
  }
6894
- result = {
6895
- action: "signal",
6896
- accountId: task.accountId,
6897
- workspaceId: task.workspaceId,
6898
- sessionId: session.id,
6899
- triggerEventId: bundled.wakeEventId,
6900
- workflowId: workflowIdForSession(session.id),
6901
- workflowWakeRevision: bundled.workflowWakeRevision
8371
+ } catch (error) {
8372
+ const terminalError = scheduledRunRecoveryTerminalError(error);
8373
+ if (!terminalError) throw error;
8374
+ await markScheduledTaskRunFailedIfQueued(
8375
+ dispatchDb,
8376
+ task.workspaceId,
8377
+ run.id,
8378
+ terminalError.code
8379
+ );
8380
+ return {
8381
+ kind: "blocked",
8382
+ result: scheduledRunTerminalResult(terminalError.code),
8383
+ run: { ...run, status: "failed", error: terminalError.code }
6902
8384
  };
6903
8385
  }
8386
+ await recordUsageEvent3(dispatchDb, {
8387
+ accountId: task.accountId,
8388
+ workspaceId: task.workspaceId,
8389
+ eventType: "agent_run.created",
8390
+ quantity: 1,
8391
+ unit: "run",
8392
+ sourceResourceType: "scheduled_task_run",
8393
+ sourceResourceId: run.id,
8394
+ sessionId: result2.sessionId,
8395
+ initiator: input.initiator ?? {
8396
+ kind: "service",
8397
+ subjectId: "scheduler"
8398
+ },
8399
+ initiatorContext: {
8400
+ scheduledTaskId: task.id,
8401
+ scheduledTaskRunId: run.id
8402
+ },
8403
+ origin: "scheduled_task",
8404
+ idempotencyKey: input.agentRunUsageIdempotencyKey ?? `usage:agent_run.created:scheduled:${run.id}`
8405
+ });
8406
+ return { kind: "complete", result: result2, run };
8407
+ };
8408
+ let dispatchOutcome;
8409
+ try {
8410
+ dispatchOutcome = incidentPreflightRequired ? await withSessionActivityRlsContext(
8411
+ db,
8412
+ { accountId: task.accountId, workspaceId: task.workspaceId },
8413
+ async (tx) => await executeDispatch(tx, true)
8414
+ ) : await executeDispatch(db, false);
6904
8415
  } catch (error) {
6905
- await markScheduledTaskRunFailedIfQueued(
8416
+ if (error instanceof IncidentTelemetryPreflightBlockedError) {
8417
+ return { action: "blocked", reason: error.reason };
8418
+ }
8419
+ throw error;
8420
+ }
8421
+ if (dispatchOutcome.kind === "replay") {
8422
+ return await replayScheduledTaskDispatch({
6906
8423
  db,
8424
+ wakeSessionWorkflow,
8425
+ task,
8426
+ run: dispatchOutcome.run,
8427
+ acceptedExecution: dispatchOutcome.acceptedExecution
8428
+ });
8429
+ }
8430
+ if (dispatchOutcome.kind === "blocked") return dispatchOutcome.result;
8431
+ for (const deferred of deferredEvents) {
8432
+ await publishDurableSessionEvents(
8433
+ bus,
6907
8434
  task.workspaceId,
6908
- run.id,
6909
- error instanceof Error ? error.message : String(error)
6910
- ).catch(() => void 0);
6911
- throw error;
8435
+ deferred.sessionId,
8436
+ deferred.events
8437
+ );
6912
8438
  }
6913
- await recordUsageEvent3(db, {
6914
- accountId: task.accountId,
6915
- workspaceId: task.workspaceId,
6916
- eventType: "agent_run.created",
6917
- quantity: 1,
6918
- unit: "run",
6919
- sourceResourceType: "scheduled_task_run",
6920
- sourceResourceId: run.id,
6921
- sessionId: result.sessionId,
6922
- initiator: input.initiator ?? { kind: "service", subjectId: "scheduler" },
6923
- initiatorContext: { scheduledTaskId: task.id, scheduledTaskRunId: run.id },
6924
- origin: "scheduled_task",
6925
- idempotencyKey: input.agentRunUsageIdempotencyKey ?? `usage:agent_run.created:scheduled:${run.id}`
6926
- });
8439
+ const result = dispatchOutcome.result;
6927
8440
  if (wakeSessionWorkflow && result.workflowWakeRevision !== null) {
6928
8441
  await wakeSessionWorkflow({
6929
8442
  accountId: result.accountId,
@@ -6937,6 +8450,585 @@ function createScheduledTaskActivities(services) {
6937
8450
  }
6938
8451
  };
6939
8452
  }
8453
+ var IncidentTelemetryPreflightBlockedError = class extends Error {
8454
+ constructor(reason) {
8455
+ super(reason);
8456
+ this.reason = reason;
8457
+ this.name = "IncidentTelemetryPreflightBlockedError";
8458
+ }
8459
+ };
8460
+ function assertReusableSessionBindingMatches(session, task) {
8461
+ if ((session.variableSetId ?? null) !== (task.variableSetId ?? null)) {
8462
+ throw new ScheduledRunTerminalAuthorityError(
8463
+ "scheduled_reusable_binding_changed",
8464
+ "scheduled task variableSet attachment does not match its reusable session"
8465
+ );
8466
+ }
8467
+ if (scheduledSlackBotConnectionId(session.metadata) !== (task.agentConfig.slackBotConnectionId ?? null)) {
8468
+ throw new ScheduledRunTerminalAuthorityError(
8469
+ "scheduled_reusable_binding_changed",
8470
+ "scheduled task OpenGeni Slack bot binding does not match its reusable session"
8471
+ );
8472
+ }
8473
+ }
8474
+ var ScheduledRunTerminalAuthorityError = class extends Error {
8475
+ constructor(code, message) {
8476
+ super(message);
8477
+ this.code = code;
8478
+ this.name = "ScheduledRunTerminalAuthorityError";
8479
+ }
8480
+ };
8481
+ function scheduledRunRecoveryTerminalError(error) {
8482
+ if (error instanceof Error && "cause" in error && error.cause !== void 0 && error.cause !== error) {
8483
+ const nested = scheduledRunRecoveryTerminalError(error.cause);
8484
+ if (nested) return nested;
8485
+ }
8486
+ if (error instanceof ScheduledRunTerminalAuthorityError) return error;
8487
+ if (error instanceof IncidentTelemetryPreflightBlockedError) {
8488
+ return new ScheduledRunTerminalAuthorityError(
8489
+ `scheduled_incident_${error.reason}`,
8490
+ error.message
8491
+ );
8492
+ }
8493
+ if (error instanceof SessionSpawnDeniedDbError) {
8494
+ return new ScheduledRunTerminalAuthorityError("scheduled_session_spawn_denied", error.message);
8495
+ }
8496
+ if (error instanceof Error && error.message.startsWith("Session not found:")) {
8497
+ return new ScheduledRunTerminalAuthorityError(
8498
+ "scheduled_target_session_unavailable",
8499
+ error.message
8500
+ );
8501
+ }
8502
+ if (typeof error === "object" && error !== null && "status" in error && error.status === 422) {
8503
+ return new ScheduledRunTerminalAuthorityError(
8504
+ "scheduled_slack_authority_unavailable",
8505
+ error instanceof Error ? error.message : "scheduled Slack authority is unavailable"
8506
+ );
8507
+ }
8508
+ if (typeof error === "object" && error !== null && "code" in error && error.code === "42501") {
8509
+ return new ScheduledRunTerminalAuthorityError(
8510
+ "scheduled_run_authority_proof_rejected",
8511
+ error instanceof Error ? error.message : "scheduled run authority proof was rejected"
8512
+ );
8513
+ }
8514
+ return null;
8515
+ }
8516
+ function scheduledRunTerminalResult(error) {
8517
+ if (error === "scheduled_authority_exhausted") {
8518
+ return { action: "blocked", reason: "scheduled_authority_exhausted" };
8519
+ }
8520
+ if (error === "scheduled_incident_incident_preflight_metadata_missing") {
8521
+ return { action: "blocked", reason: "incident_preflight_metadata_missing" };
8522
+ }
8523
+ if (error === "scheduled_incident_incident_responder_under_capable") {
8524
+ return { action: "blocked", reason: "incident_responder_under_capable" };
8525
+ }
8526
+ if (error === "scheduled_incident_incident_data_source_unsuitable") {
8527
+ return { action: "blocked", reason: "incident_data_source_unsuitable" };
8528
+ }
8529
+ return { action: "blocked", reason: "scheduled_run_terminal" };
8530
+ }
8531
+ async function prepareIncidentTelemetrySource(input) {
8532
+ const session = await requireSession(input.tx, input.workspaceId, input.sessionId);
8533
+ const { task, personalConnectionDelegations } = input.acceptedExecution ? {
8534
+ task: input.acceptedExecution.task,
8535
+ personalConnectionDelegations: input.acceptedExecution.personalConnectionDelegations
8536
+ } : await requireScheduledTaskIncidentAuthorityInTransaction(input.tx, {
8537
+ workspaceId: input.workspaceId,
8538
+ taskId: input.taskId
8539
+ });
8540
+ const responder = await resolveIncidentTelemetryResponderMetadata({
8541
+ db: input.tx,
8542
+ settings: input.settings,
8543
+ task,
8544
+ session,
8545
+ personalConnectionDelegations,
8546
+ personalResourceAuthoritySubjectId: input.acceptedExecution?.personalResourceAuthoritySubjectId ?? null,
8547
+ ...input.acceptedExecution ? {
8548
+ executionPolicy: input.acceptedExecution.targetSessionExecution ? {
8549
+ tools: input.acceptedExecution.targetSessionExecution.tools,
8550
+ firstPartyMcpTools: input.acceptedExecution.targetSessionExecution.firstPartyMcpTools,
8551
+ firstPartyMcpPermissions: input.acceptedExecution.targetSessionExecution.firstPartyMcpPermissions,
8552
+ variableSetId: input.acceptedExecution.targetSessionExecution.variableSetId,
8553
+ rigId: input.acceptedExecution.targetSessionExecution.rigId,
8554
+ rigVersionId: input.acceptedExecution.targetSessionExecution.rigVersionId,
8555
+ toolPolicy: input.acceptedExecution.targetSessionExecution.toolPolicy,
8556
+ mcpServerIds: input.acceptedExecution.targetSessionExecution.mcpServerIds,
8557
+ toolPolicyVersion: input.acceptedExecution.targetSessionExecution.toolPolicyVersion
8558
+ } : {
8559
+ tools: input.acceptedExecution.resolvedTools,
8560
+ firstPartyMcpTools: input.acceptedExecution.resolvedFirstPartyMcpTools,
8561
+ firstPartyMcpPermissions: input.acceptedExecution.resolvedFirstPartyMcpPermissions,
8562
+ variableSetId: input.acceptedExecution.resolvedVariableSet?.id ?? null,
8563
+ rigId: input.acceptedExecution.resolvedRig?.id ?? null,
8564
+ rigVersionId: input.acceptedExecution.resolvedRig?.versionId ?? null,
8565
+ toolPolicy: session.toolPolicy,
8566
+ mcpServerIds: [],
8567
+ toolPolicyVersion: session.toolPolicyVersion
8568
+ }
8569
+ } : {}
8570
+ });
8571
+ const preflight = evaluateIncidentTelemetryPreflight({
8572
+ agentConfig: task.agentConfig,
8573
+ incidentTriggered: input.alertOccurrenceLabels !== null,
8574
+ alertOccurrenceLabels: input.alertOccurrenceLabels,
8575
+ responder
8576
+ });
8577
+ if (preflight.action === "blocked") {
8578
+ throw new IncidentTelemetryPreflightBlockedError(preflight.reason);
8579
+ }
8580
+ const fence = incidentTelemetryAuthorityFence({
8581
+ task,
8582
+ responder,
8583
+ alertOccurrenceLabels: input.alertOccurrenceLabels
8584
+ });
8585
+ if (!fence) {
8586
+ throw new IncidentTelemetryPreflightBlockedError("incident_preflight_metadata_missing");
8587
+ }
8588
+ return {
8589
+ lineage: {
8590
+ [INCIDENT_TELEMETRY_AUTHORITY_FENCE_LINEAGE_KEY]: fence
8591
+ }
8592
+ };
8593
+ }
8594
+ async function recordScheduledTaskFiredUsage(db, acceptedExecution, run) {
8595
+ const task = acceptedExecution.task;
8596
+ await recordUsageEvent3(db, {
8597
+ accountId: task.accountId,
8598
+ workspaceId: task.workspaceId,
8599
+ eventType: "scheduled_task.fired",
8600
+ quantity: 1,
8601
+ unit: "run",
8602
+ sourceResourceType: "scheduled_task_run",
8603
+ sourceResourceId: run.id,
8604
+ initiator: acceptedExecution.triggerInitiator,
8605
+ initiatorContext: { scheduledTaskId: task.id, scheduledTaskRunId: run.id },
8606
+ origin: "scheduled_task",
8607
+ idempotencyKey: `usage:scheduled_task.fired:${run.id}`
8608
+ });
8609
+ }
8610
+ async function recoverBoundScheduledTaskDispatch(input) {
8611
+ const { task } = input.acceptedExecution;
8612
+ if (input.run.status !== "queued") {
8613
+ throw new Error("scheduled run is not an exact queued recovery");
8614
+ }
8615
+ const generatedSession = task.runMode === "new_session_per_run" || task.runMode === "reusable_session" && task.reusableSessionId === null;
8616
+ const frozenSlack = input.acceptedExecution.resolvedSlackBotConnection;
8617
+ if (frozenSlack) {
8618
+ const currentSlack = await requireOpenGeniSlackBotConnection(
8619
+ input.db,
8620
+ task.workspaceId,
8621
+ frozenSlack.id
8622
+ );
8623
+ const currentMetadata = openGeniSlackBotMetadata(currentSlack.metadata);
8624
+ if (currentSlack.version !== frozenSlack.version || currentSlack.verifiedInstallVersion !== frozenSlack.verifiedInstallVersion || stableJson(currentMetadata) !== stableJson(frozenSlack.metadata)) {
8625
+ throw new ScheduledRunTerminalAuthorityError(
8626
+ "scheduled_slack_authority_changed",
8627
+ "scheduled Slack bot authority changed after run admission"
8628
+ );
8629
+ }
8630
+ }
8631
+ let session;
8632
+ if (input.run.sessionId) {
8633
+ session = await requireSession(input.db, task.workspaceId, input.run.sessionId);
8634
+ if (!generatedSession) assertReusableSessionBindingMatches(session, task);
8635
+ } else if (generatedSession) {
8636
+ const variableSet = input.acceptedExecution.resolvedVariableSet;
8637
+ if (variableSet) {
8638
+ const current = await getVariableSet(
8639
+ input.db,
8640
+ {
8641
+ accountId: task.accountId,
8642
+ workspaceId: task.workspaceId,
8643
+ subjectId: input.acceptedExecution.personalResourceAuthoritySubjectId ?? task.createdBy.subjectId
8644
+ },
8645
+ variableSet.id
8646
+ );
8647
+ if (!current || current.generation !== variableSet.generation) {
8648
+ throw new ScheduledRunTerminalAuthorityError(
8649
+ "scheduled_variable_set_authority_changed",
8650
+ "scheduled variable set changed after run admission"
8651
+ );
8652
+ }
8653
+ }
8654
+ const rig = input.acceptedExecution.resolvedRig;
8655
+ if (rig) {
8656
+ const retained = await getScheduledScopedRigVersionMetadata(
8657
+ input.db,
8658
+ {
8659
+ accountId: task.accountId,
8660
+ workspaceId: task.workspaceId,
8661
+ subjectId: input.acceptedExecution.personalResourceAuthoritySubjectId ?? task.createdBy.subjectId
8662
+ },
8663
+ rig.id,
8664
+ rig.versionId
8665
+ );
8666
+ if (!retained) {
8667
+ throw new ScheduledRunTerminalAuthorityError(
8668
+ "scheduled_rig_authority_changed",
8669
+ "scheduled rig changed after run admission"
8670
+ );
8671
+ }
8672
+ }
8673
+ const taskMetadata = { ...task.agentConfig.metadata };
8674
+ delete taskMetadata[OPENGENI_SLACK_BOT_SESSION_METADATA_KEY];
8675
+ const created = await createSessionWithIdempotencyKeyResult(input.db, {
8676
+ accountId: task.accountId,
8677
+ workspaceId: task.workspaceId,
8678
+ initialMessage: task.agentConfig.prompt,
8679
+ resources: task.agentConfig.resources,
8680
+ tools: input.acceptedExecution.resolvedTools,
8681
+ firstPartyMcpTools: input.acceptedExecution.resolvedFirstPartyMcpTools,
8682
+ firstPartyMcpPermissions: input.acceptedExecution.resolvedFirstPartyMcpPermissions,
8683
+ metadata: {
8684
+ ...taskMetadata,
8685
+ model: input.acceptedExecution.resolvedModel,
8686
+ reasoningEffort: input.acceptedExecution.resolvedReasoningEffort,
8687
+ scheduledTaskId: task.id,
8688
+ scheduledTaskRunMode: task.runMode,
8689
+ ...task.agentConfig.goal ? { scheduledTaskGoal: task.agentConfig.goal } : {},
8690
+ scheduledTaskRunId: input.run.id,
8691
+ ...frozenSlack ? { [OPENGENI_SLACK_BOT_SESSION_METADATA_KEY]: frozenSlack.id } : {}
8692
+ },
8693
+ createdBy: {
8694
+ kind: "service",
8695
+ subjectId: "scheduler",
8696
+ label: "OpenGeni scheduler"
8697
+ },
8698
+ createdByContext: {
8699
+ scheduledTaskId: task.id,
8700
+ scheduledTaskRunId: input.run.id
8701
+ },
8702
+ model: input.acceptedExecution.resolvedModel,
8703
+ reasoningEffort: input.acceptedExecution.resolvedReasoningEffort,
8704
+ latencyMode: input.acceptedExecution.resolvedLatencyMode,
8705
+ sandboxBackend: input.acceptedExecution.resolvedSandboxBackend,
8706
+ sandboxOs: input.acceptedExecution.resolvedSandboxOs,
8707
+ variableSetId: variableSet?.id ?? null,
8708
+ rigId: rig?.id ?? null,
8709
+ rigVersionId: rig?.versionId ?? null,
8710
+ personalConnectionDelegations: [],
8711
+ initialXaiProviderAccountAuthoritySnapshot: input.acceptedExecution.xaiProviderAccountAuthoritySnapshot,
8712
+ maxNestedAgentDepthOverride: task.agentConfig.maxNestedAgentDepth ?? null,
8713
+ frozenNestedAgentDepthPolicy: {
8714
+ effectiveMaxNestedAgentDepth: input.acceptedExecution.generatedSessionBinding.effectiveMaxNestedAgentDepth,
8715
+ nestedAgentDepthPolicySource: input.acceptedExecution.generatedSessionBinding.nestedAgentDepthPolicySource
8716
+ },
8717
+ frozenCodexCompactionMode: input.acceptedExecution.generatedSessionBinding.codexCompactionMode,
8718
+ allowNestedAgentDepthIncrease: true,
8719
+ subjectId: `scheduled_task:${task.id}`,
8720
+ createIdempotencyKey: input.acceptedExecution.generatedSessionBinding?.createIdempotencyKey ?? (() => {
8721
+ throw new ScheduledRunTerminalAuthorityError(
8722
+ "scheduled_generated_session_binding_missing",
8723
+ "scheduled generated session binding is missing"
8724
+ );
8725
+ })(),
8726
+ beforeCreateCommit: async (tx, sessionId) => {
8727
+ await bindScheduledTaskRunSessionInTransaction(tx, {
8728
+ accountId: task.accountId,
8729
+ workspaceId: task.workspaceId,
8730
+ runId: input.run.id,
8731
+ sessionId
8732
+ });
8733
+ }
8734
+ });
8735
+ if (created.denied) throw new SessionSpawnDeniedDbError(created.denial);
8736
+ session = created.session;
8737
+ if (!created.created) {
8738
+ await bindScheduledTaskRunSessionInTransaction(input.db, {
8739
+ accountId: task.accountId,
8740
+ workspaceId: task.workspaceId,
8741
+ runId: input.run.id,
8742
+ sessionId: session.id
8743
+ });
8744
+ }
8745
+ } else {
8746
+ const targetSessionId = task.runMode === "existing_session" ? task.targetSessionId : task.reusableSessionId;
8747
+ if (!targetSessionId) {
8748
+ throw new ScheduledRunTerminalAuthorityError(
8749
+ "scheduled_target_session_unavailable",
8750
+ "scheduled accepted target session is unavailable"
8751
+ );
8752
+ }
8753
+ session = await requireSession(input.db, task.workspaceId, targetSessionId);
8754
+ assertReusableSessionBindingMatches(session, task);
8755
+ await bindScheduledTaskRunSessionInTransaction(input.db, {
8756
+ accountId: task.accountId,
8757
+ workspaceId: task.workspaceId,
8758
+ runId: input.run.id,
8759
+ sessionId: session.id
8760
+ });
8761
+ }
8762
+ if (session.status === "cancelled") {
8763
+ await markScheduledTaskRunSkippedIfQueued(input.db, {
8764
+ workspaceId: task.workspaceId,
8765
+ runId: input.run.id,
8766
+ sessionId: session.id,
8767
+ error: "session_cancelled"
8768
+ });
8769
+ return { action: "blocked", reason: "scheduled_run_terminal" };
8770
+ }
8771
+ if (generatedSession) {
8772
+ const expectedCreateKey = input.acceptedExecution.generatedSessionBinding?.createIdempotencyKey;
8773
+ if (!expectedCreateKey) {
8774
+ throw new ScheduledRunTerminalAuthorityError(
8775
+ "scheduled_generated_session_binding_missing",
8776
+ "scheduled generated session binding is missing"
8777
+ );
8778
+ }
8779
+ const generatedBinding = input.acceptedExecution.generatedSessionBinding;
8780
+ const canonicalGeneratedRunId = task.runMode === "reusable_session" && typeof session.metadata.scheduledTaskRunId === "string" ? session.metadata.scheduledTaskRunId : input.run.id;
8781
+ const expectedTaskMetadata = { ...task.agentConfig.metadata };
8782
+ delete expectedTaskMetadata[OPENGENI_SLACK_BOT_SESSION_METADATA_KEY];
8783
+ const expectedMetadata = {
8784
+ ...expectedTaskMetadata,
8785
+ model: input.acceptedExecution.resolvedModel,
8786
+ reasoningEffort: input.acceptedExecution.resolvedReasoningEffort,
8787
+ scheduledTaskId: task.id,
8788
+ scheduledTaskRunMode: task.runMode,
8789
+ ...task.agentConfig.goal ? { scheduledTaskGoal: task.agentConfig.goal } : {},
8790
+ scheduledTaskRunId: canonicalGeneratedRunId,
8791
+ ...frozenSlack ? { [OPENGENI_SLACK_BOT_SESSION_METADATA_KEY]: frozenSlack.id } : {}
8792
+ };
8793
+ if (session.createIdempotencyKey !== expectedCreateKey || session.initialMessage !== task.agentConfig.prompt || session.instructions !== null || session.policyRole !== null || stableJson(session.skills) !== "[]" || stableJson(session.toolPolicy) !== stableJson({ mode: "explicit", inheritedFromSessionId: null }) || session.toolPolicyVersion !== 1 || session.mcpServers.length !== 0 || session.parentSessionId !== null || session.rootSessionId !== session.id || session.nestedAgentDepth !== 0 || session.effectiveMaxNestedAgentDepth !== generatedBinding.effectiveMaxNestedAgentDepth || session.nestedAgentDepthPolicySource !== generatedBinding.nestedAgentDepthPolicySource || session.nestedAgentDepthPolicySessionId !== (generatedBinding.nestedAgentDepthPolicySource === "session" ? session.id : null) || session.sandboxGroupId !== session.id || session.channelId !== null || session.codexCompactionMode !== generatedBinding.codexCompactionMode || stableJson(session.metadata) !== stableJson(expectedMetadata) || session.createdBy.kind !== "service" || session.createdBy.subjectId !== "scheduler" || stableJson(session.createdByContext) !== stableJson({
8794
+ label: "OpenGeni scheduler",
8795
+ scheduledTaskId: task.id,
8796
+ scheduledTaskRunId: canonicalGeneratedRunId
8797
+ }) || task.runMode === "new_session_per_run" && input.acceptedExecution.alertOccurrenceLabels === null && (session.metadata.scheduledTaskRunId !== input.run.id || session.createdByContext.scheduledTaskRunId !== input.run.id) || session.model !== input.acceptedExecution.resolvedModel || session.reasoningEffort !== input.acceptedExecution.resolvedReasoningEffort || session.latencyMode !== input.acceptedExecution.resolvedLatencyMode || session.sandboxBackend !== input.acceptedExecution.resolvedSandboxBackend || session.sandboxOs !== input.acceptedExecution.resolvedSandboxOs || stableJson(session.resources) !== stableJson(task.agentConfig.resources) || stableJson(session.tools) !== stableJson(input.acceptedExecution.resolvedTools) || stableJson(session.firstPartyMcpTools) !== stableJson(input.acceptedExecution.resolvedFirstPartyMcpTools) || stableJson(session.firstPartyMcpPermissions) !== stableJson(input.acceptedExecution.resolvedFirstPartyMcpPermissions) || session.maxNestedAgentDepthOverride !== (task.agentConfig.maxNestedAgentDepth ?? null) || (session.variableSetId ?? null) !== (input.acceptedExecution.resolvedVariableSet?.id ?? null) || (session.rigId ?? null) !== (input.acceptedExecution.resolvedRig?.id ?? null) || (session.rigVersionId ?? null) !== (input.acceptedExecution.resolvedRig?.versionId ?? null) || scheduledSlackBotConnectionId(session.metadata) !== (frozenSlack?.id ?? null)) {
8798
+ throw new ScheduledRunTerminalAuthorityError(
8799
+ "scheduled_generated_session_changed",
8800
+ "scheduled generated session differs from accepted execution"
8801
+ );
8802
+ }
8803
+ }
8804
+ let workflowId = workflowIdForSession(session.id);
8805
+ if (generatedSession) {
8806
+ const goalSpec = task.agentConfig.goal ?? null;
8807
+ const started = await initializeSessionStartAtomically(input.db, {
8808
+ accountId: task.accountId,
8809
+ workspaceId: task.workspaceId,
8810
+ sessionId: session.id,
8811
+ reasoningEffortFallback: input.acceptedExecution.resolvedReasoningEffort,
8812
+ createdEventPayload: {
8813
+ scheduledTaskId: task.id,
8814
+ scheduledTaskRunId: input.run.id
8815
+ },
8816
+ goal: goalSpec ? {
8817
+ text: goalSpec.text,
8818
+ successCriteria: goalSpec.successCriteria ?? null,
8819
+ maxAutoContinuations: goalSpec.maxAutoContinuations ?? null,
8820
+ ...goalSpec.mutationPolicy ? { mutationPolicy: goalSpec.mutationPolicy } : {},
8821
+ createdBy: "scheduled_task"
8822
+ } : null,
8823
+ deferInitialTurn: true,
8824
+ deferredStatus: "queued"
8825
+ });
8826
+ workflowId = started.temporalWorkflowId;
8827
+ if (started.events.length > 0) {
8828
+ await publishDurableSessionEvents(input.bus, task.workspaceId, session.id, started.events);
8829
+ }
8830
+ if (task.runMode === "reusable_session") {
8831
+ await materializeScheduledTaskReusableSessionFromRun(input.db, {
8832
+ accountId: task.accountId,
8833
+ workspaceId: task.workspaceId,
8834
+ taskId: task.id,
8835
+ runId: input.run.id,
8836
+ sessionId: session.id,
8837
+ sourceTaskAuthorityRevision: input.run.taskAuthorityRevision,
8838
+ sourceExecutionDigest: input.run.taskExecutionDigest
8839
+ });
8840
+ }
8841
+ const titleEvents = await stampScheduledSessionTitle(input.db, {
8842
+ workspaceId: task.workspaceId,
8843
+ session,
8844
+ taskName: task.name
8845
+ });
8846
+ if (titleEvents.length > 0) {
8847
+ await publishDurableSessionEvents(input.bus, task.workspaceId, session.id, titleEvents);
8848
+ }
8849
+ }
8850
+ if (!generatedSession && task.runMode === "reusable_session" && task.agentConfig.goal) {
8851
+ const goalEvents = await upsertScheduledSessionGoalForRun(input.db, {
8852
+ accountId: task.accountId,
8853
+ workspaceId: task.workspaceId,
8854
+ sessionId: session.id,
8855
+ runId: input.run.id,
8856
+ text: task.agentConfig.goal.text,
8857
+ successCriteria: task.agentConfig.goal.successCriteria ?? null,
8858
+ maxAutoContinuations: task.agentConfig.goal.maxAutoContinuations ?? null,
8859
+ ...task.agentConfig.goal.mutationPolicy ? { mutationPolicy: task.agentConfig.goal.mutationPolicy } : {}
8860
+ });
8861
+ if (goalEvents.length > 0) {
8862
+ await publishDurableSessionEvents(input.bus, task.workspaceId, session.id, goalEvents);
8863
+ }
8864
+ }
8865
+ const scheduledUpdate = await addSessionSystemUpdateWithSourceMutation(
8866
+ input.db,
8867
+ {
8868
+ accountId: task.accountId,
8869
+ workspaceId: task.workspaceId,
8870
+ sessionId: session.id,
8871
+ kind: "scheduled_occurrence",
8872
+ classification: "info",
8873
+ sourceId: input.run.id,
8874
+ dedupeKey: `scheduled-task-run:${input.run.id}`,
8875
+ summary: task.agentConfig.prompt,
8876
+ payload: scheduledUserMessagePayload(
8877
+ task.agentConfig.prompt,
8878
+ task.agentConfig.resources,
8879
+ input.acceptedExecution.resolvedTools,
8880
+ task.id,
8881
+ input.run.id
8882
+ ),
8883
+ lineage: {
8884
+ scheduledTaskId: task.id,
8885
+ scheduledTaskRunId: input.run.id,
8886
+ ...input.acceptedExecution.causalHumanSubjectId ? { causalHumanSubjectId: input.acceptedExecution.causalHumanSubjectId } : {},
8887
+ ...input.acceptedExecution.connectionAuthoritySubjectId ? {
8888
+ connectionAuthoritySubjectId: input.acceptedExecution.connectionAuthoritySubjectId
8889
+ } : {},
8890
+ ...input.acceptedExecution.xaiAuthoritySubjectId ? { xaiAuthoritySubjectId: input.acceptedExecution.xaiAuthoritySubjectId } : {}
8891
+ },
8892
+ personalConnectionDelegations: input.acceptedExecution.personalConnectionDelegations,
8893
+ xaiProviderAccountAuthoritySnapshot: input.acceptedExecution.xaiProviderAccountAuthoritySnapshot,
8894
+ scheduledTaskRunId: input.run.id
8895
+ },
8896
+ async (tx, wakeEventId) => {
8897
+ if (!wakeEventId) {
8898
+ await settleScheduledTaskRunInTransaction(tx, {
8899
+ workspaceId: task.workspaceId,
8900
+ runId: input.run.id,
8901
+ sessionId: session.id,
8902
+ triggerEventId: null,
8903
+ status: "skipped",
8904
+ error: "session_cancelled"
8905
+ });
8906
+ return;
8907
+ }
8908
+ await settleScheduledTaskRunInTransaction(tx, {
8909
+ workspaceId: task.workspaceId,
8910
+ runId: input.run.id,
8911
+ sessionId: session.id,
8912
+ triggerEventId: wakeEventId,
8913
+ status: "dispatched"
8914
+ });
8915
+ },
8916
+ input.acceptedExecution.incidentPreflightRequired ? {
8917
+ prepareSource: async (tx, sessionId) => await prepareIncidentTelemetrySource({
8918
+ tx,
8919
+ settings: input.settings,
8920
+ taskId: task.id,
8921
+ workspaceId: task.workspaceId,
8922
+ sessionId,
8923
+ alertOccurrenceLabels: input.acceptedExecution.alertOccurrenceLabels,
8924
+ acceptedExecution: input.acceptedExecution
8925
+ })
8926
+ } : void 0
8927
+ );
8928
+ if (scheduledUpdate.reason === "session_cancelled") {
8929
+ return { action: "blocked", reason: "scheduled_run_terminal" };
8930
+ }
8931
+ if (scheduledUpdate.added && scheduledUpdate.events.length > 0) {
8932
+ await publishDurableSessionEvents(
8933
+ input.bus,
8934
+ task.workspaceId,
8935
+ session.id,
8936
+ scheduledUpdate.events
8937
+ );
8938
+ }
8939
+ await recordUsageEvent3(input.db, {
8940
+ accountId: task.accountId,
8941
+ workspaceId: task.workspaceId,
8942
+ eventType: "agent_run.created",
8943
+ quantity: 1,
8944
+ unit: "run",
8945
+ sourceResourceType: "scheduled_task_run",
8946
+ sourceResourceId: input.run.id,
8947
+ sessionId: session.id,
8948
+ initiator: input.acceptedExecution.triggerInitiator,
8949
+ initiatorContext: { scheduledTaskId: task.id, scheduledTaskRunId: input.run.id },
8950
+ origin: "scheduled_task",
8951
+ idempotencyKey: input.acceptedExecution.agentRunUsageIdempotencyKey ?? `usage:agent_run.created:scheduled:${input.run.id}`
8952
+ });
8953
+ const workflowWakeRevision = await enqueueSessionWorkflowWakeIfRunnable(input.db, {
8954
+ accountId: task.accountId,
8955
+ workspaceId: task.workspaceId,
8956
+ sessionId: session.id,
8957
+ temporalWorkflowId: workflowId,
8958
+ reason: "scheduled_retry"
8959
+ });
8960
+ const result = {
8961
+ action: generatedSession ? "start" : "signal",
8962
+ accountId: task.accountId,
8963
+ workspaceId: task.workspaceId,
8964
+ sessionId: session.id,
8965
+ triggerEventId: scheduledUpdate.wakeEventId,
8966
+ workflowId,
8967
+ workflowWakeRevision
8968
+ };
8969
+ if (input.wakeSessionWorkflow && workflowWakeRevision !== null) {
8970
+ await input.wakeSessionWorkflow({
8971
+ accountId: result.accountId,
8972
+ workspaceId: result.workspaceId,
8973
+ sessionId: result.sessionId,
8974
+ workflowId: result.workflowId,
8975
+ wakeRevision: workflowWakeRevision
8976
+ });
8977
+ }
8978
+ return result;
8979
+ }
8980
+ async function replayScheduledTaskDispatch(input) {
8981
+ if (!input.run.sessionId || !input.run.triggerEventId) {
8982
+ throw new Error("dispatched scheduled task run is missing its delivery identity");
8983
+ }
8984
+ const sessionId = input.run.sessionId;
8985
+ const triggerEventId = input.run.triggerEventId;
8986
+ await requireSession(input.db, input.task.workspaceId, sessionId);
8987
+ await recordUsageEvent3(input.db, {
8988
+ accountId: input.task.accountId,
8989
+ workspaceId: input.task.workspaceId,
8990
+ eventType: "agent_run.created",
8991
+ quantity: 1,
8992
+ unit: "run",
8993
+ sourceResourceType: "scheduled_task_run",
8994
+ sourceResourceId: input.run.id,
8995
+ sessionId,
8996
+ initiator: input.acceptedExecution.triggerInitiator,
8997
+ initiatorContext: {
8998
+ scheduledTaskId: input.task.id,
8999
+ scheduledTaskRunId: input.run.id
9000
+ },
9001
+ origin: "scheduled_task",
9002
+ idempotencyKey: input.acceptedExecution.agentRunUsageIdempotencyKey ?? `usage:agent_run.created:scheduled:${input.run.id}`
9003
+ });
9004
+ const workflowId = workflowIdForSession(sessionId);
9005
+ const workflowWakeRevision = await enqueueSessionWorkflowWakeIfRunnable(input.db, {
9006
+ accountId: input.task.accountId,
9007
+ workspaceId: input.task.workspaceId,
9008
+ sessionId,
9009
+ temporalWorkflowId: workflowId,
9010
+ reason: "scheduled_retry"
9011
+ });
9012
+ const result = {
9013
+ action: input.acceptedExecution.task.runMode === "new_session_per_run" ? "start" : "signal",
9014
+ accountId: input.task.accountId,
9015
+ workspaceId: input.task.workspaceId,
9016
+ sessionId,
9017
+ triggerEventId,
9018
+ workflowId,
9019
+ workflowWakeRevision
9020
+ };
9021
+ if (input.wakeSessionWorkflow && workflowWakeRevision !== null) {
9022
+ await input.wakeSessionWorkflow({
9023
+ accountId: result.accountId,
9024
+ workspaceId: result.workspaceId,
9025
+ sessionId: result.sessionId,
9026
+ workflowId: result.workflowId,
9027
+ wakeRevision: workflowWakeRevision
9028
+ });
9029
+ }
9030
+ return result;
9031
+ }
6940
9032
  function knowledgeSourceSyncEffectivelyPaused(task) {
6941
9033
  if (task.status === "paused") return true;
6942
9034
  const value = task.metadata.knowledgeSourceSync;
@@ -6946,17 +9038,17 @@ function knowledgeSourceSyncEffectivelyPaused(task) {
6946
9038
  }
6947
9039
 
6948
9040
  // src/activities/site-auth-maintenance.ts
6949
- import { resolveTurnExecutionPolicyV1 } from "@opengeni/config";
9041
+ import { resolveTurnExecutionPolicyV1 as resolveTurnExecutionPolicyV12 } from "@opengeni/config";
6950
9042
  import {
6951
- DEFAULT_FIRST_PARTY_MCP_PERMISSIONS,
9043
+ DEFAULT_FIRST_PARTY_MCP_PERMISSIONS as DEFAULT_FIRST_PARTY_MCP_PERMISSIONS2,
6952
9044
  SITE_AUTH_MAINTENANCE_CONNECTION_CONTEXT_KEY,
6953
9045
  SITE_AUTH_MAINTENANCE_OPERATION_CONTEXT_KEY
6954
9046
  } from "@opengeni/contracts";
6955
9047
  import {
6956
- assertWorkspaceModelPolicyAllows,
6957
- canonicalConfiguredModel,
6958
- createAndStartSessionWithOutcome,
6959
- recordWorkspaceUsage
9048
+ assertWorkspaceModelPolicyAllows as assertWorkspaceModelPolicyAllows2,
9049
+ canonicalConfiguredModel as canonicalConfiguredModel2,
9050
+ createAndStartSessionWithOutcome as createAndStartSessionWithOutcome2,
9051
+ recordWorkspaceUsage as recordWorkspaceUsage2
6960
9052
  } from "@opengeni/core";
6961
9053
  import {
6962
9054
  claimSiteAuthMaintenance,
@@ -6983,10 +9075,10 @@ function createSiteAuthMaintenanceActivities(services, options = {}) {
6983
9075
  const claim = options.claim ?? claimSiteAuthMaintenance;
6984
9076
  const confirm = options.confirm ?? confirmSiteAuthMaintenanceSessionInTransaction;
6985
9077
  const defer = options.defer ?? deferSiteAuthMaintenance;
6986
- const createSession2 = options.createSession ?? createAndStartSessionWithOutcome;
9078
+ const createSession2 = options.createSession ?? createAndStartSessionWithOutcome2;
6987
9079
  const admit = options.admit ?? agentRunAdmissionDenial;
6988
- const recordUsage = options.recordUsage ?? recordWorkspaceUsage;
6989
- const assertModelPolicy = options.assertModelPolicy ?? assertWorkspaceModelPolicyAllows;
9080
+ const recordUsage = options.recordUsage ?? recordWorkspaceUsage2;
9081
+ const assertModelPolicy = options.assertModelPolicy ?? assertWorkspaceModelPolicyAllows2;
6990
9082
  async function maintainSiteAuthConnections() {
6991
9083
  const service = await services();
6992
9084
  const claims = await claim(service.db, { claimTimeoutMs, limit: batchSize });
@@ -7032,7 +9124,7 @@ function createSiteAuthMaintenanceActivities(services, options = {}) {
7032
9124
  return { maintainSiteAuthConnections };
7033
9125
  }
7034
9126
  async function dispatchMaintenanceSession(service, maintenance, dependencies) {
7035
- const model = canonicalConfiguredModel(service.settings, service.settings.openaiModel);
9127
+ const model = canonicalConfiguredModel2(service.settings, service.settings.openaiModel);
7036
9128
  if (!model) throw new Error("site auth maintenance has no configured model");
7037
9129
  await dependencies.assertModelPolicy(
7038
9130
  service.db,
@@ -7048,7 +9140,7 @@ async function dispatchMaintenanceSession(service, maintenance, dependencies) {
7048
9140
  });
7049
9141
  if (denial) throw new Error(`site auth maintenance admission denied: ${denial}`);
7050
9142
  const reasoningEffort = service.settings.openaiReasoningEffort;
7051
- const turnExecutionPolicy = resolveTurnExecutionPolicyV1(service.settings, {
9143
+ const turnExecutionPolicy = resolveTurnExecutionPolicyV12(service.settings, {
7052
9144
  modelId: model,
7053
9145
  requestedModelId: null,
7054
9146
  modelSource: "deployment",
@@ -7102,7 +9194,7 @@ async function dispatchMaintenanceSession(service, maintenance, dependencies) {
7102
9194
  },
7103
9195
  instructions: "Complete only the exact browser-auth maintenance run in the initial request. Use the canonical browser/auth tools, preserve their causal fences, expose no secrets, never publish a BrowserIdentity revision, and do not perform unrelated work.",
7104
9196
  policyRole: "site_auth_maintenance",
7105
- firstPartyMcpPermissions: [...DEFAULT_FIRST_PARTY_MCP_PERMISSIONS],
9197
+ firstPartyMcpPermissions: [...DEFAULT_FIRST_PARTY_MCP_PERMISSIONS2],
7106
9198
  firstPartyMcpTools: [...MAINTENANCE_TOOLS],
7107
9199
  createIdempotencyKey: `site-auth-maintenance:${maintenance.operationId}`,
7108
9200
  subjectId: "site-auth-maintenance"
@@ -7247,14 +9339,14 @@ function createSessionStateActivities(services, overrides = {}) {
7247
9339
  return { action: "failed" };
7248
9340
  }
7249
9341
  const requestedRetryDelayMs = input.retryDelayMs;
7250
- const retryDelayMs = typeof requestedRetryDelayMs === "number" && Number.isFinite(requestedRetryDelayMs) ? Math.max(1e3, Math.min(6e4, Math.trunc(requestedRetryDelayMs))) : 6e4;
9342
+ const retryDelayMs2 = typeof requestedRetryDelayMs === "number" && Number.isFinite(requestedRetryDelayMs) ? Math.max(1e3, Math.min(6e4, Math.trunc(requestedRetryDelayMs))) : 6e4;
7251
9343
  await enqueueSessionWorkflowWakeFn(db, {
7252
9344
  accountId: input.accountId,
7253
9345
  workspaceId: input.workspaceId,
7254
9346
  sessionId: input.sessionId,
7255
9347
  temporalWorkflowId: workflowId,
7256
9348
  reason: "turn_activity_failed_before_attempt_claim",
7257
- notBefore: new Date(Date.now() + retryDelayMs)
9349
+ notBefore: new Date(Date.now() + retryDelayMs2)
7258
9350
  });
7259
9351
  return { action: "unclaimed" };
7260
9352
  }
@@ -7410,7 +9502,7 @@ function createSessionStateActivities(services, overrides = {}) {
7410
9502
  input.attemptId
7411
9503
  );
7412
9504
  if (!turn) return { action: "stale" };
7413
- if (turn.id !== input.turnId || turn.triggerEventId !== input.triggerEventId || turn.executionGeneration !== input.executionGeneration || input.executionGeneration <= 1) {
9505
+ if (turn.id !== input.turnId || turn.triggerEventId !== input.triggerEventId || turn.executionGeneration !== input.executionGeneration || input.executionGeneration <= 1 || input.providerRecoveryCount !== providerRecoveryCountFromMetadata(turn.metadata) + 1 || input.providerRecoveryCount > MAX_AUTOMATIC_PROVIDER_RECOVERIES) {
7414
9506
  return { action: "ineligible" };
7415
9507
  }
7416
9508
  const recovery = await requestSessionTurnRecoveryFn(db, input.workspaceId, {
@@ -7419,10 +9511,12 @@ function createSessionStateActivities(services, overrides = {}) {
7419
9511
  triggerEventId: input.triggerEventId,
7420
9512
  attemptId: input.attemptId,
7421
9513
  reason: "mcp_transport_timeout",
9514
+ providerRecoveryCount: input.providerRecoveryCount,
7422
9515
  detail: {
7423
9516
  code: "mcp_transport_timeout",
7424
9517
  retryable: true,
7425
- continueDelayMs: 6e4,
9518
+ continueDelayMs: input.continueDelayMs,
9519
+ providerRecoveryCount: input.providerRecoveryCount,
7426
9520
  recoverySource: "workflow_activity_failure"
7427
9521
  },
7428
9522
  fromStatuses: ["running"]
@@ -7537,6 +9631,7 @@ function createControlActivityServices(dependencies) {
7537
9631
  }
7538
9632
  function createControlActivitiesFromServices(services, resolveDocumentServices) {
7539
9633
  return {
9634
+ ...createAutomationActivities(services),
7540
9635
  ...createDocumentActivities(services, resolveDocumentServices),
7541
9636
  ...createKnowledgeSourceSyncActivities(services, resolveDocumentServices),
7542
9637
  ...createSessionStateActivities(services),
@@ -7567,4 +9662,4 @@ export {
7567
9662
  createControlActivities,
7568
9663
  createControlActivitiesFromServices
7569
9664
  };
7570
- //# sourceMappingURL=activities-control-BJSDRSIP.js.map
9665
+ //# sourceMappingURL=activities-control-DKG22ONJ.js.map