@opengeni/worker-bundle 0.20.1 → 0.22.0-canary.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (189) hide show
  1. package/dist/activities/agent-turn/admission.d.ts +52 -0
  2. package/dist/activities/agent-turn/agent-build.d.ts +89 -0
  3. package/dist/activities/agent-turn/claim.d.ts +67 -0
  4. package/dist/activities/agent-turn/codex-capacity.d.ts +40 -0
  5. package/dist/activities/agent-turn/codex.d.ts +32 -0
  6. package/dist/activities/agent-turn/compaction-prep.d.ts +88 -0
  7. package/dist/activities/agent-turn/credential-leases.d.ts +56 -0
  8. package/dist/activities/agent-turn/errors.d.ts +253 -0
  9. package/dist/activities/agent-turn/failure-settlement.d.ts +33 -0
  10. package/dist/activities/agent-turn/file-resources.d.ts +37 -0
  11. package/dist/activities/agent-turn/finalization.d.ts +42 -0
  12. package/dist/activities/agent-turn/github-repository-bindings.d.ts +105 -0
  13. package/dist/activities/agent-turn/governance-model.d.ts +66 -0
  14. package/dist/activities/agent-turn/history-sink.d.ts +52 -0
  15. package/dist/activities/agent-turn/history.d.ts +63 -0
  16. package/dist/activities/agent-turn/media-artifacts.d.ts +145 -0
  17. package/dist/activities/agent-turn/model-usage.d.ts +218 -0
  18. package/dist/activities/agent-turn/quiescence.d.ts +135 -0
  19. package/dist/activities/agent-turn/run-credentials.d.ts +84 -0
  20. package/dist/activities/agent-turn/run.d.ts +3 -0
  21. package/dist/activities/agent-turn/sandbox-establish.d.ts +93 -0
  22. package/dist/activities/agent-turn/sandbox-provision.d.ts +89 -0
  23. package/dist/activities/agent-turn/sandbox-route.d.ts +151 -0
  24. package/dist/activities/agent-turn/sandbox-runtime.d.ts +44 -0
  25. package/dist/activities/agent-turn/stream-attempt.d.ts +107 -0
  26. package/dist/activities/agent-turn/tool-environment.d.ts +142 -0
  27. package/dist/activities/agent-turn/tool-policy.d.ts +127 -0
  28. package/dist/activities/agent-turn/tool-result-spill.d.ts +59 -0
  29. package/dist/activities/agent-turn/turn-context.d.ts +144 -0
  30. package/dist/activities/agent-turn/xai-capacity.d.ts +2 -0
  31. package/dist/activities/agent-turn.d.ts +12 -954
  32. package/dist/activities/automations.d.ts +16 -0
  33. package/dist/activities/common.d.ts +1 -1
  34. package/dist/activities/connector-attachments.d.ts +13 -0
  35. package/dist/activities/documents.d.ts +1 -0
  36. package/dist/activities/environment.d.ts +26 -15
  37. package/dist/activities/generated-images.d.ts +1 -1
  38. package/dist/activities/git-credential-renewal.d.ts +3 -1
  39. package/dist/activities/goals.d.ts +4 -1
  40. package/dist/activities/google-drive-provider.d.ts +29 -0
  41. package/dist/activities/google-drive-publication.d.ts +66 -0
  42. package/dist/activities/image-generation-references.d.ts +2 -0
  43. package/dist/activities/incident-telemetry-authority.d.ts +52 -0
  44. package/dist/activities/incident-telemetry-preflight.d.ts +71 -0
  45. package/dist/activities/knowledge-source-sync-driver.d.ts +21 -0
  46. package/dist/activities/knowledge-source-sync.d.ts +33 -1
  47. package/dist/activities/mcp-credentials.d.ts +9 -3
  48. package/dist/activities/open-suffix-resume.d.ts +77 -0
  49. package/dist/activities/packs.d.ts +3 -11
  50. package/dist/activities/parent-wake.d.ts +19 -0
  51. package/dist/activities/retained-screenshot-reaper.d.ts +1 -1
  52. package/dist/activities/retained-screenshots.d.ts +1 -1
  53. package/dist/activities/rig-verification.d.ts +1 -1
  54. package/dist/activities/run-input.d.ts +16 -7
  55. package/dist/activities/sandbox-images.d.ts +1 -14
  56. package/dist/activities/sandbox-lease.d.ts +21 -3
  57. package/dist/activities/scheduled-tasks.d.ts +47 -0
  58. package/dist/activities/types.d.ts +23 -2
  59. package/dist/activities/video-media-validation.d.ts +1 -1
  60. package/dist/activities/video-output-retention.d.ts +1 -2
  61. package/dist/activities/xai-auth.d.ts +3 -0
  62. package/dist/{activities-control-SMKF2OBE.js → activities-control-DKG22ONJ.js} +2946 -549
  63. package/dist/activities-control-DKG22ONJ.js.map +1 -0
  64. package/dist/activities-control.d.ts +6 -2
  65. package/dist/activities-turn-4IMC3MAA.js +21326 -0
  66. package/dist/activities-turn-4IMC3MAA.js.map +1 -0
  67. package/dist/activities.d.ts +7 -3
  68. package/dist/artifact-materializer-entry.js +41 -22
  69. package/dist/artifact-materializer-entry.js.map +1 -1
  70. package/dist/{chunk-N5TJIFCE.js → chunk-CE7TPKBS.js} +468 -1
  71. package/dist/chunk-CE7TPKBS.js.map +1 -0
  72. package/dist/chunk-CNYGVRKF.js +4698 -0
  73. package/dist/chunk-CNYGVRKF.js.map +1 -0
  74. package/dist/connection-provider-use.d.ts +18 -0
  75. package/dist/editable-artifact-materializer.d.ts +4 -3
  76. package/dist/github-rest-mcp.d.ts +26 -0
  77. package/dist/index.d.ts +10 -2
  78. package/dist/index.js +58 -14
  79. package/dist/index.js.map +1 -1
  80. package/dist/knowledge-source-sync-activity-policy.d.ts +2 -0
  81. package/dist/model-context-contributions.d.ts +8 -4
  82. package/dist/observability-metrics.d.ts +63 -4
  83. package/dist/opensandbox-kubernetes-inventory.d.ts +21 -0
  84. package/dist/personal-github-git-credentials.d.ts +9 -0
  85. package/dist/pr-review-credentials.d.ts +6 -0
  86. package/dist/sandbox-archive-storage.d.ts +45 -0
  87. package/dist/sandbox-resume.d.ts +65 -6
  88. package/dist/sandbox-routing.d.ts +38 -2
  89. package/dist/scheduled-alert-occurrence.d.ts +33 -0
  90. package/dist/workflow-bundle.js +87 -28
  91. package/dist/workflows/activities.d.ts +1 -0
  92. package/dist/workflows/automations.d.ts +6 -0
  93. package/dist/workflows/document-index.d.ts +1 -0
  94. package/dist/workflows.d.ts +1 -0
  95. package/package.json +17 -17
  96. package/src/activities/agent-turn/admission.ts +292 -0
  97. package/src/activities/agent-turn/agent-build.ts +802 -0
  98. package/src/activities/agent-turn/claim.ts +590 -0
  99. package/src/activities/agent-turn/codex-capacity.ts +720 -0
  100. package/src/activities/agent-turn/codex.ts +94 -0
  101. package/src/activities/agent-turn/compaction-prep.ts +722 -0
  102. package/src/activities/agent-turn/credential-leases.ts +240 -0
  103. package/src/activities/agent-turn/errors.ts +1068 -0
  104. package/src/activities/agent-turn/failure-settlement.ts +1436 -0
  105. package/src/activities/agent-turn/file-resources.ts +227 -0
  106. package/src/activities/agent-turn/finalization.ts +754 -0
  107. package/src/activities/agent-turn/github-repository-bindings.ts +300 -0
  108. package/src/activities/agent-turn/governance-model.ts +465 -0
  109. package/src/activities/agent-turn/history-sink.ts +139 -0
  110. package/src/activities/agent-turn/history.ts +200 -0
  111. package/src/activities/agent-turn/media-artifacts.ts +359 -0
  112. package/src/activities/agent-turn/model-usage.ts +823 -0
  113. package/src/activities/agent-turn/quiescence.ts +353 -0
  114. package/src/activities/agent-turn/run-credentials.ts +761 -0
  115. package/src/activities/agent-turn/run.ts +1530 -0
  116. package/src/activities/agent-turn/sandbox-establish.ts +1207 -0
  117. package/src/activities/agent-turn/sandbox-provision.ts +569 -0
  118. package/src/activities/agent-turn/sandbox-route.ts +358 -0
  119. package/src/activities/agent-turn/sandbox-runtime.ts +731 -0
  120. package/src/activities/agent-turn/stream-attempt.ts +1670 -0
  121. package/src/activities/agent-turn/tool-environment.ts +729 -0
  122. package/src/activities/agent-turn/tool-policy.ts +216 -0
  123. package/src/activities/agent-turn/tool-result-spill.ts +265 -0
  124. package/src/activities/agent-turn/turn-context.ts +281 -0
  125. package/src/activities/agent-turn/xai-capacity.ts +229 -0
  126. package/src/activities/agent-turn.ts +12 -12575
  127. package/src/activities/api-integrations.ts +3 -0
  128. package/src/activities/automations.ts +214 -0
  129. package/src/activities/codemode-dispatcher.ts +12 -1
  130. package/src/activities/common.ts +1 -5
  131. package/src/activities/connector-attachments.ts +168 -0
  132. package/src/activities/context-compaction.ts +18 -0
  133. package/src/activities/documents.ts +6 -4
  134. package/src/activities/environment.ts +151 -55
  135. package/src/activities/generated-images.ts +26 -11
  136. package/src/activities/git-credential-renewal.ts +13 -3
  137. package/src/activities/goals.ts +85 -19
  138. package/src/activities/google-drive-provider.ts +178 -0
  139. package/src/activities/google-drive-publication.ts +938 -0
  140. package/src/activities/image-generation-references.ts +11 -2
  141. package/src/activities/incident-telemetry-authority.ts +327 -0
  142. package/src/activities/incident-telemetry-preflight.ts +317 -0
  143. package/src/activities/knowledge-source-sync-driver.ts +23 -0
  144. package/src/activities/knowledge-source-sync.ts +556 -40
  145. package/src/activities/mcp-credentials.ts +191 -17
  146. package/src/activities/open-suffix-resume.ts +422 -0
  147. package/src/activities/packs.ts +12 -60
  148. package/src/activities/parent-wake.ts +64 -14
  149. package/src/activities/retained-screenshot-reaper.ts +6 -3
  150. package/src/activities/retained-screenshots.ts +15 -47
  151. package/src/activities/rig-verification.ts +1 -1
  152. package/src/activities/run-input.ts +128 -136
  153. package/src/activities/sandbox-images.ts +7 -166
  154. package/src/activities/sandbox-lease.ts +406 -15
  155. package/src/activities/scheduled-tasks.ts +2103 -411
  156. package/src/activities/session-state.ts +10 -2
  157. package/src/activities/types.ts +32 -2
  158. package/src/activities/video-media-validation.ts +3 -3
  159. package/src/activities/video-output-retention.ts +2 -57
  160. package/src/activities/xai-auth.ts +8 -0
  161. package/src/activities-control.ts +2 -0
  162. package/src/activities.ts +3 -0
  163. package/src/activity-services.ts +13 -4
  164. package/src/connection-provider-use.ts +36 -0
  165. package/src/editable-artifact-materializer-subprocess.ts +25 -14
  166. package/src/editable-artifact-materializer.ts +29 -18
  167. package/src/github-rest-mcp.ts +713 -0
  168. package/src/http.ts +16 -6
  169. package/src/index.ts +40 -5
  170. package/src/knowledge-source-sync-activity-policy.ts +2 -0
  171. package/src/model-context-contributions.ts +33 -10
  172. package/src/observability-metrics.ts +357 -0
  173. package/src/opensandbox-kubernetes-inventory.ts +292 -0
  174. package/src/personal-github-git-credentials.ts +331 -0
  175. package/src/pr-review-credentials.ts +163 -0
  176. package/src/sandbox-archive-storage.ts +159 -0
  177. package/src/sandbox-resume.ts +449 -63
  178. package/src/sandbox-routing.ts +314 -16
  179. package/src/scheduled-alert-occurrence.ts +176 -0
  180. package/src/workflows/activities.ts +18 -2
  181. package/src/workflows/automations.ts +11 -0
  182. package/src/workflows/session.ts +13 -3
  183. package/src/workflows.ts +1 -0
  184. package/dist/activities-control-SMKF2OBE.js.map +0 -1
  185. package/dist/activities-turn-IDS3WEHA.js +0 -16532
  186. package/dist/activities-turn-IDS3WEHA.js.map +0 -1
  187. package/dist/chunk-N5TJIFCE.js.map +0 -1
  188. package/dist/chunk-P6Z2SLAA.js +0 -3092
  189. package/dist/chunk-P6Z2SLAA.js.map +0 -1
@@ -6,16 +6,26 @@ import {
6
6
  sandboxDrainTiming
7
7
  } from "./chunk-GRFQTTSQ.js";
8
8
  import {
9
+ INCIDENT_TELEMETRY_AUTHORITY_FENCE_LINEAGE_KEY,
10
+ MAX_AUTOMATIC_PROVIDER_RECOVERIES,
9
11
  assertReusableSessionRevivable,
12
+ collectWorkspaceArchiveObjectKeys,
10
13
  createCodexCapacityActivities,
11
14
  createGoalActivities,
12
15
  createSharedActivityServices,
13
16
  currentActivityContext,
17
+ deleteUnpublishedWorkspaceArchiveObject,
18
+ deleteWorkspaceArchiveObjectKeys,
14
19
  deliverFailedChildTurnToParent,
20
+ evaluateIncidentTelemetryPreflight,
21
+ incidentTelemetryAuthorityFence,
22
+ incidentTelemetryPreflightDeclaration,
15
23
  notifyParentOfChildIdle,
16
- providerIdentityFromResumeState,
24
+ providerRecoveryCountFromMetadata,
25
+ putVersion1TarArchiveOrInline,
17
26
  reconcilePendingParentSystemUpdates,
18
27
  reconcilePendingSessionWorkflowWakes,
28
+ resolveIncidentTelemetryResponderMetadata,
19
29
  resolveWorkspacePackRuntime,
20
30
  rigProviderImageSourceImage,
21
31
  scheduledUserMessagePayload,
@@ -23,11 +33,13 @@ import {
23
33
  settingsWithRigImage,
24
34
  withFirstPartyTools,
25
35
  workflowIdForSession
26
- } from "./chunk-P6Z2SLAA.js";
36
+ } from "./chunk-CNYGVRKF.js";
27
37
  import {
38
+ inspectOpenSandboxKubernetesInventory,
28
39
  recordCreditBalanceGauges,
29
40
  recordCreditMicros,
30
41
  recordExpiredDrainingSandboxLeaseGauges,
42
+ recordOpenSandboxKubernetesInventoryGauges,
31
43
  recordRetainedProcessInventoryGauges,
32
44
  recordRetainedProcessReconciliation,
33
45
  recordSandboxCheckpointArtifactGauges,
@@ -38,8 +50,9 @@ import {
38
50
  recordSandboxLeaseGauges,
39
51
  recordSandboxOrphansTerminated,
40
52
  recordSandboxRotationBacklogGauges,
41
- recordTurnsQueuedGauge
42
- } from "./chunk-N5TJIFCE.js";
53
+ recordTurnsQueuedGauge,
54
+ runtimeMetricsHooksForObservability
55
+ } from "./chunk-CE7TPKBS.js";
43
56
 
44
57
  // src/activities/browser-state-artifact-reaper.ts
45
58
  import {
@@ -125,14 +138,255 @@ function cleanupErrorCategory(error) {
125
138
  return typeof code === "string" && SAFE_PROVIDER_ERROR_CODES.has(code) ? code : "unknown";
126
139
  }
127
140
 
128
- // src/activities/documents.ts
141
+ // src/activities/automations.ts
142
+ import { resolveTurnExecutionPolicyV1 } from "@opengeni/config";
143
+ import {
144
+ assertWorkspaceModelPolicyAllows,
145
+ canonicalConfiguredModel,
146
+ createAndStartSessionWithOutcome,
147
+ recordWorkspaceUsage
148
+ } from "@opengeni/core";
149
+ import {
150
+ AutomationAuthorityRevokedError,
151
+ assertAutomationRunAuthorityInTransaction,
152
+ claimAutomationRun,
153
+ settleAutomationRun
154
+ } from "@opengeni/db";
155
+
156
+ // src/activities/agent-run-admission.ts
129
157
  import { configuredStaticUsageLimits } from "@opengeni/config";
158
+ import { getBillingBalance, isCodexBilledTurn, sumUsageQuantity } from "@opengeni/db";
159
+ async function agentRunAdmissionDenial(services, input) {
160
+ const externallyBilled = await isCodexBilledTurn({
161
+ db: services.db,
162
+ settings: services.settings,
163
+ workspaceId: input.workspaceId,
164
+ model: input.model
165
+ });
166
+ if (!externallyBilled && (services.settings.billingMode === "stripe" || services.settings.usageLimitsMode === "managed")) {
167
+ if (services.entitlements) {
168
+ const decision = await services.entitlements.admitRun({
169
+ accountId: input.accountId,
170
+ workspaceId: input.workspaceId,
171
+ action: "agent_run:create",
172
+ quantity: input.requestedAgentRuns
173
+ });
174
+ if (!decision.allowed) return "insufficient_credits";
175
+ } else {
176
+ const balance = await getBillingBalance(services.db, input.accountId);
177
+ if (balance.balanceMicros <= 0) return "insufficient_credits";
178
+ }
179
+ }
180
+ if (services.settings.usageLimitsMode !== "static" && services.settings.usageLimitsMode !== "managed") {
181
+ return null;
182
+ }
183
+ const limits = configuredStaticUsageLimits(services.settings);
184
+ if (!externallyBilled && limits.maxMonthlyCostMicrosPerAccount) {
185
+ const used = await sumUsageQuantity(services.db, {
186
+ accountId: input.accountId,
187
+ eventType: "model.cost",
188
+ since: startOfUtcMonth()
189
+ });
190
+ if (used >= limits.maxMonthlyCostMicrosPerAccount) {
191
+ return "monthly_model_cost_limit";
192
+ }
193
+ }
194
+ if (limits.maxMonthlyAgentRunsPerWorkspace) {
195
+ const used = await sumUsageQuantity(services.db, {
196
+ workspaceId: input.workspaceId,
197
+ eventType: "agent_run.created",
198
+ since: startOfUtcMonth()
199
+ });
200
+ if (used + input.requestedAgentRuns > limits.maxMonthlyAgentRunsPerWorkspace) {
201
+ return "monthly_agent_run_limit";
202
+ }
203
+ }
204
+ return null;
205
+ }
206
+ function startOfUtcMonth() {
207
+ const now = /* @__PURE__ */ new Date();
208
+ return new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), 1));
209
+ }
210
+
211
+ // src/activities/automations.ts
212
+ function createAutomationActivities(services, options = {}) {
213
+ const claim = options.claim ?? claimAutomationRun;
214
+ const settle = options.settle ?? settleAutomationRun;
215
+ const assertAuthority = options.assertAuthority ?? assertAutomationRunAuthorityInTransaction;
216
+ const createSession2 = options.createSession ?? createAndStartSessionWithOutcome;
217
+ const admit = options.admit ?? agentRunAdmissionDenial;
218
+ const assertModelPolicy = options.assertModelPolicy ?? assertWorkspaceModelPolicyAllows;
219
+ const recordUsage = options.recordUsage ?? recordWorkspaceUsage;
220
+ return {
221
+ dispatchAutomationRun: async (input) => {
222
+ const service = await services();
223
+ const run = await claim(service.db, input);
224
+ if (!run) return { action: "not_found" };
225
+ if (run.status === "dispatched") {
226
+ if (!run.sessionId) throw new Error("dispatched automation run is missing its session");
227
+ return { action: "already_dispatched", sessionId: run.sessionId };
228
+ }
229
+ if (run.status === "skipped") {
230
+ return { action: "skipped", reason: run.errorCode ?? "authority_revoked" };
231
+ }
232
+ const accepted = run.acceptedExecution;
233
+ if (accepted.accountId !== input.accountId || accepted.workspaceId !== input.workspaceId || accepted.sourceId !== run.sourceId || accepted.triggerId !== run.triggerId || accepted.triggerRevision !== run.triggerRevision || accepted.eventId !== run.eventId) {
234
+ await settle(service.db, {
235
+ workspaceId: input.workspaceId,
236
+ runId: run.id,
237
+ status: "skipped",
238
+ errorCode: "accepted_execution_mismatch"
239
+ });
240
+ return { action: "skipped", reason: "accepted_execution_mismatch" };
241
+ }
242
+ const template = accepted.sessionTemplate;
243
+ const model = canonicalConfiguredModel(
244
+ service.settings,
245
+ template.model ?? service.settings.openaiModel
246
+ );
247
+ if (!model) throw new Error("automation has no configured model");
248
+ await assertModelPolicy(service.db, service.settings, input.workspaceId, model);
249
+ const denial = await admit(service, {
250
+ accountId: input.accountId,
251
+ workspaceId: input.workspaceId,
252
+ model,
253
+ requestedAgentRuns: 1
254
+ });
255
+ if (denial) {
256
+ await settle(service.db, {
257
+ workspaceId: input.workspaceId,
258
+ runId: run.id,
259
+ status: "failed",
260
+ errorCode: denial
261
+ });
262
+ throw new Error(`automation admission denied: ${denial}`);
263
+ }
264
+ const reasoningEffort = template.reasoningEffort ?? service.settings.openaiReasoningEffort;
265
+ const sandboxBackend = template.sandboxBackend ?? service.settings.sandboxBackend;
266
+ if (sandboxBackend === "selfhosted") {
267
+ await settle(service.db, {
268
+ workspaceId: input.workspaceId,
269
+ runId: run.id,
270
+ status: "skipped",
271
+ errorCode: "interactive_compute_not_allowed"
272
+ });
273
+ return { action: "skipped", reason: "interactive_compute_not_allowed" };
274
+ }
275
+ const turnExecutionPolicy = resolveTurnExecutionPolicyV1(service.settings, {
276
+ modelId: model,
277
+ requestedModelId: template.model,
278
+ modelSource: template.model ? "explicit" : "deployment",
279
+ reasoningEffort,
280
+ reasoningSource: template.reasoningEffort ? "explicit" : "deployment",
281
+ latencyMode: "standard",
282
+ latencyModeSource: "deployment"
283
+ });
284
+ try {
285
+ const created = await createSession2({
286
+ db: service.db,
287
+ bus: service.bus,
288
+ workflowClient: {
289
+ wakeSessionWorkflow: async (wake) => {
290
+ await service.wakeSessionWorkflow?.(wake);
291
+ }
292
+ },
293
+ beforeCreateCommit: async (tx, sessionId) => {
294
+ await assertAuthority(tx, {
295
+ workspaceId: input.workspaceId,
296
+ runId: run.id,
297
+ triggerId: accepted.triggerId,
298
+ triggerRevision: accepted.triggerRevision,
299
+ sourceId: accepted.sourceId,
300
+ sourceVersion: accepted.sourceVersion,
301
+ sessionId
302
+ });
303
+ },
304
+ accountId: input.accountId,
305
+ workspaceId: input.workspaceId,
306
+ initialMessage: accepted.initialMessage,
307
+ resources: template.resources,
308
+ skills: template.skills,
309
+ tools: template.tools,
310
+ toolPolicy: { mode: "explicit", inheritedFromSessionId: null },
311
+ model,
312
+ reasoningEffort,
313
+ turnExecutionPolicy,
314
+ sandboxBackend,
315
+ metadata: {
316
+ ...template.metadata,
317
+ automationRunId: run.id,
318
+ automationSourceId: run.sourceId,
319
+ automationTriggerId: run.triggerId,
320
+ automationTriggerRevision: run.triggerRevision
321
+ },
322
+ createdBy: {
323
+ kind: "service",
324
+ subjectId: accepted.serviceSubjectId,
325
+ label: accepted.serviceLabel
326
+ },
327
+ createdByContext: accepted.provenance,
328
+ instructions: template.instructions,
329
+ policyRole: template.policyRole,
330
+ firstPartyMcpPermissions: template.firstPartyMcpPermissions,
331
+ firstPartyMcpTools: template.firstPartyMcpTools,
332
+ createIdempotencyKey: `automation-run:${run.id}`,
333
+ subjectId: accepted.serviceSubjectId
334
+ });
335
+ await recordUsage(
336
+ { db: service.db, settings: service.settings },
337
+ {
338
+ accountId: input.accountId,
339
+ workspaceId: input.workspaceId,
340
+ subjectId: accepted.serviceSubjectId,
341
+ eventType: "agent_run.created",
342
+ quantity: 1,
343
+ unit: "run",
344
+ sourceResourceType: "automation_run",
345
+ sourceResourceId: run.id,
346
+ sessionId: created.session.id,
347
+ initiator: created.session.createdBy,
348
+ initiatorContext: created.session.createdByContext,
349
+ origin: "system",
350
+ idempotencyKey: `agent_run.created:automation:${run.id}`
351
+ }
352
+ ).catch((error) => {
353
+ service.observability.warn("automation usage recording failed", {
354
+ workspaceId: input.workspaceId,
355
+ runId: run.id,
356
+ errorCategory: error instanceof Error ? error.name : "unknown"
357
+ });
358
+ });
359
+ return { action: "started", sessionId: created.session.id };
360
+ } catch (error) {
361
+ if (error instanceof AutomationAuthorityRevokedError) {
362
+ await settle(service.db, {
363
+ workspaceId: input.workspaceId,
364
+ runId: run.id,
365
+ status: "skipped",
366
+ errorCode: "authority_revoked"
367
+ });
368
+ return { action: "skipped", reason: "authority_revoked" };
369
+ }
370
+ await settle(service.db, {
371
+ workspaceId: input.workspaceId,
372
+ runId: run.id,
373
+ status: "failed",
374
+ errorCode: "dispatch_failed"
375
+ });
376
+ throw error;
377
+ }
378
+ }
379
+ };
380
+ }
381
+
382
+ // src/activities/documents.ts
383
+ import { configuredStaticUsageLimits as configuredStaticUsageLimits2 } from "@opengeni/config";
130
384
  import {
131
- getBillingBalance,
385
+ getBillingBalance as getBillingBalance2,
132
386
  recordUsageEvent,
133
387
  resolveDocumentIndexAuthority,
134
388
  rlsContextForWorkspace,
135
- sumUsageQuantity,
389
+ sumUsageQuantity as sumUsageQuantity2,
136
390
  withWorkspaceUsageLock
137
391
  } from "@opengeni/db";
138
392
  function createDocumentActivities(services, resolveDocumentServices) {
@@ -145,7 +399,7 @@ function createDocumentActivities(services, resolveDocumentServices) {
145
399
  if (!resolveDocumentServices) {
146
400
  throw new Error("document services are not configured");
147
401
  }
148
- const [{ getDocument, indexDocumentNow }, documentServices] = await Promise.all([
402
+ const [{ getDocumentForIndexing, indexDocumentNow }, documentServices] = await Promise.all([
149
403
  import("@opengeni/documents"),
150
404
  resolveDocumentServices()
151
405
  ]);
@@ -173,9 +427,11 @@ function createDocumentActivities(services, resolveDocumentServices) {
173
427
  if (suppliedCount !== 0 && suppliedCount !== suppliedAuthorityFields.length) {
174
428
  throw new Error("document authority tuple is partial");
175
429
  }
176
- const claimedDocument = await getDocument(lockedDb, input.workspaceId, input.documentId, {
177
- viewerSubjectId: storedAuthority.authoritySubjectId
178
- });
430
+ const claimedDocument = await getDocumentForIndexing(
431
+ lockedDb,
432
+ input.workspaceId,
433
+ input.documentId
434
+ );
179
435
  if (!claimedDocument || claimedDocument.authorityKind !== storedAuthority.authorityKind || claimedDocument.authorityWorkspaceId !== storedAuthority.authorityWorkspaceId || claimedDocument.authoritySubjectId !== storedAuthority.authoritySubjectId) {
180
436
  throw new Error("document authority changed before indexing");
181
437
  }
@@ -188,7 +444,7 @@ function createDocumentActivities(services, resolveDocumentServices) {
188
444
  {
189
445
  beforeEmbed: async ({ chunkCount }) => {
190
446
  if (settings.billingMode === "stripe" || settings.usageLimitsMode === "managed") {
191
- const balance = await getBillingBalance(lockedDb, input.accountId);
447
+ const balance = await getBillingBalance2(lockedDb, input.accountId);
192
448
  if (balance.balanceMicros <= 0) {
193
449
  throw new Error("insufficient OpenGeni credits");
194
450
  }
@@ -196,14 +452,14 @@ function createDocumentActivities(services, resolveDocumentServices) {
196
452
  if (settings.usageLimitsMode !== "static" && settings.usageLimitsMode !== "managed") {
197
453
  return;
198
454
  }
199
- const limit = configuredStaticUsageLimits(settings).maxDocumentIndexedChunksPerWorkspace;
455
+ const limit = configuredStaticUsageLimits2(settings).maxDocumentIndexedChunksPerWorkspace;
200
456
  if (!limit) {
201
457
  return;
202
458
  }
203
- const used = await sumUsageQuantity(lockedDb, {
459
+ const used = await sumUsageQuantity2(lockedDb, {
204
460
  workspaceId: input.workspaceId,
205
461
  eventType: "document.indexed",
206
- since: startOfUtcMonth()
462
+ since: startOfUtcMonth2()
207
463
  });
208
464
  if (used + chunkCount > limit) {
209
465
  throw new Error(`monthly document indexing limit reached (${limit} chunks)`);
@@ -232,7 +488,7 @@ function createDocumentActivities(services, resolveDocumentServices) {
232
488
  }
233
489
  };
234
490
  }
235
- function startOfUtcMonth(date = /* @__PURE__ */ new Date()) {
491
+ function startOfUtcMonth2(date = /* @__PURE__ */ new Date()) {
236
492
  return new Date(Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), 1));
237
493
  }
238
494
 
@@ -405,6 +661,9 @@ function createFileUploadReaperActivities(services, options = {}) {
405
661
  // src/activities/knowledge-source-sync.ts
406
662
  import { createHash as createHash2 } from "crypto";
407
663
  import { heartbeat } from "@temporalio/activity";
664
+ import {
665
+ googleDriveProviderRetryOptions
666
+ } from "@opengeni/config";
408
667
  import {
409
668
  KnowledgeSourceSyncRunSummary
410
669
  } from "@opengeni/contracts";
@@ -431,6 +690,7 @@ import {
431
690
  appendKnowledgeDocumentVersion,
432
691
  appendKnowledgeSourceAclVersion,
433
692
  beginKnowledgeSyncRun,
693
+ beginGoogleDriveObjectAclRefresh,
434
694
  buildConnectionTokenResolver,
435
695
  checkpointKnowledgeSourceSync,
436
696
  claimKnowledgeSourceSyncLease,
@@ -452,6 +712,7 @@ import {
452
712
  KnowledgeSourceSyncObservationFenceError,
453
713
  reconcileKnowledgeSourceSyncCompleteScan,
454
714
  reconcileKnowledgeSourceSyncLiveGeneration,
715
+ recordGoogleDriveObjectAclEvidence,
455
716
  recordKnowledgeSourceSyncItemOutcomes,
456
717
  recordKnowledgeSourceSyncObjectObservations,
457
718
  recordUsageEvent as recordUsageEvent2,
@@ -467,6 +728,121 @@ import {
467
728
  } from "@opengeni/db";
468
729
  import { readResponseJsonBounded } from "@opengeni/network";
469
730
 
731
+ // src/activities/google-drive-provider.ts
732
+ var GoogleDriveProviderTransportError = class extends Error {
733
+ constructor() {
734
+ super("google_drive_provider_transport_failed");
735
+ this.name = "GoogleDriveProviderTransportError";
736
+ }
737
+ };
738
+ async function fetchGoogleDriveProvider(input) {
739
+ const now = input.now ?? Date.now;
740
+ const sleep = input.sleep ?? Bun.sleep;
741
+ let retryDelaySpentMs = 0;
742
+ for (let attempt = 1; attempt <= input.policy.attempts; attempt += 1) {
743
+ let response;
744
+ input.heartbeat?.({ operation: input.operation, attempt, phase: "request_start" });
745
+ try {
746
+ response = await input.fetchImpl(input.url, {
747
+ ...input.init,
748
+ redirect: "error",
749
+ signal: AbortSignal.timeout(input.policy.requestTimeoutMs)
750
+ });
751
+ } catch {
752
+ input.heartbeat?.({ operation: input.operation, attempt, phase: "request_end" });
753
+ recordRequest(input.observability, input.operation, "retryable_error");
754
+ const delayMs2 = retryDelayMs(input.policy, attempt, null);
755
+ if (!canRetry(input.policy, attempt, retryDelaySpentMs, delayMs2)) {
756
+ throw new GoogleDriveProviderTransportError();
757
+ }
758
+ recordRetry(input.observability, input.operation, "transport", attempt, delayMs2);
759
+ retryDelaySpentMs += delayMs2;
760
+ input.heartbeat?.({ operation: input.operation, attempt, phase: "retry_delay_start" });
761
+ await sleep(delayMs2);
762
+ input.heartbeat?.({ operation: input.operation, attempt, phase: "retry_delay_end" });
763
+ continue;
764
+ }
765
+ input.heartbeat?.({ operation: input.operation, attempt, phase: "request_end" });
766
+ if (!isRetryableStatus(response.status)) {
767
+ recordRequest(
768
+ input.observability,
769
+ input.operation,
770
+ response.ok ? "succeeded" : "terminal_error"
771
+ );
772
+ return response;
773
+ }
774
+ recordRequest(input.observability, input.operation, "retryable_error");
775
+ const delayMs = retryDelayMs(
776
+ input.policy,
777
+ attempt,
778
+ retryAfterDelayMs(response.headers.get("retry-after"), now())
779
+ );
780
+ if (!canRetry(input.policy, attempt, retryDelaySpentMs, delayMs)) return response;
781
+ await response.body?.cancel().catch(() => void 0);
782
+ recordRetry(
783
+ input.observability,
784
+ input.operation,
785
+ response.status === 429 ? "rate_limited" : "provider_unavailable",
786
+ attempt,
787
+ delayMs
788
+ );
789
+ retryDelaySpentMs += delayMs;
790
+ input.heartbeat?.({ operation: input.operation, attempt, phase: "retry_delay_start" });
791
+ await sleep(delayMs);
792
+ input.heartbeat?.({ operation: input.operation, attempt, phase: "retry_delay_end" });
793
+ }
794
+ throw new GoogleDriveProviderTransportError();
795
+ }
796
+ function canRetry(policy, attempt, retryDelaySpentMs, delayMs) {
797
+ return attempt < policy.attempts && retryDelaySpentMs + delayMs <= policy.budgetMs;
798
+ }
799
+ function retryDelayMs(policy, attempt, retryAfterMs) {
800
+ const exponential = Math.min(
801
+ policy.maxDelayMs,
802
+ policy.initialDelayMs * 2 ** Math.max(0, attempt - 1)
803
+ );
804
+ return Math.min(policy.maxDelayMs, Math.max(exponential, retryAfterMs ?? 0));
805
+ }
806
+ function retryAfterDelayMs(value, nowMs) {
807
+ if (!value) return null;
808
+ const seconds = Number(value);
809
+ if (Number.isFinite(seconds) && seconds >= 0) return Math.round(seconds * 1e3);
810
+ const timestamp = Date.parse(value);
811
+ return Number.isFinite(timestamp) ? Math.max(0, timestamp - nowMs) : null;
812
+ }
813
+ function isRetryableStatus(status) {
814
+ return status === 429 || status >= 500;
815
+ }
816
+ function recordRequest(observability, operation, outcome) {
817
+ observability.incrementCounter({
818
+ name: "opengeni_google_drive_provider_requests_total",
819
+ help: "Google Drive provider request attempts by operation and bounded outcome.",
820
+ labels: { operation, outcome }
821
+ });
822
+ }
823
+ function recordRetry(observability, operation, reason, attempt, delayMs) {
824
+ observability.incrementCounter({
825
+ name: "opengeni_google_drive_provider_retries_total",
826
+ help: "Bounded Google Drive provider retries by operation and reason.",
827
+ labels: { operation, reason }
828
+ });
829
+ observability.observeHistogram({
830
+ name: "opengeni_google_drive_provider_retry_delay_seconds",
831
+ help: "Delay applied before a bounded Google Drive provider retry.",
832
+ labels: { operation, reason },
833
+ value: delayMs / 1e3,
834
+ buckets: [0.1, 0.25, 0.5, 1, 2, 5, 10, 30, 60]
835
+ });
836
+ observability.warn("Google Drive provider request retry scheduled", {
837
+ provider: "google_drive",
838
+ operation,
839
+ outcome: "retry_scheduled",
840
+ reason,
841
+ attempt,
842
+ delayMs
843
+ });
844
+ }
845
+
470
846
  // src/activities/google-drive-changes.ts
471
847
  import { createHash } from "crypto";
472
848
  import {
@@ -747,6 +1123,8 @@ var DRIVE_JSON_MAX_BYTES = 2 * 1024 * 1024;
747
1123
  var GOOGLE_DRIVE_FULL_RECONCILIATION_INTERVAL_MS = 24 * 60 * 60 * 1e3;
748
1124
  var GOOGLE_DRIVE_SYNC_INVOCATION_SLICE_MS = 3e4;
749
1125
  var GOOGLE_DRIVE_EXECUTION_CHECKPOINT_MAX_BYTES = 2 * 1024 * 1024;
1126
+ var GOOGLE_DRIVE_ACL_FRESHNESS_MS = 26 * 60 * 60 * 1e3;
1127
+ var GOOGLE_DRIVE_ACL_MAX_PRINCIPALS = 1e3;
750
1128
  var ATLASSIAN_API_BASE = "https://api.atlassian.com";
751
1129
  var ATLASSIAN_JSON_MAX_BYTES = 4 * 1024 * 1024;
752
1130
  function knowledgeSyncObservationPolicy(kind) {
@@ -799,6 +1177,10 @@ function createKnowledgeSourceSyncActivities(services, resolveDocumentServices)
799
1177
  reconnectRequired: false,
800
1178
  failures: []
801
1179
  };
1180
+ observability.info("Knowledge source sync started", {
1181
+ provider: providerLabel,
1182
+ outcome: "started"
1183
+ });
802
1184
  try {
803
1185
  summary.phase = "inventory";
804
1186
  const resolved = await getKnowledgeSourceForSyncAuthority(db, {
@@ -879,6 +1261,7 @@ function createKnowledgeSourceSyncActivities(services, resolveDocumentServices)
879
1261
  const provider = await resolveKnowledgeSyncProvider({
880
1262
  db,
881
1263
  settings,
1264
+ observability,
882
1265
  accountId: input.accountId,
883
1266
  workspaceId: input.workspaceId,
884
1267
  action,
@@ -1008,7 +1391,26 @@ function createKnowledgeSourceSyncActivities(services, resolveDocumentServices)
1008
1391
  if (obligation.documentId !== details.version.documentId) {
1009
1392
  throw new SyncFailure("authority_changed", false);
1010
1393
  }
1011
- if (obligation.status === "indexed") return obligation;
1394
+ if (obligation.status === "indexed" && !details.indexRequired) return obligation;
1395
+ if (obligation.status === "indexed" && details.indexRequired) {
1396
+ summary.phase = "index";
1397
+ try {
1398
+ const indexedDocument = await documentActivities.indexDocument({
1399
+ accountId: input.accountId,
1400
+ workspaceId: input.workspaceId,
1401
+ documentId: details.version.documentId,
1402
+ authorityKind: action.destination.kind,
1403
+ authorityWorkspaceId: action.destination.workspaceId,
1404
+ authoritySubjectId: action.destination.subjectId
1405
+ });
1406
+ if (indexedDocument.status !== "ready") {
1407
+ throw new Error("document indexing did not complete");
1408
+ }
1409
+ } catch {
1410
+ throw new SyncFailure("indexing_failed", true);
1411
+ }
1412
+ return obligation;
1413
+ }
1012
1414
  if (obligation.status === "failed") {
1013
1415
  const retried = await retryKnowledgeSourceSyncIndexObligation(db, {
1014
1416
  accountId: input.accountId,
@@ -1061,6 +1463,64 @@ function createKnowledgeSourceSyncActivities(services, resolveDocumentServices)
1061
1463
  }
1062
1464
  return { ...obligation, status: "indexed" };
1063
1465
  };
1466
+ const recordEntryAcl = async (details) => {
1467
+ if (!driver.readAcl || !provider.aclAuthority) {
1468
+ return { aclEligibility: "pending", aclEvidence: null };
1469
+ }
1470
+ await beginGoogleDriveObjectAclRefresh(db, {
1471
+ accountId: input.accountId,
1472
+ workspaceId: input.workspaceId,
1473
+ obligationId: details.obligation.id,
1474
+ sourceSyncGeneration: details.obligation.sourceSyncGeneration,
1475
+ sourceConfigGeneration: action.sourceConfigGeneration,
1476
+ sourceLifecycleGeneration: action.sourceLifecycleGeneration,
1477
+ objectLifecycleGeneration: details.object.lifecycleGeneration,
1478
+ objectVersionGeneration: details.version.versionGeneration
1479
+ });
1480
+ const remainingProviderRequests = action.limits.maxProviderRequests - summary.providerRequests;
1481
+ if (remainingProviderRequests < 1) {
1482
+ throw new SyncFailure("resource_limit", false);
1483
+ }
1484
+ const evidence = await driver.readAcl(details.entry, remainingProviderRequests);
1485
+ summary.providerRequests += evidence.providerRequests;
1486
+ if (summary.providerRequests > action.limits.maxProviderRequests) {
1487
+ throw new SyncFailure("resource_limit", false);
1488
+ }
1489
+ const recorded = await recordGoogleDriveObjectAclEvidence(db, {
1490
+ accountId: input.accountId,
1491
+ workspaceId: input.workspaceId,
1492
+ obligationId: details.obligation.id,
1493
+ connectionId: provider.aclAuthority.connectionId,
1494
+ connectionVersion: provider.aclAuthority.connectionVersion,
1495
+ sourceGooglePermissionId: provider.aclAuthority.sourceGooglePermissionId,
1496
+ sourceSyncGeneration: details.obligation.sourceSyncGeneration,
1497
+ sourceConfigGeneration: action.sourceConfigGeneration,
1498
+ sourceLifecycleGeneration: action.sourceLifecycleGeneration,
1499
+ objectLifecycleGeneration: details.object.lifecycleGeneration,
1500
+ objectVersionGeneration: details.version.versionGeneration,
1501
+ providerRevision: evidence.providerRevision,
1502
+ driveId: evidence.driveId,
1503
+ aclRevision: evidence.aclRevision,
1504
+ eligibility: evidence.eligibility,
1505
+ observedAt: evidence.observedAt,
1506
+ expiresAt: evidence.expiresAt,
1507
+ citationLocator: driver.citationLocator(details.entry),
1508
+ operationId: `drive-acl:${details.obligation.id}:${evidence.aclRevision}`,
1509
+ principals: evidence.principals
1510
+ });
1511
+ return {
1512
+ aclEligibility: evidence.eligibility,
1513
+ aclEvidence: {
1514
+ version: 1,
1515
+ provider: "google_drive",
1516
+ evidenceId: recorded.evidenceId,
1517
+ aclRevision: evidence.aclRevision,
1518
+ aclHash: recorded.aclHash,
1519
+ observedAt: evidence.observedAt,
1520
+ expiresAt: evidence.expiresAt
1521
+ }
1522
+ };
1523
+ };
1064
1524
  const outcomes = [];
1065
1525
  for (const entry of entriesToProcess) {
1066
1526
  heartbeat({ sourceId: input.sourceId, externalObjectId: entry.externalObjectId });
@@ -1158,9 +1618,15 @@ function createKnowledgeSourceSyncActivities(services, resolveDocumentServices)
1158
1618
  entry,
1159
1619
  object,
1160
1620
  version: currentVersion,
1161
- indexRequired: true,
1621
+ indexRequired: (currentObservation?.documentChunkCount ?? 0) === 0,
1162
1622
  obligation: currentObligation
1163
1623
  });
1624
+ const objectAcl2 = await recordEntryAcl({
1625
+ entry,
1626
+ object,
1627
+ version: currentVersion,
1628
+ obligation: obligation2
1629
+ });
1164
1630
  summary.unchanged += 1;
1165
1631
  outcomes.push({
1166
1632
  externalObjectId: entry.externalObjectId,
@@ -1168,7 +1634,8 @@ function createKnowledgeSourceSyncActivities(services, resolveDocumentServices)
1168
1634
  contentSha256: currentVersion.contentSha256,
1169
1635
  providerRevision: entry.externalVersionId,
1170
1636
  metadataHash,
1171
- aclEligibility: "pending",
1637
+ aclEligibility: objectAcl2.aclEligibility,
1638
+ aclEvidence: objectAcl2.aclEvidence,
1172
1639
  indexObligationId: obligation2.id
1173
1640
  });
1174
1641
  continue;
@@ -1249,9 +1716,10 @@ function createKnowledgeSourceSyncActivities(services, resolveDocumentServices)
1249
1716
  version: version2,
1250
1717
  indexRequired: (currentObservation?.documentChunkCount ?? 0) === 0
1251
1718
  });
1719
+ const objectAcl2 = await recordEntryAcl({ entry, object, version: version2, obligation: obligation2 });
1252
1720
  summary.unchanged += 1;
1253
1721
  summary.indexed += 1;
1254
- summary.aclPending += 1;
1722
+ if (objectAcl2.aclEligibility === "pending") summary.aclPending += 1;
1255
1723
  outcomes.push({
1256
1724
  externalObjectId: entry.externalObjectId,
1257
1725
  outcome: "unchanged",
@@ -1259,7 +1727,8 @@ function createKnowledgeSourceSyncActivities(services, resolveDocumentServices)
1259
1727
  sizeBytes: bytes.byteLength,
1260
1728
  providerRevision: entry.externalVersionId,
1261
1729
  metadataHash,
1262
- aclEligibility: "pending",
1730
+ aclEligibility: objectAcl2.aclEligibility,
1731
+ aclEvidence: objectAcl2.aclEvidence,
1263
1732
  indexObligationId: obligation2.id
1264
1733
  });
1265
1734
  continue;
@@ -1349,9 +1818,10 @@ function createKnowledgeSourceSyncActivities(services, resolveDocumentServices)
1349
1818
  version,
1350
1819
  indexRequired: true
1351
1820
  });
1821
+ const objectAcl = await recordEntryAcl({ entry, object, version, obligation });
1352
1822
  summary.indexed += 1;
1353
1823
  summary.imported += 1;
1354
- summary.aclPending += 1;
1824
+ if (objectAcl.aclEligibility === "pending") summary.aclPending += 1;
1355
1825
  summary.bytes += bytes.byteLength;
1356
1826
  outcomes.push({
1357
1827
  externalObjectId: entry.externalObjectId,
@@ -1360,7 +1830,8 @@ function createKnowledgeSourceSyncActivities(services, resolveDocumentServices)
1360
1830
  sizeBytes: bytes.byteLength,
1361
1831
  providerRevision: entry.externalVersionId,
1362
1832
  metadataHash,
1363
- aclEligibility: "pending",
1833
+ aclEligibility: objectAcl.aclEligibility,
1834
+ aclEvidence: objectAcl.aclEvidence,
1364
1835
  indexObligationId: obligation.id
1365
1836
  });
1366
1837
  } catch (error) {
@@ -1520,6 +1991,7 @@ function createKnowledgeSourceSyncActivities(services, resolveDocumentServices)
1520
1991
  labels: { provider: providerLabel },
1521
1992
  amount: summary.bytes
1522
1993
  });
1994
+ recordSyncHealthTelemetry(observability, providerLabel, summary, "succeeded");
1523
1995
  return {
1524
1996
  action: "complete",
1525
1997
  bufferedWake: settled.bufferedWake,
@@ -1604,6 +2076,7 @@ function createKnowledgeSourceSyncActivities(services, resolveDocumentServices)
1604
2076
  labels: { provider: providerLabel },
1605
2077
  amount: summary.bytes
1606
2078
  });
2079
+ recordSyncHealthTelemetry(observability, providerLabel, summary, "failed", failure);
1607
2080
  return {
1608
2081
  action: "failed",
1609
2082
  bufferedWake: settled.bufferedWake,
@@ -1771,6 +2244,11 @@ async function resolveKnowledgeSyncProvider(input) {
1771
2244
  return {
1772
2245
  kind: "google_drive",
1773
2246
  selectedDestination: source.destination,
2247
+ aclAuthority: {
2248
+ connectionId: resolvedConnection.id,
2249
+ connectionVersion: resolvedConnection.version,
2250
+ sourceGooglePermissionId: metadata.googlePermissionId
2251
+ },
1774
2252
  driver: googleDriveSyncDriver({
1775
2253
  actionProviderCoordinationKey: action.providerCoordinationKey,
1776
2254
  metadata,
@@ -1779,6 +2257,8 @@ async function resolveKnowledgeSyncProvider(input) {
1779
2257
  workspaceId: input.workspaceId,
1780
2258
  initiatingSubjectId: action.initiatingSubjectId,
1781
2259
  authorization,
2260
+ retry: googleDriveProviderRetryOptions(input.settings),
2261
+ observability: input.observability,
1782
2262
  limits: action.limits,
1783
2263
  connectionId: resolvedConnection.id,
1784
2264
  fullReconciliationIntervalMs: GOOGLE_DRIVE_FULL_RECONCILIATION_INTERVAL_MS,
@@ -1803,6 +2283,7 @@ async function resolveKnowledgeSyncProvider(input) {
1803
2283
  return {
1804
2284
  kind: "atlassian",
1805
2285
  selectedDestination: source.destination,
2286
+ aclAuthority: null,
1806
2287
  driver: atlassianSyncDriver({
1807
2288
  actionProviderCoordinationKey: action.providerCoordinationKey,
1808
2289
  metadata,
@@ -1863,10 +2344,18 @@ function atlassianSyncDriver(input) {
1863
2344
  function googleDriveSyncDriver(input) {
1864
2345
  const maxElapsedMs = input.limits.maxElapsedSeconds * 1e3;
1865
2346
  const provider = input.provider ?? {
1866
- getStartPageToken: async (driveId) => await getDriveStartPageToken(driveId, input.authorization),
1867
- listChanges: async (pageToken, driveId, pageSize) => await listDriveChanges(pageToken, driveId, pageSize, input.authorization),
1868
- getFile: async (fileId) => await getDriveFile(fileId, input.authorization),
1869
- listChildren: async (request) => await listDriveChildren(request, input.authorization)
2347
+ getStartPageToken: async (driveId) => await getDriveStartPageToken(driveId, input.authorization, input.retry, input.observability),
2348
+ listChanges: async (pageToken, driveId, pageSize) => await listDriveChanges(
2349
+ pageToken,
2350
+ driveId,
2351
+ pageSize,
2352
+ input.authorization,
2353
+ input.retry,
2354
+ input.observability
2355
+ ),
2356
+ getFile: async (fileId) => await getDriveFile(fileId, input.authorization, input.retry, input.observability),
2357
+ listChildren: async (request) => await listDriveChildren(request, input.authorization, input.retry, input.observability),
2358
+ listPermissions: async (request) => await listDrivePermissions(request, input.authorization, input.retry, input.observability)
1870
2359
  };
1871
2360
  const clock = provider.now ?? Date.now;
1872
2361
  const budgetLimits = {
@@ -2177,12 +2666,26 @@ function googleDriveSyncDriver(input) {
2177
2666
  };
2178
2667
  }
2179
2668
  },
2180
- fetchContent: async (entry, maxBytes) => await fetchDriveBytes(entry, input.authorization, maxBytes),
2669
+ fetchContent: async (entry, maxBytes) => await fetchDriveBytes(entry, input.authorization, maxBytes, input.retry, input.observability),
2670
+ readAcl: async (entry, maxProviderRequests) => {
2671
+ if (!provider.listPermissions) {
2672
+ throw new SyncFailure("provider_rejected", false);
2673
+ }
2674
+ return await collectGoogleDriveAclEvidence({
2675
+ entry,
2676
+ maxProviderRequests,
2677
+ listPermissions: provider.listPermissions,
2678
+ getFile: provider.getFile,
2679
+ now: clock
2680
+ });
2681
+ },
2181
2682
  citationLocator: (entry) => ({
2182
- version: 1,
2683
+ version: 2,
2183
2684
  providerKey: "google-drive",
2184
2685
  providerCoordinationKey: input.actionProviderCoordinationKey,
2185
2686
  externalObjectId: entry.externalObjectId,
2687
+ providerRevision: entry.externalVersionId,
2688
+ driveId: entry.driveId,
2186
2689
  sourceUri: entry.sourceUri
2187
2690
  })
2188
2691
  };
@@ -2351,16 +2854,18 @@ function googleDriveInventoryStopIsHard(reason, budget, limits) {
2351
2854
  }
2352
2855
  return reason === "elapsed_time_limit" && budget.elapsedMs >= limits.maxElapsedMs;
2353
2856
  }
2354
- async function getDriveStartPageToken(driveId, authorization) {
2857
+ async function getDriveStartPageToken(driveId, authorization, retry, observability) {
2355
2858
  if (!authorization) throw new SyncFailure("connection_reconnect_required", true, true);
2356
2859
  const url = new URL(`${DRIVE_API_BASE}/changes/startPageToken`);
2357
2860
  url.searchParams.set("supportsAllDrives", "true");
2358
2861
  if (driveId) url.searchParams.set("driveId", driveId);
2359
- const response = await fetch(url, {
2360
- headers: { authorization, accept: "application/json" },
2361
- redirect: "error",
2362
- signal: AbortSignal.timeout(15e3)
2363
- });
2862
+ const response = await requestGoogleDrive(
2863
+ url,
2864
+ { headers: { authorization, accept: "application/json" } },
2865
+ "start_page_token",
2866
+ retry,
2867
+ observability
2868
+ );
2364
2869
  if (!response.ok) throwGoogleDriveResponse(response);
2365
2870
  const payload = await readResponseJsonBounded(
2366
2871
  response,
@@ -2369,7 +2874,7 @@ async function getDriveStartPageToken(driveId, authorization) {
2369
2874
  );
2370
2875
  return requiredString(payload.startPageToken, "start_page_token");
2371
2876
  }
2372
- async function listDriveChanges(pageToken, driveId, pageSize, authorization) {
2877
+ async function listDriveChanges(pageToken, driveId, pageSize, authorization, retry, observability) {
2373
2878
  if (!authorization) throw new SyncFailure("connection_reconnect_required", true, true);
2374
2879
  const url = new URL(`${DRIVE_API_BASE}/changes`);
2375
2880
  url.searchParams.set("pageToken", pageToken);
@@ -2382,11 +2887,13 @@ async function listDriveChanges(pageToken, driveId, pageSize, authorization) {
2382
2887
  "nextPageToken,newStartPageToken,changes(fileId,removed,time,driveId,file(id,name,mimeType,driveId,parents,modifiedTime,createdTime,version,md5Checksum,size,webViewLink,trashed))"
2383
2888
  );
2384
2889
  if (driveId) url.searchParams.set("driveId", driveId);
2385
- const response = await fetch(url, {
2386
- headers: { authorization, accept: "application/json" },
2387
- redirect: "error",
2388
- signal: AbortSignal.timeout(15e3)
2389
- });
2890
+ const response = await requestGoogleDrive(
2891
+ url,
2892
+ { headers: { authorization, accept: "application/json" } },
2893
+ "list_changes",
2894
+ retry,
2895
+ observability
2896
+ );
2390
2897
  if (response.status === 410) throw new GoogleDriveCursorInvalidError();
2391
2898
  if (!response.ok) throwGoogleDriveResponse(response);
2392
2899
  const payload = await readResponseJsonBounded(
@@ -2409,7 +2916,7 @@ async function listDriveChanges(pageToken, driveId, pageSize, authorization) {
2409
2916
  newStartPageToken: optionalString(payload.newStartPageToken)
2410
2917
  };
2411
2918
  }
2412
- async function getDriveFile(fileId, authorization) {
2919
+ async function getDriveFile(fileId, authorization, retry, observability) {
2413
2920
  if (!authorization) throw new SyncFailure("connection_reconnect_required", true, true);
2414
2921
  const url = new URL(`${DRIVE_API_BASE}/files/${encodeURIComponent(fileId)}`);
2415
2922
  url.searchParams.set("supportsAllDrives", "true");
@@ -2417,65 +2924,276 @@ async function getDriveFile(fileId, authorization) {
2417
2924
  "fields",
2418
2925
  "id,name,mimeType,driveId,parents,modifiedTime,createdTime,version,md5Checksum,size,webViewLink,trashed"
2419
2926
  );
2420
- const response = await fetch(url, {
2421
- headers: { authorization, accept: "application/json" },
2422
- redirect: "error",
2423
- signal: AbortSignal.timeout(15e3)
2424
- });
2927
+ const response = await requestGoogleDrive(
2928
+ url,
2929
+ { headers: { authorization, accept: "application/json" } },
2930
+ "get_file",
2931
+ retry,
2932
+ observability
2933
+ );
2425
2934
  if (response.status === 404) return null;
2426
2935
  if (!response.ok) throwGoogleDriveResponse(response);
2427
2936
  return parseDriveProviderItem(
2428
2937
  await readResponseJsonBounded(response, DRIVE_JSON_MAX_BYTES, "Google Drive file metadata")
2429
2938
  );
2430
2939
  }
2431
- function throwGoogleDriveResponse(response) {
2432
- if (response.status === 401 || response.status === 403) {
2433
- throw new SyncFailure("connection_reconnect_required", true, true);
2434
- }
2435
- throw new SyncFailure(
2436
- response.status >= 500 || response.status === 429 ? "provider_unavailable" : "provider_rejected",
2437
- response.status >= 500 || response.status === 429
2438
- );
2439
- }
2440
- function selectedDriveSource(metadata, externalSourceId) {
2441
- const sources = metadata.selectedSources ?? (metadata.selectedSource ? [metadata.selectedSource] : []);
2442
- return sources.find((source) => source.id === externalSourceId) ?? null;
2443
- }
2444
- function googleDriveSyncProviderAccessAllowed(input) {
2445
- return input.resolvedConnectionVersion !== void 0 && input.connectionVersion === input.resolvedConnectionVersion && input.connectionStatus === "active" && (input.lifecycleState === void 0 || input.lifecycleState === "active") && googleDriveScopesAllowCapability(input.grantedScopes, "recursive_source_sync");
2446
- }
2447
- async function listDriveChildren(input, authorization) {
2940
+ async function listDrivePermissions(input, authorization, retry, observability) {
2448
2941
  if (!authorization) throw new SyncFailure("connection_reconnect_required", true, true);
2449
- const url = new URL(`${DRIVE_API_BASE}/files`);
2450
- url.searchParams.set(
2451
- "q",
2452
- `'${input.folderId.replaceAll("'", "\\'")}' in parents and trashed = false`
2453
- );
2454
- url.searchParams.set("pageSize", String(input.pageSize));
2942
+ const url = new URL(`${DRIVE_API_BASE}/files/${encodeURIComponent(input.fileId)}/permissions`);
2455
2943
  url.searchParams.set("supportsAllDrives", "true");
2456
- url.searchParams.set("includeItemsFromAllDrives", "true");
2944
+ url.searchParams.set("pageSize", String(input.pageSize));
2457
2945
  url.searchParams.set(
2458
2946
  "fields",
2459
- "nextPageToken,incompleteSearch,files(id,name,mimeType,driveId,parents,modifiedTime,createdTime,version,md5Checksum,size,webViewLink,trashed)"
2947
+ "nextPageToken,permissions(id,type,emailAddress,domain,role,allowFileDiscovery,expirationTime,deleted,permissionDetails(inherited))"
2460
2948
  );
2461
2949
  if (input.pageToken) url.searchParams.set("pageToken", input.pageToken);
2462
- if (input.driveId) {
2463
- url.searchParams.set("corpora", "drive");
2464
- url.searchParams.set("driveId", input.driveId);
2950
+ const response = await requestGoogleDrive(
2951
+ url,
2952
+ { headers: { authorization, accept: "application/json" } },
2953
+ "list_permissions",
2954
+ retry,
2955
+ observability
2956
+ );
2957
+ if (response.status === 403 || response.status === 404) {
2958
+ return { permissions: [], nextPageToken: null, denied: true };
2465
2959
  }
2466
- const response = await fetch(url, {
2467
- headers: { authorization, accept: "application/json" },
2468
- redirect: "error",
2469
- signal: AbortSignal.timeout(15e3)
2470
- });
2471
- if (!response.ok) {
2472
- if (response.status === 401 || response.status === 403) {
2473
- throw new SyncFailure("connection_reconnect_required", true, true);
2474
- }
2475
- throw new SyncFailure(
2476
- "provider_unavailable",
2477
- response.status >= 500 || response.status === 429
2478
- );
2960
+ if (!response.ok) throwGoogleDriveResponse(response);
2961
+ const payload = await readResponseJsonBounded(
2962
+ response,
2963
+ DRIVE_JSON_MAX_BYTES,
2964
+ "Google Drive permission list"
2965
+ );
2966
+ const rawPermissions = Array.isArray(payload.permissions) ? payload.permissions : null;
2967
+ if (!rawPermissions) throw new SyncFailure("provider_payload_invalid", false);
2968
+ return {
2969
+ permissions: rawPermissions.map(parseDrivePermission),
2970
+ nextPageToken: optionalString(payload.nextPageToken),
2971
+ denied: false
2972
+ };
2973
+ }
2974
+ async function collectGoogleDriveAclEvidence(input) {
2975
+ if (!Number.isSafeInteger(input.maxProviderRequests) || input.maxProviderRequests < 1) {
2976
+ throw new SyncFailure("resource_limit", false);
2977
+ }
2978
+ const now = input.now ?? Date.now;
2979
+ const observedAt = new Date(now());
2980
+ if (!Number.isFinite(observedAt.getTime())) {
2981
+ throw new SyncFailure("provider_payload_invalid", false);
2982
+ }
2983
+ const permissions = [];
2984
+ const seenTokens = /* @__PURE__ */ new Set();
2985
+ let pageToken = null;
2986
+ let providerRequests = 0;
2987
+ let permissionListDenied = false;
2988
+ do {
2989
+ if (providerRequests >= input.maxProviderRequests) {
2990
+ throw new SyncFailure("resource_limit", false);
2991
+ }
2992
+ providerRequests += 1;
2993
+ const page = await input.listPermissions({
2994
+ fileId: input.entry.externalObjectId,
2995
+ pageToken,
2996
+ pageSize: Math.min(100, GOOGLE_DRIVE_ACL_MAX_PRINCIPALS - permissions.length)
2997
+ });
2998
+ if (page.denied) {
2999
+ permissionListDenied = true;
3000
+ break;
3001
+ }
3002
+ permissions.push(...page.permissions);
3003
+ if (permissions.length > GOOGLE_DRIVE_ACL_MAX_PRINCIPALS) {
3004
+ throw new SyncFailure("resource_limit", false);
3005
+ }
3006
+ pageToken = page.nextPageToken;
3007
+ if (pageToken) {
3008
+ if (pageToken.length > 4096 || seenTokens.has(pageToken)) {
3009
+ throw new SyncFailure("provider_payload_invalid", false);
3010
+ }
3011
+ seenTokens.add(pageToken);
3012
+ }
3013
+ } while (pageToken);
3014
+ if (permissionListDenied) {
3015
+ if (!input.getFile || providerRequests >= input.maxProviderRequests) {
3016
+ return googleDriveDeniedAclEvidence(input.entry, observedAt, providerRequests);
3017
+ }
3018
+ providerRequests += 1;
3019
+ const current = await input.getFile(input.entry.externalObjectId);
3020
+ if (!current || current.trashed) {
3021
+ return googleDriveDeniedAclEvidence(input.entry, observedAt, providerRequests);
3022
+ }
3023
+ const aclRevision2 = createHash2("sha256").update(
3024
+ JSON.stringify({
3025
+ version: 1,
3026
+ mode: "source_owner_only",
3027
+ externalObjectId: input.entry.externalObjectId,
3028
+ providerRevision: input.entry.externalVersionId
3029
+ })
3030
+ ).digest("hex");
3031
+ return {
3032
+ eligibility: "eligible",
3033
+ providerRevision: input.entry.externalVersionId,
3034
+ driveId: input.entry.driveId,
3035
+ aclRevision: aclRevision2,
3036
+ observedAt: observedAt.toISOString(),
3037
+ expiresAt: new Date(observedAt.getTime() + GOOGLE_DRIVE_ACL_FRESHNESS_MS).toISOString(),
3038
+ providerRequests,
3039
+ principals: []
3040
+ };
3041
+ }
3042
+ const canonicalPermissions = permissions.map((permission) => ({
3043
+ id: permission.id,
3044
+ type: permission.type,
3045
+ role: permission.role,
3046
+ emailAddress: permission.emailAddress,
3047
+ domain: permission.domain,
3048
+ allowFileDiscovery: permission.allowFileDiscovery,
3049
+ expirationTime: permission.expirationTime,
3050
+ inherited: permission.inherited
3051
+ })).sort((left, right) => JSON.stringify(left).localeCompare(JSON.stringify(right)));
3052
+ const aclRevision = createHash2("sha256").update(
3053
+ JSON.stringify({
3054
+ version: 1,
3055
+ externalObjectId: input.entry.externalObjectId,
3056
+ providerRevision: input.entry.externalVersionId,
3057
+ permissions: canonicalPermissions
3058
+ })
3059
+ ).digest("hex");
3060
+ return {
3061
+ eligibility: "eligible",
3062
+ providerRevision: input.entry.externalVersionId,
3063
+ driveId: input.entry.driveId,
3064
+ aclRevision,
3065
+ observedAt: observedAt.toISOString(),
3066
+ expiresAt: new Date(observedAt.getTime() + GOOGLE_DRIVE_ACL_FRESHNESS_MS).toISOString(),
3067
+ providerRequests,
3068
+ principals: canonicalPermissions.map((permission) => ({
3069
+ type: permission.type,
3070
+ permissionId: permission.id,
3071
+ emailAddress: permission.emailAddress,
3072
+ domain: permission.domain,
3073
+ role: permission.role,
3074
+ inherited: permission.inherited,
3075
+ allowFileDiscovery: permission.allowFileDiscovery,
3076
+ expirationTime: permission.expirationTime
3077
+ }))
3078
+ };
3079
+ }
3080
+ function googleDriveDeniedAclEvidence(entry, observedAt, providerRequests) {
3081
+ return {
3082
+ eligibility: "denied",
3083
+ providerRevision: entry.externalVersionId,
3084
+ driveId: entry.driveId,
3085
+ aclRevision: createHash2("sha256").update(
3086
+ JSON.stringify({
3087
+ version: 1,
3088
+ mode: "provider_denied",
3089
+ externalObjectId: entry.externalObjectId,
3090
+ providerRevision: entry.externalVersionId
3091
+ })
3092
+ ).digest("hex"),
3093
+ observedAt: observedAt.toISOString(),
3094
+ expiresAt: new Date(observedAt.getTime() + GOOGLE_DRIVE_ACL_FRESHNESS_MS).toISOString(),
3095
+ providerRequests,
3096
+ principals: []
3097
+ };
3098
+ }
3099
+ function parseDrivePermission(value) {
3100
+ const row = objectRecord(value);
3101
+ if (row.deleted === true) throw new SyncFailure("provider_payload_invalid", false);
3102
+ const type = requiredString(row.type, "permission.type");
3103
+ if (type !== "user" && type !== "group" && type !== "domain" && type !== "anyone") {
3104
+ throw new SyncFailure("provider_payload_invalid", false);
3105
+ }
3106
+ const role = requiredString(row.role, "permission.role");
3107
+ if (role !== "owner" && role !== "organizer" && role !== "fileOrganizer" && role !== "writer" && role !== "commenter" && role !== "reader") {
3108
+ throw new SyncFailure("provider_payload_invalid", false);
3109
+ }
3110
+ const permissionDetails = Array.isArray(row.permissionDetails) ? row.permissionDetails : [];
3111
+ return {
3112
+ id: requiredString(row.id, "permission.id"),
3113
+ type,
3114
+ role,
3115
+ emailAddress: optionalString(row.emailAddress),
3116
+ domain: optionalString(row.domain),
3117
+ allowFileDiscovery: typeof row.allowFileDiscovery === "boolean" ? row.allowFileDiscovery : null,
3118
+ expirationTime: optionalExactProviderTimestamp(row.expirationTime),
3119
+ inherited: permissionDetails.some((detail) => objectRecord(detail).inherited === true)
3120
+ };
3121
+ }
3122
+ function optionalExactProviderTimestamp(value) {
3123
+ const timestamp = optionalString(value);
3124
+ if (!timestamp) return null;
3125
+ const parsed = new Date(timestamp);
3126
+ if (!Number.isFinite(parsed.getTime())) throw new SyncFailure("provider_payload_invalid", false);
3127
+ return parsed.toISOString();
3128
+ }
3129
+ function throwGoogleDriveResponse(response) {
3130
+ if (response.status === 401 || response.status === 403) {
3131
+ throw new SyncFailure("connection_reconnect_required", true, true);
3132
+ }
3133
+ throw new SyncFailure(
3134
+ response.status >= 500 || response.status === 429 ? "provider_unavailable" : "provider_rejected",
3135
+ response.status >= 500 || response.status === 429
3136
+ );
3137
+ }
3138
+ async function requestGoogleDrive(url, init, operation, retry, observability) {
3139
+ try {
3140
+ return await fetchGoogleDriveProvider({
3141
+ fetchImpl: fetch,
3142
+ url,
3143
+ init,
3144
+ operation,
3145
+ policy: retry,
3146
+ observability,
3147
+ heartbeat: (details) => heartbeat({ provider: "google_drive", ...details })
3148
+ });
3149
+ } catch (error) {
3150
+ if (error instanceof GoogleDriveProviderTransportError) {
3151
+ throw new SyncFailure("provider_unavailable", true);
3152
+ }
3153
+ throw error;
3154
+ }
3155
+ }
3156
+ function selectedDriveSource(metadata, externalSourceId) {
3157
+ const sources = metadata.selectedSources ?? (metadata.selectedSource ? [metadata.selectedSource] : []);
3158
+ return sources.find((source) => source.id === externalSourceId) ?? null;
3159
+ }
3160
+ function googleDriveSyncProviderAccessAllowed(input) {
3161
+ return input.resolvedConnectionVersion !== void 0 && input.connectionVersion === input.resolvedConnectionVersion && input.connectionStatus === "active" && (input.lifecycleState === void 0 || input.lifecycleState === "active") && googleDriveScopesAllowCapability(input.grantedScopes, "recursive_source_sync");
3162
+ }
3163
+ async function listDriveChildren(input, authorization, retry, observability) {
3164
+ if (!authorization) throw new SyncFailure("connection_reconnect_required", true, true);
3165
+ const url = new URL(`${DRIVE_API_BASE}/files`);
3166
+ url.searchParams.set(
3167
+ "q",
3168
+ `'${input.folderId.replaceAll("'", "\\'")}' in parents and trashed = false`
3169
+ );
3170
+ url.searchParams.set("pageSize", String(input.pageSize));
3171
+ url.searchParams.set("supportsAllDrives", "true");
3172
+ url.searchParams.set("includeItemsFromAllDrives", "true");
3173
+ url.searchParams.set(
3174
+ "fields",
3175
+ "nextPageToken,incompleteSearch,files(id,name,mimeType,driveId,parents,modifiedTime,createdTime,version,md5Checksum,size,webViewLink,trashed)"
3176
+ );
3177
+ if (input.pageToken) url.searchParams.set("pageToken", input.pageToken);
3178
+ if (input.driveId) {
3179
+ url.searchParams.set("corpora", "drive");
3180
+ url.searchParams.set("driveId", input.driveId);
3181
+ }
3182
+ const response = await requestGoogleDrive(
3183
+ url,
3184
+ { headers: { authorization, accept: "application/json" } },
3185
+ "list_children",
3186
+ retry,
3187
+ observability
3188
+ );
3189
+ if (!response.ok) {
3190
+ if (response.status === 401 || response.status === 403) {
3191
+ throw new SyncFailure("connection_reconnect_required", true, true);
3192
+ }
3193
+ throw new SyncFailure(
3194
+ "provider_unavailable",
3195
+ response.status >= 500 || response.status === 429
3196
+ );
2479
3197
  }
2480
3198
  const payload = await readResponseJsonBounded(
2481
3199
  response,
@@ -2506,7 +3224,7 @@ function parseDriveProviderItem(value) {
2506
3224
  trashed: row.trashed === true
2507
3225
  };
2508
3226
  }
2509
- async function fetchDriveBytes(entry, authorization, maxBytes) {
3227
+ async function fetchDriveBytes(entry, authorization, maxBytes, retry, observability) {
2510
3228
  if (!authorization) throw new SyncFailure("connection_reconnect_required", true, true);
2511
3229
  const url = entry.transfer.action === "export" ? new URL(`${DRIVE_API_BASE}/files/${encodeURIComponent(entry.externalObjectId)}/export`) : new URL(`${DRIVE_API_BASE}/files/${encodeURIComponent(entry.externalObjectId)}`);
2512
3230
  if (entry.transfer.action === "export") {
@@ -2515,11 +3233,13 @@ async function fetchDriveBytes(entry, authorization, maxBytes) {
2515
3233
  url.searchParams.set("alt", "media");
2516
3234
  url.searchParams.set("supportsAllDrives", "true");
2517
3235
  }
2518
- const response = await fetch(url, {
2519
- headers: { authorization },
2520
- redirect: "error",
2521
- signal: AbortSignal.timeout(3e4)
2522
- });
3236
+ const response = await requestGoogleDrive(
3237
+ url,
3238
+ { headers: { authorization } },
3239
+ entry.transfer.action === "export" ? "export" : "download",
3240
+ retry,
3241
+ observability
3242
+ );
2523
3243
  if (!response.ok) {
2524
3244
  if (response.status === 401 || response.status === 403) {
2525
3245
  throw new SyncFailure("connection_reconnect_required", true, true);
@@ -2891,6 +3611,59 @@ function inventoryLimit(reason) {
2891
3611
  if (reason === "known_byte_limit") return "bytes";
2892
3612
  return null;
2893
3613
  }
3614
+ function recordSyncHealthTelemetry(observability, provider, summary, outcome, failure) {
3615
+ observability.observeHistogram({
3616
+ name: "opengeni_knowledge_source_sync_terminal_batch_duration_seconds",
3617
+ help: "Terminal knowledge-source activity batch duration by provider and outcome.",
3618
+ labels: { provider, outcome },
3619
+ value: Math.max(0, summary.elapsedMs) / 1e3
3620
+ });
3621
+ if (summary.providerRequests > 0) {
3622
+ observability.incrementCounter({
3623
+ name: "opengeni_knowledge_source_sync_provider_requests_total",
3624
+ help: "Logical provider requests consumed by knowledge-source synchronization.",
3625
+ labels: { provider, outcome },
3626
+ amount: summary.providerRequests
3627
+ });
3628
+ }
3629
+ if (summary.limitReached) {
3630
+ observability.incrementCounter({
3631
+ name: "opengeni_knowledge_source_sync_limit_hits_total",
3632
+ help: "Knowledge-source synchronization runs stopped by an explicit resource budget.",
3633
+ labels: { provider, limit: summary.limitReached }
3634
+ });
3635
+ }
3636
+ if (summary.reconnectRequired) {
3637
+ observability.incrementCounter({
3638
+ name: "opengeni_knowledge_source_sync_reconnect_required_total",
3639
+ help: "Knowledge-source synchronization failures requiring connector reconnection.",
3640
+ labels: { provider }
3641
+ });
3642
+ }
3643
+ if (failure) {
3644
+ observability.incrementCounter({
3645
+ name: "opengeni_knowledge_source_sync_failures_total",
3646
+ help: "Knowledge-source synchronization terminal failures by bounded reason.",
3647
+ labels: {
3648
+ provider,
3649
+ reason: failure.code,
3650
+ retryable: failure.retryable ? "true" : "false"
3651
+ }
3652
+ });
3653
+ observability.warn("Knowledge source sync failed", {
3654
+ provider,
3655
+ outcome: failure.retryable ? "retryable_failed" : "failed",
3656
+ reason: failure.code,
3657
+ durationMs: summary.elapsedMs
3658
+ });
3659
+ return;
3660
+ }
3661
+ observability.info("Knowledge source sync completed", {
3662
+ provider,
3663
+ outcome: "succeeded",
3664
+ durationMs: summary.elapsedMs
3665
+ });
3666
+ }
2894
3667
  var SyncFailure = class extends Error {
2895
3668
  constructor(code, retryable, reconnectRequired = false) {
2896
3669
  super(code);
@@ -2907,6 +3680,7 @@ import {
2907
3680
  completeRetainedScreenshotMaintenance,
2908
3681
  promoteRetainedScreenshotMaintenanceCleanup
2909
3682
  } from "@opengeni/db";
3683
+ import { retryWhileMissing } from "@opengeni/storage";
2910
3684
  var RETAINED_SCREENSHOT_PENDING_GRACE_MS = 15 * 60 * 1e3;
2911
3685
  var RETAINED_SCREENSHOT_CLAIM_TIMEOUT_MS = 10 * 60 * 1e3;
2912
3686
  var RETAINED_SCREENSHOT_MAINTENANCE_BATCH_SIZE = 100;
@@ -3012,8 +3786,11 @@ async function reconcileClaim(db, storage, claim, provider) {
3012
3786
  }
3013
3787
  async function observeClaimObject(storage, claim, provider) {
3014
3788
  const file = claimFile(storage, claim);
3015
- if (!await provider.fileExists(storage, file)) return "missing";
3016
- const head = await provider.headFile(storage, file);
3789
+ const head = await retryWhileMissing(async () => {
3790
+ if (!await provider.fileExists(storage, file)) return null;
3791
+ return await provider.headFile(storage, file);
3792
+ });
3793
+ if (!head) return "missing";
3017
3794
  return head.ContentLength === claim.sizeBytes && head.ContentType === claim.mediaType && head.Metadata?.sha256 === claim.sha256 ? "matches" : "mismatch";
3018
3795
  }
3019
3796
  function claimFile(storage, claim) {
@@ -3361,7 +4138,7 @@ async function runWithOwnedRigVerificationSandbox(input, run, dependencies = def
3361
4138
  holderId,
3362
4139
  subjectId: null,
3363
4140
  backend: input.settings.sandboxBackend,
3364
- image: input.settings.modalImageRef ?? input.settings.dockerImage,
4141
+ image: rigProviderImageSourceImage(input.settings, input.settings.sandboxBackend),
3365
4142
  rigVersionId: input.rigVersionId,
3366
4143
  leaseTtlMs: RIG_VERIFICATION_OWNER_TTL_MS,
3367
4144
  warmingLeaseTtlMs: RIG_VERIFICATION_OWNER_TTL_MS,
@@ -4278,6 +5055,7 @@ function createRigVerificationActivities(services) {
4278
5055
  // src/activities/sandbox-lease.ts
4279
5056
  import { createHash as createHash3, randomUUID as randomUUID2 } from "crypto";
4280
5057
  import { Context } from "@temporalio/activity";
5058
+ import { OpLostReason, OpState } from "@opengeni/agent-proto";
4281
5059
  import {
4282
5060
  accrueWarmSeconds,
4283
5061
  adoptLegacyModalCheckpointArtifact,
@@ -4294,7 +5072,7 @@ import {
4294
5072
  countSandboxLeasesByLiveness,
4295
5073
  deferRetainedProcessReconciliation,
4296
5074
  forceDrainOverLimitViewerOnlyBoxes,
4297
- getBillingBalance as getBillingBalance2,
5075
+ getBillingBalance as getBillingBalance3,
4298
5076
  listCreditBalancesByAccount,
4299
5077
  listLegacyModalCheckpointSlots,
4300
5078
  listLiveModalSandboxLeaseAttributions,
@@ -4318,10 +5096,18 @@ import {
4318
5096
  rlsContextForWorkspace as rlsContextForWorkspace2,
4319
5097
  settleRetainedProcess
4320
5098
  } from "@opengeni/db";
5099
+ import {
5100
+ claimConnectedMachineSessionBackgroundCommands,
5101
+ deferConnectedMachineBackgroundCommandReconciliation,
5102
+ recordConnectedMachineBackgroundCommandProof,
5103
+ settleClaimedConnectedMachineBackgroundCommand,
5104
+ settleSessionBackgroundCommandForRetainedProcess
5105
+ } from "@opengeni/db/session-background-commands";
4321
5106
  import { sandboxWarmRateMicrosPerSecond } from "@opengeni/config";
4322
5107
  import { sandboxLeaseTelemetryKey } from "@opengeni/observability";
4323
5108
  import {
4324
5109
  captureVerifiedWorkspaceArchive,
5110
+ cancelSelfhostedOp,
4325
5111
  assertConsistentSandboxProviderIdentity,
4326
5112
  createSandboxClientForBackend,
4327
5113
  deleteModalCheckpointSnapshot as deleteModalCheckpointSnapshot2,
@@ -4330,18 +5116,22 @@ import {
4330
5116
  inspectModalSandboxLifecycle,
4331
5117
  isExecSessionLostBanner,
4332
5118
  isProviderSandboxNotFoundError,
5119
+ NatsControlRpc,
4333
5120
  parseExecBannerExitCode,
4334
5121
  prepareProviderForTeardownAfterCapture,
4335
5122
  providerWorkspaceCapturePolicy,
4336
5123
  resolveModalCheckpointProviderBindingForLiveSandbox,
4337
5124
  resolveModalCheckpointProviderBinding,
4338
5125
  resolveModalCheckpointProviderBindingForSession,
5126
+ querySelfhostedOp,
4339
5127
  resumeExactSandboxSession,
4340
5128
  sandboxProviderContinuityForState,
4341
5129
  sandboxBackendForSdkBackendId,
4342
5130
  sandboxProviderInstanceIdFromEnvelope,
4343
5131
  sandboxCommandExitCode as sandboxCommandExitCode2,
4344
5132
  sandboxCommandStillRunning,
5133
+ SelfhostedControlError,
5134
+ subjectFor,
4345
5135
  sweepModalOrphanSandboxes,
4346
5136
  terminateManagedSandboxSession as terminateManagedSandboxSession2,
4347
5137
  terminateModalSandboxById,
@@ -4351,6 +5141,8 @@ import { sandboxLeaseTelemetryKey as sandboxLeaseTelemetryKey2 } from "@opengeni
4351
5141
  var RETAINED_PROCESS_RECONCILIATION_LIMIT = 20;
4352
5142
  var RETAINED_PROCESS_RECONCILIATION_CLAIM_TTL_MS = 5 * 6e4;
4353
5143
  var RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS = 5e3;
5144
+ var CONNECTED_COMMAND_RECONCILIATION_LIMIT = 20;
5145
+ var CONNECTED_COMMAND_RECONCILIATION_CLAIM_TTL_MS = 5 * 6e4;
4354
5146
  var SandboxProviderCaptureTimeoutError = class extends Error {
4355
5147
  constructor(sandboxGroupId, backend, timeoutMs, leaseEpoch, instanceId) {
4356
5148
  super(
@@ -4438,6 +5230,7 @@ function createSandboxLeaseActivities(services, options = {}) {
4438
5230
  capturePolicy
4439
5231
  ));
4440
5232
  const sweepModalOrphans = options.sweepModalOrphans ?? sweepModalOrphansForConfiguredBackend;
5233
+ const inspectOpenSandboxInventory = options.inspectOpenSandboxKubernetesInventory ?? inspectOpenSandboxKubernetesInventory;
4441
5234
  const probeRetainedProcess = options.probeRetainedProcess ?? probeRetainedProcessAtProvider;
4442
5235
  const probeDrainableProvider = options.probeDrainableProvider ?? probeDrainableProviderReadiness;
4443
5236
  async function prepareSandboxLeaseSweep() {
@@ -4446,8 +5239,15 @@ function createSandboxLeaseActivities(services, options = {}) {
4446
5239
  const { db, settings, observability } = await services();
4447
5240
  const timing = sandboxDrainTiming(settings);
4448
5241
  if (!settings.sandboxOwnershipEnabled) {
5242
+ const drainableInventory2 = await reapStaleLeaseHoldersGlobal(db, {
5243
+ viewerHolderTtlMs: settings.sandboxViewerHolderTtlMs,
5244
+ turnHolderTtlMs: settings.sandboxLeaseTtlMs,
5245
+ interactionHolderTtlMs: settings.sandboxInteractionHolderTtlMs,
5246
+ idleGraceMs: settings.sandboxIdleGraceMs
5247
+ });
5248
+ const drainable2 = drainableInventory2.slice(0, SANDBOX_REAPER_CHILD_DISPATCH_LIMIT);
4449
5249
  return {
4450
- drainable: [],
5250
+ drainable: drainable2,
4451
5251
  timeoutClass: timing.timeoutClass,
4452
5252
  snapshotTimeoutMs: timing.snapshotTimeoutMs,
4453
5253
  captureTimeoutMs: timing.captureTimeoutMs,
@@ -4506,8 +5306,7 @@ function createSandboxLeaseActivities(services, options = {}) {
4506
5306
  instanceId: input.target.instanceId
4507
5307
  });
4508
5308
  try {
4509
- const { db, settings, observability } = await services();
4510
- if (!settings.sandboxOwnershipEnabled) return { status: "skipped" };
5309
+ const { db, settings, observability, objectStorage } = await services();
4511
5310
  assertSandboxDrainInputTiming(input);
4512
5311
  const drainSettings = settings.sandboxSnapshotTimeoutMs === input.snapshotTimeoutMs ? settings : { ...settings, sandboxSnapshotTimeoutMs: input.snapshotTimeoutMs };
4513
5312
  const row = input.target;
@@ -4526,7 +5325,8 @@ function createSandboxLeaseActivities(services, options = {}) {
4526
5325
  observability,
4527
5326
  terminateBox,
4528
5327
  probeDrainableProvider,
4529
- captureAttempt
5328
+ captureAttempt,
5329
+ objectStorage
4530
5330
  );
4531
5331
  return { status: drainedCold ? "terminated" : "skipped" };
4532
5332
  } catch (error) {
@@ -4592,8 +5392,14 @@ function createSandboxLeaseActivities(services, options = {}) {
4592
5392
  let forceDrained = 0;
4593
5393
  const rotationsRequested = input.rotationsRequested;
4594
5394
  let modalOrphansTerminated = 0;
5395
+ await reconcileConnectedMachineBackgroundCommands(db, settings, observability, service.bus);
4595
5396
  if (!settings.sandboxOwnershipEnabled) {
4596
- await refreshQueueLeaseAndCreditGauges(db, observability);
5397
+ await refreshQueueLeaseAndCreditGauges(
5398
+ db,
5399
+ settings,
5400
+ observability,
5401
+ inspectOpenSandboxInventory
5402
+ );
4597
5403
  return {
4598
5404
  metered,
4599
5405
  forceDrained,
@@ -4647,7 +5453,12 @@ function createSandboxLeaseActivities(services, options = {}) {
4647
5453
  return { claimed: 0, deleted: 0, failed: 0 };
4648
5454
  }
4649
5455
  );
4650
- await refreshQueueLeaseAndCreditGauges(db, observability);
5456
+ await refreshQueueLeaseAndCreditGauges(
5457
+ db,
5458
+ settings,
5459
+ observability,
5460
+ inspectOpenSandboxInventory
5461
+ );
4651
5462
  if (input.examined > 0 || metered > 0 || forceDrained > 0 || modalOrphansTerminated > 0 || rotationsRequested > 0 || checkpointGc.claimed > 0) {
4652
5463
  observability.info("sandbox reaper dispatched", {
4653
5464
  drainable: input.examined,
@@ -4734,6 +5545,13 @@ var CHECKPOINT_GC_CLAIM_TTL_MS = 10 * 6e4;
4734
5545
  var CHECKPOINT_TOMBSTONE_RETENTION_MS = 30 * 24 * 60 * 6e4;
4735
5546
  var CHECKPOINT_TOMBSTONE_PRUNE_LIMIT = 500;
4736
5547
  var SANDBOX_MAINTENANCE_ITEM_CONCURRENCY = 8;
5548
+ function recordConnectedCommandReconciliation(observability, outcome) {
5549
+ observability.incrementCounter({
5550
+ name: "opengeni_connected_background_command_reconciliation_total",
5551
+ help: "Exact-locator Connected Machine background-command reconciliation outcomes.",
5552
+ labels: { outcome }
5553
+ });
5554
+ }
4737
5555
  async function forEachWithConcurrency(values, concurrency, visit) {
4738
5556
  if (values.length === 0) return;
4739
5557
  const workerCount = Math.min(values.length, Math.max(1, Math.floor(concurrency)));
@@ -4856,6 +5674,154 @@ async function adoptLegacyModalCheckpointReceipts(db, settings, observability) {
4856
5674
  });
4857
5675
  return adopted;
4858
5676
  }
5677
+ async function reconcileConnectedMachineBackgroundCommands(db, settings, observability, bus) {
5678
+ const claimId = crypto.randomUUID();
5679
+ let claims;
5680
+ try {
5681
+ claims = await claimConnectedMachineSessionBackgroundCommands(db, {
5682
+ claimId,
5683
+ limit: CONNECTED_COMMAND_RECONCILIATION_LIMIT,
5684
+ claimTtlMs: CONNECTED_COMMAND_RECONCILIATION_CLAIM_TTL_MS
5685
+ });
5686
+ } catch (error) {
5687
+ recordConnectedCommandReconciliation(observability, "claim_failed");
5688
+ observability.warn("sandbox reaper: connected-command claim failed", {
5689
+ error: error instanceof Error ? error.message : String(error)
5690
+ });
5691
+ return;
5692
+ }
5693
+ const controlRpc = new NatsControlRpc(
5694
+ async () => bus.getRequestConnection()
5695
+ );
5696
+ await forEachWithConcurrency(claims, SANDBOX_MAINTENANCE_ITEM_CONCURRENCY, async (claim) => {
5697
+ let proof = claim.proof;
5698
+ if (!proof) {
5699
+ try {
5700
+ proof = await probeConnectedMachineBackgroundCommand(
5701
+ claim,
5702
+ controlRpc,
5703
+ Math.min(
5704
+ settings.sandboxSelfhostedControlTimeoutMs,
5705
+ RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS
5706
+ )
5707
+ );
5708
+ if (!proof) {
5709
+ await deferConnectedCommandClaim(db, settings, observability, claim, "provider_running");
5710
+ return;
5711
+ }
5712
+ } catch (error) {
5713
+ const offline = error instanceof SelfhostedControlError && (error.agentOffline || error.reason === "agent_reconnecting" || error.draining);
5714
+ if (!offline) {
5715
+ observability.warn("sandbox reaper: connected-command provider probe failed", {
5716
+ commandId: claim.commandId,
5717
+ enrollmentId: claim.enrollmentId,
5718
+ connectionInstanceId: claim.connectionInstanceId,
5719
+ error: error instanceof Error ? error.message : String(error)
5720
+ });
5721
+ }
5722
+ await deferConnectedCommandClaim(
5723
+ db,
5724
+ settings,
5725
+ observability,
5726
+ claim,
5727
+ offline ? "provider_offline" : "provider_error"
5728
+ );
5729
+ return;
5730
+ }
5731
+ try {
5732
+ await recordConnectedMachineBackgroundCommandProof(db, { claim, proof });
5733
+ recordConnectedCommandReconciliation(observability, `proof_${proof.outcome}`);
5734
+ } catch (error) {
5735
+ recordConnectedCommandReconciliation(observability, "proof_checkpoint_failed");
5736
+ observability.warn("sandbox reaper: connected-command proof checkpoint failed", {
5737
+ commandId: claim.commandId,
5738
+ error: error instanceof Error ? error.message : String(error)
5739
+ });
5740
+ return;
5741
+ }
5742
+ }
5743
+ try {
5744
+ if (!await settleClaimedConnectedMachineBackgroundCommand(db, { claim })) {
5745
+ throw new Error("Connected command settlement lost its exact claim or proof");
5746
+ }
5747
+ recordConnectedCommandReconciliation(observability, `settled_${proof.outcome}`);
5748
+ } catch (error) {
5749
+ recordConnectedCommandReconciliation(observability, "settlement_failed");
5750
+ observability.warn("sandbox reaper: connected-command settlement failed", {
5751
+ commandId: claim.commandId,
5752
+ error: error instanceof Error ? error.message : String(error)
5753
+ });
5754
+ await deferConnectedCommandClaim(db, settings, observability, claim, "settlement_failed");
5755
+ }
5756
+ });
5757
+ }
5758
+ async function probeConnectedMachineBackgroundCommand(claim, controlRpc, controlTimeoutMs, observedAt = /* @__PURE__ */ new Date()) {
5759
+ const input = {
5760
+ controlRpc,
5761
+ rpcSubject: subjectFor(
5762
+ claim.controlWorkspaceId,
5763
+ claim.enrollmentId,
5764
+ claim.connectionInstanceId
5765
+ ),
5766
+ opId: claim.opId,
5767
+ controlTimeoutMs
5768
+ };
5769
+ const status = claim.state === "stopping" ? await cancelSelfhostedOp(input) : await querySelfhostedOp(input);
5770
+ return connectedCommandProofFromStatus(status, observedAt);
5771
+ }
5772
+ function connectedCommandProofFromStatus(status, observedAt) {
5773
+ switch (status.state) {
5774
+ case OpState.OP_STATE_ACCEPTED:
5775
+ case OpState.OP_STATE_RUNNING:
5776
+ return null;
5777
+ case OpState.OP_STATE_COMPLETE: {
5778
+ const exit = status.exit;
5779
+ if (!exit) {
5780
+ throw new Error("Connected command completed without an exit record");
5781
+ }
5782
+ const failureCode = exit.failureCode.replace(/[^A-Za-z0-9_-]/g, "_").slice(0, 128);
5783
+ const reason = exit.cancelled ? "op_cancelled" : exit.timedOut ? "op_timed_out" : failureCode ? `op_failure_${failureCode}` : "op_exit";
5784
+ return {
5785
+ outcome: "exited",
5786
+ exitCode: exit.exitCode,
5787
+ reason,
5788
+ observedAt
5789
+ };
5790
+ }
5791
+ case OpState.OP_STATE_LOST:
5792
+ return {
5793
+ outcome: "lost",
5794
+ exitCode: null,
5795
+ reason: status.lostReason === OpLostReason.OP_LOST_REASON_EVICTED ? "op_lost_evicted" : status.lostReason === OpLostReason.OP_LOST_REASON_AGENT_RESTARTED ? "op_lost_agent_restarted" : "op_lost_unspecified",
5796
+ observedAt
5797
+ };
5798
+ default:
5799
+ throw new Error(`Connected command returned invalid op state ${status.state}`);
5800
+ }
5801
+ }
5802
+ async function deferConnectedCommandClaim(db, settings, observability, claim, outcome) {
5803
+ const exponent = Math.min(4, Math.max(0, claim.reconcileAttempts - 1));
5804
+ const retryAfterMs = Math.min(
5805
+ 5 * 6e4,
5806
+ Math.max(settings.sandboxLeaseReaperPeriodMs, 3e4) * 2 ** exponent
5807
+ );
5808
+ try {
5809
+ if (!await deferConnectedMachineBackgroundCommandReconciliation(db, {
5810
+ claim,
5811
+ outcome,
5812
+ retryAfterMs
5813
+ })) {
5814
+ throw new Error("Connected command defer lost its exact claim");
5815
+ }
5816
+ recordConnectedCommandReconciliation(observability, outcome);
5817
+ } catch (error) {
5818
+ recordConnectedCommandReconciliation(observability, "defer_failed");
5819
+ observability.warn("sandbox reaper: connected-command defer failed", {
5820
+ commandId: claim.commandId,
5821
+ error: error instanceof Error ? error.message : String(error)
5822
+ });
5823
+ }
5824
+ }
4859
5825
  async function reconcileTerminalRetainedProcesses(db, settings, observability, probe, inspectHistoricalModalSandbox) {
4860
5826
  const claimId = crypto.randomUUID();
4861
5827
  let claims;
@@ -4959,7 +5925,12 @@ async function reconcileTerminalRetainedProcesses(db, settings, observability, p
4959
5925
  }
4960
5926
  } else {
4961
5927
  try {
4962
- observation = await probe(settings, lease, process);
5928
+ observation = await probe(
5929
+ settings,
5930
+ lease,
5931
+ process,
5932
+ claim.ownerState === "background_stopping" ? "cancel" : "observe"
5933
+ );
4963
5934
  } catch (error) {
4964
5935
  observability.warn("sandbox reaper: retained-process provider probe failed", {
4965
5936
  processId: process.id,
@@ -5038,6 +6009,15 @@ async function reconcileTerminalRetainedProcesses(db, settings, observability, p
5038
6009
  reason: proof.reason,
5039
6010
  idleGraceMs: settings.sandboxIdleGraceMs
5040
6011
  });
6012
+ await settleSessionBackgroundCommandForRetainedProcess(db, {
6013
+ accountId: process.accountId,
6014
+ workspaceId: process.workspaceId,
6015
+ sessionId: process.sessionId,
6016
+ retainedProcessId: process.id,
6017
+ outcome: proof.outcome,
6018
+ exitCode: proof.exitCode,
6019
+ reason: proof.reason
6020
+ });
5041
6021
  recordRetainedProcessReconciliation(observability, `settled_${proof.outcome}`);
5042
6022
  } catch (error) {
5043
6023
  recordRetainedProcessReconciliation(observability, "settlement_failed");
@@ -5101,7 +6081,7 @@ async function withRetainedProcessProbeTimeout(promise) {
5101
6081
  if (timer) clearTimeout(timer);
5102
6082
  }
5103
6083
  }
5104
- async function probeRetainedProcessAtProvider(settings, lease, process) {
6084
+ async function probeRetainedProcessAtProvider(settings, lease, process, mode = "observe") {
5105
6085
  if (lease.id !== process.leaseId || lease.sandboxGroupId !== process.sandboxGroupId || lease.leaseEpoch !== process.leaseEpoch || lease.backend !== process.providerBackend || lease.instanceId !== process.providerInstanceId || lease.resumeBackendId !== process.providerBackend || process.routeTargetId !== null) {
5106
6086
  return { status: "deferred", reason: "identity_mismatch" };
5107
6087
  }
@@ -5112,7 +6092,7 @@ async function probeRetainedProcessAtProvider(settings, lease, process) {
5112
6092
  if (envelopeBackend !== void 0 && envelopeBackend !== process.providerBackend) {
5113
6093
  return { status: "deferred", reason: "identity_mismatch" };
5114
6094
  }
5115
- if (providerIdentityFromResumeState(lease.resumeState) !== process.providerInstanceId) {
6095
+ if (sandboxProviderInstanceIdFromEnvelope(lease.resumeState) !== process.providerInstanceId) {
5116
6096
  return { status: "deferred", reason: "identity_mismatch" };
5117
6097
  }
5118
6098
  let configuredModalBinding = null;
@@ -5215,7 +6195,7 @@ async function probeRetainedProcessAtProvider(settings, lease, process) {
5215
6195
  result = await withRetainedProcessProbeTimeout(
5216
6196
  session.writeStdin({
5217
6197
  sessionId: process.providerSessionId,
5218
- chars: "",
6198
+ chars: mode === "cancel" ? "" : "",
5219
6199
  yieldTimeMs: 1e3,
5220
6200
  maxOutputTokens: 2e3
5221
6201
  })
@@ -5339,7 +6319,7 @@ async function accrueWarmTick(db, settings, observability) {
5339
6319
  });
5340
6320
  return { accrued, workspaceIds };
5341
6321
  }
5342
- async function refreshQueueLeaseAndCreditGauges(db, observability) {
6322
+ async function refreshQueueLeaseAndCreditGauges(db, settings, observability, inspectOpenSandboxInventory) {
5343
6323
  await Promise.all([
5344
6324
  (async () => {
5345
6325
  try {
@@ -5389,6 +6369,21 @@ async function refreshQueueLeaseAndCreditGauges(db, observability) {
5389
6369
  await countExpiredDrainingSandboxLeases(db)
5390
6370
  );
5391
6371
  }),
6372
+ ...settings.sandboxBackend === "opensandbox" && settings.openSandboxKubernetesInventoryNamespace ? [
6373
+ refreshSandboxInventoryGauge(
6374
+ observability,
6375
+ "opensandbox_kubernetes",
6376
+ "OpenSandbox Kubernetes",
6377
+ async () => {
6378
+ recordOpenSandboxKubernetesInventoryGauges(
6379
+ observability,
6380
+ await inspectOpenSandboxInventory({
6381
+ namespace: settings.openSandboxKubernetesInventoryNamespace
6382
+ })
6383
+ );
6384
+ }
6385
+ )
6386
+ ] : [],
5392
6387
  (async () => {
5393
6388
  try {
5394
6389
  recordCreditBalanceGauges(observability, await listCreditBalancesByAccount(db));
@@ -5421,7 +6416,7 @@ async function forceDrainOverLimitWorkspaces(db, settings, workspaceIds, observa
5421
6416
  async (workspaceId) => {
5422
6417
  try {
5423
6418
  const { accountId } = await rlsContextForWorkspace2(db, workspaceId);
5424
- const balance = enforceBalance ? await getBillingBalance2(db, accountId) : { balanceMicros: 1 };
6419
+ const balance = enforceBalance ? await getBillingBalance3(db, accountId) : { balanceMicros: 1 };
5425
6420
  const result = await forceDrainOverLimitViewerOnlyBoxes(db, {
5426
6421
  workspaceId,
5427
6422
  balanceMicros: balance.balanceMicros,
@@ -5553,7 +6548,11 @@ async function probeDrainableProviderReadiness(settings, lease) {
5553
6548
  throw error;
5554
6549
  }
5555
6550
  }
5556
- async function terminateDrainableBox(db, settings, row, observability, terminateBox, probeDrainableProvider, attempt) {
6551
+ function sandboxDrainReason(lease) {
6552
+ if (lease.rotationRequestedAt === null) return "idle";
6553
+ return lease.rotationReason ?? "rotation_requested";
6554
+ }
6555
+ async function terminateDrainableBox(db, settings, row, observability, terminateBox, probeDrainableProvider, attempt, objectStorage) {
5557
6556
  const { accountId } = await rlsContextForWorkspace2(db, row.workspaceId);
5558
6557
  const lease = await readLease(db, row.workspaceId, row.sandboxGroupId);
5559
6558
  if (!lease) {
@@ -5716,6 +6715,7 @@ async function terminateDrainableBox(db, settings, row, observability, terminate
5716
6715
  if (!lease.instanceId || !captureClaim) {
5717
6716
  return { wrote: false, archiveRevision: null };
5718
6717
  }
6718
+ const archiveMetrics = runtimeMetricsHooksForObservability(observability);
5719
6719
  let checkpointArtifactId = null;
5720
6720
  if (archiveBase64 && archiveMetadata?.version === 2 && (archiveMetadata.provider === "modal_snapshot_filesystem" || archiveMetadata.provider === "modal_snapshot_directory")) {
5721
6721
  if (!providerBinding) {
@@ -5757,12 +6757,57 @@ async function terminateDrainableBox(db, settings, row, observability, terminate
5757
6757
  if (!archiveMetadata) {
5758
6758
  throw new Error("Sandbox snapshot publication requires a verified archive descriptor");
5759
6759
  }
5760
- result = await persistDrainSnapshot(db, {
5761
- ...baseInput,
5762
- workspaceArchive: archiveBase64,
5763
- workspaceArchiveMeta: archiveMetadata,
5764
- ...checkpointArtifactId ? { checkpointArtifactId } : {}
5765
- });
6760
+ const priorKeys = collectWorkspaceArchiveObjectKeys(
6761
+ lease.resumeState ?? null
6762
+ );
6763
+ let workspaceArchiveRef;
6764
+ try {
6765
+ const published = await putVersion1TarArchiveOrInline({
6766
+ backend: lease.backend,
6767
+ objectStorage,
6768
+ accountId,
6769
+ workspaceId: row.workspaceId,
6770
+ sandboxGroupId: row.sandboxGroupId,
6771
+ archive: {
6772
+ bytes: Buffer.from(archiveBase64, "base64"),
6773
+ descriptor: archiveMetadata,
6774
+ base64: archiveBase64
6775
+ },
6776
+ metrics: archiveMetrics
6777
+ });
6778
+ workspaceArchiveRef = published.workspaceArchiveRef;
6779
+ result = await persistDrainSnapshot(db, {
6780
+ ...baseInput,
6781
+ workspaceArchiveMeta: archiveMetadata,
6782
+ ...published,
6783
+ ...checkpointArtifactId ? { checkpointArtifactId } : {}
6784
+ });
6785
+ if (published.workspaceArchiveRef && objectStorage) {
6786
+ if (!result.wrote) {
6787
+ await deleteUnpublishedWorkspaceArchiveObject(
6788
+ objectStorage,
6789
+ published.workspaceArchiveRef,
6790
+ archiveMetrics
6791
+ );
6792
+ } else {
6793
+ const afterLease = await readLease(db, row.workspaceId, row.sandboxGroupId);
6794
+ const afterKeys = collectWorkspaceArchiveObjectKeys(
6795
+ afterLease?.resumeState ?? null
6796
+ );
6797
+ await deleteWorkspaceArchiveObjectKeys(
6798
+ objectStorage,
6799
+ [...priorKeys].filter((key) => !afterKeys.has(key))
6800
+ ).catch(() => void 0);
6801
+ }
6802
+ }
6803
+ } catch (error) {
6804
+ await deleteUnpublishedWorkspaceArchiveObject(
6805
+ objectStorage,
6806
+ workspaceArchiveRef,
6807
+ archiveMetrics
6808
+ );
6809
+ throw error;
6810
+ }
5766
6811
  }
5767
6812
  if (!result.wrote && checkpointArtifactId) {
5768
6813
  await markSandboxCheckpointArtifactDeletePending2(db, {
@@ -5809,7 +6854,12 @@ async function terminateDrainableBox(db, settings, row, observability, terminate
5809
6854
  persisted,
5810
6855
  archiveRevision,
5811
6856
  instanceId: lease.instanceId,
5812
- providerMissingBeforeCapture: providerMissing
6857
+ providerMissingBeforeCapture: providerMissing,
6858
+ // Why the box drained: the lease's bounded rotation reason when one
6859
+ // was requested (provider deadline, operator/image conflict, teardown
6860
+ // claim), otherwise the ordinary idle drain. Additive diagnostic
6861
+ // fact; no identifiers beyond the ones already carried above.
6862
+ drainReason: sandboxDrainReason(lease)
5813
6863
  }
5814
6864
  });
5815
6865
  } catch (eventError) {
@@ -6062,99 +7112,222 @@ async function terminateProviderBox(settings, lease, observability, persistArchi
6062
7112
 
6063
7113
  // src/activities/scheduled-tasks.ts
6064
7114
  import {
7115
+ DEFAULT_FIRST_PARTY_MCP_PERMISSIONS,
6065
7116
  OPENGENI_SLACK_BOT_CREDENTIAL_LABEL,
6066
7117
  OPENGENI_SLACK_BOT_CREDENTIAL_ROLE,
6067
- OPENGENI_SLACK_BOT_SESSION_METADATA_KEY
7118
+ OPENGENI_SLACK_BOT_SESSION_METADATA_KEY,
7119
+ resolveWorkspaceCodexCompactionDefault,
7120
+ SCHEDULED_TASK_OCCURRENCE_PAYLOAD_MAX_BYTES,
7121
+ ScheduledTaskRunAcceptedExecution,
7122
+ scheduledOccurrencePayloadUtf8Bytes,
7123
+ stableJson
6068
7124
  } from "@opengeni/contracts";
6069
7125
  import {
7126
+ defaultSessionMcpServerIds,
6070
7127
  openGeniSlackBotMetadata,
6071
7128
  requireOpenGeniSlackBotConnection,
7129
+ resolveSessionToolPolicy,
6072
7130
  scheduledSlackBotConnectionId
6073
7131
  } from "@opengeni/core";
6074
7132
  import {
6075
- appendSessionEventsWithLockedSessionUpdate,
7133
+ appendSessionEvents,
6076
7134
  addSessionSystemUpdateWithSourceMutation,
7135
+ bindScheduledTaskRunSessionInTransaction,
6077
7136
  createScheduledTaskRun,
6078
- createSession,
6079
- createSessionGoal,
7137
+ createSessionWithIdempotencyKeyResult,
6080
7138
  enqueueSessionWorkflowWakeIfRunnable,
6081
7139
  getScheduledTask as getScheduledTask2,
7140
+ getScheduledTaskRunAcceptedExecution,
7141
+ getScheduledTaskRunByProducerKey,
7142
+ getScheduledTargetSessionExecution,
6082
7143
  ensureKnowledgeSourceSyncState,
6083
7144
  getScheduledTaskPersonalConnectionDelegations,
7145
+ getScheduledTaskRunPersonalResourceAuthority,
7146
+ getScheduledTaskPersonalResourceAuthoritySubject,
7147
+ getScheduledTaskRevisionAuthority,
6084
7148
  getScheduledTaskXaiProviderAccountAuthoritySnapshot,
6085
7149
  getRig as getRig2,
7150
+ getScheduledScopedRigVersionMetadata,
7151
+ getNestedAgentDepthDeploymentPolicy,
7152
+ getSessionByCreateIdempotencyKey,
6086
7153
  getVariableSet,
7154
+ isCodexBilledModel,
7155
+ initializeSessionStartAtomically,
7156
+ materializeScheduledTaskReusableSessionFromRun,
7157
+ listEnabledMcpCapabilityServerIds,
7158
+ listInstalledApiIntegrationServerIdsForDelegations,
6087
7159
  markScheduledTaskRunFailedIfQueued,
7160
+ markScheduledTaskRunSkippedIfQueued,
6088
7161
  recordKnowledgeSourceSyncWake,
6089
7162
  recordUsageEvent as recordUsageEvent3,
6090
- requireScheduledTaskTargetInTransaction,
7163
+ requireScheduledTaskIncidentAuthorityInTransaction,
6091
7164
  requireSession,
6092
- setTemporalWorkflowId,
7165
+ requireWorkspace,
6093
7166
  settleScheduledTaskRunInTransaction,
6094
7167
  SessionSpawnDeniedDbError,
6095
- updateScheduledTask,
6096
7168
  updateScheduledTaskRun as updateScheduledTaskRun2,
6097
- upsertSessionGoal
7169
+ updateSessionTitle,
7170
+ upsertScheduledSessionGoalForRun,
7171
+ withSessionActivityRlsContext
6098
7172
  } from "@opengeni/db";
6099
- import { appendAndPublishEvents, publishDurableSessionEvents } from "@opengeni/events";
7173
+ import { publishDurableSessionEvents } from "@opengeni/events";
6100
7174
  import { resolveFirstPartyMcpToolPolicy } from "@opengeni/config";
7175
+ import { Context as Context2 } from "@temporalio/activity";
6101
7176
 
6102
- // src/activities/agent-run-admission.ts
6103
- import { configuredStaticUsageLimits as configuredStaticUsageLimits2 } from "@opengeni/config";
6104
- import { getBillingBalance as getBillingBalance3, isCodexBilledTurn, sumUsageQuantity as sumUsageQuantity2 } from "@opengeni/db";
6105
- async function agentRunAdmissionDenial(services, input) {
6106
- const externallyBilled = await isCodexBilledTurn({
6107
- db: services.db,
6108
- settings: services.settings,
6109
- workspaceId: input.workspaceId,
6110
- model: input.model
6111
- });
6112
- if (!externallyBilled && (services.settings.billingMode === "stripe" || services.settings.usageLimitsMode === "managed")) {
6113
- if (services.entitlements) {
6114
- const decision = await services.entitlements.admitRun({
6115
- accountId: input.accountId,
6116
- workspaceId: input.workspaceId,
6117
- action: "agent_run:create",
6118
- quantity: input.requestedAgentRuns
6119
- });
6120
- if (!decision.allowed) return "insufficient_credits";
6121
- } else {
6122
- const balance = await getBillingBalance3(services.db, input.accountId);
6123
- if (balance.balanceMicros <= 0) return "insufficient_credits";
6124
- }
6125
- }
6126
- if (services.settings.usageLimitsMode !== "static" && services.settings.usageLimitsMode !== "managed") {
7177
+ // src/scheduled-alert-occurrence.ts
7178
+ import { createHash as createHash4 } from "crypto";
7179
+ var MAX_SCHEDULED_TASK_ID_BYTES = 256;
7180
+ var MAX_STARTS_AT_BYTES = 256;
7181
+ var MAX_PROVIDER_BYTES = 256;
7182
+ var MAX_FINGERPRINT_BYTES = 1024;
7183
+ var MAX_LABELS = 256;
7184
+ var MAX_LABEL_KEY_BYTES = 256;
7185
+ var MAX_LABEL_VALUE_BYTES = 4096;
7186
+ var OFFSET_TIMESTAMP = /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2})(?::(\d{2})(?:\.(\d+))?)?(?:Z|[+-](\d{2}):(\d{2}))$/;
7187
+ function scheduledAlertResponderSessionCreateIdempotencyKey(input) {
7188
+ const digest = createHash4("sha256").update(
7189
+ JSON.stringify({
7190
+ version: 1,
7191
+ occurrenceKey: input.occurrence.sessionCreateIdempotencyKey,
7192
+ taskAuthorityRevision: input.taskAuthorityRevision,
7193
+ taskExecutionDigest: input.taskExecutionDigest
7194
+ })
7195
+ ).digest("hex");
7196
+ return `scheduled-alert-occurrence:v1:${digest}`;
7197
+ }
7198
+ function scheduledAlertOccurrenceIdentity(input) {
7199
+ const scheduledTaskId = boundedString(input.scheduledTaskId, MAX_SCHEDULED_TASK_ID_BYTES);
7200
+ if (!scheduledTaskId) return null;
7201
+ const alert = record(input.metadata.alert);
7202
+ if (!alert) return null;
7203
+ const status = alert.status;
7204
+ if (status !== "firing" && status !== "resolved") return null;
7205
+ const startsAt = canonicalTimestamp(alert.startsAt);
7206
+ const fingerprint = boundedString(alert.fingerprint, MAX_FINGERPRINT_BYTES);
7207
+ const labels = canonicalLabels(alert.labels);
7208
+ if (!startsAt || !fingerprint || !labels) return null;
7209
+ const provider = boundedString(
7210
+ alert.provider ?? input.metadata.alertProvider ?? input.metadata.provider ?? "alertmanager",
7211
+ MAX_PROVIDER_BYTES
7212
+ );
7213
+ if (!provider) return null;
7214
+ const digest = createHash4("sha256").update(
7215
+ JSON.stringify({
7216
+ version: 1,
7217
+ workspaceId: input.workspaceId,
7218
+ scheduledTaskId,
7219
+ provider,
7220
+ fingerprint,
7221
+ startsAt,
7222
+ labels
7223
+ })
7224
+ ).digest("hex");
7225
+ return {
7226
+ status,
7227
+ sessionCreateIdempotencyKey: `scheduled-alert-occurrence:v1:${digest}`,
7228
+ labels: Object.fromEntries(labels)
7229
+ };
7230
+ }
7231
+ function record(value) {
7232
+ return value !== null && typeof value === "object" && !Array.isArray(value) ? value : null;
7233
+ }
7234
+ function boundedString(value, maxBytes) {
7235
+ if (typeof value !== "string") return null;
7236
+ const normalized = value.trim();
7237
+ if (!normalized || Buffer.byteLength(normalized, "utf8") > maxBytes) return null;
7238
+ return normalized;
7239
+ }
7240
+ function canonicalTimestamp(value) {
7241
+ if (typeof value !== "string") return null;
7242
+ const trimmed = value.trim();
7243
+ if (!trimmed || Buffer.byteLength(trimmed, "utf8") > MAX_STARTS_AT_BYTES) return null;
7244
+ const match = OFFSET_TIMESTAMP.exec(trimmed);
7245
+ if (!match) return null;
7246
+ const [
7247
+ ,
7248
+ yearText,
7249
+ monthText,
7250
+ dayText,
7251
+ hourText,
7252
+ minuteText,
7253
+ secondText,
7254
+ ,
7255
+ offsetHourText,
7256
+ offsetMinuteText
7257
+ ] = match;
7258
+ const year = Number(yearText);
7259
+ const month = Number(monthText);
7260
+ const day = Number(dayText);
7261
+ const hour = Number(hourText);
7262
+ const minute = Number(minuteText);
7263
+ const second = secondText === void 0 ? 0 : Number(secondText);
7264
+ const offsetHour = offsetHourText === void 0 ? 0 : Number(offsetHourText);
7265
+ const offsetMinute = offsetMinuteText === void 0 ? 0 : Number(offsetMinuteText);
7266
+ const leapYear = year % 4 === 0 && (year % 100 !== 0 || year % 400 === 0);
7267
+ const daysInMonth = [31, leapYear ? 29 : 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31];
7268
+ if (month < 1 || month > 12 || day < 1 || day > daysInMonth[month - 1] || hour > 23 || minute > 59 || second > 59 || offsetHour > 23 || offsetMinute > 59) {
6127
7269
  return null;
6128
7270
  }
6129
- const limits = configuredStaticUsageLimits2(services.settings);
6130
- if (!externallyBilled && limits.maxMonthlyCostMicrosPerAccount) {
6131
- const used = await sumUsageQuantity2(services.db, {
6132
- accountId: input.accountId,
6133
- eventType: "model.cost",
6134
- since: startOfUtcMonth2()
6135
- });
6136
- if (used >= limits.maxMonthlyCostMicrosPerAccount) {
6137
- return "monthly_model_cost_limit";
7271
+ const parsed = new Date(trimmed);
7272
+ return Number.isFinite(parsed.getTime()) ? trimmed : null;
7273
+ }
7274
+ function canonicalLabels(value) {
7275
+ const labels = record(value);
7276
+ if (!labels) return null;
7277
+ const entries = Object.entries(labels);
7278
+ if (entries.length === 0 || entries.length > MAX_LABELS) return null;
7279
+ const canonical = [];
7280
+ for (const [key, rawValue] of entries) {
7281
+ if (!key || Buffer.byteLength(key, "utf8") > MAX_LABEL_KEY_BYTES || typeof rawValue !== "string" || Buffer.byteLength(rawValue, "utf8") > MAX_LABEL_VALUE_BYTES) {
7282
+ return null;
6138
7283
  }
7284
+ canonical.push([key, rawValue]);
6139
7285
  }
6140
- if (limits.maxMonthlyAgentRunsPerWorkspace) {
6141
- const used = await sumUsageQuantity2(services.db, {
6142
- workspaceId: input.workspaceId,
6143
- eventType: "agent_run.created",
6144
- since: startOfUtcMonth2()
6145
- });
6146
- if (used + input.requestedAgentRuns > limits.maxMonthlyAgentRunsPerWorkspace) {
6147
- return "monthly_agent_run_limit";
7286
+ return canonical.sort(([left], [right]) => left < right ? -1 : left > right ? 1 : 0);
7287
+ }
7288
+
7289
+ // src/activities/scheduled-tasks.ts
7290
+ function scheduledTaskRunProducerKey(input, suppliedActivityIdentity) {
7291
+ if (input.producerKey) return input.producerKey;
7292
+ if (input.triggerType !== "scheduled") return input.agentRunUsageIdempotencyKey;
7293
+ let info = suppliedActivityIdentity;
7294
+ if (!info) {
7295
+ try {
7296
+ info = Context2.current().info;
7297
+ } catch {
7298
+ throw new Error(
7299
+ "scheduled task delivery without a producer key requires a Temporal activity execution"
7300
+ );
6148
7301
  }
6149
7302
  }
6150
- return null;
7303
+ if (!info.workflowExecution) {
7304
+ throw new Error("scheduled task Temporal activity is missing its workflow execution identity");
7305
+ }
7306
+ return `scheduled-temporal:${stableJson({
7307
+ namespace: info.namespace,
7308
+ workflowId: info.workflowExecution.workflowId
7309
+ })}`;
6151
7310
  }
6152
- function startOfUtcMonth2() {
6153
- const now = /* @__PURE__ */ new Date();
6154
- return new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), 1));
7311
+ var SCHEDULED_SESSION_TITLE_MAX_LENGTH = 200;
7312
+ function scheduledTaskSessionTitle(taskName) {
7313
+ const name = taskName.trim() || "Scheduled run";
7314
+ if (name.length <= SCHEDULED_SESSION_TITLE_MAX_LENGTH) return name;
7315
+ return `${name.slice(0, SCHEDULED_SESSION_TITLE_MAX_LENGTH - 1).trimEnd()}\u2026`;
7316
+ }
7317
+ async function stampScheduledSessionTitle(db, input) {
7318
+ if (input.session.title !== null) return [];
7319
+ const title = scheduledTaskSessionTitle(input.taskName);
7320
+ const result = await updateSessionTitle(db, {
7321
+ workspaceId: input.workspaceId,
7322
+ sessionId: input.session.id,
7323
+ title,
7324
+ source: "agent"
7325
+ });
7326
+ if (!result.updated) return [];
7327
+ return await appendSessionEvents(db, input.workspaceId, input.session.id, [
7328
+ { type: "session.title_set", payload: { title: result.title ?? title, source: "agent" } }
7329
+ ]);
6155
7330
  }
6156
-
6157
- // src/activities/scheduled-tasks.ts
6158
7331
  function createScheduledTaskActivities(services) {
6159
7332
  return {
6160
7333
  dispatchScheduledTaskRun: async (input) => {
@@ -6163,6 +7336,75 @@ function createScheduledTaskActivities(services) {
6163
7336
  if (input.triggerType !== "scheduled" && !input.initiator) {
6164
7337
  return { action: "blocked", reason: "malformed_manual_trigger" };
6165
7338
  }
7339
+ const stableProducerKey = scheduledTaskRunProducerKey(input);
7340
+ const priorRun = await getScheduledTaskRunByProducerKey(db, {
7341
+ workspaceId: input.workspaceId,
7342
+ taskId: input.taskId,
7343
+ triggerType: input.triggerType,
7344
+ producerKey: stableProducerKey
7345
+ });
7346
+ if (priorRun?.actionKind === "agent_turn") {
7347
+ const acceptedExecution = await getScheduledTaskRunAcceptedExecution(db, {
7348
+ workspaceId: input.workspaceId,
7349
+ runId: priorRun.id
7350
+ });
7351
+ if (!acceptedExecution) {
7352
+ if (priorRun.status === "succeeded" || priorRun.status === "failed" || priorRun.status === "skipped") {
7353
+ return scheduledRunTerminalResult(priorRun.error);
7354
+ }
7355
+ throw new Error("scheduled agent run is missing accepted execution truth");
7356
+ }
7357
+ await recordScheduledTaskFiredUsage(db, acceptedExecution, priorRun);
7358
+ if (priorRun.status === "dispatched") {
7359
+ return await replayScheduledTaskDispatch({
7360
+ db,
7361
+ wakeSessionWorkflow,
7362
+ task: acceptedExecution.task,
7363
+ run: priorRun,
7364
+ acceptedExecution
7365
+ });
7366
+ }
7367
+ if (priorRun.status === "queued") {
7368
+ try {
7369
+ return await recoverBoundScheduledTaskDispatch({
7370
+ db,
7371
+ bus,
7372
+ settings,
7373
+ wakeSessionWorkflow,
7374
+ run: priorRun,
7375
+ acceptedExecution,
7376
+ input
7377
+ });
7378
+ } catch (error) {
7379
+ const terminalError = scheduledRunRecoveryTerminalError(error);
7380
+ if (!terminalError) throw error;
7381
+ await markScheduledTaskRunFailedIfQueued(
7382
+ db,
7383
+ priorRun.workspaceId,
7384
+ priorRun.id,
7385
+ terminalError.code
7386
+ );
7387
+ return scheduledRunTerminalResult(terminalError.code);
7388
+ }
7389
+ }
7390
+ if (priorRun.status === "succeeded") {
7391
+ if (!priorRun.sessionId || !priorRun.triggerEventId) {
7392
+ throw new Error("succeeded scheduled run is missing its delivery identity");
7393
+ }
7394
+ return {
7395
+ action: acceptedExecution.task.runMode === "new_session_per_run" ? "start" : "signal",
7396
+ accountId: acceptedExecution.task.accountId,
7397
+ workspaceId: acceptedExecution.task.workspaceId,
7398
+ sessionId: priorRun.sessionId,
7399
+ triggerEventId: priorRun.triggerEventId,
7400
+ workflowId: workflowIdForSession(priorRun.sessionId),
7401
+ workflowWakeRevision: null
7402
+ };
7403
+ }
7404
+ if (priorRun.status === "failed" || priorRun.status === "skipped") {
7405
+ return scheduledRunTerminalResult(priorRun.error);
7406
+ }
7407
+ }
6166
7408
  const task = await getScheduledTask2(db, input.workspaceId, input.taskId);
6167
7409
  if (!task) {
6168
7410
  return { action: "deleted" };
@@ -6171,11 +7413,13 @@ function createScheduledTaskActivities(services) {
6171
7413
  if (knowledgeSourceSyncEffectivelyPaused(task)) {
6172
7414
  return { action: "blocked", reason: "knowledge_source_paused" };
6173
7415
  }
6174
- const run2 = await createScheduledTaskRun(db, {
7416
+ const run = await createScheduledTaskRun(db, {
6175
7417
  workspaceId: task.workspaceId,
6176
7418
  taskId: task.id,
7419
+ taskAuthorityRevision: task.authorityRevision,
7420
+ taskExecutionDigest: task.executionDigest,
6177
7421
  triggerType: input.triggerType,
6178
- producerKey: input.producerKey ?? input.agentRunUsageIdempotencyKey ?? `knowledge-source-sync:${crypto.randomUUID()}`,
7422
+ producerKey: stableProducerKey,
6179
7423
  scheduledAt: null
6180
7424
  });
6181
7425
  await ensureKnowledgeSourceSyncState(db, task);
@@ -6184,9 +7428,9 @@ function createScheduledTaskActivities(services) {
6184
7428
  workspaceId: task.workspaceId,
6185
7429
  sourceId: task.action.sourceId,
6186
7430
  scheduledTaskId: task.id,
6187
- scheduledTaskRunId: run2.id,
7431
+ scheduledTaskRunId: run.id,
6188
7432
  cause: input.triggerType,
6189
- producerKey: input.producerKey ?? input.agentRunUsageIdempotencyKey ?? `knowledge-source-sync:${run2.id}`,
7433
+ producerKey: stableProducerKey,
6190
7434
  sourceConfigGeneration: task.action.sourceConfigGeneration,
6191
7435
  sourceLifecycleGeneration: task.action.sourceLifecycleGeneration
6192
7436
  });
@@ -6197,14 +7441,20 @@ function createScheduledTaskActivities(services) {
6197
7441
  quantity: 1,
6198
7442
  unit: "run",
6199
7443
  sourceResourceType: "scheduled_task_run",
6200
- sourceResourceId: run2.id,
6201
- initiator: input.initiator ?? { kind: "service", subjectId: "scheduler" },
6202
- initiatorContext: { scheduledTaskId: task.id, scheduledTaskRunId: run2.id },
7444
+ sourceResourceId: run.id,
7445
+ initiator: input.initiator ?? {
7446
+ kind: "service",
7447
+ subjectId: "scheduler"
7448
+ },
7449
+ initiatorContext: {
7450
+ scheduledTaskId: task.id,
7451
+ scheduledTaskRunId: run.id
7452
+ },
6203
7453
  origin: "scheduled_task",
6204
- idempotencyKey: `usage:knowledge_source_sync.fired:${run2.id}`
7454
+ idempotencyKey: `usage:knowledge_source_sync.fired:${run.id}`
6205
7455
  });
6206
- if (run2.status === "queued") {
6207
- await updateScheduledTaskRun2(db, task.workspaceId, run2.id, {
7456
+ if (run.status === "queued") {
7457
+ await updateScheduledTaskRun2(db, task.workspaceId, run.id, {
6208
7458
  status: "dispatched",
6209
7459
  actionKind: "knowledge_source_sync"
6210
7460
  });
@@ -6214,16 +7464,224 @@ function createScheduledTaskActivities(services) {
6214
7464
  accountId: task.accountId,
6215
7465
  workspaceId: task.workspaceId,
6216
7466
  taskId: task.id,
6217
- scheduledTaskRunId: run2.id,
7467
+ scheduledTaskRunId: run.id,
6218
7468
  sourceId: task.action.sourceId,
6219
7469
  overlapPolicy: task.overlapPolicy
6220
7470
  };
6221
7471
  }
7472
+ if (task.status !== "active") {
7473
+ return { action: "blocked", reason: "scheduled_task_paused" };
7474
+ }
7475
+ const structuredAlertOccurrence = scheduledAlertOccurrenceIdentity({
7476
+ workspaceId: task.workspaceId,
7477
+ scheduledTaskId: task.id,
7478
+ metadata: task.metadata
7479
+ });
7480
+ const alertOccurrence = task.runMode === "new_session_per_run" ? structuredAlertOccurrence : null;
7481
+ const alertResponderSessionCreateIdempotencyKey = alertOccurrence ? scheduledAlertResponderSessionCreateIdempotencyKey({
7482
+ occurrence: alertOccurrence,
7483
+ taskAuthorityRevision: task.authorityRevision,
7484
+ taskExecutionDigest: task.executionDigest
7485
+ }) : null;
7486
+ const incidentDeclaration = incidentTelemetryPreflightDeclaration(
7487
+ task.agentConfig,
7488
+ structuredAlertOccurrence !== null
7489
+ );
7490
+ if (incidentDeclaration.action === "blocked") {
7491
+ return incidentDeclaration;
7492
+ }
6222
7493
  const taskPersonalConnectionDelegations = await getScheduledTaskPersonalConnectionDelegations(
6223
7494
  db,
6224
7495
  task.workspaceId,
6225
7496
  task.id
6226
7497
  );
7498
+ const taskConnectionAuthoritySubjects = [
7499
+ ...new Set(
7500
+ taskPersonalConnectionDelegations.map((delegation) => delegation.ownerSubjectId)
7501
+ )
7502
+ ];
7503
+ if (taskConnectionAuthoritySubjects.length > 1) {
7504
+ throw new Error("scheduled connection authority has multiple causal humans");
7505
+ }
7506
+ const taskConnectionAuthoritySubjectId = taskConnectionAuthoritySubjects[0] ?? null;
7507
+ const taskPersonalResourceAuthoritySubjectId = await getScheduledTaskPersonalResourceAuthoritySubject(db, {
7508
+ accountId: task.accountId,
7509
+ workspaceId: task.workspaceId,
7510
+ taskId: task.id,
7511
+ taskAuthorityRevision: task.authorityRevision
7512
+ });
7513
+ const taskRevisionAuthority = await getScheduledTaskRevisionAuthority(db, {
7514
+ accountId: task.accountId,
7515
+ workspaceId: task.workspaceId,
7516
+ taskId: task.id,
7517
+ taskAuthorityRevision: task.authorityRevision
7518
+ });
7519
+ if (taskConnectionAuthoritySubjectId && taskPersonalResourceAuthoritySubjectId && taskConnectionAuthoritySubjectId !== taskPersonalResourceAuthoritySubjectId) {
7520
+ throw new Error("scheduled authority classes have different causal humans");
7521
+ }
7522
+ const taskAuthoritySubjectId = taskRevisionAuthority?.subjectId ?? taskPersonalResourceAuthoritySubjectId ?? taskConnectionAuthoritySubjectId;
7523
+ const model = task.agentConfig.model ?? settings.openaiModel;
7524
+ const reasoningEffort = task.agentConfig.reasoningEffort ?? settings.openaiReasoningEffort;
7525
+ const sandboxBackend = task.agentConfig.sandboxBackend ?? settings.sandboxBackend;
7526
+ const taskTools = withFirstPartyTools(settings, task.agentConfig.tools);
7527
+ const firstPartyMcpTools = resolveFirstPartyMcpToolPolicy(settings).default;
7528
+ const generatedTarget = task.runMode === "new_session_per_run" || task.runMode === "reusable_session" && task.reusableSessionId === null;
7529
+ const generatedSessionDepthPolicy = generatedTarget ? await (async () => {
7530
+ const workspace = await requireWorkspace(db, task.workspaceId);
7531
+ const deployment = await getNestedAgentDepthDeploymentPolicy(db);
7532
+ const taskOverride = task.agentConfig.maxNestedAgentDepth ?? null;
7533
+ const workspaceOverride = typeof workspace.settings.maxNestedAgentDepth === "number" ? workspace.settings.maxNestedAgentDepth : null;
7534
+ return taskOverride !== null ? {
7535
+ effectiveMaxNestedAgentDepth: taskOverride,
7536
+ nestedAgentDepthPolicySource: "session",
7537
+ codexCompactionMode: isCodexBilledModel(model) ? resolveWorkspaceCodexCompactionDefault(workspace.settings) : "portable"
7538
+ } : workspaceOverride !== null ? {
7539
+ effectiveMaxNestedAgentDepth: workspaceOverride,
7540
+ nestedAgentDepthPolicySource: "workspace",
7541
+ codexCompactionMode: isCodexBilledModel(model) ? resolveWorkspaceCodexCompactionDefault(workspace.settings) : "portable"
7542
+ } : {
7543
+ effectiveMaxNestedAgentDepth: deployment.maxNestedAgentDepth,
7544
+ nestedAgentDepthPolicySource: deployment.policySource,
7545
+ codexCompactionMode: isCodexBilledModel(model) ? resolveWorkspaceCodexCompactionDefault(workspace.settings) : "portable"
7546
+ };
7547
+ })() : null;
7548
+ const acceptedTargetSessionId = generatedTarget ? null : task.runMode === "existing_session" ? task.targetSessionId : task.reusableSessionId;
7549
+ const targetSessionExecutionBase = acceptedTargetSessionId ? await getScheduledTargetSessionExecution(
7550
+ db,
7551
+ task.workspaceId,
7552
+ acceptedTargetSessionId,
7553
+ taskAuthoritySubjectId
7554
+ ) : null;
7555
+ const targetSessionExecution = targetSessionExecutionBase ? await (async () => {
7556
+ const [capabilityServerIds, apiIntegrationServerIds] = await Promise.all([
7557
+ listEnabledMcpCapabilityServerIds(db, task.workspaceId),
7558
+ listInstalledApiIntegrationServerIdsForDelegations(
7559
+ db,
7560
+ task.workspaceId,
7561
+ taskPersonalConnectionDelegations
7562
+ )
7563
+ ]);
7564
+ const workspaceDefaultMcpServerIds = new Set(
7565
+ settings.mcpServers.map((server) => server.id)
7566
+ );
7567
+ for (const id of capabilityServerIds) workspaceDefaultMcpServerIds.add(id);
7568
+ for (const id of apiIntegrationServerIds) workspaceDefaultMcpServerIds.add(id);
7569
+ const availableMcpServerIds = new Set(workspaceDefaultMcpServerIds);
7570
+ for (const id of targetSessionExecutionBase.mcpServerIds) {
7571
+ availableMcpServerIds.add(id);
7572
+ }
7573
+ const effective = resolveSessionToolPolicy({
7574
+ toolPolicy: targetSessionExecutionBase.toolPolicy,
7575
+ sessionTools: targetSessionExecutionBase.tools,
7576
+ availableMcpServerIds,
7577
+ defaultMcpServerIds: defaultSessionMcpServerIds(
7578
+ [...workspaceDefaultMcpServerIds].map((id) => ({ id }))
7579
+ )
7580
+ });
7581
+ return {
7582
+ ...targetSessionExecutionBase,
7583
+ effectiveMcpServerIds: [
7584
+ ...new Set(
7585
+ effective.toolRefs.flatMap((tool) => tool.kind === "mcp" ? [tool.id] : [])
7586
+ )
7587
+ ].sort()
7588
+ };
7589
+ })() : null;
7590
+ const acceptedVariableSet = generatedTarget && task.variableSetId ? await getVariableSet(
7591
+ db,
7592
+ {
7593
+ accountId: task.accountId,
7594
+ workspaceId: task.workspaceId,
7595
+ // Organization/workspace sets have no personal-resource causal
7596
+ // subject. The scoped resolver still permits those common
7597
+ // scopes for a service/legacy actor, while a user-owned set can
7598
+ // only resolve through the exact frozen 0252 subject.
7599
+ subjectId: taskAuthoritySubjectId ?? task.createdBy.subjectId
7600
+ },
7601
+ task.variableSetId
7602
+ ) : null;
7603
+ if (generatedTarget && task.variableSetId && !acceptedVariableSet) {
7604
+ throw new Error(`variable set not found: ${task.variableSetId}`);
7605
+ }
7606
+ const acceptedRig = generatedTarget && task.rigId ? await getRig2(
7607
+ db,
7608
+ {
7609
+ accountId: task.accountId,
7610
+ workspaceId: task.workspaceId,
7611
+ subjectId: taskAuthoritySubjectId ?? task.createdBy.subjectId
7612
+ },
7613
+ task.rigId
7614
+ ) : null;
7615
+ if (generatedTarget && task.rigId && (!acceptedRig || !acceptedRig.activeVersion)) {
7616
+ throw new Error(`rig has no active version to bind: ${task.rigId}`);
7617
+ }
7618
+ const acceptedRigDefaultVariableSets = acceptedRig?.activeVersion ? await Promise.all(
7619
+ acceptedRig.activeVersion.defaultVariableSetIds.map(async (variableSetId) => {
7620
+ const variableSet = await getVariableSet(
7621
+ db,
7622
+ {
7623
+ accountId: task.accountId,
7624
+ workspaceId: task.workspaceId,
7625
+ subjectId: taskAuthoritySubjectId ?? task.createdBy.subjectId
7626
+ },
7627
+ variableSetId
7628
+ );
7629
+ if (!variableSet) {
7630
+ throw new Error(`rig default Variable Set not found: ${variableSetId}`);
7631
+ }
7632
+ return { id: variableSet.id, generation: variableSet.generation };
7633
+ })
7634
+ ) : [];
7635
+ let incidentPreflightRequired = incidentDeclaration.action === "required";
7636
+ if (incidentPreflightRequired) {
7637
+ const existingSessionId = task.runMode === "existing_session" ? task.targetSessionId : task.reusableSessionId;
7638
+ const canonicalAlertSession = task.runMode === "new_session_per_run" && alertOccurrence ? await getSessionByCreateIdempotencyKey(
7639
+ db,
7640
+ task.workspaceId,
7641
+ alertResponderSessionCreateIdempotencyKey
7642
+ ) : null;
7643
+ const responderSession = existingSessionId ? await requireSession(db, task.workspaceId, existingSessionId) : canonicalAlertSession;
7644
+ const responder = await resolveIncidentTelemetryResponderMetadata({
7645
+ db,
7646
+ settings,
7647
+ task,
7648
+ session: responderSession,
7649
+ personalConnectionDelegations: taskPersonalConnectionDelegations,
7650
+ personalResourceAuthoritySubjectId: taskAuthoritySubjectId,
7651
+ executionPolicy: targetSessionExecution ? {
7652
+ tools: targetSessionExecution.tools,
7653
+ firstPartyMcpTools: targetSessionExecution.firstPartyMcpTools,
7654
+ firstPartyMcpPermissions: targetSessionExecution.firstPartyMcpPermissions,
7655
+ variableSetId: targetSessionExecution.variableSetId,
7656
+ rigId: targetSessionExecution.rigId,
7657
+ rigVersionId: targetSessionExecution.rigVersionId,
7658
+ toolPolicy: targetSessionExecution.toolPolicy,
7659
+ mcpServerIds: targetSessionExecution.mcpServerIds,
7660
+ toolPolicyVersion: targetSessionExecution.toolPolicyVersion
7661
+ } : {
7662
+ tools: taskTools,
7663
+ firstPartyMcpTools,
7664
+ firstPartyMcpPermissions: [...DEFAULT_FIRST_PARTY_MCP_PERMISSIONS],
7665
+ variableSetId: acceptedVariableSet?.id ?? null,
7666
+ rigId: acceptedRig?.id ?? null,
7667
+ rigVersionId: acceptedRig?.activeVersion?.id ?? null,
7668
+ toolPolicy: responderSession?.toolPolicy ?? {
7669
+ mode: "explicit",
7670
+ inheritedFromSessionId: null
7671
+ },
7672
+ mcpServerIds: [],
7673
+ toolPolicyVersion: responderSession?.toolPolicyVersion ?? null
7674
+ }
7675
+ });
7676
+ const preflight = evaluateIncidentTelemetryPreflight({
7677
+ agentConfig: task.agentConfig,
7678
+ incidentTriggered: structuredAlertOccurrence !== null,
7679
+ alertOccurrenceLabels: structuredAlertOccurrence?.labels ?? null,
7680
+ responder
7681
+ });
7682
+ if (preflight.action === "blocked") return preflight;
7683
+ incidentPreflightRequired = preflight.action === "ready";
7684
+ }
6227
7685
  const taskXaiProviderAccountAuthoritySnapshot = await getScheduledTaskXaiProviderAccountAuthoritySnapshot(db, task.workspaceId, task.id);
6228
7686
  const slackBotConnection = task.agentConfig.slackBotConnectionId ? await requireOpenGeniSlackBotConnection(
6229
7687
  db,
@@ -6234,394 +7692,751 @@ function createScheduledTaskActivities(services) {
6234
7692
  if (slackBotConnection && !slackBotMetadata) {
6235
7693
  throw new Error("OpenGeni Slack bot connection metadata is invalid");
6236
7694
  }
6237
- const model = task.agentConfig.model ?? settings.openaiModel;
7695
+ const xaiAuthoritySubjectId = taskXaiProviderAccountAuthoritySnapshot.scope === "user" && task.createdBy.kind === "subject" ? task.createdBy.subjectId : null;
7696
+ if (taskXaiProviderAccountAuthoritySnapshot.scope === "user" && !xaiAuthoritySubjectId) {
7697
+ throw new Error("scheduled user-scoped xAI authority has no causal human");
7698
+ }
7699
+ if (xaiAuthoritySubjectId && taskAuthoritySubjectId && xaiAuthoritySubjectId !== taskAuthoritySubjectId) {
7700
+ throw new Error("scheduled authority classes have different causal humans");
7701
+ }
7702
+ const causalHumanSubjectId = taskAuthoritySubjectId ?? xaiAuthoritySubjectId;
7703
+ if ((acceptedVariableSet?.scope === "user" || acceptedRig?.scope === "user") && !causalHumanSubjectId) {
7704
+ throw new Error("scheduled personal-resource execution has no causal human");
7705
+ }
6238
7706
  const admissionDenial = await agentRunAdmissionDenial(service, {
6239
7707
  accountId: task.accountId,
6240
7708
  workspaceId: task.workspaceId,
6241
- model,
7709
+ model: targetSessionExecution?.model ?? model,
6242
7710
  requestedAgentRuns: input.agentRunUsageIdempotencyKey ? 0 : 1
6243
7711
  });
6244
7712
  if (admissionDenial) {
6245
7713
  return { action: "blocked", reason: admissionDenial };
6246
7714
  }
6247
- const run = await createScheduledTaskRun(db, {
6248
- workspaceId: task.workspaceId,
6249
- taskId: task.id,
6250
- triggerType: input.triggerType,
6251
- producerKey: input.producerKey ?? input.agentRunUsageIdempotencyKey ?? `scheduled:${crypto.randomUUID()}`,
6252
- scheduledAt: null
6253
- });
6254
- await recordUsageEvent3(db, {
6255
- accountId: task.accountId,
6256
- workspaceId: task.workspaceId,
6257
- eventType: "scheduled_task.fired",
6258
- quantity: 1,
6259
- unit: "run",
6260
- sourceResourceType: "scheduled_task_run",
6261
- sourceResourceId: run.id,
6262
- initiator: input.initiator ?? { kind: "service", subjectId: "scheduler" },
6263
- initiatorContext: { scheduledTaskId: task.id, scheduledTaskRunId: run.id },
6264
- origin: "scheduled_task",
6265
- idempotencyKey: `usage:scheduled_task.fired:${run.id}`
6266
- });
6267
- if (run.status === "dispatched" && run.sessionId && run.triggerEventId) {
6268
- await recordUsageEvent3(db, {
6269
- accountId: task.accountId,
6270
- workspaceId: task.workspaceId,
6271
- eventType: "agent_run.created",
6272
- quantity: 1,
6273
- unit: "run",
6274
- sourceResourceType: "scheduled_task_run",
6275
- sourceResourceId: run.id,
6276
- sessionId: run.sessionId,
6277
- initiator: input.initiator ?? { kind: "service", subjectId: "scheduler" },
6278
- initiatorContext: { scheduledTaskId: task.id, scheduledTaskRunId: run.id },
6279
- origin: "scheduled_task",
6280
- idempotencyKey: input.agentRunUsageIdempotencyKey ?? `usage:agent_run.created:scheduled:${run.id}`
6281
- });
6282
- const workflowId = workflowIdForSession(run.sessionId);
6283
- const workflowWakeRevision = await enqueueSessionWorkflowWakeIfRunnable(db, {
6284
- accountId: task.accountId,
7715
+ const deferredEvents = [];
7716
+ const executeDispatch = async (dispatchDb, deferPublications) => {
7717
+ const admittedRunId = crypto.randomUUID();
7718
+ const generatedSessionBinding = generatedTarget ? {
7719
+ createIdempotencyKey: alertResponderSessionCreateIdempotencyKey ?? (task.runMode === "reusable_session" ? `scheduled-task-reusable:${task.id}:${task.authorityRevision}:${task.executionDigest}` : `scheduled-task-run:${admittedRunId}`),
7720
+ effectiveMaxNestedAgentDepth: generatedSessionDepthPolicy.effectiveMaxNestedAgentDepth,
7721
+ nestedAgentDepthPolicySource: generatedSessionDepthPolicy.nestedAgentDepthPolicySource,
7722
+ codexCompactionMode: generatedSessionDepthPolicy.codexCompactionMode
7723
+ } : null;
7724
+ const acceptedExecution = {
7725
+ version: 1,
7726
+ task,
7727
+ resolvedModel: model,
7728
+ resolvedReasoningEffort: reasoningEffort,
7729
+ resolvedLatencyMode: "standard",
7730
+ resolvedSandboxBackend: sandboxBackend,
7731
+ resolvedSandboxOs: "linux",
7732
+ resolvedTools: taskTools,
7733
+ resolvedFirstPartyMcpTools: firstPartyMcpTools,
7734
+ resolvedFirstPartyMcpPermissions: [...DEFAULT_FIRST_PARTY_MCP_PERMISSIONS],
7735
+ resolvedVariableSet: acceptedVariableSet ? { id: acceptedVariableSet.id, generation: acceptedVariableSet.generation } : null,
7736
+ resolvedRig: acceptedRig?.activeVersion ? {
7737
+ id: acceptedRig.id,
7738
+ versionId: acceptedRig.activeVersion.id,
7739
+ defaultVariableSets: acceptedRigDefaultVariableSets
7740
+ } : null,
7741
+ resolvedSlackBotConnection: slackBotConnection && slackBotMetadata && slackBotConnection.verifiedInstallVersion ? {
7742
+ id: slackBotConnection.id,
7743
+ version: slackBotConnection.version,
7744
+ verifiedInstallVersion: slackBotConnection.verifiedInstallVersion,
7745
+ metadata: slackBotMetadata
7746
+ } : null,
7747
+ targetSessionExecution,
7748
+ generatedSessionBinding,
7749
+ personalConnectionDelegations: taskPersonalConnectionDelegations,
7750
+ personalResourceAuthoritySubjectId: taskPersonalResourceAuthoritySubjectId,
7751
+ causalHumanSubjectId,
7752
+ causalHumanAuthority: taskRevisionAuthority,
7753
+ xaiProviderAccountAuthoritySnapshot: taskXaiProviderAccountAuthoritySnapshot,
7754
+ xaiAuthoritySubjectId,
7755
+ connectionAuthoritySubjectId: taskConnectionAuthoritySubjectId,
7756
+ triggerInitiator: input.initiator ?? { kind: "service", subjectId: "scheduler" },
7757
+ agentRunUsageIdempotencyKey: input.agentRunUsageIdempotencyKey ?? null,
7758
+ incidentPreflightRequired,
7759
+ alertOccurrenceLabels: structuredAlertOccurrence?.labels ?? null
7760
+ };
7761
+ const acceptedExecutionParse = ScheduledTaskRunAcceptedExecution.safeParse(acceptedExecution);
7762
+ if (!acceptedExecutionParse.success || scheduledOccurrencePayloadUtf8Bytes({
7763
+ prompt: task.agentConfig.prompt,
7764
+ resources: task.agentConfig.resources,
7765
+ tools: taskTools
7766
+ }) > SCHEDULED_TASK_OCCURRENCE_PAYLOAD_MAX_BYTES) {
7767
+ return {
7768
+ kind: "blocked",
7769
+ result: {
7770
+ action: "blocked",
7771
+ reason: "scheduled_execution_unrepresentable"
7772
+ }
7773
+ };
7774
+ }
7775
+ let run = await createScheduledTaskRun(dispatchDb, {
7776
+ runId: admittedRunId,
6285
7777
  workspaceId: task.workspaceId,
6286
- sessionId: run.sessionId,
6287
- temporalWorkflowId: workflowId,
6288
- reason: "scheduled_retry"
7778
+ taskId: task.id,
7779
+ taskAuthorityRevision: task.authorityRevision,
7780
+ taskExecutionDigest: task.executionDigest,
7781
+ triggerType: input.triggerType,
7782
+ producerKey: stableProducerKey,
7783
+ scheduledAt: null,
7784
+ acceptedExecutionSnapshot: acceptedExecution
6289
7785
  });
6290
- const result2 = {
6291
- action: task.runMode === "new_session_per_run" ? "start" : "signal",
6292
- accountId: task.accountId,
6293
- workspaceId: task.workspaceId,
6294
- sessionId: run.sessionId,
6295
- triggerEventId: run.triggerEventId,
6296
- workflowId,
6297
- workflowWakeRevision
6298
- };
6299
- if (wakeSessionWorkflow && workflowWakeRevision !== null) {
6300
- await wakeSessionWorkflow({
6301
- accountId: result2.accountId,
6302
- workspaceId: result2.workspaceId,
6303
- sessionId: result2.sessionId,
6304
- workflowId: result2.workflowId,
6305
- wakeRevision: workflowWakeRevision
6306
- });
7786
+ await recordScheduledTaskFiredUsage(dispatchDb, acceptedExecution, run);
7787
+ if (run.status === "failed" && run.error === "scheduled_authority_exhausted") {
7788
+ return {
7789
+ kind: "blocked",
7790
+ result: {
7791
+ action: "blocked",
7792
+ reason: "scheduled_authority_exhausted"
7793
+ },
7794
+ run
7795
+ };
6307
7796
  }
6308
- return result2;
6309
- }
6310
- let result;
6311
- try {
6312
- const reasoningEffort = task.agentConfig.reasoningEffort ?? settings.openaiReasoningEffort;
6313
- const sandboxBackend = task.agentConfig.sandboxBackend ?? settings.sandboxBackend;
6314
- const goalSpec = task.agentConfig.goal ?? null;
6315
- const taskTools = withFirstPartyTools(settings, task.agentConfig.tools);
6316
- const existingSessionId = task.runMode === "existing_session" ? task.targetSessionId : task.reusableSessionId;
6317
- if (task.runMode === "existing_session" && !existingSessionId) {
6318
- throw new Error("scheduled task target session is unavailable");
7797
+ if (run.status === "failed" || run.status === "skipped") {
7798
+ return {
7799
+ kind: "blocked",
7800
+ result: scheduledRunTerminalResult(run.error),
7801
+ run
7802
+ };
6319
7803
  }
6320
- if (task.runMode === "new_session_per_run" || task.runMode === "reusable_session" && !existingSessionId) {
6321
- const variableSet = task.variableSetId ? await getVariableSet(db, task.workspaceId, task.variableSetId) : null;
6322
- if (task.variableSetId && !variableSet) {
6323
- throw new Error(`variable set not found: ${task.variableSetId}`);
6324
- }
6325
- let frozenRigId = null;
6326
- let frozenRigVersionId = null;
6327
- if (task.rigId) {
6328
- const rig = await getRig2(db, task.workspaceId, task.rigId);
6329
- if (!rig || !rig.activeVersion) {
6330
- throw new Error(`rig has no active version to bind: ${task.rigId}`);
6331
- }
6332
- frozenRigId = rig.id;
6333
- frozenRigVersionId = rig.activeVersion.id;
7804
+ const runPersonalResourceAuthority = await getScheduledTaskRunPersonalResourceAuthority(
7805
+ dispatchDb,
7806
+ {
7807
+ accountId: task.accountId,
7808
+ workspaceId: task.workspaceId,
7809
+ runId: run.id
6334
7810
  }
6335
- let session;
6336
- const taskMetadata = { ...task.agentConfig.metadata };
6337
- delete taskMetadata[OPENGENI_SLACK_BOT_SESSION_METADATA_KEY];
6338
- try {
6339
- session = await createSession(db, {
6340
- accountId: task.accountId,
6341
- workspaceId: task.workspaceId,
6342
- initialMessage: task.agentConfig.prompt,
6343
- resources: task.agentConfig.resources,
6344
- tools: taskTools,
6345
- firstPartyMcpTools: resolveFirstPartyMcpToolPolicy(settings).default,
6346
- metadata: {
6347
- ...taskMetadata,
6348
- model,
6349
- reasoningEffort,
6350
- scheduledTaskId: task.id,
6351
- scheduledTaskRunId: run.id,
6352
- ...slackBotConnection ? {
6353
- [OPENGENI_SLACK_BOT_SESSION_METADATA_KEY]: slackBotConnection.id
6354
- } : {}
6355
- },
6356
- createdBy: {
6357
- kind: "service",
6358
- subjectId: "scheduler",
6359
- label: "OpenGeni scheduler"
6360
- },
6361
- createdByContext: {
6362
- scheduledTaskId: task.id,
6363
- scheduledTaskRunId: run.id
6364
- },
6365
- model,
6366
- sandboxBackend,
6367
- variableSetId: task.variableSetId ?? null,
6368
- rigId: frozenRigId,
6369
- rigVersionId: frozenRigVersionId,
6370
- personalConnectionDelegations: taskPersonalConnectionDelegations,
6371
- initialXaiProviderAccountAuthoritySnapshot: taskXaiProviderAccountAuthoritySnapshot,
6372
- maxNestedAgentDepthOverride: task.agentConfig.maxNestedAgentDepth ?? null,
6373
- // The durable agent config was privilege-checked when the task
6374
- // was created/updated. Preserve that explicit policy if a broader
6375
- // workspace/deployment limit is narrowed before a later fire.
6376
- allowNestedAgentDepthIncrease: true,
6377
- subjectId: `scheduled_task:${task.id}`
6378
- });
6379
- } catch (error) {
6380
- if (error instanceof SessionSpawnDeniedDbError) {
6381
- throw new Error(`${error.denial.code}: denial=${error.denial.id}`, { cause: error });
6382
- }
6383
- throw error;
7811
+ );
7812
+ if (run?.status === "dispatched" && run.sessionId && run.triggerEventId) {
7813
+ if (deferPublications) {
7814
+ return { kind: "replay", run, acceptedExecution };
6384
7815
  }
6385
- const goal = goalSpec ? await createSessionGoal(db, {
7816
+ await recordUsageEvent3(dispatchDb, {
6386
7817
  accountId: task.accountId,
6387
7818
  workspaceId: task.workspaceId,
6388
- sessionId: session.id,
6389
- text: goalSpec.text,
6390
- successCriteria: goalSpec.successCriteria ?? null,
6391
- maxAutoContinuations: goalSpec.maxAutoContinuations ?? null,
6392
- createdBy: "scheduled_task"
6393
- }) : null;
6394
- const workflowId = workflowIdForSession(session.id);
6395
- await setTemporalWorkflowId(db, task.workspaceId, session.id, workflowId);
6396
- if (task.runMode === "reusable_session") {
6397
- await updateScheduledTask(db, task.workspaceId, task.id, {
6398
- reusableSessionId: session.id
6399
- });
6400
- }
6401
- await appendAndPublishEvents(db, bus, task.workspaceId, session.id, [
6402
- {
6403
- type: "session.created",
6404
- payload: {
6405
- status: session.status,
6406
- createdBy: session.createdBy,
6407
- scheduledTaskId: task.id,
6408
- scheduledTaskRunId: run.id,
6409
- // Names/ids only; never values.
6410
- ...variableSet ? { variableSetId: variableSet.id, variableSetName: variableSet.name } : {},
6411
- ...slackBotConnection && slackBotMetadata ? {
6412
- slackBotConnection: {
6413
- credentialRole: OPENGENI_SLACK_BOT_CREDENTIAL_ROLE,
6414
- credentialLabel: OPENGENI_SLACK_BOT_CREDENTIAL_LABEL,
6415
- connectionId: slackBotConnection.id,
6416
- slackTeamId: slackBotMetadata.slackTeamId
6417
- }
6418
- } : {}
6419
- }
6420
- },
6421
- ...goal ? [
6422
- {
6423
- type: "goal.set",
6424
- payload: {
6425
- goalId: goal.id,
6426
- text: goal.text,
6427
- ...goal.successCriteria ? { successCriteria: goal.successCriteria } : {},
6428
- version: goal.version,
6429
- actor: "scheduled_task",
6430
- replaced: false
6431
- }
6432
- }
6433
- ] : [],
6434
- { type: "session.status.changed", payload: { status: session.status } }
6435
- ]);
6436
- const scheduledUpdate = await addSessionSystemUpdateWithSourceMutation(
6437
- db,
6438
- {
6439
- accountId: task.accountId,
6440
- workspaceId: task.workspaceId,
6441
- sessionId: session.id,
6442
- kind: "scheduled_occurrence",
6443
- classification: "info",
6444
- sourceId: run.id,
6445
- dedupeKey: `scheduled-wake:${run.id}`,
6446
- summary: task.agentConfig.prompt,
6447
- payload: scheduledUserMessagePayload(
6448
- task.agentConfig.prompt,
6449
- task.agentConfig.resources,
6450
- taskTools,
6451
- task.id,
6452
- run.id
6453
- ),
6454
- lineage: {
6455
- scheduledTaskId: task.id,
6456
- scheduledTaskRunId: run.id,
6457
- ...taskXaiProviderAccountAuthoritySnapshot.scope === "user" && task.createdBy.kind === "subject" ? { xaiAuthoritySubjectId: task.createdBy.subjectId } : {}
6458
- },
6459
- personalConnectionDelegations: taskPersonalConnectionDelegations,
6460
- xaiProviderAccountAuthoritySnapshot: taskXaiProviderAccountAuthoritySnapshot
7819
+ eventType: "agent_run.created",
7820
+ quantity: 1,
7821
+ unit: "run",
7822
+ sourceResourceType: "scheduled_task_run",
7823
+ sourceResourceId: run.id,
7824
+ sessionId: run.sessionId,
7825
+ initiator: acceptedExecution.triggerInitiator,
7826
+ initiatorContext: {
7827
+ scheduledTaskId: task.id,
7828
+ scheduledTaskRunId: run.id
6461
7829
  },
6462
- async (tx, wakeEventId) => {
6463
- if (!wakeEventId) throw new Error("Scheduled delivery has no wake event");
6464
- await settleScheduledTaskRunInTransaction(tx, {
6465
- workspaceId: task.workspaceId,
6466
- runId: run.id,
6467
- sessionId: session.id,
6468
- triggerEventId: wakeEventId,
6469
- status: "dispatched"
6470
- });
6471
- }
6472
- );
6473
- if (scheduledUpdate.reason === "session_cancelled") {
6474
- throw new Error("new scheduled session was cancelled during dispatch");
6475
- }
6476
- if (scheduledUpdate.added && scheduledUpdate.events.length > 0) {
6477
- await publishDurableSessionEvents(
6478
- bus,
6479
- task.workspaceId,
6480
- session.id,
6481
- scheduledUpdate.events
6482
- );
6483
- }
6484
- result = {
6485
- action: "start",
7830
+ origin: "scheduled_task",
7831
+ idempotencyKey: acceptedExecution.agentRunUsageIdempotencyKey ?? `usage:agent_run.created:scheduled:${run.id}`
7832
+ });
7833
+ const workflowId = workflowIdForSession(run.sessionId);
7834
+ const workflowWakeRevision = await enqueueSessionWorkflowWakeIfRunnable(dispatchDb, {
7835
+ accountId: task.accountId,
7836
+ workspaceId: task.workspaceId,
7837
+ sessionId: run.sessionId,
7838
+ temporalWorkflowId: workflowId,
7839
+ reason: "scheduled_retry"
7840
+ });
7841
+ const result3 = {
7842
+ action: task.runMode === "new_session_per_run" ? "start" : "signal",
6486
7843
  accountId: task.accountId,
6487
7844
  workspaceId: task.workspaceId,
6488
- sessionId: session.id,
6489
- triggerEventId: scheduledUpdate.wakeEventId,
7845
+ sessionId: run.sessionId,
7846
+ triggerEventId: run.triggerEventId,
6490
7847
  workflowId,
6491
- workflowWakeRevision: scheduledUpdate.workflowWakeRevision
7848
+ workflowWakeRevision
6492
7849
  };
6493
- } else {
6494
- const session = await requireSession(db, task.workspaceId, existingSessionId);
6495
- assertReusableSessionRevivable(session.status);
6496
- if ((session.variableSetId ?? null) !== (task.variableSetId ?? null)) {
6497
- throw new Error(
6498
- "scheduled task variableSet attachment does not match its reusable session"
7850
+ return { kind: "complete", result: result3, run };
7851
+ }
7852
+ let result2;
7853
+ try {
7854
+ if (incidentPreflightRequired) {
7855
+ const exactAuthority = await requireScheduledTaskIncidentAuthorityInTransaction(
7856
+ dispatchDb,
7857
+ {
7858
+ workspaceId: task.workspaceId,
7859
+ taskId: task.id
7860
+ }
6499
7861
  );
7862
+ if (stableJson({
7863
+ task: exactAuthority.task,
7864
+ personalConnectionDelegations: exactAuthority.personalConnectionDelegations
7865
+ }) !== stableJson({
7866
+ task,
7867
+ personalConnectionDelegations: taskPersonalConnectionDelegations
7868
+ })) {
7869
+ throw new IncidentTelemetryPreflightBlockedError(
7870
+ "incident_preflight_metadata_missing"
7871
+ );
7872
+ }
6500
7873
  }
6501
- if (scheduledSlackBotConnectionId(session.metadata) !== (task.agentConfig.slackBotConnectionId ?? null)) {
6502
- throw new Error(
6503
- "scheduled task OpenGeni Slack bot binding does not match its reusable session"
6504
- );
7874
+ const goalSpec = task.agentConfig.goal ?? null;
7875
+ const existingSessionId = task.runMode === "existing_session" ? task.targetSessionId : task.reusableSessionId;
7876
+ if (task.runMode === "existing_session" && !existingSessionId) {
7877
+ throw new Error("scheduled task target session is unavailable");
6505
7878
  }
6506
- const reusableGoal = task.runMode === "reusable_session" && goalSpec && run.status === "queued" ? await upsertSessionGoal(db, {
6507
- accountId: task.accountId,
6508
- workspaceId: task.workspaceId,
6509
- sessionId: session.id,
6510
- text: goalSpec.text,
6511
- successCriteria: goalSpec.successCriteria ?? null,
6512
- maxAutoContinuations: goalSpec.maxAutoContinuations ?? null,
6513
- createdBy: "scheduled_task"
6514
- }) : null;
6515
- if (reusableGoal) {
6516
- const goalEvents = await appendSessionEventsWithLockedSessionUpdate(
6517
- db,
6518
- task.workspaceId,
6519
- session.id,
6520
- (locked) => {
6521
- assertReusableSessionRevivable(locked.status);
6522
- return {
6523
- events: [
6524
- {
6525
- type: "goal.set",
6526
- payload: {
6527
- goalId: reusableGoal.goal.id,
6528
- text: reusableGoal.goal.text,
6529
- ...reusableGoal.goal.successCriteria ? { successCriteria: reusableGoal.goal.successCriteria } : {},
6530
- version: reusableGoal.goal.version,
6531
- actor: "scheduled_task",
6532
- replaced: reusableGoal.replaced
7879
+ if (task.runMode === "new_session_per_run" || task.runMode === "reusable_session" && !existingSessionId) {
7880
+ const variableSet = acceptedVariableSet;
7881
+ if (task.variableSetId && !variableSet) {
7882
+ throw new Error(`variable set not found: ${task.variableSetId}`);
7883
+ }
7884
+ let frozenRigId = null;
7885
+ let frozenRigVersionId = null;
7886
+ if (task.rigId) {
7887
+ const rig = acceptedRig;
7888
+ if (!rig || !rig.activeVersion) {
7889
+ throw new Error(`rig has no active version to bind: ${task.rigId}`);
7890
+ }
7891
+ frozenRigId = rig.id;
7892
+ frozenRigVersionId = runPersonalResourceAuthority?.resources.find(
7893
+ (resource) => resource.resourceKind === "rig" && resource.resourceId === rig.id
7894
+ )?.resourceVersionId ?? rig.activeVersion.id;
7895
+ if (frozenRigVersionId !== rig.activeVersion.id) {
7896
+ throw new Error("scheduled run rig version changed during admission");
7897
+ }
7898
+ }
7899
+ let session;
7900
+ let sessionCreated = true;
7901
+ const taskMetadata = { ...task.agentConfig.metadata };
7902
+ delete taskMetadata[OPENGENI_SLACK_BOT_SESSION_METADATA_KEY];
7903
+ try {
7904
+ const sessionInput = {
7905
+ accountId: task.accountId,
7906
+ workspaceId: task.workspaceId,
7907
+ initialMessage: task.agentConfig.prompt,
7908
+ resources: task.agentConfig.resources,
7909
+ tools: taskTools,
7910
+ firstPartyMcpTools,
7911
+ firstPartyMcpPermissions: [...DEFAULT_FIRST_PARTY_MCP_PERMISSIONS],
7912
+ metadata: {
7913
+ ...taskMetadata,
7914
+ model,
7915
+ reasoningEffort,
7916
+ scheduledTaskId: task.id,
7917
+ scheduledTaskRunMode: task.runMode,
7918
+ ...goalSpec ? { scheduledTaskGoal: goalSpec } : {},
7919
+ ...run ? { scheduledTaskRunId: run.id } : {},
7920
+ ...slackBotConnection ? {
7921
+ [OPENGENI_SLACK_BOT_SESSION_METADATA_KEY]: slackBotConnection.id
7922
+ } : {}
7923
+ },
7924
+ createdBy: {
7925
+ kind: "service",
7926
+ subjectId: "scheduler",
7927
+ label: "OpenGeni scheduler"
7928
+ },
7929
+ createdByContext: {
7930
+ scheduledTaskId: task.id,
7931
+ ...run ? { scheduledTaskRunId: run.id } : {}
7932
+ },
7933
+ model,
7934
+ reasoningEffort,
7935
+ latencyMode: "standard",
7936
+ sandboxBackend,
7937
+ sandboxOs: "linux",
7938
+ variableSetId: task.variableSetId ?? null,
7939
+ rigId: frozenRigId,
7940
+ rigVersionId: frozenRigVersionId,
7941
+ personalConnectionDelegations: [],
7942
+ initialXaiProviderAccountAuthoritySnapshot: taskXaiProviderAccountAuthoritySnapshot,
7943
+ maxNestedAgentDepthOverride: task.agentConfig.maxNestedAgentDepth ?? null,
7944
+ frozenNestedAgentDepthPolicy: {
7945
+ effectiveMaxNestedAgentDepth: generatedSessionBinding.effectiveMaxNestedAgentDepth,
7946
+ nestedAgentDepthPolicySource: generatedSessionBinding.nestedAgentDepthPolicySource
7947
+ },
7948
+ frozenCodexCompactionMode: generatedSessionBinding.codexCompactionMode,
7949
+ // The durable agent config was privilege-checked when the task
7950
+ // was created/updated. Preserve that explicit policy if a broader
7951
+ // workspace/deployment limit is narrowed before a later fire.
7952
+ allowNestedAgentDepthIncrease: true,
7953
+ subjectId: `scheduled_task:${task.id}`,
7954
+ beforeCreateCommit: async (tx, sessionId) => {
7955
+ if (incidentPreflightRequired) {
7956
+ const exactSession = await requireSession(tx, task.workspaceId, sessionId);
7957
+ const exactResponder = await resolveIncidentTelemetryResponderMetadata({
7958
+ db: tx,
7959
+ settings,
7960
+ task: acceptedExecution.task,
7961
+ session: exactSession,
7962
+ personalConnectionDelegations: acceptedExecution.personalConnectionDelegations,
7963
+ personalResourceAuthoritySubjectId: acceptedExecution.personalResourceAuthoritySubjectId,
7964
+ executionPolicy: {
7965
+ tools: exactSession.tools,
7966
+ firstPartyMcpTools: exactSession.firstPartyMcpTools,
7967
+ firstPartyMcpPermissions: exactSession.firstPartyMcpPermissions,
7968
+ variableSetId: exactSession.variableSetId,
7969
+ rigId: exactSession.rigId,
7970
+ rigVersionId: exactSession.rigVersionId,
7971
+ toolPolicy: exactSession.toolPolicy,
7972
+ mcpServerIds: exactSession.mcpServers.map((server) => server.id),
7973
+ toolPolicyVersion: exactSession.toolPolicyVersion
6533
7974
  }
7975
+ });
7976
+ const exactPreflight = evaluateIncidentTelemetryPreflight({
7977
+ agentConfig: acceptedExecution.task.agentConfig,
7978
+ incidentTriggered: structuredAlertOccurrence !== null,
7979
+ alertOccurrenceLabels: structuredAlertOccurrence?.labels ?? null,
7980
+ responder: exactResponder
7981
+ });
7982
+ if (exactPreflight.action === "blocked") {
7983
+ throw new IncidentTelemetryPreflightBlockedError(exactPreflight.reason);
6534
7984
  }
6535
- ]
6536
- };
6537
- },
6538
- { activity: "semantic" }
6539
- );
6540
- await publishDurableSessionEvents(bus, task.workspaceId, session.id, goalEvents);
6541
- }
6542
- const bundled = await addSessionSystemUpdateWithSourceMutation(
6543
- db,
6544
- {
6545
- accountId: task.accountId,
7985
+ }
7986
+ await bindScheduledTaskRunSessionInTransaction(tx, {
7987
+ accountId: task.accountId,
7988
+ workspaceId: task.workspaceId,
7989
+ runId: run.id,
7990
+ sessionId
7991
+ });
7992
+ }
7993
+ };
7994
+ const created = await createSessionWithIdempotencyKeyResult(dispatchDb, {
7995
+ ...sessionInput,
7996
+ createIdempotencyKey: generatedSessionBinding.createIdempotencyKey
7997
+ });
7998
+ if (created.denied) {
7999
+ throw new SessionSpawnDeniedDbError(created.denial);
8000
+ }
8001
+ session = created.session;
8002
+ sessionCreated = created.created;
8003
+ if (!sessionCreated) {
8004
+ await bindScheduledTaskRunSessionInTransaction(dispatchDb, {
8005
+ accountId: task.accountId,
8006
+ workspaceId: task.workspaceId,
8007
+ runId: run.id,
8008
+ sessionId: session.id
8009
+ });
8010
+ run = { ...run, sessionId: session.id };
8011
+ }
8012
+ } catch (error) {
8013
+ if (error instanceof IncidentTelemetryPreflightBlockedError) {
8014
+ throw error;
8015
+ }
8016
+ if (error instanceof SessionSpawnDeniedDbError) {
8017
+ throw new Error(`${error.denial.code}: denial=${error.denial.id}`, {
8018
+ cause: error
8019
+ });
8020
+ }
8021
+ throw error;
8022
+ }
8023
+ const scheduledRun = run;
8024
+ if (!sessionCreated) {
8025
+ assertReusableSessionRevivable(session.status);
8026
+ if ((session.variableSetId ?? null) !== (task.variableSetId ?? null)) {
8027
+ throw new Error(
8028
+ "scheduled alert occurrence variableSet attachment does not match its canonical session"
8029
+ );
8030
+ }
8031
+ if (scheduledSlackBotConnectionId(session.metadata) !== (task.agentConfig.slackBotConnectionId ?? null)) {
8032
+ throw new Error(
8033
+ "scheduled alert occurrence OpenGeni Slack bot binding does not match its canonical session"
8034
+ );
8035
+ }
8036
+ }
8037
+ let workflowId;
8038
+ if (alertOccurrence) {
8039
+ const started = await initializeSessionStartAtomically(dispatchDb, {
8040
+ accountId: task.accountId,
8041
+ workspaceId: task.workspaceId,
8042
+ sessionId: session.id,
8043
+ reasoningEffortFallback: reasoningEffort,
8044
+ createdEventPayload: {
8045
+ scheduledTaskId: typeof session.metadata.scheduledTaskId === "string" ? session.metadata.scheduledTaskId : task.id,
8046
+ scheduledTaskRunId: typeof session.metadata.scheduledTaskRunId === "string" ? session.metadata.scheduledTaskRunId : run.id,
8047
+ // Names/ids only; never values.
8048
+ ...variableSet ? {
8049
+ variableSetId: variableSet.id,
8050
+ variableSetName: variableSet.name
8051
+ } : {},
8052
+ ...slackBotConnection && slackBotMetadata ? {
8053
+ slackBotConnection: {
8054
+ credentialRole: OPENGENI_SLACK_BOT_CREDENTIAL_ROLE,
8055
+ credentialLabel: OPENGENI_SLACK_BOT_CREDENTIAL_LABEL,
8056
+ connectionId: slackBotConnection.id,
8057
+ slackTeamId: slackBotMetadata.slackTeamId
8058
+ }
8059
+ } : {}
8060
+ },
8061
+ goal: goalSpec ? {
8062
+ text: goalSpec.text,
8063
+ successCriteria: goalSpec.successCriteria ?? null,
8064
+ maxAutoContinuations: goalSpec.maxAutoContinuations ?? null,
8065
+ ...goalSpec.mutationPolicy ? { mutationPolicy: goalSpec.mutationPolicy } : {},
8066
+ createdBy: "scheduled_task"
8067
+ } : null,
8068
+ deferInitialTurn: true,
8069
+ deferredStatus: "queued"
8070
+ });
8071
+ workflowId = started.temporalWorkflowId;
8072
+ if (started.events.length > 0) {
8073
+ if (deferPublications) {
8074
+ deferredEvents.push({
8075
+ sessionId: session.id,
8076
+ events: started.events
8077
+ });
8078
+ } else {
8079
+ await publishDurableSessionEvents(
8080
+ bus,
8081
+ task.workspaceId,
8082
+ session.id,
8083
+ started.events
8084
+ );
8085
+ }
8086
+ }
8087
+ } else {
8088
+ const started = await initializeSessionStartAtomically(dispatchDb, {
8089
+ accountId: task.accountId,
8090
+ workspaceId: task.workspaceId,
8091
+ sessionId: session.id,
8092
+ reasoningEffortFallback: reasoningEffort,
8093
+ createdEventPayload: {
8094
+ scheduledTaskId: typeof session.metadata.scheduledTaskId === "string" ? session.metadata.scheduledTaskId : task.id,
8095
+ scheduledTaskRunId: typeof session.metadata.scheduledTaskRunId === "string" ? session.metadata.scheduledTaskRunId : run.id,
8096
+ // Names/ids only; never values.
8097
+ ...variableSet ? { variableSetId: variableSet.id, variableSetName: variableSet.name } : {},
8098
+ ...slackBotConnection && slackBotMetadata ? {
8099
+ slackBotConnection: {
8100
+ credentialRole: OPENGENI_SLACK_BOT_CREDENTIAL_ROLE,
8101
+ credentialLabel: OPENGENI_SLACK_BOT_CREDENTIAL_LABEL,
8102
+ connectionId: slackBotConnection.id,
8103
+ slackTeamId: slackBotMetadata.slackTeamId
8104
+ }
8105
+ } : {}
8106
+ },
8107
+ goal: goalSpec ? {
8108
+ text: goalSpec.text,
8109
+ successCriteria: goalSpec.successCriteria ?? null,
8110
+ maxAutoContinuations: goalSpec.maxAutoContinuations ?? null,
8111
+ ...goalSpec.mutationPolicy ? { mutationPolicy: goalSpec.mutationPolicy } : {},
8112
+ createdBy: "scheduled_task"
8113
+ } : null,
8114
+ deferInitialTurn: true,
8115
+ deferredStatus: "queued"
8116
+ });
8117
+ workflowId = started.temporalWorkflowId;
8118
+ if (started.events.length > 0) {
8119
+ if (deferPublications) {
8120
+ deferredEvents.push({ sessionId: session.id, events: started.events });
8121
+ } else {
8122
+ await publishDurableSessionEvents(
8123
+ bus,
8124
+ task.workspaceId,
8125
+ session.id,
8126
+ started.events
8127
+ );
8128
+ }
8129
+ }
8130
+ if (task.runMode === "reusable_session") {
8131
+ await materializeScheduledTaskReusableSessionFromRun(dispatchDb, {
8132
+ accountId: task.accountId,
8133
+ workspaceId: task.workspaceId,
8134
+ taskId: task.id,
8135
+ runId: run.id,
8136
+ sessionId: session.id,
8137
+ sourceTaskAuthorityRevision: runPersonalResourceAuthority?.taskAuthorityRevision ?? run.taskAuthorityRevision,
8138
+ sourceExecutionDigest: runPersonalResourceAuthority?.executionDigest ?? run.taskExecutionDigest
8139
+ });
8140
+ }
8141
+ }
8142
+ const titleEvents = await stampScheduledSessionTitle(dispatchDb, {
6546
8143
  workspaceId: task.workspaceId,
6547
- sessionId: session.id,
6548
- kind: "scheduled_occurrence",
6549
- classification: "info",
6550
- sourceId: run.id,
6551
- dedupeKey: `scheduled-wake:${run.id}`,
6552
- summary: task.agentConfig.prompt,
6553
- payload: scheduledUserMessagePayload(
6554
- task.agentConfig.prompt,
6555
- task.agentConfig.resources,
6556
- taskTools,
6557
- task.id,
6558
- run.id
6559
- ),
6560
- lineage: {
6561
- scheduledTaskId: task.id,
6562
- scheduledTaskRunId: run.id,
6563
- ...taskXaiProviderAccountAuthoritySnapshot.scope === "user" && task.createdBy.kind === "subject" ? { xaiAuthoritySubjectId: task.createdBy.subjectId } : {}
8144
+ session,
8145
+ taskName: task.name
8146
+ });
8147
+ if (titleEvents.length > 0) {
8148
+ if (deferPublications) {
8149
+ deferredEvents.push({ sessionId: session.id, events: titleEvents });
8150
+ } else {
8151
+ await publishDurableSessionEvents(bus, task.workspaceId, session.id, titleEvents);
8152
+ }
8153
+ }
8154
+ const scheduledUpdate = await addSessionSystemUpdateWithSourceMutation(
8155
+ dispatchDb,
8156
+ {
8157
+ accountId: task.accountId,
8158
+ workspaceId: task.workspaceId,
8159
+ sessionId: session.id,
8160
+ kind: "scheduled_occurrence",
8161
+ classification: "info",
8162
+ sourceId: run.id,
8163
+ dedupeKey: `scheduled-task-run:${run.id}`,
8164
+ summary: task.agentConfig.prompt,
8165
+ payload: scheduledUserMessagePayload(
8166
+ task.agentConfig.prompt,
8167
+ task.agentConfig.resources,
8168
+ taskTools,
8169
+ task.id,
8170
+ run.id
8171
+ ),
8172
+ lineage: {
8173
+ scheduledTaskId: task.id,
8174
+ scheduledTaskRunId: run.id,
8175
+ ...causalHumanSubjectId ? { causalHumanSubjectId } : {},
8176
+ ...taskConnectionAuthoritySubjectId ? { connectionAuthoritySubjectId: taskConnectionAuthoritySubjectId } : {},
8177
+ ...taskXaiProviderAccountAuthoritySnapshot.scope === "user" && task.createdBy.kind === "subject" ? { xaiAuthoritySubjectId: task.createdBy.subjectId } : {}
8178
+ },
8179
+ personalConnectionDelegations: taskPersonalConnectionDelegations,
8180
+ xaiProviderAccountAuthoritySnapshot: taskXaiProviderAccountAuthoritySnapshot,
8181
+ scheduledTaskRunId: run.id
6564
8182
  },
6565
- personalConnectionDelegations: taskPersonalConnectionDelegations,
6566
- xaiProviderAccountAuthoritySnapshot: taskXaiProviderAccountAuthoritySnapshot
6567
- },
6568
- async (tx, wakeEventId) => {
6569
- if (task.runMode === "existing_session") {
6570
- await requireScheduledTaskTargetInTransaction(tx, {
8183
+ async (tx, wakeEventId) => {
8184
+ if (!wakeEventId) {
8185
+ await settleScheduledTaskRunInTransaction(tx, {
8186
+ workspaceId: task.workspaceId,
8187
+ runId: scheduledRun.id,
8188
+ sessionId: session.id,
8189
+ triggerEventId: null,
8190
+ status: "skipped",
8191
+ error: "session_cancelled"
8192
+ });
8193
+ return;
8194
+ }
8195
+ await settleScheduledTaskRunInTransaction(tx, {
6571
8196
  workspaceId: task.workspaceId,
8197
+ runId: scheduledRun.id,
8198
+ sessionId: session.id,
8199
+ triggerEventId: wakeEventId,
8200
+ status: "dispatched"
8201
+ });
8202
+ },
8203
+ incidentPreflightRequired ? {
8204
+ prepareSource: async (tx, sessionId) => await prepareIncidentTelemetrySource({
8205
+ tx,
8206
+ settings,
6572
8207
  taskId: task.id,
6573
- targetSessionId: session.id
8208
+ workspaceId: task.workspaceId,
8209
+ sessionId,
8210
+ alertOccurrenceLabels: structuredAlertOccurrence?.labels ?? null,
8211
+ acceptedExecution
8212
+ })
8213
+ } : void 0
8214
+ );
8215
+ if (scheduledUpdate.reason === "session_cancelled") {
8216
+ return {
8217
+ kind: "blocked",
8218
+ result: { action: "blocked", reason: "scheduled_run_terminal" },
8219
+ run: { ...run, status: "skipped", error: "session_cancelled" }
8220
+ };
8221
+ }
8222
+ if (scheduledUpdate.added && scheduledUpdate.events.length > 0) {
8223
+ if (deferPublications) {
8224
+ deferredEvents.push({
8225
+ sessionId: session.id,
8226
+ events: scheduledUpdate.events
6574
8227
  });
8228
+ } else {
8229
+ await publishDurableSessionEvents(
8230
+ bus,
8231
+ task.workspaceId,
8232
+ session.id,
8233
+ scheduledUpdate.events
8234
+ );
6575
8235
  }
6576
- if (!wakeEventId) throw new Error("Scheduled delivery has no wake event");
6577
- await settleScheduledTaskRunInTransaction(tx, {
8236
+ }
8237
+ result2 = {
8238
+ action: sessionCreated ? "start" : "signal",
8239
+ accountId: task.accountId,
8240
+ workspaceId: task.workspaceId,
8241
+ sessionId: session.id,
8242
+ triggerEventId: scheduledUpdate.wakeEventId,
8243
+ workflowId,
8244
+ workflowWakeRevision: scheduledUpdate.workflowWakeRevision
8245
+ };
8246
+ } else {
8247
+ if (!run) throw new Error("scheduled task run was not created");
8248
+ const scheduledRun = run;
8249
+ const session = await requireSession(dispatchDb, task.workspaceId, existingSessionId);
8250
+ await bindScheduledTaskRunSessionInTransaction(dispatchDb, {
8251
+ accountId: task.accountId,
8252
+ workspaceId: task.workspaceId,
8253
+ runId: run.id,
8254
+ sessionId: session.id
8255
+ });
8256
+ assertReusableSessionRevivable(session.status);
8257
+ assertReusableSessionBindingMatches(session, task);
8258
+ const goalEvents = task.runMode === "reusable_session" && goalSpec && run.status === "queued" ? await upsertScheduledSessionGoalForRun(dispatchDb, {
8259
+ accountId: task.accountId,
8260
+ workspaceId: task.workspaceId,
8261
+ sessionId: session.id,
8262
+ runId: run.id,
8263
+ text: goalSpec.text,
8264
+ successCriteria: goalSpec.successCriteria ?? null,
8265
+ maxAutoContinuations: goalSpec.maxAutoContinuations ?? null,
8266
+ ...goalSpec.mutationPolicy ? { mutationPolicy: goalSpec.mutationPolicy } : {}
8267
+ }) : [];
8268
+ if (goalEvents.length > 0) {
8269
+ if (deferPublications) {
8270
+ deferredEvents.push({
8271
+ sessionId: session.id,
8272
+ events: goalEvents
8273
+ });
8274
+ } else {
8275
+ await publishDurableSessionEvents(bus, task.workspaceId, session.id, goalEvents);
8276
+ }
8277
+ }
8278
+ const bundled = await addSessionSystemUpdateWithSourceMutation(
8279
+ dispatchDb,
8280
+ {
8281
+ accountId: task.accountId,
6578
8282
  workspaceId: task.workspaceId,
6579
- runId: run.id,
6580
8283
  sessionId: session.id,
6581
- triggerEventId: wakeEventId,
6582
- status: "dispatched"
6583
- });
8284
+ kind: "scheduled_occurrence",
8285
+ classification: "info",
8286
+ sourceId: run.id,
8287
+ dedupeKey: `scheduled-task-run:${run.id}`,
8288
+ summary: task.agentConfig.prompt,
8289
+ payload: scheduledUserMessagePayload(
8290
+ task.agentConfig.prompt,
8291
+ task.agentConfig.resources,
8292
+ taskTools,
8293
+ task.id,
8294
+ run.id
8295
+ ),
8296
+ lineage: {
8297
+ scheduledTaskId: task.id,
8298
+ scheduledTaskRunId: run.id,
8299
+ ...causalHumanSubjectId ? { causalHumanSubjectId } : {},
8300
+ ...taskConnectionAuthoritySubjectId ? { connectionAuthoritySubjectId: taskConnectionAuthoritySubjectId } : {},
8301
+ ...taskXaiProviderAccountAuthoritySnapshot.scope === "user" && task.createdBy.kind === "subject" ? { xaiAuthoritySubjectId: task.createdBy.subjectId } : {}
8302
+ },
8303
+ personalConnectionDelegations: taskPersonalConnectionDelegations,
8304
+ xaiProviderAccountAuthoritySnapshot: taskXaiProviderAccountAuthoritySnapshot,
8305
+ scheduledTaskRunId: run.id
8306
+ },
8307
+ async (tx, wakeEventId) => {
8308
+ if (!wakeEventId) {
8309
+ await settleScheduledTaskRunInTransaction(tx, {
8310
+ workspaceId: task.workspaceId,
8311
+ runId: scheduledRun.id,
8312
+ sessionId: session.id,
8313
+ triggerEventId: null,
8314
+ status: "skipped",
8315
+ error: "session_cancelled"
8316
+ });
8317
+ return;
8318
+ }
8319
+ await settleScheduledTaskRunInTransaction(tx, {
8320
+ workspaceId: task.workspaceId,
8321
+ runId: scheduledRun.id,
8322
+ sessionId: session.id,
8323
+ triggerEventId: wakeEventId,
8324
+ status: "dispatched"
8325
+ });
8326
+ },
8327
+ incidentPreflightRequired ? {
8328
+ prepareSource: async (tx, sessionId) => await prepareIncidentTelemetrySource({
8329
+ tx,
8330
+ settings,
8331
+ taskId: task.id,
8332
+ workspaceId: task.workspaceId,
8333
+ sessionId,
8334
+ alertOccurrenceLabels: structuredAlertOccurrence?.labels ?? null,
8335
+ acceptedExecution
8336
+ })
8337
+ } : void 0
8338
+ );
8339
+ if (bundled.reason === "session_cancelled") {
8340
+ return {
8341
+ kind: "blocked",
8342
+ result: { action: "blocked", reason: "scheduled_run_terminal" },
8343
+ run: { ...run, status: "skipped", error: "session_cancelled" }
8344
+ };
6584
8345
  }
6585
- );
6586
- if (bundled.reason === "session_cancelled") {
6587
- throw new Error(`scheduled wake was not added: ${bundled.reason}`);
6588
- }
6589
- if (bundled.added && bundled.events.length > 0) {
6590
- await publishDurableSessionEvents(bus, task.workspaceId, session.id, bundled.events);
8346
+ if (bundled.added && bundled.events.length > 0) {
8347
+ if (deferPublications) {
8348
+ deferredEvents.push({
8349
+ sessionId: session.id,
8350
+ events: bundled.events
8351
+ });
8352
+ } else {
8353
+ await publishDurableSessionEvents(
8354
+ bus,
8355
+ task.workspaceId,
8356
+ session.id,
8357
+ bundled.events
8358
+ );
8359
+ }
8360
+ }
8361
+ result2 = {
8362
+ action: "signal",
8363
+ accountId: task.accountId,
8364
+ workspaceId: task.workspaceId,
8365
+ sessionId: session.id,
8366
+ triggerEventId: bundled.wakeEventId,
8367
+ workflowId: workflowIdForSession(session.id),
8368
+ workflowWakeRevision: bundled.workflowWakeRevision
8369
+ };
6591
8370
  }
6592
- result = {
6593
- action: "signal",
6594
- accountId: task.accountId,
6595
- workspaceId: task.workspaceId,
6596
- sessionId: session.id,
6597
- triggerEventId: bundled.wakeEventId,
6598
- workflowId: workflowIdForSession(session.id),
6599
- workflowWakeRevision: bundled.workflowWakeRevision
8371
+ } catch (error) {
8372
+ const terminalError = scheduledRunRecoveryTerminalError(error);
8373
+ if (!terminalError) throw error;
8374
+ await markScheduledTaskRunFailedIfQueued(
8375
+ dispatchDb,
8376
+ task.workspaceId,
8377
+ run.id,
8378
+ terminalError.code
8379
+ );
8380
+ return {
8381
+ kind: "blocked",
8382
+ result: scheduledRunTerminalResult(terminalError.code),
8383
+ run: { ...run, status: "failed", error: terminalError.code }
6600
8384
  };
6601
8385
  }
8386
+ await recordUsageEvent3(dispatchDb, {
8387
+ accountId: task.accountId,
8388
+ workspaceId: task.workspaceId,
8389
+ eventType: "agent_run.created",
8390
+ quantity: 1,
8391
+ unit: "run",
8392
+ sourceResourceType: "scheduled_task_run",
8393
+ sourceResourceId: run.id,
8394
+ sessionId: result2.sessionId,
8395
+ initiator: input.initiator ?? {
8396
+ kind: "service",
8397
+ subjectId: "scheduler"
8398
+ },
8399
+ initiatorContext: {
8400
+ scheduledTaskId: task.id,
8401
+ scheduledTaskRunId: run.id
8402
+ },
8403
+ origin: "scheduled_task",
8404
+ idempotencyKey: input.agentRunUsageIdempotencyKey ?? `usage:agent_run.created:scheduled:${run.id}`
8405
+ });
8406
+ return { kind: "complete", result: result2, run };
8407
+ };
8408
+ let dispatchOutcome;
8409
+ try {
8410
+ dispatchOutcome = incidentPreflightRequired ? await withSessionActivityRlsContext(
8411
+ db,
8412
+ { accountId: task.accountId, workspaceId: task.workspaceId },
8413
+ async (tx) => await executeDispatch(tx, true)
8414
+ ) : await executeDispatch(db, false);
6602
8415
  } catch (error) {
6603
- await markScheduledTaskRunFailedIfQueued(
8416
+ if (error instanceof IncidentTelemetryPreflightBlockedError) {
8417
+ return { action: "blocked", reason: error.reason };
8418
+ }
8419
+ throw error;
8420
+ }
8421
+ if (dispatchOutcome.kind === "replay") {
8422
+ return await replayScheduledTaskDispatch({
6604
8423
  db,
8424
+ wakeSessionWorkflow,
8425
+ task,
8426
+ run: dispatchOutcome.run,
8427
+ acceptedExecution: dispatchOutcome.acceptedExecution
8428
+ });
8429
+ }
8430
+ if (dispatchOutcome.kind === "blocked") return dispatchOutcome.result;
8431
+ for (const deferred of deferredEvents) {
8432
+ await publishDurableSessionEvents(
8433
+ bus,
6605
8434
  task.workspaceId,
6606
- run.id,
6607
- error instanceof Error ? error.message : String(error)
6608
- ).catch(() => void 0);
6609
- throw error;
8435
+ deferred.sessionId,
8436
+ deferred.events
8437
+ );
6610
8438
  }
6611
- await recordUsageEvent3(db, {
6612
- accountId: task.accountId,
6613
- workspaceId: task.workspaceId,
6614
- eventType: "agent_run.created",
6615
- quantity: 1,
6616
- unit: "run",
6617
- sourceResourceType: "scheduled_task_run",
6618
- sourceResourceId: run.id,
6619
- sessionId: result.sessionId,
6620
- initiator: input.initiator ?? { kind: "service", subjectId: "scheduler" },
6621
- initiatorContext: { scheduledTaskId: task.id, scheduledTaskRunId: run.id },
6622
- origin: "scheduled_task",
6623
- idempotencyKey: input.agentRunUsageIdempotencyKey ?? `usage:agent_run.created:scheduled:${run.id}`
6624
- });
8439
+ const result = dispatchOutcome.result;
6625
8440
  if (wakeSessionWorkflow && result.workflowWakeRevision !== null) {
6626
8441
  await wakeSessionWorkflow({
6627
8442
  accountId: result.accountId,
@@ -6635,6 +8450,585 @@ function createScheduledTaskActivities(services) {
6635
8450
  }
6636
8451
  };
6637
8452
  }
8453
+ var IncidentTelemetryPreflightBlockedError = class extends Error {
8454
+ constructor(reason) {
8455
+ super(reason);
8456
+ this.reason = reason;
8457
+ this.name = "IncidentTelemetryPreflightBlockedError";
8458
+ }
8459
+ };
8460
+ function assertReusableSessionBindingMatches(session, task) {
8461
+ if ((session.variableSetId ?? null) !== (task.variableSetId ?? null)) {
8462
+ throw new ScheduledRunTerminalAuthorityError(
8463
+ "scheduled_reusable_binding_changed",
8464
+ "scheduled task variableSet attachment does not match its reusable session"
8465
+ );
8466
+ }
8467
+ if (scheduledSlackBotConnectionId(session.metadata) !== (task.agentConfig.slackBotConnectionId ?? null)) {
8468
+ throw new ScheduledRunTerminalAuthorityError(
8469
+ "scheduled_reusable_binding_changed",
8470
+ "scheduled task OpenGeni Slack bot binding does not match its reusable session"
8471
+ );
8472
+ }
8473
+ }
8474
+ var ScheduledRunTerminalAuthorityError = class extends Error {
8475
+ constructor(code, message) {
8476
+ super(message);
8477
+ this.code = code;
8478
+ this.name = "ScheduledRunTerminalAuthorityError";
8479
+ }
8480
+ };
8481
+ function scheduledRunRecoveryTerminalError(error) {
8482
+ if (error instanceof Error && "cause" in error && error.cause !== void 0 && error.cause !== error) {
8483
+ const nested = scheduledRunRecoveryTerminalError(error.cause);
8484
+ if (nested) return nested;
8485
+ }
8486
+ if (error instanceof ScheduledRunTerminalAuthorityError) return error;
8487
+ if (error instanceof IncidentTelemetryPreflightBlockedError) {
8488
+ return new ScheduledRunTerminalAuthorityError(
8489
+ `scheduled_incident_${error.reason}`,
8490
+ error.message
8491
+ );
8492
+ }
8493
+ if (error instanceof SessionSpawnDeniedDbError) {
8494
+ return new ScheduledRunTerminalAuthorityError("scheduled_session_spawn_denied", error.message);
8495
+ }
8496
+ if (error instanceof Error && error.message.startsWith("Session not found:")) {
8497
+ return new ScheduledRunTerminalAuthorityError(
8498
+ "scheduled_target_session_unavailable",
8499
+ error.message
8500
+ );
8501
+ }
8502
+ if (typeof error === "object" && error !== null && "status" in error && error.status === 422) {
8503
+ return new ScheduledRunTerminalAuthorityError(
8504
+ "scheduled_slack_authority_unavailable",
8505
+ error instanceof Error ? error.message : "scheduled Slack authority is unavailable"
8506
+ );
8507
+ }
8508
+ if (typeof error === "object" && error !== null && "code" in error && error.code === "42501") {
8509
+ return new ScheduledRunTerminalAuthorityError(
8510
+ "scheduled_run_authority_proof_rejected",
8511
+ error instanceof Error ? error.message : "scheduled run authority proof was rejected"
8512
+ );
8513
+ }
8514
+ return null;
8515
+ }
8516
+ function scheduledRunTerminalResult(error) {
8517
+ if (error === "scheduled_authority_exhausted") {
8518
+ return { action: "blocked", reason: "scheduled_authority_exhausted" };
8519
+ }
8520
+ if (error === "scheduled_incident_incident_preflight_metadata_missing") {
8521
+ return { action: "blocked", reason: "incident_preflight_metadata_missing" };
8522
+ }
8523
+ if (error === "scheduled_incident_incident_responder_under_capable") {
8524
+ return { action: "blocked", reason: "incident_responder_under_capable" };
8525
+ }
8526
+ if (error === "scheduled_incident_incident_data_source_unsuitable") {
8527
+ return { action: "blocked", reason: "incident_data_source_unsuitable" };
8528
+ }
8529
+ return { action: "blocked", reason: "scheduled_run_terminal" };
8530
+ }
8531
+ async function prepareIncidentTelemetrySource(input) {
8532
+ const session = await requireSession(input.tx, input.workspaceId, input.sessionId);
8533
+ const { task, personalConnectionDelegations } = input.acceptedExecution ? {
8534
+ task: input.acceptedExecution.task,
8535
+ personalConnectionDelegations: input.acceptedExecution.personalConnectionDelegations
8536
+ } : await requireScheduledTaskIncidentAuthorityInTransaction(input.tx, {
8537
+ workspaceId: input.workspaceId,
8538
+ taskId: input.taskId
8539
+ });
8540
+ const responder = await resolveIncidentTelemetryResponderMetadata({
8541
+ db: input.tx,
8542
+ settings: input.settings,
8543
+ task,
8544
+ session,
8545
+ personalConnectionDelegations,
8546
+ personalResourceAuthoritySubjectId: input.acceptedExecution?.personalResourceAuthoritySubjectId ?? null,
8547
+ ...input.acceptedExecution ? {
8548
+ executionPolicy: input.acceptedExecution.targetSessionExecution ? {
8549
+ tools: input.acceptedExecution.targetSessionExecution.tools,
8550
+ firstPartyMcpTools: input.acceptedExecution.targetSessionExecution.firstPartyMcpTools,
8551
+ firstPartyMcpPermissions: input.acceptedExecution.targetSessionExecution.firstPartyMcpPermissions,
8552
+ variableSetId: input.acceptedExecution.targetSessionExecution.variableSetId,
8553
+ rigId: input.acceptedExecution.targetSessionExecution.rigId,
8554
+ rigVersionId: input.acceptedExecution.targetSessionExecution.rigVersionId,
8555
+ toolPolicy: input.acceptedExecution.targetSessionExecution.toolPolicy,
8556
+ mcpServerIds: input.acceptedExecution.targetSessionExecution.mcpServerIds,
8557
+ toolPolicyVersion: input.acceptedExecution.targetSessionExecution.toolPolicyVersion
8558
+ } : {
8559
+ tools: input.acceptedExecution.resolvedTools,
8560
+ firstPartyMcpTools: input.acceptedExecution.resolvedFirstPartyMcpTools,
8561
+ firstPartyMcpPermissions: input.acceptedExecution.resolvedFirstPartyMcpPermissions,
8562
+ variableSetId: input.acceptedExecution.resolvedVariableSet?.id ?? null,
8563
+ rigId: input.acceptedExecution.resolvedRig?.id ?? null,
8564
+ rigVersionId: input.acceptedExecution.resolvedRig?.versionId ?? null,
8565
+ toolPolicy: session.toolPolicy,
8566
+ mcpServerIds: [],
8567
+ toolPolicyVersion: session.toolPolicyVersion
8568
+ }
8569
+ } : {}
8570
+ });
8571
+ const preflight = evaluateIncidentTelemetryPreflight({
8572
+ agentConfig: task.agentConfig,
8573
+ incidentTriggered: input.alertOccurrenceLabels !== null,
8574
+ alertOccurrenceLabels: input.alertOccurrenceLabels,
8575
+ responder
8576
+ });
8577
+ if (preflight.action === "blocked") {
8578
+ throw new IncidentTelemetryPreflightBlockedError(preflight.reason);
8579
+ }
8580
+ const fence = incidentTelemetryAuthorityFence({
8581
+ task,
8582
+ responder,
8583
+ alertOccurrenceLabels: input.alertOccurrenceLabels
8584
+ });
8585
+ if (!fence) {
8586
+ throw new IncidentTelemetryPreflightBlockedError("incident_preflight_metadata_missing");
8587
+ }
8588
+ return {
8589
+ lineage: {
8590
+ [INCIDENT_TELEMETRY_AUTHORITY_FENCE_LINEAGE_KEY]: fence
8591
+ }
8592
+ };
8593
+ }
8594
+ async function recordScheduledTaskFiredUsage(db, acceptedExecution, run) {
8595
+ const task = acceptedExecution.task;
8596
+ await recordUsageEvent3(db, {
8597
+ accountId: task.accountId,
8598
+ workspaceId: task.workspaceId,
8599
+ eventType: "scheduled_task.fired",
8600
+ quantity: 1,
8601
+ unit: "run",
8602
+ sourceResourceType: "scheduled_task_run",
8603
+ sourceResourceId: run.id,
8604
+ initiator: acceptedExecution.triggerInitiator,
8605
+ initiatorContext: { scheduledTaskId: task.id, scheduledTaskRunId: run.id },
8606
+ origin: "scheduled_task",
8607
+ idempotencyKey: `usage:scheduled_task.fired:${run.id}`
8608
+ });
8609
+ }
8610
+ async function recoverBoundScheduledTaskDispatch(input) {
8611
+ const { task } = input.acceptedExecution;
8612
+ if (input.run.status !== "queued") {
8613
+ throw new Error("scheduled run is not an exact queued recovery");
8614
+ }
8615
+ const generatedSession = task.runMode === "new_session_per_run" || task.runMode === "reusable_session" && task.reusableSessionId === null;
8616
+ const frozenSlack = input.acceptedExecution.resolvedSlackBotConnection;
8617
+ if (frozenSlack) {
8618
+ const currentSlack = await requireOpenGeniSlackBotConnection(
8619
+ input.db,
8620
+ task.workspaceId,
8621
+ frozenSlack.id
8622
+ );
8623
+ const currentMetadata = openGeniSlackBotMetadata(currentSlack.metadata);
8624
+ if (currentSlack.version !== frozenSlack.version || currentSlack.verifiedInstallVersion !== frozenSlack.verifiedInstallVersion || stableJson(currentMetadata) !== stableJson(frozenSlack.metadata)) {
8625
+ throw new ScheduledRunTerminalAuthorityError(
8626
+ "scheduled_slack_authority_changed",
8627
+ "scheduled Slack bot authority changed after run admission"
8628
+ );
8629
+ }
8630
+ }
8631
+ let session;
8632
+ if (input.run.sessionId) {
8633
+ session = await requireSession(input.db, task.workspaceId, input.run.sessionId);
8634
+ if (!generatedSession) assertReusableSessionBindingMatches(session, task);
8635
+ } else if (generatedSession) {
8636
+ const variableSet = input.acceptedExecution.resolvedVariableSet;
8637
+ if (variableSet) {
8638
+ const current = await getVariableSet(
8639
+ input.db,
8640
+ {
8641
+ accountId: task.accountId,
8642
+ workspaceId: task.workspaceId,
8643
+ subjectId: input.acceptedExecution.personalResourceAuthoritySubjectId ?? task.createdBy.subjectId
8644
+ },
8645
+ variableSet.id
8646
+ );
8647
+ if (!current || current.generation !== variableSet.generation) {
8648
+ throw new ScheduledRunTerminalAuthorityError(
8649
+ "scheduled_variable_set_authority_changed",
8650
+ "scheduled variable set changed after run admission"
8651
+ );
8652
+ }
8653
+ }
8654
+ const rig = input.acceptedExecution.resolvedRig;
8655
+ if (rig) {
8656
+ const retained = await getScheduledScopedRigVersionMetadata(
8657
+ input.db,
8658
+ {
8659
+ accountId: task.accountId,
8660
+ workspaceId: task.workspaceId,
8661
+ subjectId: input.acceptedExecution.personalResourceAuthoritySubjectId ?? task.createdBy.subjectId
8662
+ },
8663
+ rig.id,
8664
+ rig.versionId
8665
+ );
8666
+ if (!retained) {
8667
+ throw new ScheduledRunTerminalAuthorityError(
8668
+ "scheduled_rig_authority_changed",
8669
+ "scheduled rig changed after run admission"
8670
+ );
8671
+ }
8672
+ }
8673
+ const taskMetadata = { ...task.agentConfig.metadata };
8674
+ delete taskMetadata[OPENGENI_SLACK_BOT_SESSION_METADATA_KEY];
8675
+ const created = await createSessionWithIdempotencyKeyResult(input.db, {
8676
+ accountId: task.accountId,
8677
+ workspaceId: task.workspaceId,
8678
+ initialMessage: task.agentConfig.prompt,
8679
+ resources: task.agentConfig.resources,
8680
+ tools: input.acceptedExecution.resolvedTools,
8681
+ firstPartyMcpTools: input.acceptedExecution.resolvedFirstPartyMcpTools,
8682
+ firstPartyMcpPermissions: input.acceptedExecution.resolvedFirstPartyMcpPermissions,
8683
+ metadata: {
8684
+ ...taskMetadata,
8685
+ model: input.acceptedExecution.resolvedModel,
8686
+ reasoningEffort: input.acceptedExecution.resolvedReasoningEffort,
8687
+ scheduledTaskId: task.id,
8688
+ scheduledTaskRunMode: task.runMode,
8689
+ ...task.agentConfig.goal ? { scheduledTaskGoal: task.agentConfig.goal } : {},
8690
+ scheduledTaskRunId: input.run.id,
8691
+ ...frozenSlack ? { [OPENGENI_SLACK_BOT_SESSION_METADATA_KEY]: frozenSlack.id } : {}
8692
+ },
8693
+ createdBy: {
8694
+ kind: "service",
8695
+ subjectId: "scheduler",
8696
+ label: "OpenGeni scheduler"
8697
+ },
8698
+ createdByContext: {
8699
+ scheduledTaskId: task.id,
8700
+ scheduledTaskRunId: input.run.id
8701
+ },
8702
+ model: input.acceptedExecution.resolvedModel,
8703
+ reasoningEffort: input.acceptedExecution.resolvedReasoningEffort,
8704
+ latencyMode: input.acceptedExecution.resolvedLatencyMode,
8705
+ sandboxBackend: input.acceptedExecution.resolvedSandboxBackend,
8706
+ sandboxOs: input.acceptedExecution.resolvedSandboxOs,
8707
+ variableSetId: variableSet?.id ?? null,
8708
+ rigId: rig?.id ?? null,
8709
+ rigVersionId: rig?.versionId ?? null,
8710
+ personalConnectionDelegations: [],
8711
+ initialXaiProviderAccountAuthoritySnapshot: input.acceptedExecution.xaiProviderAccountAuthoritySnapshot,
8712
+ maxNestedAgentDepthOverride: task.agentConfig.maxNestedAgentDepth ?? null,
8713
+ frozenNestedAgentDepthPolicy: {
8714
+ effectiveMaxNestedAgentDepth: input.acceptedExecution.generatedSessionBinding.effectiveMaxNestedAgentDepth,
8715
+ nestedAgentDepthPolicySource: input.acceptedExecution.generatedSessionBinding.nestedAgentDepthPolicySource
8716
+ },
8717
+ frozenCodexCompactionMode: input.acceptedExecution.generatedSessionBinding.codexCompactionMode,
8718
+ allowNestedAgentDepthIncrease: true,
8719
+ subjectId: `scheduled_task:${task.id}`,
8720
+ createIdempotencyKey: input.acceptedExecution.generatedSessionBinding?.createIdempotencyKey ?? (() => {
8721
+ throw new ScheduledRunTerminalAuthorityError(
8722
+ "scheduled_generated_session_binding_missing",
8723
+ "scheduled generated session binding is missing"
8724
+ );
8725
+ })(),
8726
+ beforeCreateCommit: async (tx, sessionId) => {
8727
+ await bindScheduledTaskRunSessionInTransaction(tx, {
8728
+ accountId: task.accountId,
8729
+ workspaceId: task.workspaceId,
8730
+ runId: input.run.id,
8731
+ sessionId
8732
+ });
8733
+ }
8734
+ });
8735
+ if (created.denied) throw new SessionSpawnDeniedDbError(created.denial);
8736
+ session = created.session;
8737
+ if (!created.created) {
8738
+ await bindScheduledTaskRunSessionInTransaction(input.db, {
8739
+ accountId: task.accountId,
8740
+ workspaceId: task.workspaceId,
8741
+ runId: input.run.id,
8742
+ sessionId: session.id
8743
+ });
8744
+ }
8745
+ } else {
8746
+ const targetSessionId = task.runMode === "existing_session" ? task.targetSessionId : task.reusableSessionId;
8747
+ if (!targetSessionId) {
8748
+ throw new ScheduledRunTerminalAuthorityError(
8749
+ "scheduled_target_session_unavailable",
8750
+ "scheduled accepted target session is unavailable"
8751
+ );
8752
+ }
8753
+ session = await requireSession(input.db, task.workspaceId, targetSessionId);
8754
+ assertReusableSessionBindingMatches(session, task);
8755
+ await bindScheduledTaskRunSessionInTransaction(input.db, {
8756
+ accountId: task.accountId,
8757
+ workspaceId: task.workspaceId,
8758
+ runId: input.run.id,
8759
+ sessionId: session.id
8760
+ });
8761
+ }
8762
+ if (session.status === "cancelled") {
8763
+ await markScheduledTaskRunSkippedIfQueued(input.db, {
8764
+ workspaceId: task.workspaceId,
8765
+ runId: input.run.id,
8766
+ sessionId: session.id,
8767
+ error: "session_cancelled"
8768
+ });
8769
+ return { action: "blocked", reason: "scheduled_run_terminal" };
8770
+ }
8771
+ if (generatedSession) {
8772
+ const expectedCreateKey = input.acceptedExecution.generatedSessionBinding?.createIdempotencyKey;
8773
+ if (!expectedCreateKey) {
8774
+ throw new ScheduledRunTerminalAuthorityError(
8775
+ "scheduled_generated_session_binding_missing",
8776
+ "scheduled generated session binding is missing"
8777
+ );
8778
+ }
8779
+ const generatedBinding = input.acceptedExecution.generatedSessionBinding;
8780
+ const canonicalGeneratedRunId = task.runMode === "reusable_session" && typeof session.metadata.scheduledTaskRunId === "string" ? session.metadata.scheduledTaskRunId : input.run.id;
8781
+ const expectedTaskMetadata = { ...task.agentConfig.metadata };
8782
+ delete expectedTaskMetadata[OPENGENI_SLACK_BOT_SESSION_METADATA_KEY];
8783
+ const expectedMetadata = {
8784
+ ...expectedTaskMetadata,
8785
+ model: input.acceptedExecution.resolvedModel,
8786
+ reasoningEffort: input.acceptedExecution.resolvedReasoningEffort,
8787
+ scheduledTaskId: task.id,
8788
+ scheduledTaskRunMode: task.runMode,
8789
+ ...task.agentConfig.goal ? { scheduledTaskGoal: task.agentConfig.goal } : {},
8790
+ scheduledTaskRunId: canonicalGeneratedRunId,
8791
+ ...frozenSlack ? { [OPENGENI_SLACK_BOT_SESSION_METADATA_KEY]: frozenSlack.id } : {}
8792
+ };
8793
+ if (session.createIdempotencyKey !== expectedCreateKey || session.initialMessage !== task.agentConfig.prompt || session.instructions !== null || session.policyRole !== null || stableJson(session.skills) !== "[]" || stableJson(session.toolPolicy) !== stableJson({ mode: "explicit", inheritedFromSessionId: null }) || session.toolPolicyVersion !== 1 || session.mcpServers.length !== 0 || session.parentSessionId !== null || session.rootSessionId !== session.id || session.nestedAgentDepth !== 0 || session.effectiveMaxNestedAgentDepth !== generatedBinding.effectiveMaxNestedAgentDepth || session.nestedAgentDepthPolicySource !== generatedBinding.nestedAgentDepthPolicySource || session.nestedAgentDepthPolicySessionId !== (generatedBinding.nestedAgentDepthPolicySource === "session" ? session.id : null) || session.sandboxGroupId !== session.id || session.channelId !== null || session.codexCompactionMode !== generatedBinding.codexCompactionMode || stableJson(session.metadata) !== stableJson(expectedMetadata) || session.createdBy.kind !== "service" || session.createdBy.subjectId !== "scheduler" || stableJson(session.createdByContext) !== stableJson({
8794
+ label: "OpenGeni scheduler",
8795
+ scheduledTaskId: task.id,
8796
+ scheduledTaskRunId: canonicalGeneratedRunId
8797
+ }) || task.runMode === "new_session_per_run" && input.acceptedExecution.alertOccurrenceLabels === null && (session.metadata.scheduledTaskRunId !== input.run.id || session.createdByContext.scheduledTaskRunId !== input.run.id) || session.model !== input.acceptedExecution.resolvedModel || session.reasoningEffort !== input.acceptedExecution.resolvedReasoningEffort || session.latencyMode !== input.acceptedExecution.resolvedLatencyMode || session.sandboxBackend !== input.acceptedExecution.resolvedSandboxBackend || session.sandboxOs !== input.acceptedExecution.resolvedSandboxOs || stableJson(session.resources) !== stableJson(task.agentConfig.resources) || stableJson(session.tools) !== stableJson(input.acceptedExecution.resolvedTools) || stableJson(session.firstPartyMcpTools) !== stableJson(input.acceptedExecution.resolvedFirstPartyMcpTools) || stableJson(session.firstPartyMcpPermissions) !== stableJson(input.acceptedExecution.resolvedFirstPartyMcpPermissions) || session.maxNestedAgentDepthOverride !== (task.agentConfig.maxNestedAgentDepth ?? null) || (session.variableSetId ?? null) !== (input.acceptedExecution.resolvedVariableSet?.id ?? null) || (session.rigId ?? null) !== (input.acceptedExecution.resolvedRig?.id ?? null) || (session.rigVersionId ?? null) !== (input.acceptedExecution.resolvedRig?.versionId ?? null) || scheduledSlackBotConnectionId(session.metadata) !== (frozenSlack?.id ?? null)) {
8798
+ throw new ScheduledRunTerminalAuthorityError(
8799
+ "scheduled_generated_session_changed",
8800
+ "scheduled generated session differs from accepted execution"
8801
+ );
8802
+ }
8803
+ }
8804
+ let workflowId = workflowIdForSession(session.id);
8805
+ if (generatedSession) {
8806
+ const goalSpec = task.agentConfig.goal ?? null;
8807
+ const started = await initializeSessionStartAtomically(input.db, {
8808
+ accountId: task.accountId,
8809
+ workspaceId: task.workspaceId,
8810
+ sessionId: session.id,
8811
+ reasoningEffortFallback: input.acceptedExecution.resolvedReasoningEffort,
8812
+ createdEventPayload: {
8813
+ scheduledTaskId: task.id,
8814
+ scheduledTaskRunId: input.run.id
8815
+ },
8816
+ goal: goalSpec ? {
8817
+ text: goalSpec.text,
8818
+ successCriteria: goalSpec.successCriteria ?? null,
8819
+ maxAutoContinuations: goalSpec.maxAutoContinuations ?? null,
8820
+ ...goalSpec.mutationPolicy ? { mutationPolicy: goalSpec.mutationPolicy } : {},
8821
+ createdBy: "scheduled_task"
8822
+ } : null,
8823
+ deferInitialTurn: true,
8824
+ deferredStatus: "queued"
8825
+ });
8826
+ workflowId = started.temporalWorkflowId;
8827
+ if (started.events.length > 0) {
8828
+ await publishDurableSessionEvents(input.bus, task.workspaceId, session.id, started.events);
8829
+ }
8830
+ if (task.runMode === "reusable_session") {
8831
+ await materializeScheduledTaskReusableSessionFromRun(input.db, {
8832
+ accountId: task.accountId,
8833
+ workspaceId: task.workspaceId,
8834
+ taskId: task.id,
8835
+ runId: input.run.id,
8836
+ sessionId: session.id,
8837
+ sourceTaskAuthorityRevision: input.run.taskAuthorityRevision,
8838
+ sourceExecutionDigest: input.run.taskExecutionDigest
8839
+ });
8840
+ }
8841
+ const titleEvents = await stampScheduledSessionTitle(input.db, {
8842
+ workspaceId: task.workspaceId,
8843
+ session,
8844
+ taskName: task.name
8845
+ });
8846
+ if (titleEvents.length > 0) {
8847
+ await publishDurableSessionEvents(input.bus, task.workspaceId, session.id, titleEvents);
8848
+ }
8849
+ }
8850
+ if (!generatedSession && task.runMode === "reusable_session" && task.agentConfig.goal) {
8851
+ const goalEvents = await upsertScheduledSessionGoalForRun(input.db, {
8852
+ accountId: task.accountId,
8853
+ workspaceId: task.workspaceId,
8854
+ sessionId: session.id,
8855
+ runId: input.run.id,
8856
+ text: task.agentConfig.goal.text,
8857
+ successCriteria: task.agentConfig.goal.successCriteria ?? null,
8858
+ maxAutoContinuations: task.agentConfig.goal.maxAutoContinuations ?? null,
8859
+ ...task.agentConfig.goal.mutationPolicy ? { mutationPolicy: task.agentConfig.goal.mutationPolicy } : {}
8860
+ });
8861
+ if (goalEvents.length > 0) {
8862
+ await publishDurableSessionEvents(input.bus, task.workspaceId, session.id, goalEvents);
8863
+ }
8864
+ }
8865
+ const scheduledUpdate = await addSessionSystemUpdateWithSourceMutation(
8866
+ input.db,
8867
+ {
8868
+ accountId: task.accountId,
8869
+ workspaceId: task.workspaceId,
8870
+ sessionId: session.id,
8871
+ kind: "scheduled_occurrence",
8872
+ classification: "info",
8873
+ sourceId: input.run.id,
8874
+ dedupeKey: `scheduled-task-run:${input.run.id}`,
8875
+ summary: task.agentConfig.prompt,
8876
+ payload: scheduledUserMessagePayload(
8877
+ task.agentConfig.prompt,
8878
+ task.agentConfig.resources,
8879
+ input.acceptedExecution.resolvedTools,
8880
+ task.id,
8881
+ input.run.id
8882
+ ),
8883
+ lineage: {
8884
+ scheduledTaskId: task.id,
8885
+ scheduledTaskRunId: input.run.id,
8886
+ ...input.acceptedExecution.causalHumanSubjectId ? { causalHumanSubjectId: input.acceptedExecution.causalHumanSubjectId } : {},
8887
+ ...input.acceptedExecution.connectionAuthoritySubjectId ? {
8888
+ connectionAuthoritySubjectId: input.acceptedExecution.connectionAuthoritySubjectId
8889
+ } : {},
8890
+ ...input.acceptedExecution.xaiAuthoritySubjectId ? { xaiAuthoritySubjectId: input.acceptedExecution.xaiAuthoritySubjectId } : {}
8891
+ },
8892
+ personalConnectionDelegations: input.acceptedExecution.personalConnectionDelegations,
8893
+ xaiProviderAccountAuthoritySnapshot: input.acceptedExecution.xaiProviderAccountAuthoritySnapshot,
8894
+ scheduledTaskRunId: input.run.id
8895
+ },
8896
+ async (tx, wakeEventId) => {
8897
+ if (!wakeEventId) {
8898
+ await settleScheduledTaskRunInTransaction(tx, {
8899
+ workspaceId: task.workspaceId,
8900
+ runId: input.run.id,
8901
+ sessionId: session.id,
8902
+ triggerEventId: null,
8903
+ status: "skipped",
8904
+ error: "session_cancelled"
8905
+ });
8906
+ return;
8907
+ }
8908
+ await settleScheduledTaskRunInTransaction(tx, {
8909
+ workspaceId: task.workspaceId,
8910
+ runId: input.run.id,
8911
+ sessionId: session.id,
8912
+ triggerEventId: wakeEventId,
8913
+ status: "dispatched"
8914
+ });
8915
+ },
8916
+ input.acceptedExecution.incidentPreflightRequired ? {
8917
+ prepareSource: async (tx, sessionId) => await prepareIncidentTelemetrySource({
8918
+ tx,
8919
+ settings: input.settings,
8920
+ taskId: task.id,
8921
+ workspaceId: task.workspaceId,
8922
+ sessionId,
8923
+ alertOccurrenceLabels: input.acceptedExecution.alertOccurrenceLabels,
8924
+ acceptedExecution: input.acceptedExecution
8925
+ })
8926
+ } : void 0
8927
+ );
8928
+ if (scheduledUpdate.reason === "session_cancelled") {
8929
+ return { action: "blocked", reason: "scheduled_run_terminal" };
8930
+ }
8931
+ if (scheduledUpdate.added && scheduledUpdate.events.length > 0) {
8932
+ await publishDurableSessionEvents(
8933
+ input.bus,
8934
+ task.workspaceId,
8935
+ session.id,
8936
+ scheduledUpdate.events
8937
+ );
8938
+ }
8939
+ await recordUsageEvent3(input.db, {
8940
+ accountId: task.accountId,
8941
+ workspaceId: task.workspaceId,
8942
+ eventType: "agent_run.created",
8943
+ quantity: 1,
8944
+ unit: "run",
8945
+ sourceResourceType: "scheduled_task_run",
8946
+ sourceResourceId: input.run.id,
8947
+ sessionId: session.id,
8948
+ initiator: input.acceptedExecution.triggerInitiator,
8949
+ initiatorContext: { scheduledTaskId: task.id, scheduledTaskRunId: input.run.id },
8950
+ origin: "scheduled_task",
8951
+ idempotencyKey: input.acceptedExecution.agentRunUsageIdempotencyKey ?? `usage:agent_run.created:scheduled:${input.run.id}`
8952
+ });
8953
+ const workflowWakeRevision = await enqueueSessionWorkflowWakeIfRunnable(input.db, {
8954
+ accountId: task.accountId,
8955
+ workspaceId: task.workspaceId,
8956
+ sessionId: session.id,
8957
+ temporalWorkflowId: workflowId,
8958
+ reason: "scheduled_retry"
8959
+ });
8960
+ const result = {
8961
+ action: generatedSession ? "start" : "signal",
8962
+ accountId: task.accountId,
8963
+ workspaceId: task.workspaceId,
8964
+ sessionId: session.id,
8965
+ triggerEventId: scheduledUpdate.wakeEventId,
8966
+ workflowId,
8967
+ workflowWakeRevision
8968
+ };
8969
+ if (input.wakeSessionWorkflow && workflowWakeRevision !== null) {
8970
+ await input.wakeSessionWorkflow({
8971
+ accountId: result.accountId,
8972
+ workspaceId: result.workspaceId,
8973
+ sessionId: result.sessionId,
8974
+ workflowId: result.workflowId,
8975
+ wakeRevision: workflowWakeRevision
8976
+ });
8977
+ }
8978
+ return result;
8979
+ }
8980
+ async function replayScheduledTaskDispatch(input) {
8981
+ if (!input.run.sessionId || !input.run.triggerEventId) {
8982
+ throw new Error("dispatched scheduled task run is missing its delivery identity");
8983
+ }
8984
+ const sessionId = input.run.sessionId;
8985
+ const triggerEventId = input.run.triggerEventId;
8986
+ await requireSession(input.db, input.task.workspaceId, sessionId);
8987
+ await recordUsageEvent3(input.db, {
8988
+ accountId: input.task.accountId,
8989
+ workspaceId: input.task.workspaceId,
8990
+ eventType: "agent_run.created",
8991
+ quantity: 1,
8992
+ unit: "run",
8993
+ sourceResourceType: "scheduled_task_run",
8994
+ sourceResourceId: input.run.id,
8995
+ sessionId,
8996
+ initiator: input.acceptedExecution.triggerInitiator,
8997
+ initiatorContext: {
8998
+ scheduledTaskId: input.task.id,
8999
+ scheduledTaskRunId: input.run.id
9000
+ },
9001
+ origin: "scheduled_task",
9002
+ idempotencyKey: input.acceptedExecution.agentRunUsageIdempotencyKey ?? `usage:agent_run.created:scheduled:${input.run.id}`
9003
+ });
9004
+ const workflowId = workflowIdForSession(sessionId);
9005
+ const workflowWakeRevision = await enqueueSessionWorkflowWakeIfRunnable(input.db, {
9006
+ accountId: input.task.accountId,
9007
+ workspaceId: input.task.workspaceId,
9008
+ sessionId,
9009
+ temporalWorkflowId: workflowId,
9010
+ reason: "scheduled_retry"
9011
+ });
9012
+ const result = {
9013
+ action: input.acceptedExecution.task.runMode === "new_session_per_run" ? "start" : "signal",
9014
+ accountId: input.task.accountId,
9015
+ workspaceId: input.task.workspaceId,
9016
+ sessionId,
9017
+ triggerEventId,
9018
+ workflowId,
9019
+ workflowWakeRevision
9020
+ };
9021
+ if (input.wakeSessionWorkflow && workflowWakeRevision !== null) {
9022
+ await input.wakeSessionWorkflow({
9023
+ accountId: result.accountId,
9024
+ workspaceId: result.workspaceId,
9025
+ sessionId: result.sessionId,
9026
+ workflowId: result.workflowId,
9027
+ wakeRevision: workflowWakeRevision
9028
+ });
9029
+ }
9030
+ return result;
9031
+ }
6638
9032
  function knowledgeSourceSyncEffectivelyPaused(task) {
6639
9033
  if (task.status === "paused") return true;
6640
9034
  const value = task.metadata.knowledgeSourceSync;
@@ -6644,17 +9038,17 @@ function knowledgeSourceSyncEffectivelyPaused(task) {
6644
9038
  }
6645
9039
 
6646
9040
  // src/activities/site-auth-maintenance.ts
6647
- import { resolveTurnExecutionPolicyV1 } from "@opengeni/config";
9041
+ import { resolveTurnExecutionPolicyV1 as resolveTurnExecutionPolicyV12 } from "@opengeni/config";
6648
9042
  import {
6649
- DEFAULT_FIRST_PARTY_MCP_PERMISSIONS,
9043
+ DEFAULT_FIRST_PARTY_MCP_PERMISSIONS as DEFAULT_FIRST_PARTY_MCP_PERMISSIONS2,
6650
9044
  SITE_AUTH_MAINTENANCE_CONNECTION_CONTEXT_KEY,
6651
9045
  SITE_AUTH_MAINTENANCE_OPERATION_CONTEXT_KEY
6652
9046
  } from "@opengeni/contracts";
6653
9047
  import {
6654
- assertWorkspaceModelPolicyAllows,
6655
- canonicalConfiguredModel,
6656
- createAndStartSessionWithOutcome,
6657
- recordWorkspaceUsage
9048
+ assertWorkspaceModelPolicyAllows as assertWorkspaceModelPolicyAllows2,
9049
+ canonicalConfiguredModel as canonicalConfiguredModel2,
9050
+ createAndStartSessionWithOutcome as createAndStartSessionWithOutcome2,
9051
+ recordWorkspaceUsage as recordWorkspaceUsage2
6658
9052
  } from "@opengeni/core";
6659
9053
  import {
6660
9054
  claimSiteAuthMaintenance,
@@ -6681,10 +9075,10 @@ function createSiteAuthMaintenanceActivities(services, options = {}) {
6681
9075
  const claim = options.claim ?? claimSiteAuthMaintenance;
6682
9076
  const confirm = options.confirm ?? confirmSiteAuthMaintenanceSessionInTransaction;
6683
9077
  const defer = options.defer ?? deferSiteAuthMaintenance;
6684
- const createSession2 = options.createSession ?? createAndStartSessionWithOutcome;
9078
+ const createSession2 = options.createSession ?? createAndStartSessionWithOutcome2;
6685
9079
  const admit = options.admit ?? agentRunAdmissionDenial;
6686
- const recordUsage = options.recordUsage ?? recordWorkspaceUsage;
6687
- const assertModelPolicy = options.assertModelPolicy ?? assertWorkspaceModelPolicyAllows;
9080
+ const recordUsage = options.recordUsage ?? recordWorkspaceUsage2;
9081
+ const assertModelPolicy = options.assertModelPolicy ?? assertWorkspaceModelPolicyAllows2;
6688
9082
  async function maintainSiteAuthConnections() {
6689
9083
  const service = await services();
6690
9084
  const claims = await claim(service.db, { claimTimeoutMs, limit: batchSize });
@@ -6730,7 +9124,7 @@ function createSiteAuthMaintenanceActivities(services, options = {}) {
6730
9124
  return { maintainSiteAuthConnections };
6731
9125
  }
6732
9126
  async function dispatchMaintenanceSession(service, maintenance, dependencies) {
6733
- const model = canonicalConfiguredModel(service.settings, service.settings.openaiModel);
9127
+ const model = canonicalConfiguredModel2(service.settings, service.settings.openaiModel);
6734
9128
  if (!model) throw new Error("site auth maintenance has no configured model");
6735
9129
  await dependencies.assertModelPolicy(
6736
9130
  service.db,
@@ -6746,7 +9140,7 @@ async function dispatchMaintenanceSession(service, maintenance, dependencies) {
6746
9140
  });
6747
9141
  if (denial) throw new Error(`site auth maintenance admission denied: ${denial}`);
6748
9142
  const reasoningEffort = service.settings.openaiReasoningEffort;
6749
- const turnExecutionPolicy = resolveTurnExecutionPolicyV1(service.settings, {
9143
+ const turnExecutionPolicy = resolveTurnExecutionPolicyV12(service.settings, {
6750
9144
  modelId: model,
6751
9145
  requestedModelId: null,
6752
9146
  modelSource: "deployment",
@@ -6800,7 +9194,7 @@ async function dispatchMaintenanceSession(service, maintenance, dependencies) {
6800
9194
  },
6801
9195
  instructions: "Complete only the exact browser-auth maintenance run in the initial request. Use the canonical browser/auth tools, preserve their causal fences, expose no secrets, never publish a BrowserIdentity revision, and do not perform unrelated work.",
6802
9196
  policyRole: "site_auth_maintenance",
6803
- firstPartyMcpPermissions: [...DEFAULT_FIRST_PARTY_MCP_PERMISSIONS],
9197
+ firstPartyMcpPermissions: [...DEFAULT_FIRST_PARTY_MCP_PERMISSIONS2],
6804
9198
  firstPartyMcpTools: [...MAINTENANCE_TOOLS],
6805
9199
  createIdempotencyKey: `site-auth-maintenance:${maintenance.operationId}`,
6806
9200
  subjectId: "site-auth-maintenance"
@@ -6945,14 +9339,14 @@ function createSessionStateActivities(services, overrides = {}) {
6945
9339
  return { action: "failed" };
6946
9340
  }
6947
9341
  const requestedRetryDelayMs = input.retryDelayMs;
6948
- const retryDelayMs = typeof requestedRetryDelayMs === "number" && Number.isFinite(requestedRetryDelayMs) ? Math.max(1e3, Math.min(6e4, Math.trunc(requestedRetryDelayMs))) : 6e4;
9342
+ const retryDelayMs2 = typeof requestedRetryDelayMs === "number" && Number.isFinite(requestedRetryDelayMs) ? Math.max(1e3, Math.min(6e4, Math.trunc(requestedRetryDelayMs))) : 6e4;
6949
9343
  await enqueueSessionWorkflowWakeFn(db, {
6950
9344
  accountId: input.accountId,
6951
9345
  workspaceId: input.workspaceId,
6952
9346
  sessionId: input.sessionId,
6953
9347
  temporalWorkflowId: workflowId,
6954
9348
  reason: "turn_activity_failed_before_attempt_claim",
6955
- notBefore: new Date(Date.now() + retryDelayMs)
9349
+ notBefore: new Date(Date.now() + retryDelayMs2)
6956
9350
  });
6957
9351
  return { action: "unclaimed" };
6958
9352
  }
@@ -7108,7 +9502,7 @@ function createSessionStateActivities(services, overrides = {}) {
7108
9502
  input.attemptId
7109
9503
  );
7110
9504
  if (!turn) return { action: "stale" };
7111
- if (turn.id !== input.turnId || turn.triggerEventId !== input.triggerEventId || turn.executionGeneration !== input.executionGeneration || input.executionGeneration <= 1) {
9505
+ if (turn.id !== input.turnId || turn.triggerEventId !== input.triggerEventId || turn.executionGeneration !== input.executionGeneration || input.executionGeneration <= 1 || input.providerRecoveryCount !== providerRecoveryCountFromMetadata(turn.metadata) + 1 || input.providerRecoveryCount > MAX_AUTOMATIC_PROVIDER_RECOVERIES) {
7112
9506
  return { action: "ineligible" };
7113
9507
  }
7114
9508
  const recovery = await requestSessionTurnRecoveryFn(db, input.workspaceId, {
@@ -7117,10 +9511,12 @@ function createSessionStateActivities(services, overrides = {}) {
7117
9511
  triggerEventId: input.triggerEventId,
7118
9512
  attemptId: input.attemptId,
7119
9513
  reason: "mcp_transport_timeout",
9514
+ providerRecoveryCount: input.providerRecoveryCount,
7120
9515
  detail: {
7121
9516
  code: "mcp_transport_timeout",
7122
9517
  retryable: true,
7123
- continueDelayMs: 6e4,
9518
+ continueDelayMs: input.continueDelayMs,
9519
+ providerRecoveryCount: input.providerRecoveryCount,
7124
9520
  recoverySource: "workflow_activity_failure"
7125
9521
  },
7126
9522
  fromStatuses: ["running"]
@@ -7235,6 +9631,7 @@ function createControlActivityServices(dependencies) {
7235
9631
  }
7236
9632
  function createControlActivitiesFromServices(services, resolveDocumentServices) {
7237
9633
  return {
9634
+ ...createAutomationActivities(services),
7238
9635
  ...createDocumentActivities(services, resolveDocumentServices),
7239
9636
  ...createKnowledgeSourceSyncActivities(services, resolveDocumentServices),
7240
9637
  ...createSessionStateActivities(services),
@@ -7265,4 +9662,4 @@ export {
7265
9662
  createControlActivities,
7266
9663
  createControlActivitiesFromServices
7267
9664
  };
7268
- //# sourceMappingURL=activities-control-SMKF2OBE.js.map
9665
+ //# sourceMappingURL=activities-control-DKG22ONJ.js.map