@opengeni/worker-bundle 1.0.0-canary.0 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (244) hide show
  1. package/dist/activities/agent-run-admission.d.ts +3 -1
  2. package/dist/activities/agent-turn/admission.d.ts +24 -3
  3. package/dist/activities/agent-turn/agent-build.d.ts +4 -0
  4. package/dist/activities/agent-turn/claude-usage-observer.d.ts +58 -0
  5. package/dist/activities/agent-turn/code-search.d.ts +60 -0
  6. package/dist/activities/agent-turn/codex-capacity.d.ts +13 -0
  7. package/dist/activities/agent-turn/codex-plan-entitlement.d.ts +105 -0
  8. package/dist/activities/agent-turn/codex.d.ts +3 -2
  9. package/dist/activities/agent-turn/compaction-prep.d.ts +21 -5
  10. package/dist/activities/agent-turn/credential-leases.d.ts +19 -33
  11. package/dist/activities/agent-turn/credential-restriction.d.ts +3 -0
  12. package/dist/activities/agent-turn/errors.d.ts +82 -11
  13. package/dist/activities/agent-turn/failure-settlement.d.ts +6 -3
  14. package/dist/activities/agent-turn/final-reply.d.ts +18 -0
  15. package/dist/activities/agent-turn/finalization-monitor.d.ts +17 -0
  16. package/dist/activities/agent-turn/finalization.d.ts +4 -2
  17. package/dist/activities/agent-turn/github-repository-bindings.d.ts +2 -0
  18. package/dist/activities/agent-turn/governance-model.d.ts +9 -2
  19. package/dist/activities/agent-turn/history-sink.d.ts +2 -0
  20. package/dist/activities/agent-turn/history.d.ts +2 -0
  21. package/dist/activities/agent-turn/input-wait-reply.d.ts +41 -0
  22. package/dist/activities/agent-turn/media-artifacts.d.ts +5 -3
  23. package/dist/activities/agent-turn/model-call-admission.d.ts +15 -0
  24. package/dist/activities/agent-turn/model-usage.d.ts +6 -0
  25. package/dist/activities/agent-turn/model-wire-dispatch.d.ts +15 -0
  26. package/dist/activities/agent-turn/optional-repositories.d.ts +59 -0
  27. package/dist/activities/agent-turn/provider-dispatch-barrier.d.ts +12 -1
  28. package/dist/activities/agent-turn/quiescence.d.ts +7 -7
  29. package/dist/activities/agent-turn/recovery-warning.d.ts +9 -0
  30. package/dist/activities/agent-turn/refresh-credentials.d.ts +6 -0
  31. package/dist/activities/agent-turn/run-credentials.d.ts +8 -3
  32. package/dist/activities/agent-turn/sandbox-establish.d.ts +2 -1
  33. package/dist/activities/agent-turn/sandbox-provision.d.ts +3 -1
  34. package/dist/activities/agent-turn/sandbox-route.d.ts +3 -20
  35. package/dist/activities/agent-turn/sandbox-runtime.d.ts +2 -2
  36. package/dist/activities/agent-turn/session-title.d.ts +34 -2
  37. package/dist/activities/agent-turn/skill-checkout.d.ts +25 -6
  38. package/dist/activities/agent-turn/skill-read.d.ts +70 -2
  39. package/dist/activities/agent-turn/skill-remove.d.ts +14 -0
  40. package/dist/activities/agent-turn/skill-search.d.ts +3 -1
  41. package/dist/activities/agent-turn/skill-tools.d.ts +15 -2
  42. package/dist/activities/agent-turn/skill-transfer.d.ts +18 -6
  43. package/dist/activities/agent-turn/stream-attempt.d.ts +7 -3
  44. package/dist/activities/agent-turn/subscription-lease.d.ts +38 -0
  45. package/dist/activities/agent-turn/tool-environment.d.ts +56 -6
  46. package/dist/activities/agent-turn/tool-policy.d.ts +43 -0
  47. package/dist/activities/agent-turn/turn-context.d.ts +29 -0
  48. package/dist/activities/agent-turn/turn-sandbox-access.d.ts +3 -0
  49. package/dist/activities/agent-turn/xai-capacity.d.ts +2 -1
  50. package/dist/activities/automations.d.ts +3 -1
  51. package/dist/activities/browser-deadline-checkpoint.d.ts +29 -0
  52. package/dist/activities/codemode-dispatcher.d.ts +9 -2
  53. package/dist/activities/codex-capacity.d.ts +18 -2
  54. package/dist/activities/codex-rotation.d.ts +7 -1
  55. package/dist/activities/context-compaction.d.ts +16 -2
  56. package/dist/activities/gmail-files.d.ts +32 -0
  57. package/dist/activities/goals.d.ts +8 -0
  58. package/dist/activities/knowledge-indexing.d.ts +19 -0
  59. package/dist/activities/mcp-account-routes.d.ts +3 -0
  60. package/dist/activities/mcp-credentials.d.ts +12 -13
  61. package/dist/activities/mcp-operation-authority.d.ts +0 -5
  62. package/dist/activities/model-call-fact-reconciler.d.ts +34 -0
  63. package/dist/activities/open-suffix-resume.d.ts +13 -0
  64. package/dist/activities/parent-wake.d.ts +6 -12
  65. package/dist/activities/retained-screenshots.d.ts +7 -2
  66. package/dist/activities/rig-verification.d.ts +2 -3
  67. package/dist/activities/run-credential-renewal.d.ts +1 -0
  68. package/dist/activities/run-credentials.d.ts +15 -3
  69. package/dist/activities/sandbox-images.d.ts +1 -1
  70. package/dist/activities/sandbox-lease.d.ts +47 -9
  71. package/dist/activities/scheduled-tasks.d.ts +4 -35
  72. package/dist/activities/session-state.d.ts +70 -3
  73. package/dist/activities/types.d.ts +55 -6
  74. package/dist/activities/workspace-capture.d.ts +2 -0
  75. package/dist/activities/workspace-credential-provider.d.ts +46 -0
  76. package/dist/{activities-control-HMFGSKC5.js → activities-control-BNPTRTOE.js} +1896 -418
  77. package/dist/activities-control-BNPTRTOE.js.map +1 -0
  78. package/dist/activities-control.d.ts +182 -4
  79. package/dist/{activities-turn-AP67JADS.js → activities-turn-5LGZOSVU.js} +4476 -1327
  80. package/dist/activities-turn-5LGZOSVU.js.map +1 -0
  81. package/dist/activities.d.ts +182 -4
  82. package/dist/artifact-materializer-entry.js +5 -2
  83. package/dist/artifact-materializer-entry.js.map +1 -1
  84. package/dist/artifact-outbox-entry.js +5 -2
  85. package/dist/artifact-outbox-entry.js.map +1 -1
  86. package/dist/browser-deadline-checkpoint-contract.d.ts +2 -0
  87. package/dist/{chunk-A4XQWVZK.js → chunk-3ZTRWDMN.js} +307 -4
  88. package/dist/chunk-3ZTRWDMN.js.map +1 -0
  89. package/dist/{chunk-RAMNYSAW.js → chunk-4ER3YXOT.js} +22 -4
  90. package/dist/chunk-4ER3YXOT.js.map +1 -0
  91. package/dist/{chunk-CYITU5UH.js → chunk-LJX7Z2KP.js} +1445 -343
  92. package/dist/chunk-LJX7Z2KP.js.map +1 -0
  93. package/dist/editable-artifact-materializer-service.d.ts +2 -2
  94. package/dist/editable-artifact-outbox-service.d.ts +2 -2
  95. package/dist/host-export-pump.d.ts +3 -1
  96. package/dist/host-export.d.ts +1 -1
  97. package/dist/host-export.js +1 -1
  98. package/dist/http.d.ts +1 -1
  99. package/dist/index.d.ts +95 -3
  100. package/dist/index.js +137 -43
  101. package/dist/index.js.map +1 -1
  102. package/dist/observability-metrics.d.ts +118 -8
  103. package/dist/retained-process-diagnostics.d.ts +4 -0
  104. package/dist/retained-process-retry.d.ts +7 -0
  105. package/dist/sandbox-resume.d.ts +55 -13
  106. package/dist/sandbox-routing.d.ts +5 -3
  107. package/dist/sandbox-snapshot-diagnostics.d.ts +20 -0
  108. package/dist/session-event-fanout.d.ts +6 -0
  109. package/dist/warm-capture-holder.d.ts +3 -0
  110. package/dist/worker-service-lifecycle.d.ts +12 -0
  111. package/dist/workflow-bundle.js +273 -41
  112. package/dist/workflows/activities.d.ts +1 -1
  113. package/dist/workflows/browser-deadline-checkpoint.d.ts +9 -0
  114. package/dist/workflows/scheduled-tasks.d.ts +4 -0
  115. package/dist/workflows/session.d.ts +36 -0
  116. package/dist/workflows.d.ts +1 -0
  117. package/package.json +21 -19
  118. package/src/activities/agent-run-admission.ts +17 -1
  119. package/src/activities/agent-turn/admission.ts +66 -3
  120. package/src/activities/agent-turn/agent-build.ts +84 -10
  121. package/src/activities/agent-turn/claim.ts +186 -43
  122. package/src/activities/agent-turn/claude-usage-observer.ts +221 -0
  123. package/src/activities/agent-turn/code-search.ts +275 -0
  124. package/src/activities/agent-turn/codex-capacity.ts +172 -8
  125. package/src/activities/agent-turn/codex-plan-entitlement.ts +254 -0
  126. package/src/activities/agent-turn/codex.ts +13 -5
  127. package/src/activities/agent-turn/compaction-prep.ts +170 -102
  128. package/src/activities/agent-turn/credential-leases.ts +122 -216
  129. package/src/activities/agent-turn/credential-restriction.ts +14 -0
  130. package/src/activities/agent-turn/errors.ts +647 -48
  131. package/src/activities/agent-turn/failure-settlement.ts +682 -130
  132. package/src/activities/agent-turn/final-reply.ts +46 -0
  133. package/src/activities/agent-turn/finalization-monitor.ts +120 -0
  134. package/src/activities/agent-turn/finalization.ts +146 -43
  135. package/src/activities/agent-turn/github-repository-bindings.ts +3 -1
  136. package/src/activities/agent-turn/governance-model.ts +69 -28
  137. package/src/activities/agent-turn/history-sink.ts +16 -2
  138. package/src/activities/agent-turn/history.ts +38 -11
  139. package/src/activities/agent-turn/input-wait-reply.ts +103 -0
  140. package/src/activities/agent-turn/knowledge-source-tools.ts +2 -0
  141. package/src/activities/agent-turn/media-artifacts.ts +2 -1
  142. package/src/activities/agent-turn/model-call-admission.ts +94 -0
  143. package/src/activities/agent-turn/model-usage.ts +64 -3
  144. package/src/activities/agent-turn/model-wire-dispatch.ts +27 -0
  145. package/src/activities/agent-turn/optional-repositories.ts +224 -0
  146. package/src/activities/agent-turn/provider-dispatch-barrier.ts +22 -0
  147. package/src/activities/agent-turn/quiescence.ts +10 -7
  148. package/src/activities/agent-turn/recovery-warning.ts +16 -0
  149. package/src/activities/agent-turn/refresh-credentials.ts +44 -0
  150. package/src/activities/agent-turn/run-credentials.ts +116 -55
  151. package/src/activities/agent-turn/run.ts +320 -93
  152. package/src/activities/agent-turn/sandbox-establish.ts +84 -8
  153. package/src/activities/agent-turn/sandbox-provision.ts +3 -1
  154. package/src/activities/agent-turn/sandbox-route.ts +4 -21
  155. package/src/activities/agent-turn/sandbox-runtime.ts +343 -18
  156. package/src/activities/agent-turn/session-title.ts +81 -5
  157. package/src/activities/agent-turn/skill-checkout.ts +111 -24
  158. package/src/activities/agent-turn/skill-install.ts +1 -1
  159. package/src/activities/agent-turn/skill-read.ts +374 -28
  160. package/src/activities/agent-turn/skill-remove.ts +46 -0
  161. package/src/activities/agent-turn/skill-save.ts +1 -1
  162. package/src/activities/agent-turn/skill-search.ts +10 -1
  163. package/src/activities/agent-turn/skill-selection.ts +4 -0
  164. package/src/activities/agent-turn/skill-tools.ts +112 -18
  165. package/src/activities/agent-turn/skill-transfer.ts +68 -23
  166. package/src/activities/agent-turn/stream-attempt.ts +313 -77
  167. package/src/activities/agent-turn/subscription-lease.ts +115 -0
  168. package/src/activities/agent-turn/tool-environment.ts +316 -167
  169. package/src/activities/agent-turn/tool-policy.ts +80 -1
  170. package/src/activities/agent-turn/turn-context.ts +35 -0
  171. package/src/activities/agent-turn/turn-sandbox-access.ts +7 -1
  172. package/src/activities/agent-turn/xai-capacity.ts +94 -51
  173. package/src/activities/automations.ts +76 -4
  174. package/src/activities/browser-deadline-checkpoint.ts +279 -0
  175. package/src/activities/capabilities.ts +108 -11
  176. package/src/activities/codemode-dispatcher.ts +36 -8
  177. package/src/activities/codex-capacity.ts +60 -25
  178. package/src/activities/codex-rotation.ts +39 -17
  179. package/src/activities/context-compaction.ts +75 -18
  180. package/src/activities/gmail-files.ts +88 -0
  181. package/src/activities/goals.ts +116 -121
  182. package/src/activities/google-drive-publication.ts +1 -3
  183. package/src/activities/image-generation-operation.ts +1 -1
  184. package/src/activities/knowledge-indexing.ts +269 -15
  185. package/src/activities/knowledge-source-sync.ts +10 -0
  186. package/src/activities/mcp-account-routes.ts +17 -0
  187. package/src/activities/mcp-credentials.ts +83 -194
  188. package/src/activities/mcp-operation-authority.ts +16 -41
  189. package/src/activities/model-call-fact-reconciler.ts +112 -0
  190. package/src/activities/open-suffix-resume.ts +15 -2
  191. package/src/activities/parent-wake.ts +46 -21
  192. package/src/activities/retained-screenshots.ts +82 -11
  193. package/src/activities/rig-verification.ts +38 -37
  194. package/src/activities/run-credential-renewal.ts +13 -1
  195. package/src/activities/run-credentials.ts +59 -17
  196. package/src/activities/run-input.ts +25 -16
  197. package/src/activities/sandbox-images.ts +0 -1
  198. package/src/activities/sandbox-lease.ts +798 -82
  199. package/src/activities/scheduled-tasks.ts +550 -178
  200. package/src/activities/session-state.ts +245 -6
  201. package/src/activities/site-auth-maintenance.ts +3 -0
  202. package/src/activities/types.ts +72 -5
  203. package/src/activities/workflow-wake.ts +2 -17
  204. package/src/activities/workspace-capture.ts +13 -2
  205. package/src/activities/workspace-credential-provider.ts +283 -0
  206. package/src/activities-control.ts +4 -0
  207. package/src/activities-turn.ts +5 -0
  208. package/src/activities.ts +13 -0
  209. package/src/activity-services.ts +16 -4
  210. package/src/browser-deadline-checkpoint-contract.ts +5 -0
  211. package/src/editable-artifact-hint-broker.ts +3 -0
  212. package/src/editable-artifact-materializer-service.ts +7 -3
  213. package/src/editable-artifact-outbox-service.ts +4 -4
  214. package/src/github-rest-mcp.ts +1 -2
  215. package/src/host-export-pump.ts +26 -4
  216. package/src/host-export.ts +3 -0
  217. package/src/http.ts +4 -2
  218. package/src/index.ts +99 -37
  219. package/src/observability-metrics.ts +510 -5
  220. package/src/personal-github-git-credentials.ts +2 -1
  221. package/src/pr-review-credentials.ts +1 -3
  222. package/src/retained-process-diagnostics.ts +50 -0
  223. package/src/retained-process-retry.ts +36 -0
  224. package/src/sandbox-resume.ts +388 -101
  225. package/src/sandbox-routing.ts +150 -25
  226. package/src/sandbox-snapshot-diagnostics.ts +112 -0
  227. package/src/session-event-fanout.ts +23 -0
  228. package/src/warm-capture-holder.ts +15 -0
  229. package/src/worker-service-lifecycle.ts +53 -0
  230. package/src/workflows/activities.ts +16 -1
  231. package/src/workflows/browser-deadline-checkpoint.ts +50 -0
  232. package/src/workflows/file-upload-reaper.ts +2 -0
  233. package/src/workflows/scheduled-tasks.ts +26 -13
  234. package/src/workflows/session.ts +242 -22
  235. package/src/workflows.ts +4 -0
  236. package/dist/activities/agent-turn/knowledge-sources.d.ts +0 -20
  237. package/dist/activities/packs.d.ts +0 -55
  238. package/dist/activities-control-HMFGSKC5.js.map +0 -1
  239. package/dist/activities-turn-AP67JADS.js.map +0 -1
  240. package/dist/chunk-A4XQWVZK.js.map +0 -1
  241. package/dist/chunk-CYITU5UH.js.map +0 -1
  242. package/dist/chunk-RAMNYSAW.js.map +0 -1
  243. package/src/activities/agent-turn/knowledge-sources.ts +0 -86
  244. package/src/activities/packs.ts +0 -166
@@ -8,6 +8,7 @@ import {
8
8
  import {
9
9
  INCIDENT_TELEMETRY_AUTHORITY_FENCE_LINEAGE_KEY,
10
10
  MAX_AUTOMATIC_PROVIDER_RECOVERIES,
11
+ agentRunAdmissionDenial,
11
12
  assertReusableSessionRevivable,
12
13
  captureConnectedCommandOutput,
13
14
  createCodexCapacityActivities,
@@ -23,59 +24,147 @@ import {
23
24
  notifyParentOfChildIdle,
24
25
  persistWorkspaceArchiveCandidate,
25
26
  providerRecoveryCountFromMetadata,
27
+ publishDurableSessionEvents,
26
28
  putVersion1TarArchiveOrInline,
27
29
  reconcileAutomaticSessionTitleFanout,
28
30
  reconcilePendingParentSystemUpdates,
29
31
  reconcilePendingSessionWorkflowWakes,
30
32
  resolveIncidentTelemetryResponderMetadata,
31
- resolveWorkspacePackRuntime,
32
33
  rigProviderImageSourceImage,
33
34
  scheduledUserMessagePayload,
35
+ warnDrainSnapshotFailure,
34
36
  withFirstPartyTools,
35
37
  workflowIdForSession
36
- } from "./chunk-CYITU5UH.js";
38
+ } from "./chunk-LJX7Z2KP.js";
37
39
  import {
38
40
  inspectOpenSandboxKubernetesInventory,
41
+ recordActiveUserGauges,
39
42
  recordCreditBalanceGauges,
43
+ recordCreditGrantGauges,
40
44
  recordCreditMicros,
41
45
  recordExpiredDrainingSandboxLeaseGauges,
46
+ recordManagedAuthNewSignupsSwitchGauge,
42
47
  recordOpenSandboxKubernetesInventoryGauges,
43
48
  recordRetainedProcessInventoryGauges,
44
49
  recordRetainedProcessReconciliation,
45
50
  recordSandboxCheckpointArtifactGauges,
46
51
  recordSandboxCheckpointArtifactOutcome,
52
+ recordSandboxCommandContainment,
47
53
  recordSandboxDeadlineRotationsRequested,
48
54
  recordSandboxInventoryProjectionFailure,
49
55
  recordSandboxInventoryProjectionSuccess,
50
56
  recordSandboxLeaseGauges,
51
57
  recordSandboxOrphansTerminated,
58
+ recordSandboxProviderMissingBeforeCapture,
59
+ recordSandboxRecoveryObservationGauges,
52
60
  recordSandboxRotationBacklogGauges,
53
61
  recordTurnsQueuedGauge,
62
+ recordVerifiedSignupTrialDeploymentFlagGauge,
63
+ recordVerifiedSignupTrialSwitchGauge,
54
64
  recordWorkerDeathRecoveryMetrics,
55
65
  runtimeMetricsHooksForObservability
56
- } from "./chunk-A4XQWVZK.js";
66
+ } from "./chunk-3ZTRWDMN.js";
57
67
  import "./chunk-PZ5AY32C.js";
58
68
 
59
69
  // src/activities/knowledge-indexing.ts
60
70
  import { configuredStaticUsageLimits } from "@opengeni/config";
71
+ import { documentEmbeddingCostMicros, paidDocumentEmbedding } from "@opengeni/core";
61
72
  import {
73
+ applyCreditDebitAfterUse,
62
74
  appendKnowledgeIndexChunks,
63
75
  claimKnowledgeIndexJobs,
76
+ checkWorkspaceAllowance,
64
77
  completeKnowledgeIndexJob,
65
78
  continueKnowledgeIndexJob,
79
+ creditDebitAttributionMetadata,
66
80
  deferKnowledgeIndexJob,
81
+ freezeKnowledgeIndexBillingMode,
67
82
  getBillingBalance,
83
+ guardPaidKnowledgeIndexPublication,
84
+ knowledgeIndexBillingActivationTime,
68
85
  readKnowledgeIndexSource,
69
86
  recordUsageEvent,
70
87
  sumUsageQuantity,
71
- withWorkspaceUsageLock
88
+ withWorkspaceUsageLock,
89
+ waitKnowledgeIndexForFunding
72
90
  } from "@opengeni/db";
91
+ var KnowledgeIndexUsageLimitError = class extends Error {
92
+ constructor() {
93
+ super("monthly document indexing limit reached");
94
+ this.name = "KnowledgeIndexUsageLimitError";
95
+ }
96
+ };
97
+ function knowledgeIndexFailureDiagnostic(stage, error) {
98
+ if (error instanceof KnowledgeIndexUsageLimitError) {
99
+ return {
100
+ errorClass: "KnowledgeIndexOperationError",
101
+ errorCode: "knowledge_index_usage_limit_reached",
102
+ origin: "worker"
103
+ };
104
+ }
105
+ if (stage === "embedding") {
106
+ const status = ownValue(error, "status");
107
+ return {
108
+ errorClass: "KnowledgeIndexOperationError",
109
+ errorCode: "knowledge_index_embedding_failed",
110
+ origin: "worker",
111
+ ...typeof status === "number" && Number.isInteger(status) && status >= 100 && status <= 599 ? { status } : {}
112
+ };
113
+ }
114
+ const postgres = postgresErrorState(error);
115
+ if (!postgres) {
116
+ return {
117
+ errorClass: "KnowledgeIndexOperationError",
118
+ errorCode: "knowledge_index_failed",
119
+ origin: "worker"
120
+ };
121
+ }
122
+ return {
123
+ errorClass: "KnowledgeIndexOperationError",
124
+ errorCode: "knowledge_index_persistence_failed",
125
+ origin: "db",
126
+ ...postgres.sqlState ? { sqlState: postgres.sqlState } : {}
127
+ };
128
+ }
129
+ function ownValue(value, key) {
130
+ try {
131
+ if (!value || typeof value !== "object") return void 0;
132
+ const descriptor = Object.getOwnPropertyDescriptor(value, key);
133
+ return descriptor && "value" in descriptor ? descriptor.value : void 0;
134
+ } catch {
135
+ return void 0;
136
+ }
137
+ }
138
+ function postgresErrorState(error) {
139
+ let current = error;
140
+ for (let depth = 0; depth < 4 && current && typeof current === "object"; depth += 1) {
141
+ if (ownValue(current, "name") === "PostgresError") {
142
+ const code = ownValue(current, "code");
143
+ return typeof code === "string" && /^[0-9A-Z]{5}$/.test(code) ? { sqlState: code } : {};
144
+ }
145
+ current = ownValue(current, "cause");
146
+ }
147
+ return void 0;
148
+ }
73
149
  function createKnowledgeIndexingActivities(services, resolveDocumentServices) {
150
+ let activationTime;
74
151
  return {
75
152
  indexKnowledge: async () => {
76
153
  const result = { completed: 0, advanced: 0, deferred: 0, unavailable: 0 };
77
154
  if (!resolveDocumentServices) return result;
78
155
  const { db, settings, observability } = await services();
156
+ let policyActivatedAt;
157
+ if (paidDocumentEmbedding(settings)) {
158
+ if (!settings.documentEmbeddingCreditsActivatedAt)
159
+ throw new Error("paid Knowledge embedding requires an activation cutoff");
160
+ policyActivatedAt = new Date(settings.documentEmbeddingCreditsActivatedAt);
161
+ } else {
162
+ activationTime ??= knowledgeIndexBillingActivationTime(db).catch((error) => {
163
+ activationTime = void 0;
164
+ throw error;
165
+ });
166
+ policyActivatedAt = await activationTime;
167
+ }
79
168
  const { embedder } = await resolveDocumentServices();
80
169
  const { knowledgeIndexChunks } = await import("@opengeni/documents");
81
170
  const claims = await claimKnowledgeIndexJobs(db, {
@@ -84,6 +173,7 @@ function createKnowledgeIndexingActivities(services, resolveDocumentServices) {
84
173
  limit: 2
85
174
  });
86
175
  for (const claim of claims) {
176
+ let stage = "processing";
87
177
  try {
88
178
  const source = await readKnowledgeIndexSource(db, claim);
89
179
  if (!source) {
@@ -107,9 +197,57 @@ function createKnowledgeIndexingActivities(services, resolveDocumentServices) {
107
197
  chunks.push(chunk);
108
198
  }
109
199
  if (chunks.length) {
110
- if (settings.billingMode === "stripe" || settings.usageLimitsMode === "managed") {
200
+ const frozenPolicy = await freezeKnowledgeIndexBillingMode(
201
+ lockedDb,
202
+ claim,
203
+ // Deterministic embeddings incur no provider charge; a valid
204
+ // credits-mode config with zero tariff must still index them.
205
+ // OpenAI shadow keeps its price snapshot for internal estimates.
206
+ settings.documentEmbeddingProvider === "openai" ? settings.documentEmbeddingBillingMode ?? "usage_only" : "usage_only",
207
+ policyActivatedAt,
208
+ settings.documentEmbeddingRateMicrosPerMillionBytes ?? 0
209
+ );
210
+ if (frozenPolicy.mode === "awaiting_review") {
211
+ result.deferred++;
212
+ return;
213
+ }
214
+ if (frozenPolicy.mode === "obsolete") {
215
+ result.unavailable++;
216
+ return;
217
+ }
218
+ if (frozenPolicy.mode === "credits" && !paidDocumentEmbedding(settings)) {
219
+ await deferKnowledgeIndexJob(lockedDb, claim);
220
+ result.deferred++;
221
+ return;
222
+ }
223
+ const paid = frozenPolicy.mode === "credits";
224
+ if (paid && claim.billingAttribution.kind === "unknown") {
225
+ observability.warn("Paid Knowledge indexing awaits initiating attribution", {
226
+ errorCode: "knowledge_index_attribution_unavailable"
227
+ });
228
+ await deferKnowledgeIndexJob(lockedDb, claim);
229
+ result.deferred++;
230
+ return;
231
+ }
232
+ if (paid && current.nextIndex === 0) {
111
233
  const balance = await getBillingBalance(lockedDb, claim.accountId);
112
- if (balance.balanceMicros <= 0) throw new Error("insufficient OpenGeni credits");
234
+ if (balance.balanceMicros <= 0) {
235
+ await waitKnowledgeIndexForFunding(lockedDb, claim);
236
+ result.deferred++;
237
+ return;
238
+ }
239
+ }
240
+ if (paid) {
241
+ const refusal = await checkWorkspaceAllowance(lockedDb, {
242
+ accountId: claim.accountId,
243
+ workspaceId: current.billingWorkspaceId,
244
+ subjectId: claim.billingAttribution.kind === "turn" || claim.billingAttribution.kind === "human" ? claim.billingAttribution.initiatingHumanSubjectId : null
245
+ });
246
+ if (refusal) {
247
+ await deferKnowledgeIndexJob(lockedDb, claim);
248
+ result.deferred++;
249
+ return;
250
+ }
113
251
  }
114
252
  if (settings.usageLimitsMode === "static" || settings.usageLimitsMode === "managed") {
115
253
  const limit = configuredStaticUsageLimits(settings).maxDocumentIndexedChunksPerWorkspace;
@@ -120,13 +258,27 @@ function createKnowledgeIndexingActivities(services, resolveDocumentServices) {
120
258
  eventType: "document.indexed",
121
259
  since: new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), 1))
122
260
  });
123
- if (used + chunks.length > limit)
124
- throw new Error("monthly document indexing limit reached");
261
+ if (used + chunks.length > limit) throw new KnowledgeIndexUsageLimitError();
125
262
  }
126
263
  }
127
- const vectors = await embedder.embedMany(chunks.map((chunk) => chunk.embeddingInput));
264
+ const inputs = chunks.map((chunk) => chunk.embeddingInput);
265
+ const bytes = inputs.reduce(
266
+ (sum, input) => sum + Buffer.byteLength(input, "utf8"),
267
+ 0
268
+ );
269
+ stage = "embedding";
270
+ const vectors = await embedder.embedMany(inputs);
128
271
  if (vectors.length !== chunks.length)
129
272
  throw new Error("Incomplete Knowledge embeddings");
273
+ stage = "processing";
274
+ if (paid) {
275
+ const publication = await guardPaidKnowledgeIndexPublication(lockedDb, claim);
276
+ if (publication !== "published") {
277
+ if (publication === "obsolete") result.unavailable++;
278
+ else result.deferred++;
279
+ return;
280
+ }
281
+ }
130
282
  const appended = await appendKnowledgeIndexChunks(
131
283
  lockedDb,
132
284
  claim,
@@ -147,6 +299,62 @@ function createKnowledgeIndexingActivities(services, resolveDocumentServices) {
147
299
  sourceResourceId: claim.revisionId,
148
300
  idempotencyKey: `knowledge.indexed:${claim.revisionId}:${claim.generation}:${current.nextIndex}`
149
301
  });
302
+ await recordUsageEvent(lockedDb, {
303
+ accountId: claim.accountId,
304
+ workspaceId: current.billingWorkspaceId,
305
+ eventType: "document.embedding_bytes",
306
+ quantity: bytes,
307
+ unit: "byte",
308
+ sourceResourceType: "knowledge_revision",
309
+ sourceResourceId: claim.revisionId,
310
+ idempotencyKey: `knowledge.embedding_bytes:${claim.revisionId}:${claim.generation}:${current.nextIndex}`
311
+ });
312
+ if (frozenPolicy.mode === "shadow" && frozenPolicy.rateMicrosPerMillionBytes > 0) {
313
+ const estimate = documentEmbeddingCostMicros(
314
+ {
315
+ ...settings,
316
+ documentEmbeddingRateMicrosPerMillionBytes: frozenPolicy.rateMicrosPerMillionBytes
317
+ },
318
+ bytes
319
+ );
320
+ if (estimate > 0)
321
+ await recordUsageEvent(lockedDb, {
322
+ accountId: claim.accountId,
323
+ workspaceId: current.billingWorkspaceId,
324
+ eventType: "document.embedding_shadow_estimate",
325
+ quantity: estimate,
326
+ unit: "micro_usd",
327
+ sourceResourceType: "knowledge_revision",
328
+ sourceResourceId: claim.revisionId,
329
+ idempotencyKey: `knowledge.embedding_shadow:${claim.revisionId}:${claim.generation}:${current.nextIndex}`
330
+ });
331
+ }
332
+ if (paid) {
333
+ const cost = documentEmbeddingCostMicros(
334
+ {
335
+ ...settings,
336
+ documentEmbeddingRateMicrosPerMillionBytes: frozenPolicy.rateMicrosPerMillionBytes
337
+ },
338
+ bytes
339
+ );
340
+ if (cost > 0)
341
+ await applyCreditDebitAfterUse(lockedDb, {
342
+ accountId: claim.accountId,
343
+ workspaceId: current.billingWorkspaceId,
344
+ type: "document_embedding_debit",
345
+ amountMicros: cost,
346
+ sourceType: "knowledge_revision",
347
+ sourceId: claim.revisionId,
348
+ idempotencyKey: `knowledge.embedding:${claim.revisionId}:${claim.generation}:${current.nextIndex}`,
349
+ metadata: {
350
+ ...creditDebitAttributionMetadata(claim.billingAttribution),
351
+ model: claim.model,
352
+ bytes,
353
+ chunks: chunks.length,
354
+ rateMicrosPerMillionBytes: frozenPolicy.rateMicrosPerMillionBytes
355
+ }
356
+ });
357
+ }
150
358
  }
151
359
  if (more) {
152
360
  await continueKnowledgeIndexJob(lockedDb, claim);
@@ -161,14 +369,19 @@ function createKnowledgeIndexingActivities(services, resolveDocumentServices) {
161
369
  else result.unavailable++;
162
370
  }
163
371
  });
164
- } catch {
165
- await deferKnowledgeIndexJob(db, claim).catch(() => void 0);
166
- result.deferred++;
167
- observability.warn("Knowledge indexing batch deferred", {
168
- accountId: claim.accountId,
169
- entryId: claim.entryId,
170
- revisionId: claim.revisionId
372
+ } catch (error) {
373
+ observability.warn(
374
+ "Knowledge indexing batch deferred",
375
+ knowledgeIndexFailureDiagnostic(stage, error)
376
+ );
377
+ await deferKnowledgeIndexJob(db, claim).catch((deferError) => {
378
+ const deferDiagnostic = knowledgeIndexFailureDiagnostic("processing", deferError);
379
+ observability.warn("Knowledge indexing batch deferral failed", {
380
+ ...deferDiagnostic,
381
+ errorCode: "knowledge_index_defer_failed"
382
+ });
171
383
  });
384
+ result.deferred++;
172
385
  }
173
386
  }
174
387
  return result;
@@ -260,6 +473,240 @@ function cleanupErrorCategory(error) {
260
473
  return typeof code === "string" && SAFE_PROVIDER_ERROR_CODES.has(code) ? code : "unknown";
261
474
  }
262
475
 
476
+ // src/activities/browser-deadline-checkpoint.ts
477
+ import { Context } from "@temporalio/activity";
478
+ import {
479
+ environmentsEncryptionKeyBytes,
480
+ resolveFirstPartyDelegationSecret
481
+ } from "@opengeni/config";
482
+ import { BROWSER_STATE_ARTIFACT_CONTENT_TYPE } from "@opengeni/contracts";
483
+ import {
484
+ browserDeadlineCheckpoint,
485
+ clearSuspendedBrowserSessionController,
486
+ commitBrowserSessionSuspension,
487
+ dispatchBrowserSessionOperation,
488
+ failBrowserSessionSuspension,
489
+ listBrowserDeadlineCheckpoints,
490
+ readLease,
491
+ releaseLeaseHolder,
492
+ withRlsContext
493
+ } from "@opengeni/db";
494
+ import { deriveBrowserControllerAdminToken } from "@opengeni/interaction/browser-controller-authority";
495
+ import {
496
+ browserStateArtifactAad,
497
+ browserStateManifestDigest,
498
+ browserStateObjectKey,
499
+ deriveBrowserStateDataKey,
500
+ wrapBrowserStateDataKey
501
+ } from "@opengeni/interaction/browser-state-authority";
502
+ import {
503
+ BrowserControlClient,
504
+ BrowserControlRequestError,
505
+ createSandboxClientForBackend,
506
+ deserializeSandboxSessionStateEnvelope,
507
+ resumeExactSandboxSession
508
+ } from "@opengeni/runtime";
509
+ var UPLOAD_CLEANUP_GRACE_MS = 24 * 60 * 6e4;
510
+ function heartbeat() {
511
+ try {
512
+ Context.current().heartbeat();
513
+ } catch {
514
+ }
515
+ }
516
+ async function withActivityHeartbeat(run) {
517
+ heartbeat();
518
+ const timer = setInterval(heartbeat, 5e3);
519
+ try {
520
+ return await run();
521
+ } finally {
522
+ clearInterval(timer);
523
+ }
524
+ }
525
+ async function resumeCheckpointController({
526
+ target,
527
+ lease,
528
+ settings,
529
+ controllerRoot
530
+ }) {
531
+ const provider = createSandboxClientForBackend("modal", settings);
532
+ const envelope = lease.resumeState;
533
+ const state = await deserializeSandboxSessionStateEnvelope(
534
+ provider,
535
+ envelope.sessionState ?? lease.resumeState,
536
+ target.instanceId
537
+ );
538
+ const resumed = await resumeExactSandboxSession(provider, "modal", state, target.instanceId);
539
+ return new BrowserControlClient(resumed.session, {
540
+ adminToken: deriveBrowserControllerAdminToken({
541
+ rootSecret: controllerRoot,
542
+ accountId: target.accountId,
543
+ workspaceId: target.workspaceId,
544
+ placement: { kind: "sandbox_group", sandboxGroupId: target.sandboxGroupId },
545
+ placementInstanceId: target.instanceId
546
+ })
547
+ });
548
+ }
549
+ function createBrowserDeadlineCheckpointActivities(services, connectController = resumeCheckpointController) {
550
+ return {
551
+ async listDueBrowserCheckpoints() {
552
+ return await withActivityHeartbeat(async () => {
553
+ const { db, settings, objectStorage } = await services();
554
+ if (!settings.sandboxOwnershipEnabled || !objectStorage || !resolveFirstPartyDelegationSecret(settings))
555
+ return [];
556
+ const root = environmentsEncryptionKeyBytes(settings);
557
+ if (!root) return [];
558
+ root.fill(0);
559
+ return await listBrowserDeadlineCheckpoints(db);
560
+ });
561
+ },
562
+ async checkpointBrowserBeforeDeadline(target) {
563
+ return await withActivityHeartbeat(async () => {
564
+ const { db, settings, objectStorage } = await services();
565
+ const rootKey = environmentsEncryptionKeyBytes(settings);
566
+ const controllerRoot = resolveFirstPartyDelegationSecret(settings);
567
+ if (!rootKey || !objectStorage || !controllerRoot) {
568
+ rootKey?.fill(0);
569
+ throw new Error("browser deadline checkpoint storage or encryption is unavailable");
570
+ }
571
+ let dataKey = null;
572
+ let aad = null;
573
+ let timer = null;
574
+ try {
575
+ const claim = await browserDeadlineCheckpoint(db, target, { prepare: true, touch: true });
576
+ if (!claim) return { status: "skipped" };
577
+ let pulseRunning = false;
578
+ let authorityLost = false;
579
+ const pulse = async () => {
580
+ if (pulseRunning) return;
581
+ pulseRunning = true;
582
+ try {
583
+ if (!await browserDeadlineCheckpoint(db, target, { touch: true }))
584
+ authorityLost = true;
585
+ } finally {
586
+ pulseRunning = false;
587
+ }
588
+ };
589
+ timer = setInterval(() => {
590
+ void pulse().catch(() => void 0);
591
+ }, 5e3);
592
+ const lease = await readLease(db, target.workspaceId, target.sandboxGroupId);
593
+ if (!lease || lease.instanceId !== target.instanceId || lease.leaseEpoch !== target.leaseEpoch || !lease.resumeState) {
594
+ throw new Error("browser deadline checkpoint placement changed");
595
+ }
596
+ const placement = {
597
+ kind: "sandbox_group",
598
+ sandboxGroupId: target.sandboxGroupId
599
+ };
600
+ const client = await connectController({ target, lease, settings, controllerRoot });
601
+ if (authorityLost || !await browserDeadlineCheckpoint(db, target, { touch: true })) {
602
+ throw new Error("browser deadline checkpoint authority changed");
603
+ }
604
+ if (claim.state !== "completed") {
605
+ const scope = { ...target, operationId: claim.operationId };
606
+ const objectKey = browserStateObjectKey(target.workspaceId, claim.operationId);
607
+ dataKey = deriveBrowserStateDataKey(rootKey, { ...scope, objectKey });
608
+ aad = browserStateArtifactAad({ ...scope, objectKey });
609
+ await dispatchBrowserSessionOperation(db, {
610
+ ...scope,
611
+ deadlineTarget: target,
612
+ stateUpload: {
613
+ objectKey,
614
+ cleanupAfter: new Date(Date.now() + UPLOAD_CLEANUP_GRACE_MS)
615
+ }
616
+ });
617
+ const signed = await objectStorage.createPutUrl({
618
+ key: objectKey,
619
+ contentType: BROWSER_STATE_ARTIFACT_CONTENT_TYPE,
620
+ audience: "sandbox"
621
+ });
622
+ const receipt = await client.captureState({
623
+ ...scope,
624
+ objectKey,
625
+ afterCapture: "stop",
626
+ dataKey,
627
+ aad,
628
+ upload: {
629
+ url: signed.url,
630
+ requiredHeaders: signed.requiredHeaders,
631
+ expiresAt: signed.expiresAt.toISOString()
632
+ }
633
+ }).catch(async (error) => {
634
+ if (error instanceof BrowserControlRequestError && (error.error.code === "outcome_unknown" || !error.error.retryable)) {
635
+ await failBrowserSessionSuspension(db, { ...scope, error: error.error });
636
+ }
637
+ throw error;
638
+ });
639
+ if (authorityLost) throw new Error("browser deadline checkpoint authority changed");
640
+ const portable = receipt.manifest.profileCrypto !== "platform_bound";
641
+ await commitBrowserSessionSuspension(db, {
642
+ ...scope,
643
+ deadlineTarget: target,
644
+ artifact: {
645
+ kind: "chromium_profile",
646
+ format: receipt.format,
647
+ objectKey,
648
+ artifactDigest: receipt.artifactDigest,
649
+ contentDigest: receipt.contentDigest,
650
+ manifestDigest: browserStateManifestDigest(receipt.manifest),
651
+ sizeBytes: receipt.sizeBytes,
652
+ encryptedDataKey: wrapBrowserStateDataKey(rootKey, dataKey, {
653
+ ...scope,
654
+ objectKey,
655
+ artifactDigest: receipt.artifactDigest,
656
+ contentDigest: receipt.contentDigest
657
+ }),
658
+ materialization: {
659
+ portability: portable ? "portable" : "placement_bound",
660
+ reason: portable ? null : "Profile encryption depends on the source operating-system credential store.",
661
+ platform: receipt.manifest.platform,
662
+ architecture: receipt.manifest.architecture,
663
+ engine: receipt.manifest.engine,
664
+ engineVersion: receipt.manifest.engineVersion,
665
+ driverId: receipt.manifest.driverId,
666
+ driverSchemaVersion: receipt.manifest.driverSchemaVersion,
667
+ profileCrypto: receipt.manifest.profileCrypto,
668
+ providerId: null,
669
+ placement: portable ? null : placement
670
+ }
671
+ }
672
+ });
673
+ }
674
+ await client.endSession(target, { removeState: true }).catch((error) => {
675
+ if (!(error instanceof BrowserControlRequestError && error.status === 404)) throw error;
676
+ });
677
+ await withRlsContext(
678
+ db,
679
+ target,
680
+ async (tx) => {
681
+ if ((await browserDeadlineCheckpoint(tx, target))?.state !== "completed") {
682
+ throw new Error("browser deadline checkpoint cleanup authority changed");
683
+ }
684
+ await clearSuspendedBrowserSessionController(tx, {
685
+ ...target,
686
+ expectedControllerGeneration: target.controllerGeneration
687
+ });
688
+ await releaseLeaseHolder(tx, {
689
+ ...target,
690
+ kind: "interaction",
691
+ holderId: `browser-session:${target.browserSessionId}`,
692
+ idleGraceMs: settings.sandboxIdleGraceMs
693
+ });
694
+ },
695
+ void 0,
696
+ "none"
697
+ );
698
+ return { status: "suspended" };
699
+ } finally {
700
+ if (timer) clearInterval(timer);
701
+ dataKey?.fill(0);
702
+ aad?.fill(0);
703
+ rootKey.fill(0);
704
+ }
705
+ });
706
+ }
707
+ };
708
+ }
709
+
263
710
  // src/activities/automations.ts
264
711
  import {
265
712
  resolveTurnExecutionPolicyV1,
@@ -267,88 +714,30 @@ import {
267
714
  WORKSPACE_OPENROUTER_MODEL_ID_PREFIX
268
715
  } from "@opengeni/config";
269
716
  import {
717
+ applySessionAgentConfigWriteThrough,
270
718
  assertWorkspaceModelPolicyAllows,
271
719
  canonicalConfiguredModel,
720
+ resolveSessionAgentConfigForCreate,
272
721
  createAndStartSessionWithOutcome,
273
722
  recordWorkspaceUsage,
274
723
  resolveCatalogSettings,
275
- resolveWorkspaceCatalogSettings
724
+ resolveWorkspaceCatalogSettings,
725
+ isDeveloperSetupDelegatedPermissionAllowed
276
726
  } from "@opengeni/core";
277
727
  import {
278
728
  AutomationAuthorityRevokedError,
279
729
  assertAutomationRunAuthorityInTransaction,
280
730
  claimAutomationRun,
281
- settleAutomationRun
731
+ settleAutomationRun,
732
+ requireWorkspace
282
733
  } from "@opengeni/db";
283
734
  import { SessionSkills } from "@opengeni/contracts";
284
-
285
- // src/activities/agent-run-admission.ts
286
- import { configuredStaticUsageLimits as configuredStaticUsageLimits2 } from "@opengeni/config";
287
- import { modelFundingForAdmission } from "@opengeni/core";
288
- import { getBillingBalance as getBillingBalance2, isCodexBilledTurn, sumUsageQuantity as sumUsageQuantity2 } from "@opengeni/db";
289
- async function agentRunAdmissionDenial(services, input) {
290
- const codexBilled = await isCodexBilledTurn({
291
- db: services.db,
292
- settings: services.settings,
293
- workspaceId: input.workspaceId,
294
- model: input.model
295
- });
296
- const externallyBilled = modelFundingForAdmission(
297
- services.settings,
298
- input.model,
299
- codexBilled
300
- ).fundedWithoutCredits;
301
- if (!externallyBilled && (services.settings.billingMode === "stripe" || services.settings.usageLimitsMode === "managed")) {
302
- if (services.entitlements) {
303
- const decision = await services.entitlements.admitRun({
304
- accountId: input.accountId,
305
- workspaceId: input.workspaceId,
306
- action: "agent_run:create",
307
- quantity: input.requestedAgentRuns
308
- });
309
- if (!decision.allowed) return "insufficient_credits";
310
- } else {
311
- const balance = await getBillingBalance2(services.db, input.accountId);
312
- if (balance.balanceMicros <= 0) return "insufficient_credits";
313
- }
314
- }
315
- if (services.settings.usageLimitsMode !== "static" && services.settings.usageLimitsMode !== "managed") {
316
- return null;
317
- }
318
- const limits = configuredStaticUsageLimits2(services.settings);
319
- if (!externallyBilled && limits.maxMonthlyCostMicrosPerAccount) {
320
- const used = await sumUsageQuantity2(services.db, {
321
- accountId: input.accountId,
322
- eventType: "model.cost",
323
- since: startOfUtcMonth()
324
- });
325
- if (used >= limits.maxMonthlyCostMicrosPerAccount) {
326
- return "monthly_model_cost_limit";
327
- }
328
- }
329
- if (limits.maxMonthlyAgentRunsPerWorkspace) {
330
- const used = await sumUsageQuantity2(services.db, {
331
- workspaceId: input.workspaceId,
332
- eventType: "agent_run.created",
333
- since: startOfUtcMonth()
334
- });
335
- if (used + input.requestedAgentRuns > limits.maxMonthlyAgentRunsPerWorkspace) {
336
- return "monthly_agent_run_limit";
337
- }
338
- }
339
- return null;
340
- }
341
- function startOfUtcMonth() {
342
- const now = /* @__PURE__ */ new Date();
343
- return new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), 1));
344
- }
345
-
346
- // src/activities/automations.ts
347
735
  function createAutomationActivities(services, options = {}) {
348
736
  const claim = options.claim ?? claimAutomationRun;
349
737
  const settle = options.settle ?? settleAutomationRun;
350
738
  const assertAuthority = options.assertAuthority ?? assertAutomationRunAuthorityInTransaction;
351
739
  const createSession2 = options.createSession ?? createAndStartSessionWithOutcome;
740
+ const readWorkspace = options.readWorkspace ?? requireWorkspace;
352
741
  const admit = options.admit ?? agentRunAdmissionDenial;
353
742
  const assertModelPolicy = options.assertModelPolicy ?? assertWorkspaceModelPolicyAllows;
354
743
  const recordUsage = options.recordUsage ?? recordWorkspaceUsage;
@@ -393,6 +782,17 @@ function createAutomationActivities(services, options = {}) {
393
782
  return { action: "skipped", reason: "accepted_execution_mismatch" };
394
783
  }
395
784
  const template = accepted.sessionTemplate;
785
+ if (template.credentialRestriction === "developer_setup" && template.firstPartyMcpPermissions.some(
786
+ (permission) => !isDeveloperSetupDelegatedPermissionAllowed(permission)
787
+ )) {
788
+ await settle(service.db, {
789
+ workspaceId: input.workspaceId,
790
+ runId: run.id,
791
+ status: "failed",
792
+ errorCode: "credential_restriction_violation"
793
+ });
794
+ return { action: "failed", reason: "credential_restriction_violation" };
795
+ }
396
796
  const requestedModel = template.model ?? deploymentCatalogSettings.openaiModel;
397
797
  const catalogSettings = requestedModel.startsWith(WORKSPACE_GATEWAY_MODEL_ID_PREFIX) || requestedModel.startsWith(WORKSPACE_OPENROUTER_MODEL_ID_PREFIX) ? (await resolveWorkspaceCatalogSettings(service.db, catalogSourceSettings, {
398
798
  accountId: input.accountId,
@@ -445,7 +845,42 @@ function createAutomationActivities(services, options = {}) {
445
845
  reason: "interactive_compute_not_allowed"
446
846
  };
447
847
  }
448
- const turnExecutionPolicy = resolveTurnExecutionPolicyV1(catalogSettings, {
848
+ let agentWriteThrough;
849
+ try {
850
+ const workspaceSettings = (await readWorkspace(service.db, input.workspaceId)).settings;
851
+ const resolution = resolveSessionAgentConfigForCreate({
852
+ settings: catalogSettings,
853
+ creator: "automation",
854
+ request: template.agent,
855
+ instructions: template.instructions ?? void 0,
856
+ workspaceSettings,
857
+ parent: null,
858
+ goal: false
859
+ });
860
+ const written = applySessionAgentConfigWriteThrough({
861
+ config: resolution.config,
862
+ firstPartyMcpTools: template.firstPartyMcpTools,
863
+ tools: template.tools,
864
+ toolPolicy: { mode: "explicit", inheritedFromSessionId: null },
865
+ productServerIds: template.tools.map((tool) => tool.id)
866
+ });
867
+ agentWriteThrough = {
868
+ config: resolution.config,
869
+ instructions: resolution.instructions,
870
+ firstPartyMcpTools: written.firstPartyMcpTools,
871
+ tools: written.tools
872
+ };
873
+ } catch (error) {
874
+ if (!isUnprocessableEntity(error)) throw error;
875
+ await settle(service.db, {
876
+ workspaceId: input.workspaceId,
877
+ runId: run.id,
878
+ status: "failed",
879
+ errorCode: "dispatch_failed"
880
+ });
881
+ return { action: "failed", reason: "dispatch_failed" };
882
+ }
883
+ const resolvedTurnExecutionPolicy = resolveTurnExecutionPolicyV1(catalogSettings, {
449
884
  modelId: model,
450
885
  requestedModelId: template.model,
451
886
  modelSource: template.model ? "explicit" : "deployment",
@@ -454,6 +889,10 @@ function createAutomationActivities(services, options = {}) {
454
889
  latencyMode: "standard",
455
890
  latencyModeSource: "deployment"
456
891
  });
892
+ const turnExecutionPolicy = template.credentialRestriction ? {
893
+ ...resolvedTurnExecutionPolicy,
894
+ credentialRestriction: template.credentialRestriction
895
+ } : resolvedTurnExecutionPolicy;
457
896
  const created = await createSession2({
458
897
  db: service.db,
459
898
  bus: service.bus,
@@ -476,10 +915,12 @@ function createAutomationActivities(services, options = {}) {
476
915
  accountId: input.accountId,
477
916
  workspaceId: input.workspaceId,
478
917
  initialMessage: accepted.initialMessage,
918
+ surface: "automation",
919
+ metrics: service.observability,
479
920
  resources: template.resources,
480
921
  skills: SessionSkills.parse(template.skills),
481
922
  bundledSkillIds: template.bundledSkillIds,
482
- tools: template.tools,
923
+ tools: agentWriteThrough.tools,
483
924
  toolPolicy: { mode: "explicit", inheritedFromSessionId: null },
484
925
  model,
485
926
  reasoningEffort,
@@ -498,10 +939,11 @@ function createAutomationActivities(services, options = {}) {
498
939
  label: accepted.serviceLabel
499
940
  },
500
941
  createdByContext: accepted.provenance,
501
- instructions: template.instructions,
942
+ instructions: agentWriteThrough.instructions ?? template.instructions,
502
943
  policyRole: template.policyRole,
503
944
  firstPartyMcpPermissions: template.firstPartyMcpPermissions,
504
- firstPartyMcpTools: template.firstPartyMcpTools,
945
+ firstPartyMcpTools: agentWriteThrough.firstPartyMcpTools,
946
+ ...agentWriteThrough.config ? { agentConfig: agentWriteThrough.config } : {},
505
947
  retainWorkspaceCustomModel: model.startsWith(WORKSPACE_GATEWAY_MODEL_ID_PREFIX) || model.startsWith(WORKSPACE_OPENROUTER_MODEL_ID_PREFIX),
506
948
  createIdempotencyKey: `automation-run:${run.id}`,
507
949
  subjectId: accepted.serviceSubjectId
@@ -716,6 +1158,85 @@ function createFileUploadReaperActivities(services, options = {}) {
716
1158
  return { reapExpiredFileUploads, recoverVideoGenerationWorkflows };
717
1159
  }
718
1160
 
1161
+ // src/activities/model-call-fact-reconciler.ts
1162
+ import { randomUUID as randomUUID2 } from "crypto";
1163
+ import { listWorkspaceIdsAfter, reconcileModelCallFacts } from "@opengeni/db";
1164
+ var MODEL_CALL_FACT_RECONCILE_LOOKBACK_MS = 2 * 60 * 60 * 1e3;
1165
+ var MODEL_CALL_FACT_RECONCILE_SETTLE_MS = 10 * 60 * 1e3;
1166
+ var MODEL_CALL_FACT_RECONCILE_WORKSPACE_LIMIT = 500;
1167
+ var MODEL_CALL_FACT_RECONCILE_RUN_BUDGET_MS = 3 * 60 * 1e3;
1168
+ var WORKSPACE_PAGE_SIZE = 200;
1169
+ function createModelCallFactReconcilerActivities(services, options = {}) {
1170
+ const now = options.now ?? Date.now;
1171
+ const runBudgetMs = options.runBudgetMs ?? MODEL_CALL_FACT_RECONCILE_RUN_BUDGET_MS;
1172
+ const startAfterWorkspaceId = options.startAfterWorkspaceId ?? randomUUID2;
1173
+ const reconcile = options.reconcile ?? reconcileModelCallFacts;
1174
+ const listWorkspaces = options.listWorkspaces ?? listWorkspaceIdsAfter;
1175
+ async function reconcileRecentModelCallFacts() {
1176
+ const { db, observability } = await services();
1177
+ const startedAt = now();
1178
+ const until = new Date(startedAt - MODEL_CALL_FACT_RECONCILE_SETTLE_MS);
1179
+ const since = new Date(until.getTime() - MODEL_CALL_FACT_RECONCILE_LOOKBACK_MS);
1180
+ const result = {
1181
+ workspaces: 0,
1182
+ missing: 0,
1183
+ repaired: 0,
1184
+ unrepaired: 0,
1185
+ truncatedWorkspaces: 0,
1186
+ failedWorkspaces: 0,
1187
+ budgetExhausted: false
1188
+ };
1189
+ const start = startAfterWorkspaceId();
1190
+ let cursor = start;
1191
+ let wrapped = false;
1192
+ while (true) {
1193
+ const page = await listWorkspaces(db, { afterId: cursor, limit: WORKSPACE_PAGE_SIZE });
1194
+ const ids = wrapped ? page.filter((id) => id <= start) : page;
1195
+ for (const workspaceId of ids) {
1196
+ if (now() - startedAt >= runBudgetMs) {
1197
+ result.budgetExhausted = true;
1198
+ break;
1199
+ }
1200
+ result.workspaces += 1;
1201
+ try {
1202
+ const outcome = await reconcile(db, {
1203
+ workspaceId,
1204
+ since,
1205
+ until,
1206
+ limit: MODEL_CALL_FACT_RECONCILE_WORKSPACE_LIMIT
1207
+ });
1208
+ result.missing += outcome.missing;
1209
+ result.repaired += outcome.repaired;
1210
+ result.unrepaired += outcome.unrepaired;
1211
+ if (outcome.truncated) result.truncatedWorkspaces += 1;
1212
+ } catch (error) {
1213
+ result.failedWorkspaces += 1;
1214
+ observability.warn("model call fact reconciliation failed for a workspace", {
1215
+ workspaceId,
1216
+ errorName: error instanceof Error ? error.name : "unknown"
1217
+ });
1218
+ }
1219
+ }
1220
+ if (result.budgetExhausted) break;
1221
+ if (wrapped && ids.length < page.length) break;
1222
+ if (page.length < WORKSPACE_PAGE_SIZE) {
1223
+ if (wrapped) break;
1224
+ wrapped = true;
1225
+ cursor = null;
1226
+ continue;
1227
+ }
1228
+ cursor = page[page.length - 1];
1229
+ }
1230
+ if (result.failedWorkspaces > 0 || result.truncatedWorkspaces > 0 || result.budgetExhausted) {
1231
+ observability.warn("model call fact reconciliation incomplete", { ...result });
1232
+ } else if (result.missing > 0) {
1233
+ observability.info("model call fact reconciliation", { ...result });
1234
+ }
1235
+ return result;
1236
+ }
1237
+ return { reconcileRecentModelCallFacts };
1238
+ }
1239
+
719
1240
  // src/activities/retained-screenshot-reaper.ts
720
1241
  import {
721
1242
  claimRetainedScreenshotMaintenance,
@@ -884,7 +1405,7 @@ import {
884
1405
  markSandboxCheckpointArtifactDeletePending,
885
1406
  recordWarmingSandboxCreated,
886
1407
  registerSandboxCheckpointArtifact,
887
- releaseLeaseHolder,
1408
+ releaseLeaseHolder as releaseLeaseHolder2,
888
1409
  touchLeaseHolder,
889
1410
  updateRigChangeStatus,
890
1411
  SandboxCheckpointArtifactRegistrationConflictError
@@ -978,13 +1499,13 @@ var RIG_VERIFICATION_CLEANUP_OPERATION_MAX_MS = 6e4;
978
1499
  var RIG_VERIFICATION_HEARTBEAT_MAX_INTERVAL_MS = 1e4;
979
1500
  var RIG_VERIFICATION_HEARTBEAT_MIN_INTERVAL_MS = 250;
980
1501
  var RIG_VERIFICATION_OWNER_TTL_MS = 20 * 6e4;
981
- var RIG_VERIFICATION_OWNERS_DISABLED_MESSAGE = "Rig verification lease ownership is disabled; refusing to create an unowned verifier sandbox";
1502
+ var RIG_VERIFICATION_OWNERS_DISABLED_MESSAGE = "Sandbox Environment verification lease ownership is disabled; refusing to create an unowned verifier sandbox";
982
1503
  var RIG_VERIFICATION_HOLDER_TOUCH_INTERVAL_MS = 1e4;
983
1504
  var RIG_VERIFICATION_RELEASE_GRACE_MS = 1;
984
1505
  var RigVerificationActivityDeadlineError = class extends Error {
985
1506
  constructor(startToCloseTimeoutMs, cleanupReserveMs) {
986
1507
  super(
987
- `Rig verification activity-local deadline reached with ${cleanupReserveMs}ms reserved for cleanup before the ${startToCloseTimeoutMs}ms Temporal start-to-close timeout`
1508
+ `Sandbox Environment verification activity-local deadline reached with ${cleanupReserveMs}ms reserved for cleanup before the ${startToCloseTimeoutMs}ms Temporal start-to-close timeout`
988
1509
  );
989
1510
  this.startToCloseTimeoutMs = startToCloseTimeoutMs;
990
1511
  this.cleanupReserveMs = cleanupReserveMs;
@@ -1003,7 +1524,9 @@ function createRigVerificationActivityLifecycle(context = currentActivityContext
1003
1524
  const temporalSignal = context?.cancellationSignal;
1004
1525
  const forwardTemporalCancellation = () => {
1005
1526
  if (!controller.signal.aborted) {
1006
- controller.abort(temporalSignal?.reason ?? new Error("Rig verification cancelled"));
1527
+ controller.abort(
1528
+ temporalSignal?.reason ?? new Error("Sandbox Environment verification cancelled")
1529
+ );
1007
1530
  }
1008
1531
  };
1009
1532
  if (temporalSignal?.aborted) {
@@ -1030,7 +1553,7 @@ function createRigVerificationActivityLifecycle(context = currentActivityContext
1030
1553
  Math.max(1, startToCloseTimeoutMs - reserveMs)
1031
1554
  );
1032
1555
  }
1033
- const heartbeat = () => {
1556
+ const heartbeat2 = () => {
1034
1557
  if (!context) return;
1035
1558
  try {
1036
1559
  context.heartbeat({
@@ -1043,13 +1566,13 @@ function createRigVerificationActivityLifecycle(context = currentActivityContext
1043
1566
  if (!controller.signal.aborted) controller.abort(error);
1044
1567
  }
1045
1568
  };
1046
- heartbeat();
1569
+ heartbeat2();
1047
1570
  const heartbeatTimeoutMs = context?.info.heartbeatTimeoutMs;
1048
1571
  const heartbeatIntervalMs = typeof heartbeatTimeoutMs === "number" && heartbeatTimeoutMs > 0 ? Math.min(
1049
1572
  RIG_VERIFICATION_HEARTBEAT_MAX_INTERVAL_MS,
1050
1573
  Math.max(RIG_VERIFICATION_HEARTBEAT_MIN_INTERVAL_MS, Math.floor(heartbeatTimeoutMs / 3))
1051
1574
  ) : RIG_VERIFICATION_HEARTBEAT_MAX_INTERVAL_MS;
1052
- const heartbeatTimer = context ? setInterval(heartbeat, heartbeatIntervalMs) : null;
1575
+ const heartbeatTimer = context ? setInterval(heartbeat2, heartbeatIntervalMs) : null;
1053
1576
  if (heartbeatTimer && "unref" in heartbeatTimer && typeof heartbeatTimer.unref === "function") {
1054
1577
  heartbeatTimer.unref();
1055
1578
  }
@@ -1065,7 +1588,7 @@ function createRigVerificationActivityLifecycle(context = currentActivityContext
1065
1588
  };
1066
1589
  }
1067
1590
  function abortReason(signal) {
1068
- return signal.reason ?? new Error("Rig verification cancelled");
1591
+ return signal.reason ?? new Error("Sandbox Environment verification cancelled");
1069
1592
  }
1070
1593
  function throwIfAborted(signal) {
1071
1594
  if (signal.aborted) throw abortReason(signal);
@@ -1134,7 +1657,7 @@ var defaultOwnershipDependencies = {
1134
1657
  recordCreated: recordWarmingSandboxCreated,
1135
1658
  commitWarm: commitWarmingToWarm,
1136
1659
  touchHolder: touchLeaseHolder,
1137
- releaseHolder: releaseLeaseHolder,
1660
+ releaseHolder: releaseLeaseHolder2,
1138
1661
  failWarming: failWarmingToCold,
1139
1662
  markWarmLost: markWarmLeaseInstanceLost,
1140
1663
  serialize: serializeEstablishedSandboxEnvelope,
@@ -1145,7 +1668,9 @@ var defaultOwnershipDependencies = {
1145
1668
  };
1146
1669
  var RigVerificationLeaseUnavailableError = class extends Error {
1147
1670
  constructor(sandboxGroupId, role) {
1148
- super(`Rig verification requires a new clean sandbox, but lease ${sandboxGroupId} was ${role}`);
1671
+ super(
1672
+ `Sandbox Environment verification requires a new clean sandbox, but lease ${sandboxGroupId} was ${role}`
1673
+ );
1149
1674
  this.sandboxGroupId = sandboxGroupId;
1150
1675
  this.role = role;
1151
1676
  }
@@ -1168,7 +1693,7 @@ async function runWithOwnedRigVerificationSandbox(input, run, dependencies = def
1168
1693
  let establishmentSettled = false;
1169
1694
  let establishmentPromise = null;
1170
1695
  const warnCleanupTimeout = (operation, instanceId) => {
1171
- input.observability.warn("rig verifier: ownership cleanup timed out", {
1696
+ input.observability.warn("sandbox environment verifier: ownership cleanup timed out", {
1172
1697
  holderId,
1173
1698
  instanceId,
1174
1699
  operation,
@@ -1176,7 +1701,7 @@ async function runWithOwnedRigVerificationSandbox(input, run, dependencies = def
1176
1701
  });
1177
1702
  };
1178
1703
  const warnCleanupFailure = (operation, instanceId, error) => {
1179
- input.observability.warn("rig verifier: ownership cleanup failed", {
1704
+ input.observability.warn("sandbox environment verifier: ownership cleanup failed", {
1180
1705
  holderId,
1181
1706
  instanceId,
1182
1707
  operation,
@@ -1302,11 +1827,14 @@ async function runWithOwnedRigVerificationSandbox(input, run, dependencies = def
1302
1827
  );
1303
1828
  } catch (error) {
1304
1829
  if (!signal.aborted) {
1305
- input.observability.warn("rig verifier: Modal ownership tag failed", {
1306
- holderId,
1307
- instanceId: created.instanceId,
1308
- error: error instanceof Error ? error.message : String(error)
1309
- });
1830
+ input.observability.warn(
1831
+ "sandbox environment verifier: Modal ownership tag failed",
1832
+ {
1833
+ holderId,
1834
+ instanceId: created.instanceId,
1835
+ error: error instanceof Error ? error.message : String(error)
1836
+ }
1837
+ );
1310
1838
  }
1311
1839
  }
1312
1840
  }
@@ -1501,10 +2029,12 @@ function rigProviderImageContentMarkerCommand(contentHash, markerRoot = "/var/op
1501
2029
  }
1502
2030
  function rigProviderImageContentMarker(contentHash, markerRoot = "/var/opengeni") {
1503
2031
  if (!/^sha256:[0-9a-f]{64}$/u.test(contentHash)) {
1504
- throw new Error("Rig provider image content marker requires a canonical SHA-256 value");
2032
+ throw new Error(
2033
+ "Sandbox Environment provider image content marker requires a canonical SHA-256 value"
2034
+ );
1505
2035
  }
1506
2036
  if (!/^\/[A-Za-z0-9._/-]+$/u.test(markerRoot)) {
1507
- throw new Error("Rig provider image marker root must be a safe absolute path");
2037
+ throw new Error("Sandbox Environment provider image marker root must be a safe absolute path");
1508
2038
  }
1509
2039
  const normalizedRoot = markerRoot.replace(/\/+$/u, "") || "/";
1510
2040
  const marker = `${normalizedRoot === "/" ? "" : normalizedRoot}/rig-setup-content-${contentHash.slice("sha256:".length)}.done`;
@@ -1542,7 +2072,9 @@ async function verifyRigProviderImageColdBoot(input, dependencies = defaultRigPr
1542
2072
  runContext.commandRunner
1543
2073
  );
1544
2074
  if (markerResult.exitCode !== 0) {
1545
- throw new Error("built provider image is missing its exact rig-content marker");
2075
+ throw new Error(
2076
+ "built provider image is missing its exact sandbox environment-content marker"
2077
+ );
1546
2078
  }
1547
2079
  for (const check of rigPlatformChecksForSettings(input.settings)) {
1548
2080
  const result = await runCommand(
@@ -1627,7 +2159,7 @@ async function buildVerifiedRigProviderImage(input) {
1627
2159
  finishedAt: (/* @__PURE__ */ new Date()).toISOString(),
1628
2160
  error: {
1629
2161
  code: "provider_image_content_conflict",
1630
- message: "this exact rig version already records a provider image for different effective content; mint and verify a new version",
2162
+ message: "this exact sandbox environment version already records a provider image for different effective content; mint and verify a new version",
1631
2163
  retryable: false
1632
2164
  }
1633
2165
  };
@@ -1801,37 +2333,36 @@ async function buildVerifiedRigProviderImage(input) {
1801
2333
  });
1802
2334
  }
1803
2335
  }
1804
- function settingsForRigVerification(settings, packRuntime, rigImage) {
1805
- void packRuntime;
2336
+ function settingsForRigVerification(settings, rigImage) {
1806
2337
  void rigImage;
1807
2338
  return settings;
1808
2339
  }
1809
2340
  async function loadChangeTarget(db, workspaceId, changeId) {
1810
2341
  const change = await getRigChange(db, workspaceId, changeId);
1811
2342
  if (!change) {
1812
- throw new Error(`Rig change not found: ${changeId}`);
2343
+ throw new Error(`Sandbox Environment change not found: ${changeId}`);
1813
2344
  }
1814
2345
  const rig = await getRig(db, workspaceId, change.rigId);
1815
2346
  if (!rig) {
1816
- throw new Error(`Rig not found for change: ${change.rigId}`);
2347
+ throw new Error(`Sandbox Environment not found for change: ${change.rigId}`);
1817
2348
  }
1818
2349
  if (!change.baseVersionId) {
1819
- throw new Error(`Rig change ${change.id} has no base version`);
2350
+ throw new Error(`Sandbox Environment change ${change.id} has no base version`);
1820
2351
  }
1821
2352
  const baseVersion = await getRigVersionById(db, workspaceId, change.baseVersionId);
1822
2353
  if (!baseVersion || baseVersion.rigId !== rig.id) {
1823
- throw new Error(`Base rig version not found: ${change.baseVersionId}`);
2354
+ throw new Error(`Base sandbox environment version not found: ${change.baseVersionId}`);
1824
2355
  }
1825
2356
  return { rig, baseVersion, change };
1826
2357
  }
1827
2358
  async function loadVersionTarget(db, workspaceId, versionId) {
1828
2359
  const version = await getRigVersionById(db, workspaceId, versionId);
1829
2360
  if (!version) {
1830
- throw new Error(`Rig version not found: ${versionId}`);
2361
+ throw new Error(`Sandbox Environment version not found: ${versionId}`);
1831
2362
  }
1832
2363
  const rig = await getRig(db, workspaceId, version.rigId);
1833
2364
  if (!rig) {
1834
- throw new Error(`Rig not found for version: ${version.rigId}`);
2365
+ throw new Error(`Sandbox Environment not found for version: ${version.rigId}`);
1835
2366
  }
1836
2367
  return { rig, version };
1837
2368
  }
@@ -1874,12 +2405,7 @@ function createRigVerificationActivities(services) {
1874
2405
  change,
1875
2406
  candidateVersion
1876
2407
  );
1877
- const packRuntime = await resolveWorkspacePackRuntime(db, input.workspaceId);
1878
- const runSettings = settingsForRigVerification(
1879
- settings,
1880
- packRuntime,
1881
- candidateVersion.image
1882
- );
2408
+ const runSettings = settingsForRigVerification(settings, candidateVersion.image);
1883
2409
  const providerImageContentHash = rigProviderImageContentHash({
1884
2410
  backend: runSettings.sandboxBackend,
1885
2411
  sourceImage: rigProviderImageSourceImage(runSettings, runSettings.sandboxBackend),
@@ -1953,7 +2479,7 @@ function createRigVerificationActivities(services) {
1953
2479
  );
1954
2480
  if (markerResult.exitCode !== 0) {
1955
2481
  throw new Error(
1956
- `Failed to seal rig provider image content marker: ${markerResult.output.slice(-2e3)}`
2482
+ `Failed to seal sandbox environment provider image content marker: ${markerResult.output.slice(-2e3)}`
1957
2483
  );
1958
2484
  }
1959
2485
  const platformCheckResults = await runRigChecks(
@@ -2064,8 +2590,7 @@ function createRigVerificationActivities(services) {
2064
2590
  metadata: { versionId: version.id }
2065
2591
  });
2066
2592
  try {
2067
- const packRuntime = await resolveWorkspacePackRuntime(db, input.workspaceId);
2068
- const runSettings = settingsForRigVerification(settings, packRuntime, version.image);
2593
+ const runSettings = settingsForRigVerification(settings, version.image);
2069
2594
  const providerImageContentHash = rigProviderImageContentHash({
2070
2595
  backend: runSettings.sandboxBackend,
2071
2596
  sourceImage: rigProviderImageSourceImage(runSettings, runSettings.sandboxBackend),
@@ -2107,7 +2632,7 @@ function createRigVerificationActivities(services) {
2107
2632
  );
2108
2633
  if (markerResult.exitCode !== 0) {
2109
2634
  throw new Error(
2110
- `Failed to seal rig provider image content marker: ${markerResult.output.slice(-2e3)}`
2635
+ `Failed to seal sandbox environment provider image content marker: ${markerResult.output.slice(-2e3)}`
2111
2636
  );
2112
2637
  }
2113
2638
  const platformCheckResults = await runRigChecks(
@@ -2180,13 +2705,60 @@ function createRigVerificationActivities(services) {
2180
2705
  };
2181
2706
  }
2182
2707
 
2708
+ // src/retained-process-diagnostics.ts
2709
+ import {
2710
+ databaseFailureCode,
2711
+ nestedPostgresSqlState,
2712
+ safeDatabaseErrorFacts,
2713
+ SandboxWorkspaceMutationFencedError
2714
+ } from "@opengeni/db";
2715
+ function warnRetainedProcessProofFailure(observability, error, process) {
2716
+ let correlationId;
2717
+ try {
2718
+ const sqlState = nestedPostgresSqlState(error);
2719
+ const constraint = safeDatabaseErrorFacts(error).constraint;
2720
+ correlationId = observability.recordFailureDiagnostic({
2721
+ code: error instanceof SandboxWorkspaceMutationFencedError ? "retained_process_fenced" : sqlState ? databaseFailureCode(sqlState) : "retained_process_proof_failed",
2722
+ stage: "sandbox_retained_processes.proof",
2723
+ error,
2724
+ processId: process.id,
2725
+ sessionId: process.sessionId,
2726
+ ...process.ownerAttemptId ? { attemptId: process.ownerAttemptId } : {},
2727
+ ...process.ownerTurnId ? { turnId: process.ownerTurnId } : {},
2728
+ attempts: process.reconcileAttempts,
2729
+ sqlState,
2730
+ ...constraint ? { constraint } : {}
2731
+ });
2732
+ } catch {
2733
+ }
2734
+ try {
2735
+ observability.warn("sandbox reaper: retained-process proof checkpoint failed", {
2736
+ ...correlationId ? { correlationId } : {}
2737
+ });
2738
+ } catch {
2739
+ }
2740
+ }
2741
+
2742
+ // src/retained-process-retry.ts
2743
+ function retainedProcessDeadlineRetryMs(process, lease, settings, retryAfterMs, now = Date.now()) {
2744
+ if (process.providerBackend !== "modal" || process.routeTargetId !== null || !lease || lease.id !== process.leaseId || lease.leaseEpoch !== process.leaseEpoch || lease.backend !== process.providerBackend || lease.instanceId !== process.providerInstanceId || !lease.providerDeadlineAt)
2745
+ return retryAfterMs;
2746
+ const untilRotation = lease.providerDeadlineAt.getTime() - settings.sandboxRotationLeadMs - now;
2747
+ if (!Number.isFinite(untilRotation)) return retryAfterMs;
2748
+ return Math.min(
2749
+ retryAfterMs,
2750
+ Math.max(1e3, settings.sandboxLeaseReaperPeriodMs, untilRotation)
2751
+ );
2752
+ }
2753
+
2183
2754
  // src/activities/sandbox-lease.ts
2184
- import { createHash, randomUUID as randomUUID2 } from "crypto";
2185
- import { retainedProviderCommandPersistence } from "@opengeni/db/retained-provider-commands";
2186
- import { Context } from "@temporalio/activity";
2755
+ import { createHash, randomUUID as randomUUID3 } from "crypto";
2756
+ import { requestRetainedProcessDeadlineCancellation } from "@opengeni/db/retained-provider-commands";
2757
+ import { Context as Context2 } from "@temporalio/activity";
2187
2758
  import { OpLostReason, OpState } from "@opengeni/agent-proto";
2188
2759
  import {
2189
2760
  accrueWarmSeconds,
2761
+ retainedProviderCommandPersistence,
2190
2762
  adoptLegacyModalCheckpointArtifact,
2191
2763
  confirmDrainCold,
2192
2764
  appendSessionEventToSandboxGroup,
@@ -2202,10 +2774,17 @@ import {
2202
2774
  countSandboxLeasesByLiveness,
2203
2775
  deferRetainedProcessReconciliation,
2204
2776
  forceDrainOverLimitViewerOnlyBoxes,
2205
- getBillingBalance as getBillingBalance3,
2206
2777
  listCreditBalancesByAccount,
2778
+ countActiveUsers,
2779
+ readCreditGrantTotals,
2780
+ readManagedAuthNewSignupsSwitch,
2781
+ readVerifiedSignupTrialSwitch,
2207
2782
  listLegacyModalCheckpointSlots,
2208
2783
  listLiveModalSandboxLeaseAttributions,
2784
+ listPendingModalProviderCreates,
2785
+ claimModalProviderCreateRecovery,
2786
+ recordRecoveredModalProviderCreate,
2787
+ markWarmBillingStopCutoff,
2209
2788
  listMeterableWarmLeases,
2210
2789
  listSandboxViewerForceDrainWorkspaceIds,
2211
2790
  markSandboxCheckpointArtifactDeletePending as markSandboxCheckpointArtifactDeletePending2,
@@ -2214,23 +2793,29 @@ import {
2214
2793
  registerSandboxCheckpointArtifact as registerSandboxCheckpointArtifact2,
2215
2794
  recordRetainedProcessReconciliationProof,
2216
2795
  replaceWorkspaceArchiveCaptureAfterProof,
2217
- readLease,
2796
+ readLease as readLease2,
2797
+ readWorkspaceArchiveCapturePreflight,
2218
2798
  reapExpiredSessionListSnapshots,
2219
2799
  reapStaleLeaseHoldersGlobal,
2220
2800
  requestDueSandboxRotationsGlobal,
2221
2801
  readSandboxRotationBacklog,
2802
+ readRecentSandboxRecoveryObservations,
2222
2803
  workspaceArchiveCaptureDeadlineElapsed,
2223
2804
  retainedProcessReconciliationProof,
2224
2805
  retainedProcessSettlementIdentity,
2806
+ recoverManagedSessionBackgroundCommand,
2225
2807
  settleClaimedConnectedMachineBackgroundCommand,
2226
2808
  settleSandboxCheckpointArtifactGc,
2227
2809
  rlsContextForWorkspace,
2228
2810
  settleRetainedProcess
2229
2811
  } from "@opengeni/db";
2230
2812
  import {
2813
+ claimConnectedCommandOutputReleases,
2231
2814
  claimConnectedMachineSessionBackgroundCommands,
2232
2815
  deferConnectedMachineBackgroundCommandReconciliation,
2233
- recordConnectedMachineBackgroundCommandProof
2816
+ recordConnectedMachineBackgroundCommandProof,
2817
+ recordConnectedCommandOutputConsumption,
2818
+ settleConnectedCommandOutputReleaseClaim
2234
2819
  } from "@opengeni/db/session-background-commands";
2235
2820
  import { sandboxWarmRateMicrosPerSecond } from "@opengeni/config";
2236
2821
  import { appendSessionCommandOutput } from "@opengeni/db/session-command-output";
@@ -2240,9 +2825,9 @@ import {
2240
2825
  disposeWorkspaceArchive,
2241
2826
  cancelSelfhostedOp,
2242
2827
  assertConsistentSandboxProviderIdentity,
2243
- createSandboxClientForBackend,
2828
+ createSandboxClientForBackend as createSandboxClientForBackend2,
2244
2829
  deleteModalCheckpointSnapshot as deleteModalCheckpointSnapshot2,
2245
- deserializeSandboxSessionStateEnvelope,
2830
+ deserializeSandboxSessionStateEnvelope as deserializeSandboxSessionStateEnvelope2,
2246
2831
  establishSandboxSessionFromEnvelope as establishSandboxSessionFromEnvelope2,
2247
2832
  inspectModalSandboxLifecycle,
2248
2833
  isExecSessionLostBanner,
@@ -2258,9 +2843,10 @@ import {
2258
2843
  providerWorkspaceCapturePolicy,
2259
2844
  resolveModalCheckpointProviderBindingForLiveSandbox,
2260
2845
  resolveModalCheckpointProviderBinding,
2846
+ findModalProviderCreateReceipt,
2261
2847
  resolveModalCheckpointProviderBindingForSession,
2262
2848
  querySelfhostedOp,
2263
- resumeExactSandboxSession,
2849
+ resumeExactSandboxSession as resumeExactSandboxSession2,
2264
2850
  sandboxProviderContinuityForState,
2265
2851
  sandboxBackendForSdkBackendId,
2266
2852
  sandboxProviderInstanceIdFromEnvelope,
@@ -2277,6 +2863,10 @@ import { sandboxLeaseTelemetryKey as sandboxLeaseTelemetryKey2 } from "@opengeni
2277
2863
  var RETAINED_PROCESS_RECONCILIATION_LIMIT = 20;
2278
2864
  var RETAINED_PROCESS_RECONCILIATION_CLAIM_TTL_MS = 5 * 6e4;
2279
2865
  var RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS = 5e3;
2866
+ var RETAINED_PROCESS_OUTPUT_DRAIN_MAX_READS = 8;
2867
+ var RETAINED_PROCESS_OUTPUT_DRAIN_BUDGET_MS = 1e4;
2868
+ var RETAINED_PROCESS_OUTPUT_DRAIN_SWEEP_BUDGET_MS = 6e4;
2869
+ var RETAINED_PROCESS_OUTPUT_BACKLOG_BYTES = 256 * 1024;
2280
2870
  var RETAINED_PROCESS_BINDING_QUARANTINE_AFTER_ATTEMPTS = 5;
2281
2871
  var RETAINED_PROCESS_BINDING_QUARANTINE_RETRY_MS = 24 * 60 * 6e4;
2282
2872
  var CONNECTED_COMMAND_RECONCILIATION_LIMIT = 20;
@@ -2315,18 +2905,18 @@ async function awaitProviderCaptureWithLatePublication(input) {
2315
2905
  function startSandboxReaperHeartbeat(details) {
2316
2906
  let context;
2317
2907
  try {
2318
- context = Context.current();
2908
+ context = Context2.current();
2319
2909
  } catch {
2320
2910
  return () => void 0;
2321
2911
  }
2322
- const heartbeat = () => {
2912
+ const heartbeat2 = () => {
2323
2913
  try {
2324
2914
  context.heartbeat({ ...details, heartbeatAt: (/* @__PURE__ */ new Date()).toISOString() });
2325
2915
  } catch {
2326
2916
  }
2327
2917
  };
2328
- heartbeat();
2329
- const timer = setInterval(heartbeat, SANDBOX_REAPER_ACTIVITY_HEARTBEAT_INTERVAL_MS);
2918
+ heartbeat2();
2919
+ const timer = setInterval(heartbeat2, SANDBOX_REAPER_ACTIVITY_HEARTBEAT_INTERVAL_MS);
2330
2920
  if ("unref" in timer && typeof timer.unref === "function") timer.unref();
2331
2921
  return () => clearInterval(timer);
2332
2922
  }
@@ -2349,25 +2939,30 @@ function sandboxDrainCaptureId(operationId, attempt) {
2349
2939
  }
2350
2940
  function sandboxDrainActivityAttempt() {
2351
2941
  try {
2352
- const attempt = Context.current().info.attempt;
2942
+ const attempt = Context2.current().info.attempt;
2353
2943
  return Number.isSafeInteger(attempt) && attempt > 0 ? attempt : 1;
2354
2944
  } catch {
2355
2945
  return 1;
2356
2946
  }
2357
2947
  }
2358
2948
  function createSandboxLeaseActivities(services, options = {}) {
2359
- const terminateBox = options.terminateBox ?? (async (settings, lease, observability, persistArchive, providerCaptureRequestId, captureDisposition, capturePolicy, diskBackedArchives, releaseFailedCapture) => await terminateProviderBox(
2949
+ const terminateBox = (options.terminateBox ? async (...args) => {
2950
+ await args[10]?.();
2951
+ return options.terminateBox(...args);
2952
+ } : null) ?? (async (settings, lease, observability, persistArchive, providerCaptureRequestId, captureDisposition, capturePolicy, diskBackedArchives, releaseFailedCapture, workspaceId, beforeProviderStop) => await terminateProviderBox(
2360
2953
  settings,
2361
2954
  lease,
2362
2955
  observability,
2363
2956
  persistArchive,
2364
- createSandboxClientForBackend,
2957
+ createSandboxClientForBackend2,
2365
2958
  terminateModalSandboxById,
2366
2959
  providerCaptureRequestId,
2367
2960
  captureDisposition,
2368
2961
  capturePolicy,
2369
2962
  diskBackedArchives,
2370
- releaseFailedCapture
2963
+ releaseFailedCapture,
2964
+ workspaceId,
2965
+ beforeProviderStop
2371
2966
  ));
2372
2967
  const sweepModalOrphans = options.sweepModalOrphans ?? sweepModalOrphansForConfiguredBackend;
2373
2968
  const inspectOpenSandboxInventory = options.inspectOpenSandboxKubernetesInventory ?? inspectOpenSandboxKubernetesInventory;
@@ -2378,14 +2973,18 @@ function createSandboxLeaseActivities(services, options = {}) {
2378
2973
  try {
2379
2974
  const { db, settings, observability } = await services();
2380
2975
  const timing = sandboxDrainTiming(settings);
2381
- const onUnobservableCommandDrainError = (error) => {
2382
- observability.warn("sandbox reaper: unobservable command drain inspection failed", {
2383
- error: error instanceof Error ? error.message : String(error)
2384
- });
2976
+ const commandContainment = {
2977
+ idleCommandContainmentMs: settings.sandboxIdleCommandContainmentMs,
2978
+ onCommandContainment: (outcome) => recordSandboxCommandContainment(observability, outcome),
2979
+ onCommandContainmentError: (error) => {
2980
+ observability.warn("sandbox reaper: retained command containment inspection failed", {
2981
+ error: error instanceof Error ? error.message : String(error)
2982
+ });
2983
+ }
2385
2984
  };
2386
2985
  if (!settings.sandboxOwnershipEnabled) {
2387
2986
  const drainableInventory2 = await reapStaleLeaseHoldersGlobal(db, {
2388
- onUnobservableCommandDrainError,
2987
+ ...commandContainment,
2389
2988
  viewerHolderTtlMs: settings.sandboxViewerHolderTtlMs,
2390
2989
  turnHolderTtlMs: settings.sandboxLeaseTtlMs,
2391
2990
  interactionHolderTtlMs: settings.sandboxInteractionHolderTtlMs,
@@ -2412,7 +3011,7 @@ function createSandboxLeaseActivities(services, options = {}) {
2412
3011
  });
2413
3012
  recordSandboxDeadlineRotationsRequested(observability, rotationsRequested);
2414
3013
  const drainableInventory = await reapStaleLeaseHoldersGlobal(db, {
2415
- onUnobservableCommandDrainError,
3014
+ ...commandContainment,
2416
3015
  viewerHolderTtlMs: settings.sandboxViewerHolderTtlMs,
2417
3016
  // Dead-worker turn holders: a live holder is touched every 10s from the
2418
3017
  // moment it is registered (resumeBoxForTurn's holder-liveness loop covers
@@ -2453,7 +3052,7 @@ function createSandboxLeaseActivities(services, options = {}) {
2453
3052
  instanceId: input.target.instanceId
2454
3053
  });
2455
3054
  try {
2456
- const { db, settings, observability, objectStorage } = await services();
3055
+ const { db, settings, observability, objectStorage, bus } = await services();
2457
3056
  assertSandboxDrainInputTiming(input);
2458
3057
  const drainSettings = settings.sandboxSnapshotTimeoutMs === input.snapshotTimeoutMs ? settings : { ...settings, sandboxSnapshotTimeoutMs: input.snapshotTimeoutMs };
2459
3058
  const row = input.target;
@@ -2473,11 +3072,12 @@ function createSandboxLeaseActivities(services, options = {}) {
2473
3072
  terminateBox,
2474
3073
  probeDrainableProvider,
2475
3074
  captureAttempt,
2476
- objectStorage
3075
+ objectStorage,
3076
+ bus
2477
3077
  );
2478
3078
  return { status: drainedCold ? "terminated" : "skipped" };
2479
3079
  } catch (error) {
2480
- const lease = await readLease(db, row.workspaceId, row.sandboxGroupId).catch(() => null);
3080
+ const lease = await readLease2(db, row.workspaceId, row.sandboxGroupId).catch(() => null);
2481
3081
  const details = {
2482
3082
  sandboxLeaseKey: sandboxLeaseTelemetryKey(row.workspaceId, row.sandboxGroupId),
2483
3083
  workspaceId: row.workspaceId,
@@ -2540,6 +3140,19 @@ function createSandboxLeaseActivities(services, options = {}) {
2540
3140
  const rotationsRequested = input.rotationsRequested;
2541
3141
  let modalOrphansTerminated = 0;
2542
3142
  await reconcileConnectedMachineBackgroundCommands(db, settings, observability, service.bus);
3143
+ await reconcileConnectedCommandOutputReleases(db, settings, observability, service.bus);
3144
+ try {
3145
+ await reconcileModalProviderCreates(
3146
+ db,
3147
+ settings,
3148
+ observability,
3149
+ options.findModalProviderCreateReceipt
3150
+ );
3151
+ } catch (error) {
3152
+ observability.warn("sandbox reaper: Modal create recovery inventory failed", {
3153
+ error: error instanceof Error ? error.message : String(error)
3154
+ });
3155
+ }
2543
3156
  if (!settings.sandboxOwnershipEnabled) {
2544
3157
  await refreshQueueLeaseAndCreditGauges(
2545
3158
  db,
@@ -2557,7 +3170,7 @@ function createSandboxLeaseActivities(services, options = {}) {
2557
3170
  const meterResult = await accrueWarmTick(db, settings, observability);
2558
3171
  metered = meterResult.accrued;
2559
3172
  const forceDrainWorkspaceIds = /* @__PURE__ */ new Set();
2560
- if (settings.billingMode === "stripe" || settings.usageLimitsMode === "managed" || settings.sandboxMaxWarmSecondsPerWorkspace > 0) {
3173
+ if (settings.sandboxWarmBillingMode === "credits" || settings.sandboxMaxWarmSecondsPerWorkspace > 0) {
2561
3174
  for (const workspaceId of meterResult.workspaceIds) {
2562
3175
  forceDrainWorkspaceIds.add(workspaceId);
2563
3176
  }
@@ -2657,7 +3270,7 @@ function createSandboxLeaseActivities(services, options = {}) {
2657
3270
  timeoutClass: plan.timeoutClass,
2658
3271
  snapshotTimeoutMs: plan.snapshotTimeoutMs,
2659
3272
  captureTimeoutMs: plan.captureTimeoutMs,
2660
- operationId: randomUUID2()
3273
+ operationId: randomUUID3()
2661
3274
  })
2662
3275
  )
2663
3276
  );
@@ -2849,6 +3462,7 @@ async function reconcileConnectedMachineBackgroundCommands(db, settings, observa
2849
3462
  );
2850
3463
  await forEachWithConcurrency(claims, SANDBOX_MAINTENANCE_ITEM_CONCURRENCY, async (claim) => {
2851
3464
  let proof = claim.proof;
3465
+ let outputReplay = null;
2852
3466
  const connectionKey = JSON.stringify([
2853
3467
  claim.controlWorkspaceId,
2854
3468
  claim.enrollmentId,
@@ -2904,18 +3518,14 @@ async function reconcileConnectedMachineBackgroundCommands(db, settings, observa
2904
3518
  journal: { attachGeneration: () => String(Date.now()), persistSettled: () => {
2905
3519
  } }
2906
3520
  });
2907
- const replay = await client.readExisting(
3521
+ outputReplay = await replayConnectedCommandOutput(
3522
+ client,
2908
3523
  claim.opId,
2909
- proof ? 5e3 : 250,
3524
+ Boolean(proof),
2910
3525
  async (frames) => {
2911
3526
  await captureConnectedCommandOutput(db, claim, bus)(claim.commandId, frames);
2912
3527
  }
2913
3528
  );
2914
- if (proof && replay.status !== "completed" && !replay.terminal) {
2915
- throw new Error(
2916
- "Connected command terminal output replay has not reached its exit frontier"
2917
- );
2918
- }
2919
3529
  }
2920
3530
  if (!proof) {
2921
3531
  await deferConnectedCommandClaim(
@@ -2966,6 +3576,15 @@ async function reconcileConnectedMachineBackgroundCommands(db, settings, observa
2966
3576
  if (!settlement.settled) {
2967
3577
  throw new Error("Connected command settlement lost its exact claim or proof");
2968
3578
  }
3579
+ const receipt = outputReplay?.outputReceipt ?? (outputReplay?.status === "running" ? outputReplay.terminal?.outputReceipt : void 0);
3580
+ const terminal = outputReplay?.status === "completed" ? outputReplay : outputReplay?.terminal;
3581
+ if (proof.outcome === "exited" && receipt && terminal) {
3582
+ await recordConnectedCommandOutputConsumption(db, {
3583
+ ...claim,
3584
+ receipt,
3585
+ exitCode: terminal.outcome.response.exitCode
3586
+ });
3587
+ }
2969
3588
  if (settlement.events.length > 0) {
2970
3589
  try {
2971
3590
  await bus.publish(claim.workspaceId, claim.sessionId, settlement.events);
@@ -2989,6 +3608,133 @@ async function reconcileConnectedMachineBackgroundCommands(db, settings, observa
2989
3608
  if (claims.length < CONNECTED_COMMAND_RECONCILIATION_LIMIT) break;
2990
3609
  } while (Date.now() < deadline);
2991
3610
  }
3611
+ async function replayConnectedCommandOutput(client, opId, terminalKnown, capture) {
3612
+ let capturedThrough = 0n;
3613
+ while (true) {
3614
+ const before = capturedThrough;
3615
+ const replay = await client.readExisting(opId, terminalKnown ? 5e3 : 250, capture);
3616
+ if (!terminalKnown || replay.status === "completed" || replay.terminal) return replay;
3617
+ capturedThrough = BigInt(replay.replaySequence);
3618
+ if (capturedThrough === before) {
3619
+ throw new Error("Connected command terminal output replay has not reached its exit frontier");
3620
+ }
3621
+ }
3622
+ }
3623
+ async function reconcileConnectedCommandOutputReleases(db, settings, observability, bus, controlRpcOverride) {
3624
+ const dueBefore = /* @__PURE__ */ new Date();
3625
+ const deadline = Date.now() + settings.sandboxLeaseReaperPeriodMs;
3626
+ const controlRpc = controlRpcOverride ?? new NatsControlRpc(async () => bus.getRequestConnection());
3627
+ do {
3628
+ let claims;
3629
+ try {
3630
+ claims = await claimConnectedCommandOutputReleases(db, {
3631
+ claimId: crypto.randomUUID(),
3632
+ dueBefore,
3633
+ limit: CONNECTED_COMMAND_RECONCILIATION_LIMIT,
3634
+ claimTtlMs: CONNECTED_COMMAND_RECONCILIATION_CLAIM_TTL_MS
3635
+ });
3636
+ } catch (error) {
3637
+ recordConnectedCommandReconciliation(observability, "claim_failed");
3638
+ observability.warn("sandbox reaper: connected-command output claim failed", {
3639
+ error: error instanceof Error ? error.message : String(error)
3640
+ });
3641
+ return;
3642
+ }
3643
+ await forEachWithConcurrency(claims, SANDBOX_MAINTENANCE_ITEM_CONCURRENCY, async (claim) => {
3644
+ try {
3645
+ if (!bus.getOpStreamConnection)
3646
+ throw new Error("Connected command output transport unavailable");
3647
+ const rpcSubject = subjectFor(
3648
+ claim.controlWorkspaceId,
3649
+ claim.enrollmentId,
3650
+ claim.connectionInstanceId
3651
+ );
3652
+ const controlTimeoutMs = Math.min(
3653
+ settings.sandboxSelfhostedControlTimeoutMs,
3654
+ RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS
3655
+ );
3656
+ const client = new OpStreamExecClient({
3657
+ workspaceId: claim.controlWorkspaceId,
3658
+ agentId: claim.enrollmentId,
3659
+ connectionInstanceId: claim.connectionInstanceId,
3660
+ epoch: 0,
3661
+ controlRpc,
3662
+ rpcSubject,
3663
+ transport: new NatsOpStreamTransport(async () => bus.getOpStreamConnection?.() ?? null),
3664
+ controlTimeoutMs,
3665
+ retryClock: defaultSelfhostedRetryClock,
3666
+ journal: {
3667
+ attachGeneration: () => String(Date.now()),
3668
+ persistSettled: () => {
3669
+ throw new Error("Background output release requires its database receipt");
3670
+ }
3671
+ }
3672
+ });
3673
+ let receipt = claim.receipt;
3674
+ if (!receipt) {
3675
+ const status = await querySelfhostedOp({
3676
+ controlRpc,
3677
+ rpcSubject,
3678
+ opId: claim.opId,
3679
+ controlTimeoutMs
3680
+ });
3681
+ if (status.state === OpState.OP_STATE_LOST) {
3682
+ if (!await settleConnectedCommandOutputReleaseClaim(db, {
3683
+ claim,
3684
+ outcome: "unavailable",
3685
+ retryAfterMs: 0
3686
+ }))
3687
+ throw new Error("Connected command output loss no longer owns its exact claim");
3688
+ recordConnectedCommandReconciliation(observability, "output_unavailable");
3689
+ return;
3690
+ }
3691
+ if (status.state !== OpState.OP_STATE_COMPLETE)
3692
+ throw new Error("Terminal command output returned a live operation");
3693
+ const replay = await replayConnectedCommandOutput(
3694
+ client,
3695
+ claim.opId,
3696
+ true,
3697
+ async (frames) => captureConnectedCommandOutput(db, claim, bus)(claim.commandId, frames)
3698
+ );
3699
+ const terminal = replay.status === "completed" ? replay : replay.terminal;
3700
+ receipt = replay.outputReceipt ?? (replay.status === "running" ? replay.terminal?.outputReceipt : void 0) ?? null;
3701
+ if (!terminal || !receipt)
3702
+ throw new Error("Connected command output replay has no complete custody receipt");
3703
+ await recordConnectedCommandOutputConsumption(db, {
3704
+ ...claim,
3705
+ receipt,
3706
+ exitCode: terminal.outcome.response.exitCode
3707
+ });
3708
+ }
3709
+ const outcome = await client.releaseCapturedOutput(claim.opId, receipt);
3710
+ if (!await settleConnectedCommandOutputReleaseClaim(db, {
3711
+ claim,
3712
+ outcome,
3713
+ retryAfterMs: outcome === "published" ? settings.sandboxLeaseReaperPeriodMs : 0
3714
+ }))
3715
+ throw new Error("Connected command output release no longer owns its exact claim");
3716
+ recordConnectedCommandReconciliation(
3717
+ observability,
3718
+ outcome === "published" ? "output_ack_published" : "output_released"
3719
+ );
3720
+ } catch (error) {
3721
+ await settleConnectedCommandOutputReleaseClaim(db, {
3722
+ claim,
3723
+ outcome: "retry",
3724
+ retryAfterMs: Math.min(
3725
+ 5 * 6e4,
3726
+ Math.max(settings.sandboxLeaseReaperPeriodMs, 3e4) * 2 ** Math.min(4, claim.reconcileAttempts - 1)
3727
+ )
3728
+ }).catch(() => false);
3729
+ observability.warn("sandbox reaper: connected-command output release deferred", {
3730
+ commandId: claim.commandId,
3731
+ error: error instanceof Error ? error.message : String(error)
3732
+ });
3733
+ }
3734
+ });
3735
+ if (claims.length < CONNECTED_COMMAND_RECONCILIATION_LIMIT) break;
3736
+ } while (Date.now() < deadline);
3737
+ }
2992
3738
  async function probeConnectedMachineBackgroundCommand(claim, controlRpc, controlTimeoutMs, observedAt = /* @__PURE__ */ new Date()) {
2993
3739
  const input = {
2994
3740
  controlRpc,
@@ -3075,10 +3821,65 @@ async function reconcileTerminalRetainedProcesses(db, settings, observability, p
3075
3821
  });
3076
3822
  return;
3077
3823
  }
3824
+ const drainBudget = {
3825
+ until: Date.now() + RETAINED_PROCESS_OUTPUT_DRAIN_SWEEP_BUDGET_MS
3826
+ };
3078
3827
  for (const claim of claims) {
3079
3828
  let process = claim.process;
3080
3829
  const expected = retainedProcessSettlementIdentity(process);
3081
3830
  let proof = retainedProcessReconciliationProof(process);
3831
+ const processScope = {
3832
+ accountId: process.accountId,
3833
+ workspaceId: process.workspaceId,
3834
+ sessionId: process.sessionId,
3835
+ processId: process.id
3836
+ };
3837
+ if (process.ownerActorKind === "turn" && process.providerBackend === "modal" && !proof && !claim.ownerState.startsWith("background_")) {
3838
+ try {
3839
+ const command = await recoverManagedSessionBackgroundCommand(db, {
3840
+ ...processScope,
3841
+ expected,
3842
+ reconciliationClaimId: claim.claimId
3843
+ });
3844
+ if (command) {
3845
+ claim.ownerState = `background_${command.state}`;
3846
+ recordRetainedProcessReconciliation(observability, "background_recovered");
3847
+ }
3848
+ } catch (error) {
3849
+ observability.warn("sandbox reaper: retained-command background recovery failed", {
3850
+ processId: process.id,
3851
+ error: error instanceof Error ? error.message : String(error)
3852
+ });
3853
+ }
3854
+ }
3855
+ const storedCommandPersistence = retainedProviderCommandPersistence(
3856
+ db,
3857
+ processScope,
3858
+ bus ? (events) => bus.publish(process.workspaceId, process.sessionId, events) : void 0
3859
+ );
3860
+ const commandPersistence = {
3861
+ ...storedCommandPersistence,
3862
+ requestCancellation: async (reason) => {
3863
+ if (reason === "provider_deadline") {
3864
+ await requestRetainedProcessDeadlineCancellation(db, processScope);
3865
+ if (!await storedCommandPersistence.cancellationRequested())
3866
+ throw new Error(
3867
+ "Supervised deadline cancellation no longer owns its original rotating lease"
3868
+ );
3869
+ } else await storedCommandPersistence.requestCancellation(reason);
3870
+ }
3871
+ };
3872
+ if (proof && process.providerBackend === "modal") {
3873
+ try {
3874
+ const command = await commandPersistence.load();
3875
+ if (command?.kind === "modal-router-v1" && command.supervision && (!await commandPersistence.loadSupervisionReceipt() || ![command.streams.stdout, command.streams.stderr].every(
3876
+ (stream) => stream.eof && stream.exitCode === 0
3877
+ )))
3878
+ proof = null;
3879
+ } catch {
3880
+ proof = null;
3881
+ }
3882
+ }
3082
3883
  if (!proof) {
3083
3884
  let observation = null;
3084
3885
  if (process.providerBackend === "modal" && process.routeTargetId === null && !process.providerBindingKey) {
@@ -3121,8 +3922,18 @@ async function reconcileTerminalRetainedProcesses(db, settings, observability, p
3121
3922
  };
3122
3923
  }
3123
3924
  }
3925
+ if (observation?.status === "deferred" && process.providerBackend === "modal" && !process.providerBindingKey && !process.deadlineCancellationRequestedAt) {
3926
+ try {
3927
+ await requestRetainedProcessDeadlineCancellation(db, processScope);
3928
+ } catch (error) {
3929
+ observability.warn("sandbox reaper: legacy deadline cancellation intent failed", {
3930
+ processId: process.id,
3931
+ error: error instanceof Error ? error.message : String(error)
3932
+ });
3933
+ }
3934
+ }
3124
3935
  if (observation === null) {
3125
- const lease = await readLease(db, process.workspaceId, process.sandboxGroupId).catch(
3936
+ const lease = await readLease2(db, process.workspaceId, process.sandboxGroupId).catch(
3126
3937
  () => null
3127
3938
  );
3128
3939
  const currentLeaseIdentityMatches = Boolean(lease) && lease.id === process.leaseId && lease.leaseEpoch === process.leaseEpoch && lease.backend === process.providerBackend && lease.instanceId === process.providerInstanceId && process.routeTargetId === null;
@@ -3162,11 +3973,25 @@ async function reconcileTerminalRetainedProcesses(db, settings, observability, p
3162
3973
  }
3163
3974
  } else {
3164
3975
  try {
3165
- observation = await probe(
3976
+ const command = await commandPersistence.load();
3977
+ const supervised = command?.kind === "modal-router-v1" && Boolean(command.supervision);
3978
+ const supervisionMetric = (outcome) => observability.incrementCounter({
3979
+ name: "opengeni_command_supervision_total",
3980
+ help: "Native retained command supervision reconciliation outcomes.",
3981
+ labels: { outcome }
3982
+ });
3983
+ if (supervised && (lease.rotationReason === "provider_deadline" || claim.ownerState === "background_stopping")) {
3984
+ await commandPersistence.requestCancellation(
3985
+ lease.rotationReason === "provider_deadline" ? "provider_deadline" : "explicit_stop"
3986
+ );
3987
+ supervisionMetric("cancellation_intent");
3988
+ }
3989
+ const legacyDeadlineStop = lease.rotationReason === "provider_deadline" && !supervised && !process.deadlineCancellationRequestedAt;
3990
+ const probeOutcome = await probe(
3166
3991
  settings,
3167
3992
  lease,
3168
3993
  process,
3169
- claim.ownerState === "background_stopping" ? "cancel" : "observe",
3994
+ claim.ownerState === "background_stopping" && !process.deadlineCancellationRequestedAt || legacyDeadlineStop ? "cancel" : "observe",
3170
3995
  async (result, chunkId, stream, streamFidelity) => {
3171
3996
  if (typeof result !== "string" || !stream && isExecSessionLostBanner(result, process.providerSessionId))
3172
3997
  return;
@@ -3183,13 +4008,25 @@ async function reconcileTerminalRetainedProcesses(db, settings, observability, p
3183
4008
  if (events.length && bus)
3184
4009
  await bus.publish(process.workspaceId, process.sessionId, events).catch(() => void 0);
3185
4010
  },
3186
- retainedProviderCommandPersistence(db, {
3187
- accountId: process.accountId,
3188
- workspaceId: process.workspaceId,
3189
- sessionId: process.sessionId,
3190
- processId: process.id
3191
- })
4011
+ commandPersistence,
4012
+ drainBudget
4013
+ ).then(
4014
+ (value) => ({ ok: true, value }),
4015
+ (error) => ({ ok: false, error })
3192
4016
  );
4017
+ if (legacyDeadlineStop && !await requestRetainedProcessDeadlineCancellation(db, processScope))
4018
+ throw new Error("Deadline cancellation no longer owns its rotating lease");
4019
+ if (!probeOutcome.ok) throw probeOutcome.error;
4020
+ observation = probeOutcome.value;
4021
+ if (supervised) {
4022
+ supervisionMetric(
4023
+ await commandPersistence.loadSupervisionReceipt() ? "proof_retained" : "proof_missing"
4024
+ );
4025
+ if (observation.status === "deferred")
4026
+ supervisionMetric(
4027
+ observation.reason === "provider_running" ? "checkpoint_blocked" : "provider_failure"
4028
+ );
4029
+ }
3193
4030
  } catch (error) {
3194
4031
  observability.warn("sandbox reaper: retained-process provider probe failed", {
3195
4032
  processId: process.id,
@@ -3230,7 +4067,8 @@ async function reconcileTerminalRetainedProcesses(db, settings, observability, p
3230
4067
  process,
3231
4068
  expected,
3232
4069
  claim.claimId,
3233
- observation.reason
4070
+ observation.reason,
4071
+ observation.outputBacklog === true
3234
4072
  );
3235
4073
  continue;
3236
4074
  }
@@ -3248,10 +4086,7 @@ async function reconcileTerminalRetainedProcesses(db, settings, observability, p
3248
4086
  recordRetainedProcessReconciliation(observability, `proof_${proof.outcome}`);
3249
4087
  } catch (error) {
3250
4088
  recordRetainedProcessReconciliation(observability, "proof_checkpoint_failed");
3251
- observability.warn("sandbox reaper: retained-process proof checkpoint failed", {
3252
- processId: process.id,
3253
- error: error instanceof Error ? error.message : String(error)
3254
- });
4089
+ warnRetainedProcessProofFailure(observability, error, process);
3255
4090
  continue;
3256
4091
  }
3257
4092
  }
@@ -3304,9 +4139,10 @@ async function reconcileTerminalRetainedProcesses(db, settings, observability, p
3304
4139
  }
3305
4140
  }
3306
4141
  }
3307
- async function deferRetainedProcessClaim(db, settings, observability, process, expected, claimId, outcome) {
3308
- const deferral = retainedProcessReconciliationDeferral(settings, process, outcome);
4142
+ async function deferRetainedProcessClaim(db, settings, observability, process, expected, claimId, outcome, outputBacklog = false) {
4143
+ const deferral = retainedProcessReconciliationDeferral(settings, process, outcome, outputBacklog);
3309
4144
  try {
4145
+ const lease = process.providerBackend === "modal" && process.routeTargetId === null ? await readLease2(db, process.workspaceId, process.sandboxGroupId) : null;
3310
4146
  await deferRetainedProcessReconciliation(db, {
3311
4147
  accountId: process.accountId,
3312
4148
  workspaceId: process.workspaceId,
@@ -3315,7 +4151,7 @@ async function deferRetainedProcessClaim(db, settings, observability, process, e
3315
4151
  expected,
3316
4152
  claimId,
3317
4153
  outcome: deferral.durableOutcome,
3318
- retryAfterMs: deferral.retryAfterMs
4154
+ retryAfterMs: retainedProcessDeadlineRetryMs(process, lease, settings, deferral.retryAfterMs)
3319
4155
  });
3320
4156
  recordRetainedProcessReconciliation(observability, deferral.metricOutcome);
3321
4157
  } catch (error) {
@@ -3326,7 +4162,7 @@ async function deferRetainedProcessClaim(db, settings, observability, process, e
3326
4162
  });
3327
4163
  }
3328
4164
  }
3329
- function retainedProcessReconciliationDeferral(settings, process, outcome) {
4165
+ function retainedProcessReconciliationDeferral(settings, process, outcome, outputBacklog = false) {
3330
4166
  const bindingQuarantine = process.reconcileAttempts >= RETAINED_PROCESS_BINDING_QUARANTINE_AFTER_ATTEMPTS && (outcome === "provider_binding_missing" || outcome === "provider_binding_mismatch" || outcome === "process_observation_unavailable");
3331
4167
  if (bindingQuarantine) {
3332
4168
  return {
@@ -3336,11 +4172,9 @@ function retainedProcessReconciliationDeferral(settings, process, outcome) {
3336
4172
  quarantined: true
3337
4173
  };
3338
4174
  }
3339
- const exponent = Math.min(4, Math.max(0, process.reconcileAttempts - 1));
3340
- const retryAfterMs = Math.min(
3341
- 5 * 6e4,
3342
- Math.max(settings.sandboxLeaseReaperPeriodMs, 3e4) * 2 ** exponent
3343
- );
4175
+ const baseRetryMs = Math.max(settings.sandboxLeaseReaperPeriodMs, 3e4);
4176
+ const exponent = outcome === "provider_running" && outputBacklog ? 0 : Math.min(4, Math.max(0, process.reconcileAttempts - 1));
4177
+ const retryAfterMs = Math.min(5 * 6e4, baseRetryMs * 2 ** exponent);
3344
4178
  return {
3345
4179
  durableOutcome: outcome,
3346
4180
  metricOutcome: outcome,
@@ -3349,6 +4183,45 @@ function retainedProcessReconciliationDeferral(settings, process, outcome) {
3349
4183
  };
3350
4184
  }
3351
4185
  var RETAINED_PROCESS_PROBE_TIMEOUT = Symbol("retained-process-provider-probe-timeout");
4186
+ async function drainRetainedCommandBacklog(session, providerSessionId, first, capturePage, budget) {
4187
+ let backlog = retainedBacklogRemains(session, first);
4188
+ await capturePage(first);
4189
+ let observation = classifyRetainedProcessPollResult(first, providerSessionId, session);
4190
+ if (typeof session.writeStdin !== "function") return observation;
4191
+ const until = Math.min(
4192
+ Date.now() + RETAINED_PROCESS_OUTPUT_DRAIN_BUDGET_MS,
4193
+ budget?.until ?? Number.POSITIVE_INFINITY
4194
+ );
4195
+ for (let reads = 0; backlog && observation.status === "deferred" && observation.reason === "provider_running" && reads < RETAINED_PROCESS_OUTPUT_DRAIN_MAX_READS && Date.now() < until; reads++) {
4196
+ let next;
4197
+ try {
4198
+ next = await withRetainedProcessProbeTimeout(
4199
+ session.writeStdin({
4200
+ sessionId: providerSessionId,
4201
+ chars: "",
4202
+ yieldTimeMs: 1e3,
4203
+ maxOutputTokens: 2e3
4204
+ })
4205
+ );
4206
+ } catch {
4207
+ break;
4208
+ }
4209
+ backlog = retainedBacklogRemains(session, next);
4210
+ await capturePage(next);
4211
+ observation = classifyRetainedProcessPollResult(next, providerSessionId, session);
4212
+ }
4213
+ return backlog && observation.status === "deferred" && observation.reason === "provider_running" ? { ...observation, outputBacklog: true } : observation;
4214
+ }
4215
+ function retainedBacklogRemains(session, result) {
4216
+ const page = session.getProviderCommandOutput?.(result);
4217
+ if (!page?.expected || page.command.kind !== "modal-router-v1") return false;
4218
+ if (page.command.supervision || page.exitCode !== null) return false;
4219
+ if (page.providerExited) return true;
4220
+ const { command, expected } = page;
4221
+ return ["stdout", "stderr"].some(
4222
+ (stream) => command.streams[stream].byteOffset - expected.streams[stream].byteOffset >= RETAINED_PROCESS_OUTPUT_BACKLOG_BYTES
4223
+ );
4224
+ }
3352
4225
  async function withRetainedProcessProbeTimeout(promise) {
3353
4226
  let timer;
3354
4227
  try {
@@ -3376,8 +4249,8 @@ async function captureRetainedProbeOutput(processId, result, capture) {
3376
4249
  await capture(pending.result, pending.chunkId);
3377
4250
  pendingRetainedProbeOutput.delete(processId);
3378
4251
  }
3379
- async function probeRetainedProcessAtProvider(settings, lease, process, mode = "observe", captureOutput, providerPersistence) {
3380
- if (lease.id !== process.leaseId || lease.sandboxGroupId !== process.sandboxGroupId || lease.leaseEpoch !== process.leaseEpoch || lease.backend !== process.providerBackend || lease.instanceId !== process.providerInstanceId || lease.resumeBackendId !== process.providerBackend || process.routeTargetId !== null) {
4252
+ async function probeRetainedProcessAtProvider(settings, lease, process, mode = "observe", captureOutput, providerPersistence, drainBudget) {
4253
+ if (lease.id !== process.leaseId || lease.sandboxGroupId !== process.sandboxGroupId || lease.leaseEpoch !== process.leaseEpoch || lease.backend !== process.providerBackend || lease.instanceId !== process.providerInstanceId || !lease.resumeBackendId || sandboxBackendForSdkBackendId(lease.resumeBackendId) !== process.providerBackend || process.routeTargetId !== null) {
3381
4254
  return { status: "deferred", reason: "identity_mismatch" };
3382
4255
  }
3383
4256
  if (!lease.resumeState) {
@@ -3387,10 +4260,14 @@ async function probeRetainedProcessAtProvider(settings, lease, process, mode = "
3387
4260
  if (pending) {
3388
4261
  if (!captureOutput) throw new Error("Pending command output requires its persistence callback");
3389
4262
  await captureRetainedProbeOutput(process.id, pending.result, captureOutput);
3390
- return classifyRetainedProcessPollResult(pending.result, process.providerSessionId);
4263
+ const pendingObservation = classifyRetainedProcessPollResult(
4264
+ pending.result,
4265
+ process.providerSessionId
4266
+ );
4267
+ if (mode !== "cancel" || pendingObservation.status === "proved") return pendingObservation;
3391
4268
  }
3392
4269
  const envelopeBackend = lease.resumeState.backendId;
3393
- if (envelopeBackend !== void 0 && envelopeBackend !== process.providerBackend) {
4270
+ if (envelopeBackend !== void 0 && (typeof envelopeBackend !== "string" || sandboxBackendForSdkBackendId(envelopeBackend) !== process.providerBackend)) {
3394
4271
  return { status: "deferred", reason: "identity_mismatch" };
3395
4272
  }
3396
4273
  if (sandboxProviderInstanceIdFromEnvelope(lease.resumeState) !== process.providerInstanceId) {
@@ -3414,17 +4291,17 @@ async function probeRetainedProcessAtProvider(settings, lease, process, mode = "
3414
4291
  }
3415
4292
  let client;
3416
4293
  try {
3417
- client = createSandboxClientForBackend(process.providerBackend, settings);
4294
+ client = createSandboxClientForBackend2(process.providerBackend, settings);
3418
4295
  } catch {
3419
4296
  return { status: "deferred", reason: "provider_error" };
3420
4297
  }
3421
- if (!client || client.backendId !== process.providerBackend || !client.resume) {
4298
+ if (!client || sandboxBackendForSdkBackendId(client.backendId) !== process.providerBackend || !client.resume) {
3422
4299
  return { status: "deferred", reason: "backend_unsupported" };
3423
4300
  }
3424
4301
  const envelopeSessionState = lease.resumeState.sessionState ?? lease.resumeState;
3425
4302
  let resumedState;
3426
4303
  try {
3427
- resumedState = await deserializeSandboxSessionStateEnvelope(
4304
+ resumedState = await deserializeSandboxSessionStateEnvelope2(
3428
4305
  client,
3429
4306
  envelopeSessionState,
3430
4307
  process.providerInstanceId
@@ -3438,7 +4315,7 @@ async function probeRetainedProcessAtProvider(settings, lease, process, mode = "
3438
4315
  let session;
3439
4316
  try {
3440
4317
  const resumed = await withRetainedProcessProbeTimeout(
3441
- resumeExactSandboxSession(
4318
+ resumeExactSandboxSession2(
3442
4319
  client,
3443
4320
  process.providerBackend,
3444
4321
  resumedState,
@@ -3468,6 +4345,7 @@ async function probeRetainedProcessAtProvider(settings, lease, process, mode = "
3468
4345
  if (typeof session.writeStdin !== "function") {
3469
4346
  return { status: "deferred", reason: "backend_unsupported" };
3470
4347
  }
4348
+ let legacyPty = false;
3471
4349
  if (process.providerBackend === "modal") {
3472
4350
  let liveBinding;
3473
4351
  try {
@@ -3494,6 +4372,7 @@ async function probeRetainedProcessAtProvider(settings, lease, process, mode = "
3494
4372
  if (!command || !providerPersistence || !session.bindProviderCommand)
3495
4373
  return { status: "deferred", reason: "process_observation_unavailable" };
3496
4374
  session.bindProviderCommand(process.providerSessionId, command, providerPersistence);
4375
+ legacyPty = command.pty === true;
3497
4376
  }
3498
4377
  const capturePage = async (value) => {
3499
4378
  if (!captureOutput) {
@@ -3502,6 +4381,11 @@ async function probeRetainedProcessAtProvider(settings, lease, process, mode = "
3502
4381
  return;
3503
4382
  }
3504
4383
  const page = session.getProviderCommandOutput?.(value);
4384
+ if (page?.command.kind === "modal-router-v1" && typeof value === "string") {
4385
+ if (!session.captureCommandOutput || !await session.captureCommandOutput(value))
4386
+ throw new Error("Byte-offset output requires atomic capture before settlement");
4387
+ return;
4388
+ }
3505
4389
  if (page) {
3506
4390
  for (const chunk of page.chunks)
3507
4391
  if (chunk.text)
@@ -3513,10 +4397,16 @@ async function probeRetainedProcessAtProvider(settings, lease, process, mode = "
3513
4397
  };
3514
4398
  let result;
3515
4399
  try {
4400
+ const supervisedCancelled = mode === "cancel" || lease.rotationReason === "provider_deadline" ? await session.cancelSupervisedCommand?.(
4401
+ process.providerSessionId,
4402
+ lease.rotationReason === "provider_deadline" ? "provider_deadline" : "explicit_stop"
4403
+ ) ?? false : false;
3516
4404
  result = await withRetainedProcessProbeTimeout(
3517
4405
  session.writeStdin({
3518
4406
  sessionId: process.providerSessionId,
3519
- chars: mode === "cancel" ? "" : "",
4407
+ // Ctrl-C is a signal only through a PTY. For a non-PTY command it is
4408
+ // ordinary stdin data and can corrupt the command's input.
4409
+ chars: mode === "cancel" && !supervisedCancelled && (process.providerBackend !== "modal" || legacyPty) ? "" : "",
3520
4410
  yieldTimeMs: 1e3,
3521
4411
  maxOutputTokens: 2e3
3522
4412
  })
@@ -3540,9 +4430,14 @@ async function probeRetainedProcessAtProvider(settings, lease, process, mode = "
3540
4430
  }
3541
4431
  return { status: "deferred", reason: "provider_error" };
3542
4432
  }
3543
- await capturePage(result);
3544
- const observation = classifyRetainedProcessPollResult(result, process.providerSessionId, session);
3545
- if (observation.status === "deferred" && observation.reason === "provider_running" && lease.rotationRequestedAt !== null) {
4433
+ const observation = await drainRetainedCommandBacklog(
4434
+ session,
4435
+ process.providerSessionId,
4436
+ result,
4437
+ capturePage,
4438
+ drainBudget
4439
+ );
4440
+ if (observation.status === "deferred" && observation.reason === "provider_running" && lease.rotationRequestedAt !== null && mode !== "cancel" && !process.cancellationRequestedAt && legacyPty && !await providerPersistence?.load().then((command) => command?.kind === "modal-router-v1" && command.supervision)) {
3546
4441
  try {
3547
4442
  const interrupted = await withRetainedProcessProbeTimeout(
3548
4443
  session.writeStdin({
@@ -3619,19 +4514,22 @@ async function accrueWarmTick(db, settings, observability) {
3619
4514
  }
3620
4515
  await forEachWithConcurrency(leases, SANDBOX_MAINTENANCE_ITEM_CONCURRENCY, async (lease) => {
3621
4516
  try {
3622
- const rate = sandboxWarmRateMicrosPerSecond(settings, lease.backend);
4517
+ const rate = settings.sandboxWarmBillingMode === "usage_only" ? 0 : sandboxWarmRateMicrosPerSecond(settings, lease.backend);
3623
4518
  const result = await accrueWarmSeconds(db, {
3624
4519
  accountId: lease.accountId,
3625
4520
  workspaceId: lease.workspaceId,
3626
4521
  sandboxGroupId: lease.sandboxGroupId,
3627
4522
  expectedEpoch: lease.leaseEpoch,
3628
4523
  warmRateMicrosPerSecond: rate,
4524
+ billingMode: settings.sandboxWarmBillingMode,
3629
4525
  subjectId: lease.sandboxGroupId
3630
4526
  });
3631
4527
  if (result.accrued) {
3632
4528
  accrued += 1;
3633
4529
  }
3634
- recordCreditMicros(observability, "usage", result.costMicros);
4530
+ if (result.costMicros > 0 && settings.sandboxWarmBillingMode === "credits") {
4531
+ recordCreditMicros(observability, "usage", result.costMicros);
4532
+ }
3635
4533
  } catch (error) {
3636
4534
  observability.warn("sandbox reaper: warm-seconds accrual failed for lease", {
3637
4535
  workspaceId: lease.workspaceId,
@@ -3667,6 +4565,17 @@ async function refreshQueueLeaseAndCreditGauges(db, settings, observability, ins
3667
4565
  );
3668
4566
  }
3669
4567
  ),
4568
+ refreshSandboxInventoryGauge(
4569
+ observability,
4570
+ "recovery_observations",
4571
+ "recovery-observations",
4572
+ async () => {
4573
+ recordSandboxRecoveryObservationGauges(
4574
+ observability,
4575
+ await readRecentSandboxRecoveryObservations(db)
4576
+ );
4577
+ }
4578
+ ),
3670
4579
  refreshSandboxInventoryGauge(
3671
4580
  observability,
3672
4581
  "rotation_backlog",
@@ -3715,6 +4624,51 @@ async function refreshQueueLeaseAndCreditGauges(db, settings, observability, ins
3715
4624
  error: error instanceof Error ? error.message : String(error)
3716
4625
  });
3717
4626
  }
4627
+ })(),
4628
+ (async () => {
4629
+ try {
4630
+ recordActiveUserGauges(observability, await countActiveUsers(db));
4631
+ } catch (error) {
4632
+ observability.warn("sandbox reaper: active-user gauge refresh failed", {
4633
+ error: error instanceof Error ? error.message : String(error)
4634
+ });
4635
+ }
4636
+ })(),
4637
+ (async () => {
4638
+ try {
4639
+ recordCreditGrantGauges(observability, await readCreditGrantTotals(db));
4640
+ } catch (error) {
4641
+ observability.warn("sandbox reaper: credit-grant gauge refresh failed", {
4642
+ error: error instanceof Error ? error.message : String(error)
4643
+ });
4644
+ }
4645
+ })(),
4646
+ (async () => {
4647
+ recordVerifiedSignupTrialDeploymentFlagGauge(
4648
+ observability,
4649
+ settings.verifiedSignupTrialCreditsEnabled
4650
+ );
4651
+ try {
4652
+ const trialSwitch = await readVerifiedSignupTrialSwitch(db);
4653
+ recordVerifiedSignupTrialSwitchGauge(observability, trialSwitch?.grantsEnabled === true);
4654
+ } catch (error) {
4655
+ observability.warn("sandbox reaper: trial-credit switch gauge refresh failed", {
4656
+ error: error instanceof Error ? error.message : String(error)
4657
+ });
4658
+ }
4659
+ })(),
4660
+ (async () => {
4661
+ try {
4662
+ const signupsSwitch = await readManagedAuthNewSignupsSwitch(db);
4663
+ recordManagedAuthNewSignupsSwitchGauge(
4664
+ observability,
4665
+ signupsSwitch?.signupsEnabled !== false
4666
+ );
4667
+ } catch (error) {
4668
+ observability.warn("sandbox reaper: new-signups switch gauge refresh failed", {
4669
+ error: error instanceof Error ? error.message : String(error)
4670
+ });
4671
+ }
3718
4672
  })()
3719
4673
  ]);
3720
4674
  }
@@ -3730,7 +4684,7 @@ async function refreshSandboxInventoryGauge(observability, domain, diagnosticNam
3730
4684
  }
3731
4685
  }
3732
4686
  async function forceDrainOverLimitWorkspaces(db, settings, workspaceIds, observability) {
3733
- const enforceBalance = settings.billingMode === "stripe" || settings.usageLimitsMode === "managed";
4687
+ const enforceBalance = false;
3734
4688
  const cap = settings.sandboxMaxWarmSecondsPerWorkspace;
3735
4689
  let forceDrained = 0;
3736
4690
  await forEachWithConcurrency(
@@ -3738,11 +4692,8 @@ async function forceDrainOverLimitWorkspaces(db, settings, workspaceIds, observa
3738
4692
  SANDBOX_MAINTENANCE_ITEM_CONCURRENCY,
3739
4693
  async (workspaceId) => {
3740
4694
  try {
3741
- const { accountId } = await rlsContextForWorkspace(db, workspaceId);
3742
- const balance = enforceBalance ? await getBillingBalance3(db, accountId) : { balanceMicros: 1 };
3743
4695
  const result = await forceDrainOverLimitViewerOnlyBoxes(db, {
3744
4696
  workspaceId,
3745
- balanceMicros: balance.balanceMicros,
3746
4697
  enforceBalance,
3747
4698
  maxWarmSecondsPerWorkspace: cap,
3748
4699
  idleGraceMs: settings.sandboxIdleGraceMs
@@ -3765,7 +4716,37 @@ async function forceDrainOverLimitWorkspaces(db, settings, workspaceIds, observa
3765
4716
  );
3766
4717
  return forceDrained;
3767
4718
  }
4719
+ async function reconcileModalProviderCreates(db, settings, observability, discover = findModalProviderCreateReceipt) {
4720
+ if (settings.sandboxBackend !== "modal" && !settings.modalTokenId && !settings.modalTokenSecret)
4721
+ return;
4722
+ const candidates = await listPendingModalProviderCreates(db);
4723
+ await forEachWithConcurrency(candidates, 2, async (candidate) => {
4724
+ try {
4725
+ const { accountId } = await rlsContextForWorkspace(db, candidate.workspaceId);
4726
+ const identity = { ...candidate, accountId };
4727
+ const attempt = await claimModalProviderCreateRecovery(db, identity);
4728
+ if (!attempt) return;
4729
+ const instanceId = await discover(settings, attempt);
4730
+ if (!instanceId) return;
4731
+ if (await recordRecoveredModalProviderCreate(db, { ...identity, attempt, instanceId })) {
4732
+ observability.info("sandbox reaper: recovered Modal create receipt", {
4733
+ ...candidate,
4734
+ operationId: attempt.operationId,
4735
+ instanceId
4736
+ });
4737
+ }
4738
+ } catch (error) {
4739
+ observability.warn("sandbox reaper: Modal create remains fenced pending discovery", {
4740
+ ...candidate,
4741
+ error: error instanceof Error ? error.message : String(error)
4742
+ });
4743
+ }
4744
+ });
4745
+ }
3768
4746
  function modalOrphanTerminationStillEligible(latest, candidate) {
4747
+ if (latest.some((lease) => lease.liveness === "warming" && lease.instanceId === null)) {
4748
+ return false;
4749
+ }
3769
4750
  if (latest.some((lease) => lease.instanceId === candidate.sandboxId)) {
3770
4751
  return false;
3771
4752
  }
@@ -3875,9 +4856,9 @@ function sandboxDrainReason(lease) {
3875
4856
  if (lease.rotationRequestedAt === null) return "idle";
3876
4857
  return lease.rotationReason ?? "rotation_requested";
3877
4858
  }
3878
- async function terminateDrainableBox(db, settings, row, observability, terminateBox, probeDrainableProvider, attempt, objectStorage) {
4859
+ async function terminateDrainableBox(db, settings, row, observability, terminateBox, probeDrainableProvider, attempt, objectStorage, bus = null) {
3879
4860
  const { accountId } = await rlsContextForWorkspace(db, row.workspaceId);
3880
- const lease = await readLease(db, row.workspaceId, row.sandboxGroupId);
4861
+ const lease = await readLease2(db, row.workspaceId, row.sandboxGroupId);
3881
4862
  if (!lease) {
3882
4863
  return false;
3883
4864
  }
@@ -4118,6 +5099,27 @@ async function terminateDrainableBox(db, settings, row, observability, terminate
4118
5099
  }
4119
5100
  return result;
4120
5101
  };
5102
+ if (!providerMissingBeforeCapture && backend === "modal" && lease.instanceId && !await readWorkspaceArchiveCapturePreflight(db, {
5103
+ accountId,
5104
+ workspaceId: row.workspaceId,
5105
+ sandboxGroupId: row.sandboxGroupId,
5106
+ expectedEpoch: row.leaseEpoch,
5107
+ expectedInstanceId: lease.instanceId,
5108
+ liveness: "draining"
5109
+ })) {
5110
+ return false;
5111
+ }
5112
+ let stopCutoffMarked = false;
5113
+ const beforeProviderStop = async () => {
5114
+ const cutoff = await markWarmBillingStopCutoff(db, {
5115
+ accountId,
5116
+ workspaceId: row.workspaceId,
5117
+ sandboxGroupId: row.sandboxGroupId,
5118
+ expectedEpoch: row.leaseEpoch
5119
+ });
5120
+ if (!cutoff) throw new Error("sandbox stop cutoff was fenced before provider termination");
5121
+ stopCutoffMarked = true;
5122
+ };
4121
5123
  const termination = providerMissingBeforeCapture ? { terminated: true, providerMissingBeforeCapture: true } : await terminateBox(
4122
5124
  settings,
4123
5125
  lease,
@@ -4137,22 +5139,62 @@ async function terminateDrainableBox(db, settings, row, observability, terminate
4137
5139
  expectedEpoch: row.leaseEpoch,
4138
5140
  expectedInstanceId: lease.instanceId
4139
5141
  });
4140
- }
5142
+ },
5143
+ row.workspaceId,
5144
+ beforeProviderStop
4141
5145
  );
4142
5146
  const terminated = typeof termination === "boolean" ? termination : termination.terminated;
4143
5147
  if (!terminated) {
4144
5148
  return false;
4145
5149
  }
5150
+ const providerMissingDuringStop = providerMissingBeforeCapture || typeof termination !== "boolean" && termination.providerMissingBeforeCapture;
5151
+ if (!stopCutoffMarked && !providerMissingDuringStop) {
5152
+ if (managedProvider && lease.instanceId) {
5153
+ throw new Error("managed sandbox stopped without a warm usage cutoff");
5154
+ }
5155
+ await beforeProviderStop();
5156
+ }
5157
+ const finalRate = settings.sandboxWarmBillingMode === "usage_only" ? 0 : sandboxWarmRateMicrosPerSecond(settings, backend);
5158
+ if (!providerMissingDuringStop) {
5159
+ const finalTick = await accrueWarmSeconds(db, {
5160
+ accountId,
5161
+ workspaceId: row.workspaceId,
5162
+ sandboxGroupId: row.sandboxGroupId,
5163
+ expectedEpoch: row.leaseEpoch,
5164
+ warmRateMicrosPerSecond: finalRate,
5165
+ billingMode: settings.sandboxWarmBillingMode,
5166
+ finalDrain: true
5167
+ });
5168
+ if (finalTick.costMicros > 0 && settings.sandboxWarmBillingMode === "credits") {
5169
+ recordCreditMicros(observability, "usage", finalTick.costMicros);
5170
+ }
5171
+ }
4146
5172
  const providerMissing = providerMissingBeforeCapture || (typeof termination === "boolean" ? false : termination.providerMissingBeforeCapture);
4147
- const { wentCold } = await confirmDrainCold(db, {
5173
+ const { wentCold, backgroundCommandEvents } = await confirmDrainCold(db, {
4148
5174
  accountId,
4149
5175
  workspaceId: row.workspaceId,
4150
5176
  sandboxGroupId: row.sandboxGroupId,
4151
5177
  expectedEpoch: row.leaseEpoch,
4152
5178
  ...captureClaim ? { expectedCaptureId: captureClaim.id } : {},
4153
- providerMissingBeforeCapture: providerMissing
5179
+ providerMissingBeforeCapture: providerMissing,
5180
+ idleCommandContainmentMs: settings.sandboxIdleCommandContainmentMs
5181
+ });
5182
+ await publishDurableSessionEvents(bus, row.workspaceId, backgroundCommandEvents, (error) => {
5183
+ observability.warn("sandbox reaper: contained command event fanout failed", {
5184
+ sandboxGroupId: row.sandboxGroupId,
5185
+ error: error instanceof Error ? error.message : String(error)
5186
+ });
4154
5187
  });
4155
5188
  if (wentCold) {
5189
+ if (providerMissing) {
5190
+ recordSandboxProviderMissingBeforeCapture(observability, backend);
5191
+ }
5192
+ if (lease.unobservableCommandDrainIds?.length) {
5193
+ recordSandboxCommandContainment(
5194
+ observability,
5195
+ providerMissing ? "provider_missing" : "contained"
5196
+ );
5197
+ }
4156
5198
  try {
4157
5199
  await appendSessionEventToSandboxGroup(db, row.workspaceId, row.sandboxGroupId, {
4158
5200
  type: "sandbox.box.terminated",
@@ -4193,18 +5235,20 @@ function providerDrainLogIdentity(lease, backend) {
4193
5235
  instanceId: lease.instanceId
4194
5236
  };
4195
5237
  }
4196
- async function terminateProviderBox(settings, lease, observability, persistArchive, createClientForBackend = createSandboxClientForBackend, terminateModalById = terminateModalSandboxById, providerCaptureRequestId, captureDisposition = "capture_required", claimedCapturePolicy, diskBackedArchives = false, releaseFailedCapture) {
5238
+ async function terminateProviderBox(settings, lease, observability, persistArchive, createClientForBackend = createSandboxClientForBackend2, terminateModalById = terminateModalSandboxById, providerCaptureRequestId, captureDisposition = "capture_required", claimedCapturePolicy, diskBackedArchives = false, releaseFailedCapture, workspaceId, beforeProviderStop) {
4197
5239
  const durableBackendId = lease.resumeBackendId ?? lease.backend;
4198
5240
  const backend = sandboxBackendForSdkBackendId(durableBackendId) ?? durableBackendId;
4199
5241
  const logIdentity = providerDrainLogIdentity(lease, backend);
5242
+ const warnCaptureFailure = (message, error) => warnDrainSnapshotFailure(observability, message, error, {
5243
+ sandboxGroupId: lease.sandboxGroupId,
5244
+ leaseEpoch: lease.leaseEpoch,
5245
+ ...workspaceId ? { workspaceId } : {}
5246
+ });
4200
5247
  const releaseAfterCaptureFailure = async () => {
4201
5248
  try {
4202
5249
  await releaseFailedCapture?.();
4203
5250
  } catch (error) {
4204
- observability.warn("sandbox reaper: failed capture claim release failed", {
4205
- ...logIdentity,
4206
- error: error instanceof Error ? error.message : String(error)
4207
- });
5251
+ warnCaptureFailure("sandbox reaper: failed capture claim release failed", error);
4208
5252
  }
4209
5253
  };
4210
5254
  if (!backend || backend === "none") {
@@ -4230,6 +5274,11 @@ async function terminateProviderBox(settings, lease, observability, persistArchi
4230
5274
  }
4231
5275
  const persistedInstanceId = assertConsistentSandboxProviderIdentity(backend, lease.resumeState);
4232
5276
  if (!lease.instanceId) {
5277
+ if (lease.providerCreateAttempt?.instanceId === null) {
5278
+ throw new Error(
5279
+ "provider_create_outcome_unknown: cannot infer termination from missing instance ID"
5280
+ );
5281
+ }
4233
5282
  if (persistedInstanceId) {
4234
5283
  throw new Error(
4235
5284
  `sandbox backend ${backend} has persisted provider identity ${persistedInstanceId} but no authoritative lease instance; refusing teardown`
@@ -4255,7 +5304,13 @@ async function terminateProviderBox(settings, lease, observability, persistArchi
4255
5304
  return { terminated: false, providerMissingBeforeCapture: false };
4256
5305
  }
4257
5306
  try {
4258
- await terminateModalById(settings, lease.instanceId);
5307
+ await beforeProviderStop?.();
5308
+ await terminateModalById(
5309
+ settings,
5310
+ lease.instanceId,
5311
+ void 0,
5312
+ lease.providerCreateAttempt?.providerBindingKey
5313
+ );
4259
5314
  } catch (error) {
4260
5315
  if (!isProviderSandboxNotFoundError("modal", error)) {
4261
5316
  throw error;
@@ -4282,7 +5337,7 @@ async function terminateProviderBox(settings, lease, observability, persistArchi
4282
5337
  throw new Error(`sandbox backend ${backend} cannot resume a drainable provider box`);
4283
5338
  }
4284
5339
  const envelopeSessionState = lease.resumeState.sessionState ?? lease.resumeState;
4285
- const resumedState = await deserializeSandboxSessionStateEnvelope(
5340
+ const resumedState = await deserializeSandboxSessionStateEnvelope2(
4286
5341
  client,
4287
5342
  envelopeSessionState,
4288
5343
  lease.instanceId
@@ -4291,7 +5346,7 @@ async function terminateProviderBox(settings, lease, observability, persistArchi
4291
5346
  throw new Error(`sandbox backend ${backend} returned no resumable provider state`);
4292
5347
  }
4293
5348
  const continuity = sandboxProviderContinuityForState(backend, resumedState, lease.instanceId);
4294
- const resumed = await resumeExactSandboxSession(
5349
+ const resumed = await resumeExactSandboxSession2(
4295
5350
  client,
4296
5351
  backend,
4297
5352
  resumedState,
@@ -4329,7 +5384,7 @@ async function terminateProviderBox(settings, lease, observability, persistArchi
4329
5384
  session,
4330
5385
  Date.now(),
4331
5386
  {
4332
- requestId: providerCaptureRequestId ?? randomUUID2(),
5387
+ requestId: providerCaptureRequestId ?? randomUUID3(),
4333
5388
  strategy: liveCapturePolicy.strategy
4334
5389
  },
4335
5390
  diskBackedArchives
@@ -4359,26 +5414,17 @@ async function terminateProviderBox(settings, lease, observability, persistArchi
4359
5414
  }
4360
5415
  );
4361
5416
  } catch (error) {
4362
- observability.warn("sandbox reaper: late workspace capture publication failed", {
4363
- ...logIdentity,
4364
- error: error instanceof Error ? error.message : String(error)
4365
- });
5417
+ warnCaptureFailure("sandbox reaper: late workspace capture publication failed", error);
4366
5418
  } finally {
4367
5419
  await disposeWorkspaceArchive(archive);
4368
5420
  }
4369
5421
  },
4370
5422
  observeLatePublicationFailure: (error) => {
4371
- observability.warn("sandbox reaper: late capture publication cleanup failed", {
4372
- ...logIdentity,
4373
- error: error instanceof Error ? error.message : String(error)
4374
- });
5423
+ warnCaptureFailure("sandbox reaper: late capture publication cleanup failed", error);
4375
5424
  },
4376
5425
  observeLateFailure: (error) => {
4377
5426
  void releaseAfterCaptureFailure();
4378
- observability.warn("sandbox reaper: timed-out provider capture later failed", {
4379
- ...logIdentity,
4380
- error: error instanceof Error ? error.message : String(error)
4381
- });
5427
+ warnCaptureFailure("sandbox reaper: timed-out provider capture later failed", error);
4382
5428
  }
4383
5429
  });
4384
5430
  }
@@ -4389,12 +5435,9 @@ async function terminateProviderBox(settings, lease, observability, persistArchi
4389
5435
  });
4390
5436
  return { terminated: true, providerMissingBeforeCapture: true };
4391
5437
  }
4392
- observability.warn(
5438
+ warnCaptureFailure(
4393
5439
  "sandbox reaper: persistWorkspace failed \u2014 leaving box draining (files NOT lost)",
4394
- {
4395
- ...logIdentity,
4396
- error: error instanceof Error ? error.message : String(error)
4397
- }
5440
+ error
4398
5441
  );
4399
5442
  if (!(error instanceof SandboxProviderCaptureTimeoutError)) {
4400
5443
  await releaseAfterCaptureFailure();
@@ -4436,6 +5479,7 @@ async function terminateProviderBox(settings, lease, observability, persistArchi
4436
5479
  throw new Error(`sandbox backend ${backend} returned no terminable session state`);
4437
5480
  }
4438
5481
  prepareProviderForTeardownAfterCapture(backend, session);
5482
+ await beforeProviderStop?.();
4439
5483
  await terminateManagedSandboxSession2(client, sessionState, session);
4440
5484
  return { terminated: true, providerMissingBeforeCapture: false };
4441
5485
  } catch (error) {
@@ -4450,6 +5494,11 @@ async function terminateProviderBox(settings, lease, observability, persistArchi
4450
5494
  }
4451
5495
 
4452
5496
  // src/activities/scheduled-tasks.ts
5497
+ import { getScheduledTaskClaudeProviderAccountAuthoritySnapshot } from "@opengeni/db";
5498
+ import {
5499
+ ATLASSIAN_NATIVE_RETIRED_REASON,
5500
+ isRetiredNativeAtlassianTask
5501
+ } from "@opengeni/contracts/atlassian-native-retirement";
4453
5502
  import {
4454
5503
  AUTOMATIC_SESSION_TITLE_FALLBACK,
4455
5504
  DEFAULT_FIRST_PARTY_MCP_PERMISSIONS,
@@ -4462,25 +5511,40 @@ import {
4462
5511
  SessionAgentAccess,
4463
5512
  SessionMemoryScope,
4464
5513
  normalizeAutomaticSessionTitle,
5514
+ metadataWithTurnExecutionPolicyV1,
5515
+ mergeResourceRefs,
4465
5516
  scheduledOccurrencePayloadUtf8Bytes,
4466
5517
  stableJson,
4467
5518
  TurnExecutionPolicyV1
4468
5519
  } from "@opengeni/contracts";
4469
5520
  import {
4470
- defaultSessionMcpServerIds,
4471
5521
  openGeniSlackBotMetadata,
4472
5522
  requireOpenGeniSlackBotConnection,
5523
+ withScheduledSlackBotPostingTools,
4473
5524
  resolveWorkspaceCatalogSettings as resolveWorkspaceCatalogSettings2,
5525
+ resolveScheduledTaskDefaultModel,
4474
5526
  resolveSessionToolPolicy,
4475
5527
  workspaceCustomModelReference,
4476
5528
  lockActiveCustomModelForAdmission,
4477
5529
  scheduledSlackBotConnectionId,
4478
- swapActiveSandbox
5530
+ freezeConnectionAccounts,
5531
+ ConnectionAccountSelectionError,
5532
+ scheduledConnectionSurfaceEligibility,
5533
+ scheduledConnectionTools,
5534
+ settingsWithEnabledCapabilityMcpServers,
5535
+ settingsWithSessionMcpServerMetadata,
5536
+ swapActiveSandbox,
5537
+ applySessionAgentConfigWriteThrough as applySessionAgentConfigWriteThrough2,
5538
+ resolveSessionAgentConfigForCreate as resolveSessionAgentConfigForCreate2,
5539
+ recordSessionCreated,
5540
+ recordUserMessageAccepted
4479
5541
  } from "@opengeni/core";
4480
5542
  import {
4481
5543
  addSessionSystemUpdateWithSourceMutation,
4482
5544
  bindScheduledTaskRunSessionInTransaction,
4483
5545
  createScheduledTaskRun,
5546
+ recordScheduledTaskAdmissionFailure,
5547
+ recordScheduledTaskAdmissionRefusal,
4484
5548
  createSessionWithIdempotencyKeyResult,
4485
5549
  enqueueSessionWorkflowWakeIfRunnable,
4486
5550
  failScheduledGeneratedSessionRoute,
@@ -4489,7 +5553,6 @@ import {
4489
5553
  getScheduledTaskRunAcceptedExecution,
4490
5554
  getScheduledTaskRunByProducerKey,
4491
5555
  getScheduledTargetSessionExecution,
4492
- getScheduledTaskPersonalConnectionDelegations,
4493
5556
  getScheduledTaskRunPersonalResourceAuthority,
4494
5557
  getScheduledTaskPersonalResourceAuthoritySubject,
4495
5558
  getScheduledTaskRevisionAuthority,
@@ -4501,31 +5564,31 @@ import {
4501
5564
  getNestedAgentDepthDeploymentPolicy,
4502
5565
  getSessionByCreateIdempotencyKey,
4503
5566
  getSessionCreationExecutionPolicy,
5567
+ metadataWithAgentConfigCreateIdentity,
4504
5568
  getVariableSet,
4505
5569
  isCodexBilledModel,
4506
5570
  initializeSessionStartAtomically,
4507
5571
  materializeScheduledTaskReusableSessionFromRun,
4508
- listEnabledMcpCapabilityServerIds,
4509
5572
  listInstalledApiIntegrationServerIdsForDelegations,
4510
5573
  markScheduledTaskRunFailedIfQueued,
4511
5574
  markScheduledTaskRunSkippedIfQueued,
4512
5575
  recordUsageEvent as recordUsageEvent2,
4513
5576
  requireScheduledTaskIncidentAuthorityInTransaction,
4514
5577
  requireSession,
4515
- requireWorkspace,
5578
+ requireWorkspace as requireWorkspace2,
4516
5579
  settleScheduledTaskRunInTransaction,
4517
5580
  SessionSpawnDeniedDbError,
4518
5581
  updateSessionTitleWithEvent,
4519
5582
  upsertScheduledSessionGoalForRun,
4520
5583
  withSessionActivityRlsContext
4521
5584
  } from "@opengeni/db";
4522
- import { publishDurableSessionEvents } from "@opengeni/events";
5585
+ import { publishDurableSessionEvents as publishDurableSessionEvents2 } from "@opengeni/events";
4523
5586
  import {
4524
5587
  allowedFirstPartyMcpToolsForSession,
4525
5588
  resolveFirstPartyMcpToolPolicy,
4526
5589
  resolveTurnExecutionPolicyV1 as resolveTurnExecutionPolicyV12
4527
5590
  } from "@opengeni/config";
4528
- import { Context as Context2 } from "@temporalio/activity";
5591
+ import { Context as Context3 } from "@temporalio/activity";
4529
5592
  import { createHash as createHash3 } from "crypto";
4530
5593
 
4531
5594
  // src/scheduled-alert-occurrence.ts
@@ -4641,13 +5704,21 @@ function canonicalLabels(value) {
4641
5704
  }
4642
5705
 
4643
5706
  // src/activities/scheduled-tasks.ts
5707
+ function scheduledTaskRunExecutionPolicy(policy, input, creatorRestriction) {
5708
+ return creatorRestriction === "developer_setup" || input.triggerType !== "scheduled" && input.credentialRestriction === "developer_setup" ? { ...policy, credentialRestriction: "developer_setup" } : policy;
5709
+ }
5710
+ function scheduledSessionExecutionPolicyMetadata(policy, creatorRestriction) {
5711
+ if (policy === void 0 || policy === null) return {};
5712
+ const acceptedPolicy = TurnExecutionPolicyV1.parse(policy);
5713
+ return creatorRestriction === "developer_setup" && acceptedPolicy.credentialRestriction === "developer_setup" ? metadataWithTurnExecutionPolicyV1({}, acceptedPolicy) : {};
5714
+ }
4644
5715
  function scheduledTaskRunProducerKey(input, suppliedActivityIdentity) {
4645
5716
  if (input.producerKey) return input.producerKey;
4646
5717
  if (input.triggerType !== "scheduled") return input.agentRunUsageIdempotencyKey;
4647
5718
  let info = suppliedActivityIdentity;
4648
5719
  if (!info) {
4649
5720
  try {
4650
- info = Context2.current().info;
5721
+ info = Context3.current().info;
4651
5722
  } catch {
4652
5723
  throw new Error(
4653
5724
  "scheduled task delivery without a producer key requires a Temporal activity execution"
@@ -4666,6 +5737,13 @@ function scheduledTaskGeneratedSessionCreateIdempotencyKey(producerKey) {
4666
5737
  const digest = createHash3("sha256").update("scheduled-task-run\0", "utf8").update(producerKey, "utf8").digest("hex");
4667
5738
  return `scheduled-task-run:${digest}`;
4668
5739
  }
5740
+ function recordScheduledSessionCreated(observability) {
5741
+ recordSessionCreated(observability, {
5742
+ surface: "scheduled",
5743
+ createdByKind: "service",
5744
+ parentSessionId: null
5745
+ });
5746
+ }
4669
5747
  function scheduledTaskSessionTitle(taskName) {
4670
5748
  return normalizeAutomaticSessionTitle(taskName) ?? "Scheduled run";
4671
5749
  }
@@ -4708,6 +5786,9 @@ function createScheduledTaskActivities(services) {
4708
5786
  producerKey: stableProducerKey
4709
5787
  });
4710
5788
  if (priorRun?.actionKind === "agent_turn") {
5789
+ if (priorRun.admissionDiagnostic || priorRun.admissionRefusal) {
5790
+ return scheduledAdmissionRefusalResult(priorRun);
5791
+ }
4711
5792
  const acceptedExecution = await getScheduledTaskRunAcceptedExecution(db, {
4712
5793
  workspaceId: input.workspaceId,
4713
5794
  runId: priorRun.id
@@ -4718,6 +5799,15 @@ function createScheduledTaskActivities(services) {
4718
5799
  }
4719
5800
  throw new Error("scheduled agent run is missing accepted execution truth");
4720
5801
  }
5802
+ if (priorRun.status === "queued" && isRetiredNativeAtlassianTask(acceptedExecution.task)) {
5803
+ await markScheduledTaskRunFailedIfQueued(
5804
+ db,
5805
+ priorRun.workspaceId,
5806
+ priorRun.id,
5807
+ ATLASSIAN_NATIVE_RETIRED_REASON
5808
+ );
5809
+ return scheduledRunTerminalResult(ATLASSIAN_NATIVE_RETIRED_REASON);
5810
+ }
4721
5811
  await recordScheduledTaskFiredUsage(db, acceptedExecution, priorRun);
4722
5812
  if (priorRun.status === "dispatched") {
4723
5813
  return await replayScheduledTaskDispatch({
@@ -4737,6 +5827,7 @@ function createScheduledTaskActivities(services) {
4737
5827
  try {
4738
5828
  return await recoverBoundScheduledTaskDispatch({
4739
5829
  db,
5830
+ observability: baseService.observability,
4740
5831
  bus,
4741
5832
  settings: settings2,
4742
5833
  wakeSessionWorkflow,
@@ -4778,6 +5869,8 @@ function createScheduledTaskActivities(services) {
4778
5869
  if (!task) {
4779
5870
  return { action: "deleted" };
4780
5871
  }
5872
+ if (isRetiredNativeAtlassianTask(task))
5873
+ return { action: "blocked", reason: ATLASSIAN_NATIVE_RETIRED_REASON };
4781
5874
  if (task.action.kind !== "agent_turn") {
4782
5875
  return { action: "blocked", reason: "legacy_source_schedule_requires_migration" };
4783
5876
  }
@@ -4805,20 +5898,6 @@ function createScheduledTaskActivities(services) {
4805
5898
  if (incidentDeclaration.action === "blocked") {
4806
5899
  return incidentDeclaration;
4807
5900
  }
4808
- const taskPersonalConnectionDelegations = await getScheduledTaskPersonalConnectionDelegations(
4809
- db,
4810
- task.workspaceId,
4811
- task.id
4812
- );
4813
- const taskConnectionAuthoritySubjects = [
4814
- ...new Set(
4815
- taskPersonalConnectionDelegations.map((delegation) => delegation.ownerSubjectId)
4816
- )
4817
- ];
4818
- if (taskConnectionAuthoritySubjects.length > 1) {
4819
- throw new Error("scheduled connection authority has multiple causal humans");
4820
- }
4821
- const taskConnectionAuthoritySubjectId = taskConnectionAuthoritySubjects[0] ?? null;
4822
5901
  const taskPersonalResourceAuthoritySubjectId = await getScheduledTaskPersonalResourceAuthoritySubject(db, {
4823
5902
  accountId: task.accountId,
4824
5903
  workspaceId: task.workspaceId,
@@ -4831,10 +5910,35 @@ function createScheduledTaskActivities(services) {
4831
5910
  taskId: task.id,
4832
5911
  taskAuthorityRevision: task.authorityRevision
4833
5912
  });
4834
- if (taskConnectionAuthoritySubjectId && taskPersonalResourceAuthoritySubjectId && taskConnectionAuthoritySubjectId !== taskPersonalResourceAuthoritySubjectId) {
4835
- throw new Error("scheduled authority classes have different causal humans");
5913
+ const taskAuthoritySubjectId = task.ownerSubjectId;
5914
+ const refuseAdmission = async (reason, retryable, detail) => {
5915
+ console.warn(
5916
+ `[scheduled-task] refused occurrence of ${task.id} (revision ${task.authorityRevision}): ${reason}: ${detail}`
5917
+ );
5918
+ const receipt = await recordScheduledTaskAdmissionRefusal(db, {
5919
+ workspaceId: task.workspaceId,
5920
+ taskId: task.id,
5921
+ taskAuthorityRevision: task.authorityRevision,
5922
+ taskExecutionDigest: task.executionDigest,
5923
+ triggerType: input.triggerType,
5924
+ producerKey: stableProducerKey,
5925
+ reason,
5926
+ retryable
5927
+ });
5928
+ if (!receipt.admissionRefusal) {
5929
+ return createScheduledTaskActivities(services).dispatchScheduledTaskRun(input);
5930
+ }
5931
+ return scheduledAdmissionRefusalResult(receipt);
5932
+ };
5933
+ if (task.action.kind === "agent_turn" && ((taskRevisionAuthority?.subjectId ?? null) !== taskAuthoritySubjectId || [taskPersonalResourceAuthoritySubjectId].some(
5934
+ (subject) => subject !== null && subject !== taskAuthoritySubjectId
5935
+ ))) {
5936
+ return await refuseAdmission(
5937
+ "scheduled_authority_unavailable",
5938
+ false,
5939
+ "scheduled authority differs from its immutable execution owner"
5940
+ );
4836
5941
  }
4837
- const taskAuthoritySubjectId = taskRevisionAuthority?.subjectId ?? taskPersonalResourceAuthoritySubjectId ?? taskConnectionAuthoritySubjectId;
4838
5942
  const acceptedTargetSessionId = task.runMode === "existing_session" ? task.targetSessionId : task.runMode === "reusable_session" ? task.reusableSessionId : null;
4839
5943
  const targetSessionExecutionBase = acceptedTargetSessionId ? await getScheduledTargetSessionExecution(
4840
5944
  db,
@@ -4843,18 +5947,113 @@ function createScheduledTaskActivities(services) {
4843
5947
  taskAuthoritySubjectId
4844
5948
  ) : null;
4845
5949
  const settings = await settingsForTask(task, targetSessionExecutionBase?.model);
4846
- const model = task.agentConfig.model ?? settings.openaiModel;
4847
- const reasoningEffort = task.agentConfig.reasoningEffort ?? settings.openaiReasoningEffort;
5950
+ const resolvedDefault = task.agentConfig.model || targetSessionExecutionBase ? null : await resolveScheduledTaskDefaultModel(db, settings, task);
5951
+ const model = task.agentConfig.model ?? resolvedDefault?.model ?? settings.openaiModel;
5952
+ const reasoningEffort = task.agentConfig.reasoningEffort ?? resolvedDefault?.reasoningEffort ?? settings.openaiReasoningEffort;
4848
5953
  let sandboxBackend = task.agentConfig.sandboxBackend ?? settings.sandboxBackend;
4849
5954
  let sandboxOs = "linux";
4850
- const taskTools = withFirstPartyTools(settings, task.agentConfig.tools);
5955
+ let taskTools = withFirstPartyTools(settings, task.agentConfig.tools);
4851
5956
  const creatorPolicy = await getScheduledTaskCreatorPolicy(db, task.workspaceId, task.id);
4852
- const firstPartyMcpTools = creatorPolicy?.firstPartyMcpTools ? allowedFirstPartyMcpToolsForSession(settings, creatorPolicy.firstPartyMcpTools) : resolveFirstPartyMcpToolPolicy(settings).default;
5957
+ let firstPartyMcpTools = withScheduledSlackBotPostingTools(
5958
+ creatorPolicy?.firstPartyMcpTools ? allowedFirstPartyMcpToolsForSession(settings, creatorPolicy.firstPartyMcpTools) : resolveFirstPartyMcpToolPolicy(settings).default,
5959
+ task.agentConfig,
5960
+ resolveFirstPartyMcpToolPolicy(settings).allowed
5961
+ );
5962
+ let resolvedAgentConfig = null;
5963
+ let resolvedAgentInstructions;
5964
+ const generatedTarget = task.runMode === "new_session_per_run" || task.runMode === "reusable_session" && task.reusableSessionId === null;
5965
+ if (generatedTarget) {
5966
+ try {
5967
+ const agentResolution = resolveSessionAgentConfigForCreate2({
5968
+ settings,
5969
+ creator: "scheduled",
5970
+ request: task.agentConfig.agent,
5971
+ instructions: void 0,
5972
+ workspaceSettings: (await requireWorkspace2(db, task.workspaceId)).settings,
5973
+ parent: null,
5974
+ goal: task.agentConfig.goal !== void 0
5975
+ });
5976
+ resolvedAgentConfig = agentResolution.config;
5977
+ resolvedAgentInstructions = agentResolution.instructions;
5978
+ const written = applySessionAgentConfigWriteThrough2({
5979
+ config: resolvedAgentConfig,
5980
+ firstPartyMcpTools,
5981
+ tools: taskTools,
5982
+ toolPolicy: { mode: "explicit", inheritedFromSessionId: null },
5983
+ productServerIds: task.agentConfig.tools.map((tool) => tool.id)
5984
+ });
5985
+ firstPartyMcpTools = written.firstPartyMcpTools;
5986
+ taskTools = written.tools;
5987
+ } catch (error) {
5988
+ const status = error instanceof Error ? error.status : void 0;
5989
+ if (!(error instanceof Error) || status !== 422) throw error;
5990
+ return await refuseAdmission(
5991
+ "scheduled_authority_unavailable",
5992
+ false,
5993
+ `scheduled agent configuration is not admissible: ${error.message}`
5994
+ );
5995
+ }
5996
+ }
4853
5997
  const firstPartyMcpPermissions = creatorPolicy?.firstPartyMcpPermissions ? [...creatorPolicy.firstPartyMcpPermissions] : [...DEFAULT_FIRST_PARTY_MCP_PERMISSIONS];
5998
+ const connectionSettings = await settingsWithEnabledCapabilityMcpServers(
5999
+ db,
6000
+ task.workspaceId,
6001
+ settings,
6002
+ taskAuthoritySubjectId ? { subjectId: taskAuthoritySubjectId } : {}
6003
+ );
6004
+ const connectionTarget = acceptedTargetSessionId && targetSessionExecutionBase ? await requireSession(db, task.workspaceId, acceptedTargetSessionId) : null;
6005
+ const connectionTools = await scheduledConnectionTools(
6006
+ db,
6007
+ task.workspaceId,
6008
+ connectionSettings,
6009
+ connectionTarget,
6010
+ taskTools,
6011
+ taskAuthoritySubjectId ?? void 0
6012
+ );
6013
+ const taskConnections = await freezeConnectionAccounts({
6014
+ db,
6015
+ accountId: task.accountId,
6016
+ workspaceId: task.workspaceId,
6017
+ settings: connectionTarget ? settingsWithSessionMcpServerMetadata(connectionSettings, connectionTarget.mcpServers) : connectionSettings,
6018
+ tools: connectionTools,
6019
+ resources: mergeResourceRefs(connectionTarget?.resources ?? [], task.agentConfig.resources),
6020
+ source: taskAuthoritySubjectId ? { kind: "subject", subjectId: taskAuthoritySubjectId, accountId: task.accountId } : { kind: "none" },
6021
+ authoritySelections: task.agentConfig.connectionAccounts ?? [],
6022
+ authoritySelectionsFrozen: task.agentConfig.connectionAccountsFrozen === true,
6023
+ ...scheduledConnectionSurfaceEligibility(
6024
+ settings,
6025
+ connectionTarget ?? {
6026
+ firstPartyMcpTools,
6027
+ firstPartyMcpPermissions
6028
+ }
6029
+ )
6030
+ }).catch((error) => {
6031
+ if (error instanceof ConnectionAccountSelectionError) return error;
6032
+ throw error;
6033
+ });
6034
+ if (taskConnections instanceof ConnectionAccountSelectionError) {
6035
+ const receipt = await recordScheduledTaskAdmissionFailure(db, {
6036
+ workspaceId: task.workspaceId,
6037
+ taskId: task.id,
6038
+ taskAuthorityRevision: task.authorityRevision,
6039
+ taskExecutionDigest: task.executionDigest,
6040
+ triggerType: input.triggerType,
6041
+ producerKey: stableProducerKey,
6042
+ diagnostic: taskConnections.diagnostic
6043
+ });
6044
+ if (!receipt.admissionDiagnostic) {
6045
+ return createScheduledTaskActivities(services).dispatchScheduledTaskRun(input);
6046
+ }
6047
+ return scheduledAdmissionRefusalResult(receipt);
6048
+ }
6049
+ const {
6050
+ personalConnectionDelegations: taskPersonalConnectionDelegations,
6051
+ mcpAccountBindings: taskMcpAccountBindings
6052
+ } = taskConnections;
6053
+ const taskConnectionAuthoritySubjectId = taskPersonalConnectionDelegations.length > 0 ? taskAuthoritySubjectId : null;
4854
6054
  const creatorSessionPolicy = scheduledCreatorSessionPolicyInput(
4855
6055
  creatorPolicy?.sessionPolicy ?? null
4856
6056
  );
4857
- const generatedTarget = task.runMode === "new_session_per_run" || task.runMode === "reusable_session" && task.reusableSessionId === null;
4858
6057
  if (generatedTarget && task.agentConfig.machineTarget) {
4859
6058
  const access = {
4860
6059
  accountId: task.accountId,
@@ -4867,19 +6066,38 @@ function createScheduledTaskActivities(services) {
4867
6066
  task.agentConfig.machineTarget.targetSandboxId
4868
6067
  );
4869
6068
  if (!machine || machine.kind !== "selfhosted" || !machine.enrollmentId || machine.scope === "user") {
4870
- throw new Error("scheduled Connected Machine target is unavailable");
6069
+ return await refuseAdmission(
6070
+ "machine_target_unavailable",
6071
+ false,
6072
+ "scheduled Connected Machine target is unavailable"
6073
+ );
4871
6074
  }
4872
6075
  const enrollment = await getEnrollment(db, access, machine.enrollmentId);
4873
- if (!enrollment || enrollment.status !== "active") {
4874
- throw new Error("scheduled Connected Machine enrollment is unavailable");
6076
+ if (!enrollment) {
6077
+ return await refuseAdmission(
6078
+ "machine_target_unavailable",
6079
+ false,
6080
+ "scheduled Connected Machine enrollment is gone"
6081
+ );
6082
+ }
6083
+ if (enrollment.status !== "active") {
6084
+ return await refuseAdmission(
6085
+ "machine_enrollment_inactive",
6086
+ true,
6087
+ `scheduled Connected Machine enrollment is ${enrollment.status}`
6088
+ );
4875
6089
  }
4876
6090
  sandboxBackend = "selfhosted";
4877
6091
  sandboxOs = enrollment.os;
4878
6092
  } else if (generatedTarget && sandboxBackend === "selfhosted") {
4879
- throw new Error("self-hosted scheduled task has no Connected Machine target");
6093
+ return await refuseAdmission(
6094
+ "machine_target_unavailable",
6095
+ false,
6096
+ "self-hosted scheduled task has no Connected Machine target"
6097
+ );
4880
6098
  }
4881
6099
  const generatedSessionDepthPolicy = generatedTarget ? await (async () => {
4882
- const workspace = await requireWorkspace(db, task.workspaceId);
6100
+ const workspace = await requireWorkspace2(db, task.workspaceId);
4883
6101
  const deployment = await getNestedAgentDepthDeploymentPolicy(db);
4884
6102
  const taskOverride = task.agentConfig.maxNestedAgentDepth ?? null;
4885
6103
  const workspaceOverride = typeof workspace.settings.maxNestedAgentDepth === "number" ? workspace.settings.maxNestedAgentDepth : null;
@@ -4898,20 +6116,15 @@ function createScheduledTaskActivities(services) {
4898
6116
  };
4899
6117
  })() : null;
4900
6118
  const targetSessionExecution = targetSessionExecutionBase ? await (async () => {
4901
- const [capabilityServerIds, apiIntegrationServerIds] = await Promise.all([
4902
- listEnabledMcpCapabilityServerIds(db, task.workspaceId),
4903
- listInstalledApiIntegrationServerIdsForDelegations(
4904
- db,
4905
- task.workspaceId,
4906
- taskPersonalConnectionDelegations
4907
- )
4908
- ]);
4909
- const workspaceDefaultMcpServerIds = new Set(
4910
- settings.mcpServers.map((server) => server.id)
6119
+ const apiIntegrationServerIds = await listInstalledApiIntegrationServerIdsForDelegations(
6120
+ db,
6121
+ task.workspaceId,
6122
+ taskPersonalConnectionDelegations
4911
6123
  );
4912
- for (const id of capabilityServerIds) workspaceDefaultMcpServerIds.add(id);
4913
- for (const id of apiIntegrationServerIds) workspaceDefaultMcpServerIds.add(id);
4914
- const availableMcpServerIds = new Set(workspaceDefaultMcpServerIds);
6124
+ const availableMcpServerIds = /* @__PURE__ */ new Set([
6125
+ ...connectionSettings.mcpServers.map((server) => server.id),
6126
+ ...apiIntegrationServerIds
6127
+ ]);
4915
6128
  for (const id of targetSessionExecutionBase.mcpServerIds) {
4916
6129
  availableMcpServerIds.add(id);
4917
6130
  }
@@ -4919,9 +6132,10 @@ function createScheduledTaskActivities(services) {
4919
6132
  toolPolicy: targetSessionExecutionBase.toolPolicy,
4920
6133
  sessionTools: targetSessionExecutionBase.tools,
4921
6134
  availableMcpServerIds,
4922
- defaultMcpServerIds: defaultSessionMcpServerIds(
4923
- [...workspaceDefaultMcpServerIds].map((id) => ({ id }))
4924
- )
6135
+ defaultMcpServerIds: [
6136
+ ...connectionTools.map((tool) => tool.id),
6137
+ ...apiIntegrationServerIds
6138
+ ]
4925
6139
  });
4926
6140
  return {
4927
6141
  ...targetSessionExecutionBase,
@@ -4946,7 +6160,11 @@ function createScheduledTaskActivities(services) {
4946
6160
  task.variableSetId
4947
6161
  ) : null;
4948
6162
  if (generatedTarget && task.variableSetId && !acceptedVariableSet) {
4949
- throw new Error(`variable set not found: ${task.variableSetId}`);
6163
+ return await refuseAdmission(
6164
+ "variable_set_unavailable",
6165
+ false,
6166
+ `variable set not found: ${task.variableSetId}`
6167
+ );
4950
6168
  }
4951
6169
  const acceptedRig = generatedTarget && task.rigId ? await getRig2(
4952
6170
  db,
@@ -4958,11 +6176,16 @@ function createScheduledTaskActivities(services) {
4958
6176
  task.rigId
4959
6177
  ) : null;
4960
6178
  if (generatedTarget && task.rigId && (!acceptedRig || !acceptedRig.activeVersion)) {
4961
- throw new Error(`rig has no active version to bind: ${task.rigId}`);
6179
+ return await refuseAdmission(
6180
+ "rig_version_unavailable",
6181
+ false,
6182
+ `sandbox environment has no active version to bind: ${task.rigId}`
6183
+ );
4962
6184
  }
4963
- const acceptedRigDefaultVariableSets = acceptedRig?.activeVersion ? await Promise.all(
4964
- acceptedRig.activeVersion.defaultVariableSetIds.map(async (variableSetId) => {
4965
- const variableSet = await getVariableSet(
6185
+ const acceptedRigDefaultVariableSetRows = acceptedRig?.activeVersion ? await Promise.all(
6186
+ acceptedRig.activeVersion.defaultVariableSetIds.map(async (variableSetId) => ({
6187
+ variableSetId,
6188
+ variableSet: await getVariableSet(
4966
6189
  db,
4967
6190
  {
4968
6191
  accountId: task.accountId,
@@ -4970,13 +6193,22 @@ function createScheduledTaskActivities(services) {
4970
6193
  subjectId: taskAuthoritySubjectId ?? task.createdBy.subjectId
4971
6194
  },
4972
6195
  variableSetId
4973
- );
4974
- if (!variableSet) {
4975
- throw new Error(`rig default Variable Set not found: ${variableSetId}`);
4976
- }
4977
- return { id: variableSet.id, generation: variableSet.generation };
4978
- })
6196
+ )
6197
+ }))
4979
6198
  ) : [];
6199
+ const missingRigDefaultVariableSet = acceptedRigDefaultVariableSetRows.find(
6200
+ (row) => !row.variableSet
6201
+ );
6202
+ if (missingRigDefaultVariableSet) {
6203
+ return await refuseAdmission(
6204
+ "variable_set_unavailable",
6205
+ false,
6206
+ `sandbox environment default Variable Set not found: ${missingRigDefaultVariableSet.variableSetId}`
6207
+ );
6208
+ }
6209
+ const acceptedRigDefaultVariableSets = acceptedRigDefaultVariableSetRows.map(
6210
+ ({ variableSet }) => ({ id: variableSet.id, generation: variableSet.generation })
6211
+ );
4980
6212
  let incidentPreflightRequired = incidentDeclaration.action === "required";
4981
6213
  if (incidentPreflightRequired) {
4982
6214
  const existingSessionId = task.runMode === "existing_session" ? task.targetSessionId : task.reusableSessionId;
@@ -5031,6 +6263,7 @@ function createScheduledTaskActivities(services) {
5031
6263
  incidentPreflightRequired = preflight.action === "ready";
5032
6264
  }
5033
6265
  const taskXaiProviderAccountAuthoritySnapshot = await getScheduledTaskXaiProviderAccountAuthoritySnapshot(db, task.workspaceId, task.id);
6266
+ const taskClaudeProviderAccountAuthoritySnapshot = await getScheduledTaskClaudeProviderAccountAuthoritySnapshot(db, task.workspaceId, task.id);
5034
6267
  const slackBotConnection = task.agentConfig.slackBotConnectionId ? await requireOpenGeniSlackBotConnection(
5035
6268
  db,
5036
6269
  task.workspaceId,
@@ -5038,23 +6271,58 @@ function createScheduledTaskActivities(services) {
5038
6271
  ) : null;
5039
6272
  const slackBotMetadata = slackBotConnection ? openGeniSlackBotMetadata(slackBotConnection.metadata) : null;
5040
6273
  if (slackBotConnection && !slackBotMetadata) {
5041
- throw new Error("OpenGeni Slack bot connection metadata is invalid");
6274
+ throw new Error("Opengeni Slack bot connection metadata is invalid");
5042
6275
  }
5043
6276
  const xaiAuthoritySubjectId = taskXaiProviderAccountAuthoritySnapshot.scope === "user" && task.createdBy.kind === "subject" ? task.createdBy.subjectId : null;
5044
6277
  if (taskXaiProviderAccountAuthoritySnapshot.scope === "user" && !xaiAuthoritySubjectId) {
5045
- throw new Error("scheduled user-scoped xAI authority has no causal human");
6278
+ return await refuseAdmission(
6279
+ "scheduled_authority_unavailable",
6280
+ false,
6281
+ "scheduled user-scoped xAI authority has no causal human"
6282
+ );
5046
6283
  }
5047
6284
  if (xaiAuthoritySubjectId && taskAuthoritySubjectId && xaiAuthoritySubjectId !== taskAuthoritySubjectId) {
5048
- throw new Error("scheduled authority classes have different causal humans");
6285
+ return await refuseAdmission(
6286
+ "scheduled_authority_unavailable",
6287
+ false,
6288
+ "scheduled authority classes have different causal humans"
6289
+ );
5049
6290
  }
5050
- const causalHumanSubjectId = taskAuthoritySubjectId ?? xaiAuthoritySubjectId;
6291
+ const claudeAuthoritySubjectId = taskClaudeProviderAccountAuthoritySnapshot.scope === "user" && task.ownerSubjectId ? task.ownerSubjectId : null;
6292
+ if (taskClaudeProviderAccountAuthoritySnapshot.scope === "user" && !claudeAuthoritySubjectId) {
6293
+ return await refuseAdmission(
6294
+ "scheduled_authority_unavailable",
6295
+ false,
6296
+ "scheduled user-scoped Claude authority has no causal human"
6297
+ );
6298
+ }
6299
+ if (claudeAuthoritySubjectId && taskAuthoritySubjectId && claudeAuthoritySubjectId !== taskAuthoritySubjectId) {
6300
+ return await refuseAdmission(
6301
+ "scheduled_authority_unavailable",
6302
+ false,
6303
+ "scheduled authority classes have different causal humans"
6304
+ );
6305
+ }
6306
+ const causalHumanSubjectId = taskAuthoritySubjectId ?? xaiAuthoritySubjectId ?? claudeAuthoritySubjectId;
6307
+ if (xaiAuthoritySubjectId && claudeAuthoritySubjectId && xaiAuthoritySubjectId !== claudeAuthoritySubjectId)
6308
+ return await refuseAdmission(
6309
+ "scheduled_authority_unavailable",
6310
+ false,
6311
+ "scheduled subscription pools have different causal humans"
6312
+ );
5051
6313
  if (task.agentConfig.knowledgeSource && causalHumanSubjectId !== task.agentConfig.knowledgeSource.initiatingSubjectId) {
5052
- throw new Error(
6314
+ return await refuseAdmission(
6315
+ "scheduled_authority_unavailable",
6316
+ false,
5053
6317
  "Source schedule requires its connection owner's current human authorization; update the source schedule to authorize it"
5054
6318
  );
5055
6319
  }
5056
6320
  if ((acceptedVariableSet?.scope === "user" || acceptedRig?.scope === "user") && !causalHumanSubjectId) {
5057
- throw new Error("scheduled personal-resource execution has no causal human");
6321
+ return await refuseAdmission(
6322
+ "scheduled_authority_unavailable",
6323
+ false,
6324
+ "scheduled personal-resource execution has no causal human"
6325
+ );
5058
6326
  }
5059
6327
  const admissionDenial = await agentRunAdmissionDenial(
5060
6328
  { ...baseService, settings },
@@ -5062,11 +6330,12 @@ function createScheduledTaskActivities(services) {
5062
6330
  accountId: task.accountId,
5063
6331
  workspaceId: task.workspaceId,
5064
6332
  model: targetSessionExecution?.model ?? model,
5065
- requestedAgentRuns: input.agentRunUsageIdempotencyKey ? 0 : 1
6333
+ requestedAgentRuns: input.agentRunUsageIdempotencyKey ? 0 : 1,
6334
+ initiatingHumanSubjectId: causalHumanSubjectId
5066
6335
  }
5067
6336
  );
5068
6337
  if (admissionDenial) {
5069
- return { action: "blocked", reason: admissionDenial };
6338
+ return await refuseAdmission(admissionDenial, true, "agent run admission denied");
5070
6339
  }
5071
6340
  const acceptedModel = targetSessionExecution?.model ?? model;
5072
6341
  const acceptedReasoningEffort = targetSessionExecution?.reasoningEffort ?? reasoningEffort;
@@ -5085,15 +6354,19 @@ function createScheduledTaskActivities(services) {
5085
6354
  );
5086
6355
  }
5087
6356
  } : void 0;
5088
- const turnExecutionPolicy = resolveTurnExecutionPolicyV12(settings, {
5089
- modelId: acceptedModel,
5090
- requestedModelId: generatedTarget && task.agentConfig.model ? task.agentConfig.model : null,
5091
- modelSource: generatedTarget ? task.agentConfig.model ? "explicit" : "deployment" : "session",
5092
- reasoningEffort: acceptedReasoningEffort,
5093
- reasoningSource: generatedTarget ? task.agentConfig.reasoningEffort ? "explicit" : "deployment" : "session",
5094
- latencyMode: acceptedLatencyMode,
5095
- latencyModeSource: generatedTarget ? "deployment" : "session"
5096
- });
6357
+ const turnExecutionPolicy = scheduledTaskRunExecutionPolicy(
6358
+ resolveTurnExecutionPolicyV12(settings, {
6359
+ modelId: acceptedModel,
6360
+ requestedModelId: generatedTarget && task.agentConfig.model ? task.agentConfig.model : null,
6361
+ modelSource: generatedTarget ? task.agentConfig.model ? "explicit" : "deployment" : "session",
6362
+ reasoningEffort: acceptedReasoningEffort,
6363
+ reasoningSource: generatedTarget ? task.agentConfig.reasoningEffort ? "explicit" : "deployment" : "session",
6364
+ latencyMode: acceptedLatencyMode,
6365
+ latencyModeSource: generatedTarget ? "deployment" : "session"
6366
+ }),
6367
+ input,
6368
+ creatorPolicy?.credentialRestriction
6369
+ );
5097
6370
  const deferredEvents = [];
5098
6371
  const executeDispatch = async (dispatchDb, deferPublications) => {
5099
6372
  const admittedRunId = crypto.randomUUID();
@@ -5115,6 +6388,10 @@ function createScheduledTaskActivities(services) {
5115
6388
  resolvedTools: taskTools,
5116
6389
  resolvedFirstPartyMcpTools: firstPartyMcpTools,
5117
6390
  resolvedFirstPartyMcpPermissions: firstPartyMcpPermissions,
6391
+ ...resolvedAgentConfig ? {
6392
+ resolvedAgentConfig,
6393
+ ...resolvedAgentInstructions !== void 0 ? { resolvedAgentInstructions } : {}
6394
+ } : {},
5118
6395
  resolvedVariableSet: acceptedVariableSet ? { id: acceptedVariableSet.id, generation: acceptedVariableSet.generation } : null,
5119
6396
  resolvedRig: acceptedRig?.activeVersion ? {
5120
6397
  id: acceptedRig.id,
@@ -5130,11 +6407,14 @@ function createScheduledTaskActivities(services) {
5130
6407
  targetSessionExecution,
5131
6408
  generatedSessionBinding,
5132
6409
  personalConnectionDelegations: taskPersonalConnectionDelegations,
6410
+ mcpAccountBindings: taskMcpAccountBindings,
5133
6411
  personalResourceAuthoritySubjectId: taskPersonalResourceAuthoritySubjectId,
5134
6412
  causalHumanSubjectId,
5135
6413
  causalHumanAuthority: taskRevisionAuthority,
5136
6414
  xaiProviderAccountAuthoritySnapshot: taskXaiProviderAccountAuthoritySnapshot,
6415
+ claudeProviderAccountAuthoritySnapshot: taskClaudeProviderAccountAuthoritySnapshot,
5137
6416
  xaiAuthoritySubjectId,
6417
+ claudeAuthoritySubjectId,
5138
6418
  connectionAuthoritySubjectId: taskConnectionAuthoritySubjectId,
5139
6419
  triggerInitiator: input.initiator ?? { kind: "service", subjectId: "scheduler" },
5140
6420
  agentRunUsageIdempotencyKey: input.agentRunUsageIdempotencyKey ?? null,
@@ -5243,13 +6523,7 @@ function createScheduledTaskActivities(services) {
5243
6523
  taskId: task.id
5244
6524
  }
5245
6525
  );
5246
- if (stableJson({
5247
- task: exactAuthority.task,
5248
- personalConnectionDelegations: exactAuthority.personalConnectionDelegations
5249
- }) !== stableJson({
5250
- task,
5251
- personalConnectionDelegations: taskPersonalConnectionDelegations
5252
- })) {
6526
+ if (stableJson(exactAuthority) !== stableJson(task)) {
5253
6527
  throw new IncidentTelemetryPreflightBlockedError(
5254
6528
  "incident_preflight_metadata_missing"
5255
6529
  );
@@ -5270,14 +6544,16 @@ function createScheduledTaskActivities(services) {
5270
6544
  if (task.rigId) {
5271
6545
  const rig = acceptedRig;
5272
6546
  if (!rig || !rig.activeVersion) {
5273
- throw new Error(`rig has no active version to bind: ${task.rigId}`);
6547
+ throw new Error(`sandbox environment has no active version to bind: ${task.rigId}`);
5274
6548
  }
5275
6549
  frozenRigId = rig.id;
5276
6550
  frozenRigVersionId = runPersonalResourceAuthority?.resources.find(
5277
6551
  (resource) => resource.resourceKind === "rig" && resource.resourceId === rig.id
5278
6552
  )?.resourceVersionId ?? rig.activeVersion.id;
5279
6553
  if (frozenRigVersionId !== rig.activeVersion.id) {
5280
- throw new Error("scheduled run rig version changed during admission");
6554
+ throw new Error(
6555
+ "scheduled run sandbox environment version changed during admission"
6556
+ );
5281
6557
  }
5282
6558
  }
5283
6559
  let session;
@@ -5294,6 +6570,8 @@ function createScheduledTaskActivities(services) {
5294
6570
  tools: taskTools,
5295
6571
  firstPartyMcpTools,
5296
6572
  firstPartyMcpPermissions,
6573
+ ...resolvedAgentConfig ? { agentConfig: resolvedAgentConfig } : {},
6574
+ ...resolvedAgentInstructions !== void 0 ? { instructions: resolvedAgentInstructions } : {},
5297
6575
  ...creatorSessionPolicy,
5298
6576
  ...task.agentConfig.knowledgeSource?.destination.kind === "personal" ? {
5299
6577
  visibility: "user_private",
@@ -5301,6 +6579,10 @@ function createScheduledTaskActivities(services) {
5301
6579
  } : {},
5302
6580
  metadata: {
5303
6581
  ...taskMetadata,
6582
+ ...scheduledSessionExecutionPolicyMetadata(
6583
+ turnExecutionPolicy,
6584
+ creatorPolicy?.credentialRestriction
6585
+ ),
5304
6586
  model,
5305
6587
  reasoningEffort,
5306
6588
  scheduledTaskId: task.id,
@@ -5330,6 +6612,7 @@ function createScheduledTaskActivities(services) {
5330
6612
  rigVersionId: frozenRigVersionId,
5331
6613
  personalConnectionDelegations: [],
5332
6614
  initialXaiProviderAccountAuthoritySnapshot: taskXaiProviderAccountAuthoritySnapshot,
6615
+ initialClaudeProviderAccountAuthoritySnapshot: taskClaudeProviderAccountAuthoritySnapshot,
5333
6616
  maxNestedAgentDepthOverride: task.agentConfig.maxNestedAgentDepth ?? null,
5334
6617
  frozenNestedAgentDepthPolicy: {
5335
6618
  effectiveMaxNestedAgentDepth: generatedSessionBinding.effectiveMaxNestedAgentDepth,
@@ -5390,6 +6673,7 @@ function createScheduledTaskActivities(services) {
5390
6673
  }
5391
6674
  session = created.session;
5392
6675
  sessionCreated = created.created;
6676
+ if (sessionCreated) recordScheduledSessionCreated(baseService.observability);
5393
6677
  if (!sessionCreated) {
5394
6678
  await bindScheduledTaskRunSessionInTransaction(dispatchDb, {
5395
6679
  accountId: task.accountId,
@@ -5420,7 +6704,7 @@ function createScheduledTaskActivities(services) {
5420
6704
  }
5421
6705
  if (scheduledSlackBotConnectionId(session.metadata) !== (task.agentConfig.slackBotConnectionId ?? null)) {
5422
6706
  throw new Error(
5423
- "scheduled alert occurrence OpenGeni Slack bot binding does not match its canonical session"
6707
+ "scheduled alert occurrence Opengeni Slack bot binding does not match its canonical session"
5424
6708
  );
5425
6709
  }
5426
6710
  }
@@ -5476,7 +6760,7 @@ function createScheduledTaskActivities(services) {
5476
6760
  events: started.events
5477
6761
  });
5478
6762
  } else {
5479
- await publishDurableSessionEvents(
6763
+ await publishDurableSessionEvents2(
5480
6764
  bus,
5481
6765
  task.workspaceId,
5482
6766
  session.id,
@@ -5519,7 +6803,7 @@ function createScheduledTaskActivities(services) {
5519
6803
  if (deferPublications) {
5520
6804
  deferredEvents.push({ sessionId: session.id, events: started.events });
5521
6805
  } else {
5522
- await publishDurableSessionEvents(
6806
+ await publishDurableSessionEvents2(
5523
6807
  bus,
5524
6808
  task.workspaceId,
5525
6809
  session.id,
@@ -5548,7 +6832,7 @@ function createScheduledTaskActivities(services) {
5548
6832
  if (deferPublications) {
5549
6833
  deferredEvents.push({ sessionId: session.id, events: titleEvents });
5550
6834
  } else {
5551
- await publishDurableSessionEvents(bus, task.workspaceId, session.id, titleEvents);
6835
+ await publishDurableSessionEvents2(bus, task.workspaceId, session.id, titleEvents);
5552
6836
  }
5553
6837
  }
5554
6838
  const scheduledUpdate = await addSessionSystemUpdateWithSourceMutation(
@@ -5574,10 +6858,13 @@ function createScheduledTaskActivities(services) {
5574
6858
  scheduledTaskRunId: run.id,
5575
6859
  ...causalHumanSubjectId ? { causalHumanSubjectId } : {},
5576
6860
  ...taskConnectionAuthoritySubjectId ? { connectionAuthoritySubjectId: taskConnectionAuthoritySubjectId } : {},
5577
- ...taskXaiProviderAccountAuthoritySnapshot.scope === "user" && task.createdBy.kind === "subject" ? { xaiAuthoritySubjectId: task.createdBy.subjectId } : {}
6861
+ ...taskXaiProviderAccountAuthoritySnapshot.scope === "user" && task.createdBy.kind === "subject" ? { xaiAuthoritySubjectId: task.createdBy.subjectId } : {},
6862
+ ...claudeAuthoritySubjectId ? { claudeAuthoritySubjectId } : {}
5578
6863
  },
5579
6864
  personalConnectionDelegations: taskPersonalConnectionDelegations,
6865
+ mcpAccountBindings: taskMcpAccountBindings,
5580
6866
  xaiProviderAccountAuthoritySnapshot: taskXaiProviderAccountAuthoritySnapshot,
6867
+ claudeProviderAccountAuthoritySnapshot: taskClaudeProviderAccountAuthoritySnapshot,
5581
6868
  scheduledTaskRunId: run.id
5582
6869
  },
5583
6870
  async (tx, wakeEventId, _updateId, rejectionReason) => {
@@ -5624,6 +6911,9 @@ function createScheduledTaskActivities(services) {
5624
6911
  run: { ...run, status: "skipped", error }
5625
6912
  };
5626
6913
  }
6914
+ if (scheduledUpdate.added) {
6915
+ recordUserMessageAccepted(baseService.observability, { surface: "scheduled" });
6916
+ }
5627
6917
  if (scheduledUpdate.added && scheduledUpdate.events.length > 0) {
5628
6918
  if (deferPublications) {
5629
6919
  deferredEvents.push({
@@ -5631,7 +6921,7 @@ function createScheduledTaskActivities(services) {
5631
6921
  events: scheduledUpdate.events
5632
6922
  });
5633
6923
  } else {
5634
- await publishDurableSessionEvents(
6924
+ await publishDurableSessionEvents2(
5635
6925
  bus,
5636
6926
  task.workspaceId,
5637
6927
  session.id,
@@ -5683,10 +6973,13 @@ function createScheduledTaskActivities(services) {
5683
6973
  scheduledTaskRunId: run.id,
5684
6974
  ...causalHumanSubjectId ? { causalHumanSubjectId } : {},
5685
6975
  ...taskConnectionAuthoritySubjectId ? { connectionAuthoritySubjectId: taskConnectionAuthoritySubjectId } : {},
5686
- ...taskXaiProviderAccountAuthoritySnapshot.scope === "user" && task.createdBy.kind === "subject" ? { xaiAuthoritySubjectId: task.createdBy.subjectId } : {}
6976
+ ...taskXaiProviderAccountAuthoritySnapshot.scope === "user" && task.createdBy.kind === "subject" ? { xaiAuthoritySubjectId: task.createdBy.subjectId } : {},
6977
+ ...claudeAuthoritySubjectId ? { claudeAuthoritySubjectId } : {}
5687
6978
  },
5688
6979
  personalConnectionDelegations: taskPersonalConnectionDelegations,
6980
+ mcpAccountBindings: taskMcpAccountBindings,
5689
6981
  xaiProviderAccountAuthoritySnapshot: taskXaiProviderAccountAuthoritySnapshot,
6982
+ claudeProviderAccountAuthoritySnapshot: taskClaudeProviderAccountAuthoritySnapshot,
5690
6983
  scheduledTaskRunId: run.id
5691
6984
  },
5692
6985
  async (tx, wakeEventId, _updateId, rejectionReason) => {
@@ -5747,6 +7040,9 @@ function createScheduledTaskActivities(services) {
5747
7040
  run: { ...run, status: "skipped", error }
5748
7041
  };
5749
7042
  }
7043
+ if (bundled.added) {
7044
+ recordUserMessageAccepted(baseService.observability, { surface: "scheduled" });
7045
+ }
5750
7046
  if (bundled.added && bundled.events.length > 0) {
5751
7047
  if (deferPublications) {
5752
7048
  deferredEvents.push({
@@ -5754,7 +7050,7 @@ function createScheduledTaskActivities(services) {
5754
7050
  events: bundled.events
5755
7051
  });
5756
7052
  } else {
5757
- await publishDurableSessionEvents(
7053
+ await publishDurableSessionEvents2(
5758
7054
  bus,
5759
7055
  task.workspaceId,
5760
7056
  session.id,
@@ -5836,7 +7132,7 @@ function createScheduledTaskActivities(services) {
5836
7132
  });
5837
7133
  }
5838
7134
  for (const deferred of deferredEvents) {
5839
- await publishDurableSessionEvents(
7135
+ await publishDurableSessionEvents2(
5840
7136
  bus,
5841
7137
  task.workspaceId,
5842
7138
  deferred.sessionId,
@@ -5866,16 +7162,11 @@ var IncidentTelemetryPreflightBlockedError = class extends Error {
5866
7162
  }
5867
7163
  };
5868
7164
  function assertReusableSessionBindingMatches(session, task) {
5869
- if ((session.variableSetId ?? null) !== (task.variableSetId ?? null)) {
5870
- throw new ScheduledRunTerminalAuthorityError(
5871
- "scheduled_reusable_binding_changed",
5872
- "scheduled task variableSet attachment does not match its reusable session"
5873
- );
5874
- }
7165
+ if (task.runMode === "existing_session") return;
5875
7166
  if (scheduledSlackBotConnectionId(session.metadata) !== (task.agentConfig.slackBotConnectionId ?? null)) {
5876
7167
  throw new ScheduledRunTerminalAuthorityError(
5877
7168
  "scheduled_reusable_binding_changed",
5878
- "scheduled task OpenGeni Slack bot binding does not match its reusable session"
7169
+ "scheduled task Opengeni Slack bot binding does not match its reusable session"
5879
7170
  );
5880
7171
  }
5881
7172
  }
@@ -5921,7 +7212,25 @@ function scheduledRunRecoveryTerminalError(error) {
5921
7212
  }
5922
7213
  return null;
5923
7214
  }
7215
+ function scheduledAdmissionRefusalResult(run) {
7216
+ if (run.admissionRefusal) {
7217
+ return {
7218
+ action: "blocked",
7219
+ reason: run.admissionRefusal.reason,
7220
+ runId: run.id,
7221
+ refusal: run.admissionRefusal
7222
+ };
7223
+ }
7224
+ return {
7225
+ action: "blocked",
7226
+ reason: "connection_account_unavailable",
7227
+ runId: run.id,
7228
+ ...run.admissionDiagnostic ? { diagnostic: run.admissionDiagnostic } : {}
7229
+ };
7230
+ }
5924
7231
  function scheduledRunTerminalResult(error) {
7232
+ if (error === ATLASSIAN_NATIVE_RETIRED_REASON)
7233
+ return { action: "blocked", reason: ATLASSIAN_NATIVE_RETIRED_REASON };
5925
7234
  if (error === "scheduled_authority_exhausted") {
5926
7235
  return { action: "blocked", reason: "scheduled_authority_exhausted" };
5927
7236
  }
@@ -5941,46 +7250,38 @@ function scheduledSessionRejectionError(rejectionReason) {
5941
7250
  }
5942
7251
  async function prepareIncidentTelemetrySource(input) {
5943
7252
  const session = await requireSession(input.tx, input.workspaceId, input.sessionId);
5944
- const { task, personalConnectionDelegations } = input.acceptedExecution ? {
5945
- task: input.acceptedExecution.task,
5946
- personalConnectionDelegations: input.acceptedExecution.personalConnectionDelegations
5947
- } : await requireScheduledTaskIncidentAuthorityInTransaction(input.tx, {
5948
- workspaceId: input.workspaceId,
5949
- taskId: input.taskId
5950
- });
7253
+ const { task, personalConnectionDelegations } = input.acceptedExecution;
5951
7254
  const responder = await resolveIncidentTelemetryResponderMetadata({
5952
7255
  db: input.tx,
5953
7256
  settings: input.settings,
5954
7257
  task,
5955
7258
  session,
5956
7259
  personalConnectionDelegations,
5957
- personalResourceAuthoritySubjectId: input.acceptedExecution?.personalResourceAuthoritySubjectId ?? null,
5958
- ...input.acceptedExecution ? {
5959
- executionPolicy: input.acceptedExecution.targetSessionExecution ? {
5960
- tools: input.acceptedExecution.targetSessionExecution.tools,
5961
- firstPartyMcpTools: input.acceptedExecution.targetSessionExecution.firstPartyMcpTools,
5962
- firstPartyMcpPermissions: input.acceptedExecution.targetSessionExecution.firstPartyMcpPermissions,
5963
- variableSetIds: input.acceptedExecution.targetSessionExecution.variableSets.map(
5964
- (variableSet) => variableSet.id
5965
- ),
5966
- variableSetId: input.acceptedExecution.targetSessionExecution.variableSetId,
5967
- rigId: input.acceptedExecution.targetSessionExecution.rigId,
5968
- rigVersionId: input.acceptedExecution.targetSessionExecution.rigVersionId,
5969
- toolPolicy: input.acceptedExecution.targetSessionExecution.toolPolicy,
5970
- mcpServerIds: input.acceptedExecution.targetSessionExecution.mcpServerIds,
5971
- toolPolicyVersion: input.acceptedExecution.targetSessionExecution.toolPolicyVersion
5972
- } : {
5973
- tools: input.acceptedExecution.resolvedTools,
5974
- firstPartyMcpTools: input.acceptedExecution.resolvedFirstPartyMcpTools,
5975
- firstPartyMcpPermissions: input.acceptedExecution.resolvedFirstPartyMcpPermissions,
5976
- variableSetId: input.acceptedExecution.resolvedVariableSet?.id ?? null,
5977
- rigId: input.acceptedExecution.resolvedRig?.id ?? null,
5978
- rigVersionId: input.acceptedExecution.resolvedRig?.versionId ?? null,
5979
- toolPolicy: session.toolPolicy,
5980
- mcpServerIds: [],
5981
- toolPolicyVersion: session.toolPolicyVersion
5982
- }
5983
- } : {}
7260
+ personalResourceAuthoritySubjectId: input.acceptedExecution.personalResourceAuthoritySubjectId ?? null,
7261
+ executionPolicy: input.acceptedExecution.targetSessionExecution ? {
7262
+ tools: input.acceptedExecution.targetSessionExecution.tools,
7263
+ firstPartyMcpTools: input.acceptedExecution.targetSessionExecution.firstPartyMcpTools,
7264
+ firstPartyMcpPermissions: input.acceptedExecution.targetSessionExecution.firstPartyMcpPermissions,
7265
+ variableSetIds: input.acceptedExecution.targetSessionExecution.variableSets.map(
7266
+ (variableSet) => variableSet.id
7267
+ ),
7268
+ variableSetId: input.acceptedExecution.targetSessionExecution.variableSetId,
7269
+ rigId: input.acceptedExecution.targetSessionExecution.rigId,
7270
+ rigVersionId: input.acceptedExecution.targetSessionExecution.rigVersionId,
7271
+ toolPolicy: input.acceptedExecution.targetSessionExecution.toolPolicy,
7272
+ mcpServerIds: input.acceptedExecution.targetSessionExecution.mcpServerIds,
7273
+ toolPolicyVersion: input.acceptedExecution.targetSessionExecution.toolPolicyVersion
7274
+ } : {
7275
+ tools: input.acceptedExecution.resolvedTools,
7276
+ firstPartyMcpTools: input.acceptedExecution.resolvedFirstPartyMcpTools,
7277
+ firstPartyMcpPermissions: input.acceptedExecution.resolvedFirstPartyMcpPermissions,
7278
+ variableSetId: input.acceptedExecution.resolvedVariableSet?.id ?? null,
7279
+ rigId: input.acceptedExecution.resolvedRig?.id ?? null,
7280
+ rigVersionId: input.acceptedExecution.resolvedRig?.versionId ?? null,
7281
+ toolPolicy: session.toolPolicy,
7282
+ mcpServerIds: [],
7283
+ toolPolicyVersion: session.toolPolicyVersion
7284
+ }
5984
7285
  });
5985
7286
  const preflight = evaluateIncidentTelemetryPreflight({
5986
7287
  agentConfig: task.agentConfig,
@@ -6060,7 +7361,7 @@ async function seedScheduledGeneratedSessionRoute(input) {
6060
7361
  if (input.deferPublications) {
6061
7362
  input.deferredEvents.push({ sessionId: input.session.id, events: failed.events });
6062
7363
  } else {
6063
- await publishDurableSessionEvents(
7364
+ await publishDurableSessionEvents2(
6064
7365
  input.bus,
6065
7366
  input.task.workspaceId,
6066
7367
  input.session.id,
@@ -6080,6 +7381,7 @@ async function recoverBoundScheduledTaskDispatch(input) {
6080
7381
  throw new Error("scheduled run is not an exact queued recovery");
6081
7382
  }
6082
7383
  const generatedSession = task.runMode === "new_session_per_run" || task.runMode === "reusable_session" && task.reusableSessionId === null;
7384
+ const recoveredCreatorPolicy = generatedSession ? await getScheduledTaskCreatorPolicy(input.db, task.workspaceId, task.id) : null;
6083
7385
  const frozenSlack = input.acceptedExecution.resolvedSlackBotConnection;
6084
7386
  if (frozenSlack) {
6085
7387
  const currentSlack = await requireOpenGeniSlackBotConnection(
@@ -6139,11 +7441,6 @@ async function recoverBoundScheduledTaskDispatch(input) {
6139
7441
  }
6140
7442
  const taskMetadata = { ...task.agentConfig.metadata };
6141
7443
  delete taskMetadata[OPENGENI_SLACK_BOT_SESSION_METADATA_KEY];
6142
- const recoveredCreatorPolicy = await getScheduledTaskCreatorPolicy(
6143
- input.db,
6144
- task.workspaceId,
6145
- task.id
6146
- );
6147
7444
  const created = await createSessionWithIdempotencyKeyResult(input.db, {
6148
7445
  accountId: task.accountId,
6149
7446
  workspaceId: task.workspaceId,
@@ -6153,6 +7450,8 @@ async function recoverBoundScheduledTaskDispatch(input) {
6153
7450
  tools: input.acceptedExecution.resolvedTools,
6154
7451
  firstPartyMcpTools: input.acceptedExecution.resolvedFirstPartyMcpTools,
6155
7452
  firstPartyMcpPermissions: input.acceptedExecution.resolvedFirstPartyMcpPermissions,
7453
+ ...input.acceptedExecution.resolvedAgentConfig ? { agentConfig: input.acceptedExecution.resolvedAgentConfig } : {},
7454
+ ...input.acceptedExecution.resolvedAgentInstructions !== void 0 ? { instructions: input.acceptedExecution.resolvedAgentInstructions } : {},
6156
7455
  ...scheduledCreatorSessionPolicyInput(recoveredCreatorPolicy?.sessionPolicy ?? null),
6157
7456
  ...task.agentConfig.knowledgeSource?.destination.kind === "personal" ? {
6158
7457
  visibility: "user_private",
@@ -6160,6 +7459,10 @@ async function recoverBoundScheduledTaskDispatch(input) {
6160
7459
  } : {},
6161
7460
  metadata: {
6162
7461
  ...taskMetadata,
7462
+ ...scheduledSessionExecutionPolicyMetadata(
7463
+ input.acceptedExecution.turnExecutionPolicy,
7464
+ recoveredCreatorPolicy?.credentialRestriction
7465
+ ),
6163
7466
  model: input.acceptedExecution.resolvedModel,
6164
7467
  reasoningEffort: input.acceptedExecution.resolvedReasoningEffort,
6165
7468
  scheduledTaskId: task.id,
@@ -6187,6 +7490,7 @@ async function recoverBoundScheduledTaskDispatch(input) {
6187
7490
  rigVersionId: rig?.versionId ?? null,
6188
7491
  personalConnectionDelegations: [],
6189
7492
  initialXaiProviderAccountAuthoritySnapshot: input.acceptedExecution.xaiProviderAccountAuthoritySnapshot,
7493
+ initialClaudeProviderAccountAuthoritySnapshot: input.acceptedExecution.claudeProviderAccountAuthoritySnapshot,
6190
7494
  maxNestedAgentDepthOverride: task.agentConfig.maxNestedAgentDepth ?? null,
6191
7495
  frozenNestedAgentDepthPolicy: {
6192
7496
  effectiveMaxNestedAgentDepth: input.acceptedExecution.generatedSessionBinding.effectiveMaxNestedAgentDepth,
@@ -6212,6 +7516,7 @@ async function recoverBoundScheduledTaskDispatch(input) {
6212
7516
  });
6213
7517
  if (created.denied) throw new SessionSpawnDeniedDbError(created.denial);
6214
7518
  session = created.session;
7519
+ if (created.created) recordScheduledSessionCreated(input.observability);
6215
7520
  if (!created.created) {
6216
7521
  await bindScheduledTaskRunSessionInTransaction(input.db, {
6217
7522
  accountId: task.accountId,
@@ -6275,21 +7580,28 @@ async function recoverBoundScheduledTaskDispatch(input) {
6275
7580
  );
6276
7581
  const expectedTaskMetadata = { ...task.agentConfig.metadata };
6277
7582
  delete expectedTaskMetadata[OPENGENI_SLACK_BOT_SESSION_METADATA_KEY];
6278
- const expectedMetadata = {
6279
- ...expectedTaskMetadata,
6280
- model: input.acceptedExecution.resolvedModel,
6281
- reasoningEffort: input.acceptedExecution.resolvedReasoningEffort,
6282
- scheduledTaskId: task.id,
6283
- scheduledTaskRunMode: task.runMode,
6284
- ...task.agentConfig.goal ? { scheduledTaskGoal: task.agentConfig.goal } : {},
6285
- scheduledTaskRunId: canonicalGeneratedRunId,
6286
- ...frozenSlack ? { [OPENGENI_SLACK_BOT_SESSION_METADATA_KEY]: frozenSlack.id } : {}
6287
- };
6288
- if (session.createIdempotencyKey !== expectedCreateKey || session.initialMessage !== task.agentConfig.prompt || session.instructions !== null || session.policyRole !== null || stableJson(session.skills) !== "[]" || stableJson(session.toolPolicy) !== stableJson({ mode: "explicit", inheritedFromSessionId: null }) || session.toolPolicyVersion !== 1 || session.mcpServers.length !== 0 || session.parentSessionId !== null || session.rootSessionId !== session.id || session.nestedAgentDepth !== 0 || session.effectiveMaxNestedAgentDepth !== generatedBinding.effectiveMaxNestedAgentDepth || session.nestedAgentDepthPolicySource !== generatedBinding.nestedAgentDepthPolicySource || session.nestedAgentDepthPolicySessionId !== (generatedBinding.nestedAgentDepthPolicySource === "session" ? session.id : null) || session.sandboxGroupId !== session.id || session.channelId !== null || session.codexCompactionMode !== generatedBinding.codexCompactionMode || stableJson(session.metadata) !== stableJson(expectedMetadata) || session.createdBy.kind !== "service" || session.createdBy.subjectId !== "scheduler" || stableJson(session.createdByContext) !== stableJson({
7583
+ const expectedMetadata = metadataWithAgentConfigCreateIdentity(
7584
+ {
7585
+ ...expectedTaskMetadata,
7586
+ ...scheduledSessionExecutionPolicyMetadata(
7587
+ input.acceptedExecution.turnExecutionPolicy,
7588
+ recoveredCreatorPolicy?.credentialRestriction
7589
+ ),
7590
+ model: input.acceptedExecution.resolvedModel,
7591
+ reasoningEffort: input.acceptedExecution.resolvedReasoningEffort,
7592
+ scheduledTaskId: task.id,
7593
+ scheduledTaskRunMode: task.runMode,
7594
+ ...task.agentConfig.goal ? { scheduledTaskGoal: task.agentConfig.goal } : {},
7595
+ scheduledTaskRunId: canonicalGeneratedRunId,
7596
+ ...frozenSlack ? { [OPENGENI_SLACK_BOT_SESSION_METADATA_KEY]: frozenSlack.id } : {}
7597
+ },
7598
+ input.acceptedExecution.resolvedAgentConfig
7599
+ );
7600
+ if (session.createIdempotencyKey !== expectedCreateKey || session.initialMessage !== task.agentConfig.prompt || session.instructions !== (input.acceptedExecution.resolvedAgentInstructions ?? null) || session.policyRole !== null || stableJson(session.skills) !== "[]" || stableJson(session.toolPolicy) !== stableJson({ mode: "explicit", inheritedFromSessionId: null }) || session.toolPolicyVersion !== 1 || session.mcpServers.length !== 0 || session.parentSessionId !== null || session.rootSessionId !== session.id || session.nestedAgentDepth !== 0 || session.effectiveMaxNestedAgentDepth !== generatedBinding.effectiveMaxNestedAgentDepth || session.nestedAgentDepthPolicySource !== generatedBinding.nestedAgentDepthPolicySource || session.nestedAgentDepthPolicySessionId !== (generatedBinding.nestedAgentDepthPolicySource === "session" ? session.id : null) || session.sandboxGroupId !== session.id || session.channelId !== null || session.codexCompactionMode !== generatedBinding.codexCompactionMode || stableJson(session.metadata) !== stableJson(expectedMetadata) || session.createdBy.kind !== "service" || session.createdBy.subjectId !== "scheduler" || stableJson(session.createdByContext) !== stableJson({
6289
7601
  label: "OpenGeni scheduler",
6290
7602
  scheduledTaskId: task.id,
6291
7603
  scheduledTaskRunId: canonicalGeneratedRunId
6292
- }) || task.runMode === "new_session_per_run" && input.acceptedExecution.alertOccurrenceLabels === null && (session.metadata.scheduledTaskRunId !== input.run.id || session.createdByContext.scheduledTaskRunId !== input.run.id) || creationPolicy?.model !== input.acceptedExecution.resolvedModel || creationPolicy.reasoningEffort !== input.acceptedExecution.resolvedReasoningEffort || creationPolicy.latencyMode !== input.acceptedExecution.resolvedLatencyMode || session.sandboxBackend !== input.acceptedExecution.resolvedSandboxBackend || session.sandboxOs !== input.acceptedExecution.resolvedSandboxOs || session.activeSandboxId !== (task.agentConfig.machineTarget?.targetSandboxId ?? null) || stableJson(session.resources) !== stableJson(task.agentConfig.resources) || stableJson(session.tools) !== stableJson(input.acceptedExecution.resolvedTools) || stableJson(session.firstPartyMcpTools) !== stableJson(input.acceptedExecution.resolvedFirstPartyMcpTools) || stableJson(session.firstPartyMcpPermissions) !== stableJson(input.acceptedExecution.resolvedFirstPartyMcpPermissions) || session.maxNestedAgentDepthOverride !== (task.agentConfig.maxNestedAgentDepth ?? null) || (session.variableSetId ?? null) !== (input.acceptedExecution.resolvedVariableSet?.id ?? null) || (session.rigId ?? null) !== (input.acceptedExecution.resolvedRig?.id ?? null) || (session.rigVersionId ?? null) !== (input.acceptedExecution.resolvedRig?.versionId ?? null) || scheduledSlackBotConnectionId(session.metadata) !== (frozenSlack?.id ?? null)) {
7604
+ }) || task.runMode === "new_session_per_run" && input.acceptedExecution.alertOccurrenceLabels === null && (session.metadata.scheduledTaskRunId !== input.run.id || session.createdByContext.scheduledTaskRunId !== input.run.id) || creationPolicy?.model !== input.acceptedExecution.resolvedModel || creationPolicy.reasoningEffort !== input.acceptedExecution.resolvedReasoningEffort || creationPolicy.latencyMode !== input.acceptedExecution.resolvedLatencyMode || session.sandboxBackend !== input.acceptedExecution.resolvedSandboxBackend || session.sandboxOs !== input.acceptedExecution.resolvedSandboxOs || session.activeSandboxId !== (task.agentConfig.machineTarget?.targetSandboxId ?? null) || stableJson(session.resources) !== stableJson(task.agentConfig.resources) || stableJson(session.tools) !== stableJson(input.acceptedExecution.resolvedTools) || stableJson(session.firstPartyMcpTools) !== stableJson(input.acceptedExecution.resolvedFirstPartyMcpTools) || stableJson(session.firstPartyMcpPermissions) !== stableJson(input.acceptedExecution.resolvedFirstPartyMcpPermissions) || stableJson(session.agent) !== stableJson(input.acceptedExecution.resolvedAgentConfig ?? null) || session.maxNestedAgentDepthOverride !== (task.agentConfig.maxNestedAgentDepth ?? null) || (session.variableSetId ?? null) !== (input.acceptedExecution.resolvedVariableSet?.id ?? null) || (session.rigId ?? null) !== (input.acceptedExecution.resolvedRig?.id ?? null) || (session.rigVersionId ?? null) !== (input.acceptedExecution.resolvedRig?.versionId ?? null) || scheduledSlackBotConnectionId(session.metadata) !== (frozenSlack?.id ?? null)) {
6293
7605
  throw new ScheduledRunTerminalAuthorityError(
6294
7606
  "scheduled_generated_session_changed",
6295
7607
  "scheduled generated session differs from accepted execution"
@@ -6320,7 +7632,7 @@ async function recoverBoundScheduledTaskDispatch(input) {
6320
7632
  });
6321
7633
  workflowId = started.temporalWorkflowId;
6322
7634
  if (started.events.length > 0) {
6323
- await publishDurableSessionEvents(input.bus, task.workspaceId, session.id, started.events);
7635
+ await publishDurableSessionEvents2(input.bus, task.workspaceId, session.id, started.events);
6324
7636
  }
6325
7637
  if (task.runMode === "reusable_session") {
6326
7638
  await materializeScheduledTaskReusableSessionFromRun(input.db, {
@@ -6339,7 +7651,7 @@ async function recoverBoundScheduledTaskDispatch(input) {
6339
7651
  taskName: task.name
6340
7652
  });
6341
7653
  if (titleEvents.length > 0) {
6342
- await publishDurableSessionEvents(input.bus, task.workspaceId, session.id, titleEvents);
7654
+ await publishDurableSessionEvents2(input.bus, task.workspaceId, session.id, titleEvents);
6343
7655
  }
6344
7656
  }
6345
7657
  const scheduledUpdate = await addSessionSystemUpdateWithSourceMutation(
@@ -6367,10 +7679,13 @@ async function recoverBoundScheduledTaskDispatch(input) {
6367
7679
  ...input.acceptedExecution.connectionAuthoritySubjectId ? {
6368
7680
  connectionAuthoritySubjectId: input.acceptedExecution.connectionAuthoritySubjectId
6369
7681
  } : {},
6370
- ...input.acceptedExecution.xaiAuthoritySubjectId ? { xaiAuthoritySubjectId: input.acceptedExecution.xaiAuthoritySubjectId } : {}
7682
+ ...input.acceptedExecution.xaiAuthoritySubjectId ? { xaiAuthoritySubjectId: input.acceptedExecution.xaiAuthoritySubjectId } : {},
7683
+ ...input.acceptedExecution.claudeAuthoritySubjectId ? { claudeAuthoritySubjectId: input.acceptedExecution.claudeAuthoritySubjectId } : {}
6371
7684
  },
6372
7685
  personalConnectionDelegations: input.acceptedExecution.personalConnectionDelegations,
7686
+ mcpAccountBindings: input.acceptedExecution.mcpAccountBindings ?? null,
6373
7687
  xaiProviderAccountAuthoritySnapshot: input.acceptedExecution.xaiProviderAccountAuthoritySnapshot,
7688
+ claudeProviderAccountAuthoritySnapshot: input.acceptedExecution.claudeProviderAccountAuthoritySnapshot,
6374
7689
  scheduledTaskRunId: input.run.id
6375
7690
  },
6376
7691
  async (tx, wakeEventId, _updateId, rejectionReason) => {
@@ -6427,8 +7742,11 @@ async function recoverBoundScheduledTaskDispatch(input) {
6427
7742
  if (scheduledUpdate.reason === "session_cancelled" || scheduledUpdate.reason === "session_not_idle") {
6428
7743
  return { action: "blocked", reason: "scheduled_run_terminal" };
6429
7744
  }
7745
+ if (scheduledUpdate.added) {
7746
+ recordUserMessageAccepted(input.observability, { surface: "scheduled" });
7747
+ }
6430
7748
  if (scheduledUpdate.added && scheduledUpdate.events.length > 0) {
6431
- await publishDurableSessionEvents(
7749
+ await publishDurableSessionEvents2(
6432
7750
  input.bus,
6433
7751
  task.workspaceId,
6434
7752
  session.id,
@@ -6680,6 +7998,9 @@ async function dispatchMaintenanceSession(service, maintenance, dependencies) {
6680
7998
  accountId: maintenance.accountId,
6681
7999
  workspaceId: maintenance.workspaceId,
6682
8000
  initialMessage: maintenancePrompt(maintenance),
8001
+ // OpenGeni's own periodic check: no human or product surface entered it.
8002
+ surface: "system",
8003
+ metrics: service.observability,
6683
8004
  resources: [],
6684
8005
  tools: [{ kind: "mcp", id: "opengeni" }],
6685
8006
  toolPolicy: { mode: "explicit", inheritedFromSessionId: null },
@@ -6766,43 +8087,55 @@ import {
6766
8087
  applySessionTurnSettlement,
6767
8088
  enqueueSessionWorkflowWake,
6768
8089
  failSessionWorkBeforeAttemptClaim,
8090
+ blockSessionWorkBeforeAttemptClaim,
6769
8091
  requestSessionTurnRecovery,
6770
8092
  recoverSessionDispatch,
8093
+ reconcileSettledSessionAttempt,
6771
8094
  reconcileSessionAttemptQuiescence,
8095
+ reconcileCompletedSandboxSetup,
6772
8096
  peekSessionWork as peekSessionWorkDb,
6773
8097
  settleSessionInputWait as settleSessionInputWaitDb,
6774
8098
  countQueuedTurns as countQueuedTurns2,
6775
8099
  getSessionAttemptActivityRef,
6776
8100
  getSessionEvent,
6777
8101
  getSessionTurnForAttempt,
8102
+ getWorkspace,
6778
8103
  expireSessionInteractionIntervention as expireSessionInteractionInterventionDb,
8104
+ expireScheduledRunHumanWait as expireScheduledRunHumanWaitDb,
6779
8105
  expireSessionHumanInputRequest,
6780
8106
  markSessionAttemptQuiesced,
6781
8107
  requireSession as requireSession2,
6782
- settleSessionIdleWithParentOutbox
8108
+ settleSessionIdleWithParentOutbox,
8109
+ SANDBOX_SETUP_RECOVERY_LIMIT
6783
8110
  } from "@opengeni/db";
6784
- import { publishDurableSessionEvents as publishDurableSessionEvents2 } from "@opengeni/events";
8111
+ import { publishDurableSessionEvents as publishDurableSessionEvents3 } from "@opengeni/events";
8112
+ import { CancelledFailure } from "@temporalio/activity";
6785
8113
  var WORKER_DEATH_MAX_REDISPATCHES = 3;
6786
8114
  function createSessionStateActivities(services, overrides = {}) {
6787
8115
  const settleSessionAttemptInterruptionsFn = overrides.settleSessionAttemptInterruptions ?? settleSessionAttemptInterruptions;
6788
8116
  const applySessionTurnSettlementFn = overrides.applySessionTurnSettlement ?? applySessionTurnSettlement;
6789
8117
  const enqueueSessionWorkflowWakeFn = overrides.enqueueSessionWorkflowWake ?? enqueueSessionWorkflowWake;
6790
8118
  const failSessionWorkBeforeAttemptClaimFn = overrides.failSessionWorkBeforeAttemptClaim ?? failSessionWorkBeforeAttemptClaim;
8119
+ const blockSessionWorkBeforeAttemptClaimFn = overrides.blockSessionWorkBeforeAttemptClaim ?? blockSessionWorkBeforeAttemptClaim;
6791
8120
  const requestSessionTurnRecoveryFn = overrides.requestSessionTurnRecovery ?? requestSessionTurnRecovery;
6792
8121
  const recoverSessionDispatchFn = overrides.recoverSessionDispatch ?? recoverSessionDispatch;
8122
+ const reconcileSettledSessionAttemptFn = overrides.reconcileSettledSessionAttempt ?? reconcileSettledSessionAttempt;
6793
8123
  const reconcileSessionAttemptQuiescenceFn = overrides.reconcileSessionAttemptQuiescence ?? reconcileSessionAttemptQuiescence;
6794
8124
  const peekSessionWorkFn = overrides.peekSessionWork ?? peekSessionWorkDb;
8125
+ const reconcileCompletedSandboxSetupFn = overrides.reconcileCompletedSandboxSetup ?? reconcileCompletedSandboxSetup;
6795
8126
  const settleSessionInputWaitFn = overrides.settleSessionInputWait ?? settleSessionInputWaitDb;
6796
8127
  const countQueuedTurnsFn = overrides.countQueuedTurns ?? countQueuedTurns2;
6797
8128
  const getSessionAttemptActivityRefFn = overrides.getSessionAttemptActivityRef ?? getSessionAttemptActivityRef;
6798
8129
  const getSessionEventFn = overrides.getSessionEvent ?? getSessionEvent;
6799
8130
  const getSessionTurnForAttemptFn = overrides.getSessionTurnForAttempt ?? getSessionTurnForAttempt;
8131
+ const getWorkspaceFn = overrides.getWorkspace ?? getWorkspace;
6800
8132
  const expireSessionHumanInputRequestFn = overrides.expireSessionHumanInputRequest ?? expireSessionHumanInputRequest;
6801
8133
  const expireSessionInteractionInterventionFn = overrides.expireSessionInteractionIntervention ?? expireSessionInteractionInterventionDb;
8134
+ const expireScheduledRunHumanWaitFn = overrides.expireScheduledRunHumanWait ?? expireScheduledRunHumanWaitDb;
6802
8135
  const requireSessionFn = overrides.requireSession ?? requireSession2;
6803
8136
  const settleSessionIdleWithParentOutboxFn = overrides.settleSessionIdleWithParentOutbox ?? settleSessionIdleWithParentOutbox;
6804
8137
  const markSessionAttemptQuiescedFn = overrides.markSessionAttemptQuiesced ?? markSessionAttemptQuiesced;
6805
- const publishDurableSessionEventsFn = overrides.publishDurableSessionEvents ?? publishDurableSessionEvents2;
8138
+ const publishDurableSessionEventsFn = overrides.publishDurableSessionEvents ?? publishDurableSessionEvents3;
6806
8139
  const deliverFailedChildTurnToParentFn = overrides.deliverFailedChildTurnToParent ?? deliverFailedChildTurnToParent;
6807
8140
  const notifyParentOfChildIdleFn = overrides.notifyParentOfChildIdle ?? notifyParentOfChildIdle;
6808
8141
  const recordTurnsQueuedGaugeFn = overrides.recordTurnsQueuedGauge ?? recordTurnsQueuedGauge;
@@ -6830,6 +8163,24 @@ function createSessionStateActivities(services, overrides = {}) {
6830
8163
  });
6831
8164
  if (attempt) return { action: "stale" };
6832
8165
  const preClaimFailureDisposition = input.preClaimFailure?.disposition ?? input.preClaimFailureDisposition;
8166
+ if (preClaimFailureDisposition === "blocked" && input.admissionFence && input.preClaimFailure?.reason) {
8167
+ const blocked = await blockSessionWorkBeforeAttemptClaimFn(db, input.workspaceId, {
8168
+ accountId: input.accountId,
8169
+ sessionId: input.sessionId,
8170
+ workflowId,
8171
+ attemptId: input.attemptId,
8172
+ fence: input.admissionFence,
8173
+ reason: input.preClaimFailure.reason,
8174
+ sqlState: input.preClaimFailure.sqlState ?? null
8175
+ });
8176
+ await publishDurableSessionEventsFn(
8177
+ bus,
8178
+ input.workspaceId,
8179
+ input.sessionId,
8180
+ blocked.events
8181
+ );
8182
+ return { action: blocked.action };
8183
+ }
6833
8184
  if (preClaimFailureDisposition === "permanent" && input.trigger) {
6834
8185
  const failed = await failSessionWorkBeforeAttemptClaimFn(db, input.workspaceId, {
6835
8186
  accountId: input.accountId,
@@ -6837,7 +8188,12 @@ function createSessionStateActivities(services, overrides = {}) {
6837
8188
  workflowId,
6838
8189
  trigger: input.trigger,
6839
8190
  error: input.error ?? "Agent turn admission failed before attempt claim.",
6840
- ...input.preClaimFailure ? { admissionFailure: input.preClaimFailure } : {}
8191
+ ...input.preClaimFailure?.disposition === "permanent" ? {
8192
+ admissionFailure: {
8193
+ disposition: "permanent",
8194
+ code: input.preClaimFailure.code
8195
+ }
8196
+ } : {}
6841
8197
  });
6842
8198
  if (failed.action === "terminal") return { action: "terminal" };
6843
8199
  if (failed.action === "stale") return { action: "stale" };
@@ -6868,10 +8224,18 @@ function createSessionStateActivities(services, overrides = {}) {
6868
8224
  const postClaimIdentityMatches = Boolean(
6869
8225
  postClaimRecovery && turn.id === postClaimRecovery.turnId && turn.triggerEventId === postClaimRecovery.triggerEventId && turn.executionGeneration === postClaimRecovery.executionGeneration
6870
8226
  );
8227
+ if (postClaimRecovery?.sandboxSetupRecoveryExhausted && !postClaimIdentityMatches) {
8228
+ return { action: "stale" };
8229
+ }
6871
8230
  const providerRecoveryCount = postClaimIdentityMatches ? postClaimRecovery?.providerRecoveryCount : void 0;
6872
8231
  const providerFailureCode = postClaimIdentityMatches ? postClaimRecovery?.providerFailureCode : void 0;
6873
8232
  const hasProviderRecoveryCount = providerRecoveryCount !== void 0;
6874
8233
  const hasProviderFailureCode = providerFailureCode !== void 0;
8234
+ const setupOutcomeUnknown = postClaimIdentityMatches && postClaimRecovery?.sandboxSetupOutcomeUnknown === true;
8235
+ const setupRecoveryExhausted = postClaimIdentityMatches && postClaimRecovery?.sandboxSetupRecoveryExhausted === true;
8236
+ if (setupOutcomeUnknown && setupRecoveryExhausted || (setupOutcomeUnknown || setupRecoveryExhausted) && (hasProviderRecoveryCount || hasProviderFailureCode) || setupRecoveryExhausted && turn.metadata?.providerRecoveryCount !== SANDBOX_SETUP_RECOVERY_LIMIT) {
8237
+ return { action: "stale" };
8238
+ }
6875
8239
  if (hasProviderRecoveryCount !== hasProviderFailureCode) {
6876
8240
  return { action: "stale" };
6877
8241
  }
@@ -6885,11 +8249,20 @@ function createSessionStateActivities(services, overrides = {}) {
6885
8249
  turnId: turn.id,
6886
8250
  triggerEventId: turn.triggerEventId,
6887
8251
  attemptId: input.attemptId,
6888
- reason: providerFailureCode ?? "claimed_attempt_database_failure",
8252
+ reason: setupOutcomeUnknown ? "sandbox_command_start_outcome_unknown" : setupRecoveryExhausted ? "sandbox_command_start_recovery_exhausted" : providerFailureCode ?? "claimed_attempt_database_failure",
8253
+ ...setupOutcomeUnknown ? { sandboxSetupOutcomeUnknown: true } : {},
8254
+ ...setupRecoveryExhausted ? { sandboxSetupRecoveryExhausted: true } : {},
6889
8255
  ...providerRecoveryCount !== void 0 ? { providerRecoveryCount } : {},
6890
8256
  detail: {
6891
- code: providerFailureCode ?? recoveredClaimCode,
6892
- retryable: true,
8257
+ code: setupOutcomeUnknown ? "sandbox_command_start_outcome_unknown" : setupRecoveryExhausted ? "sandbox_command_start_recovery_exhausted" : providerFailureCode ?? recoveredClaimCode,
8258
+ retryable: !setupOutcomeUnknown && !setupRecoveryExhausted,
8259
+ ...setupOutcomeUnknown ? { setupOutcome: "unknown", replay: "blocked" } : {},
8260
+ ...setupRecoveryExhausted ? {
8261
+ setupOutcome: "not_started",
8262
+ replay: "blocked",
8263
+ recoveryExhausted: true,
8264
+ providerRecoveryCount: SANDBOX_SETUP_RECOVERY_LIMIT
8265
+ } : {},
6893
8266
  ...providerRecoveryCount !== void 0 ? {
6894
8267
  databaseFailureCode: recoveredClaimCode,
6895
8268
  providerRecoveryCount
@@ -7019,6 +8392,54 @@ function createSessionStateActivities(services, overrides = {}) {
7019
8392
  }
7020
8393
  return { action: result.action };
7021
8394
  }
8395
+ async function reconcileSettledSessionAttemptActivity(input) {
8396
+ const { db, bus, inspectSessionAttemptActivity, settings, observability, wakeSessionWorkflow } = await services();
8397
+ const ref = await getSessionAttemptActivityRefFn(db, {
8398
+ ...input,
8399
+ temporalWorkflowId: input.workflowId
8400
+ });
8401
+ if (!ref || ref.workflowRunId !== input.workflowRunId || ref.activityId !== input.activityId)
8402
+ return { action: "stale" };
8403
+ if (!inspectSessionAttemptActivity) return { action: "pending" };
8404
+ try {
8405
+ if (await inspectSessionAttemptActivity(ref) !== "settled") return { action: "pending" };
8406
+ } catch (error) {
8407
+ if (error instanceof CancelledFailure) throw error;
8408
+ if (currentActivityContext()?.cancellationSignal.aborted)
8409
+ throw new CancelledFailure("Settled-owner reconciliation cancelled");
8410
+ return { action: "pending" };
8411
+ }
8412
+ if (currentActivityContext()?.cancellationSignal.aborted)
8413
+ throw new CancelledFailure("Settled-owner reconciliation cancelled");
8414
+ const result = await reconcileSettledSessionAttemptFn(db, {
8415
+ accountId: input.accountId,
8416
+ workspaceId: input.workspaceId,
8417
+ sessionId: input.sessionId,
8418
+ turnId: input.turnId,
8419
+ attemptId: input.attemptId,
8420
+ executionGeneration: input.executionGeneration,
8421
+ temporalWorkflowId: ref.workflowId,
8422
+ temporalWorkflowRunId: ref.workflowRunId,
8423
+ temporalActivityId: ref.activityId,
8424
+ activitySettled: true,
8425
+ maxRedispatches: WORKER_DEATH_MAX_REDISPATCHES
8426
+ });
8427
+ if (result.events.length > 0)
8428
+ await publishDurableSessionEventsFn(
8429
+ bus,
8430
+ input.workspaceId,
8431
+ input.sessionId,
8432
+ result.events
8433
+ ).catch(() => void 0);
8434
+ if (result.action === "exceeded")
8435
+ await deliverFailedChildTurnToParentFn(
8436
+ { db, bus, settings, observability, wakeSessionWorkflow },
8437
+ input.workspaceId,
8438
+ input.sessionId,
8439
+ result.turnId
8440
+ );
8441
+ return { action: result.action };
8442
+ }
7022
8443
  async function recoverDispatch(input) {
7023
8444
  const { settings, db, bus, observability, wakeSessionWorkflow } = await services();
7024
8445
  const result = await recoverSessionDispatchFn(db, input.workspaceId, {
@@ -7091,8 +8512,68 @@ function createSessionStateActivities(services, overrides = {}) {
7091
8512
  return { action: "recovering" };
7092
8513
  }
7093
8514
  async function peekSessionWork(input) {
7094
- const { db, observability } = await services();
7095
- const peek = await peekSessionWorkFn(db, input.workspaceId, input.sessionId);
8515
+ const { db, bus, observability, inspectSessionAttemptActivity } = await services();
8516
+ const observerAccountId = input.observerAccountId ?? (await getWorkspaceFn(db, input.workspaceId))?.accountId;
8517
+ if (!observerAccountId) return { kind: "unavailable" };
8518
+ let peek = await peekSessionWorkFn(
8519
+ db,
8520
+ input.workspaceId,
8521
+ input.sessionId,
8522
+ input.includeAdmissionFence,
8523
+ observerAccountId
8524
+ );
8525
+ if (peek.kind === "unavailable") return peek;
8526
+ if (peek.kind === "admission-blocked" && peek.reason === "sandbox_setup_outcome_unknown") {
8527
+ const ref = peek.ref;
8528
+ if (ref && "turnId" in ref && "attemptId" in ref) {
8529
+ const settled = await reconcileCompletedSandboxSetupFn(db, {
8530
+ accountId: observerAccountId,
8531
+ workspaceId: input.workspaceId,
8532
+ sessionId: input.sessionId,
8533
+ turnId: ref.turnId,
8534
+ attemptId: ref.attemptId
8535
+ });
8536
+ if (settled.events.length > 0)
8537
+ await publishDurableSessionEventsFn(
8538
+ bus,
8539
+ input.workspaceId,
8540
+ input.sessionId,
8541
+ settled.events
8542
+ );
8543
+ if (settled.reconciled)
8544
+ peek = await peekSessionWorkFn(
8545
+ db,
8546
+ input.workspaceId,
8547
+ input.sessionId,
8548
+ input.includeAdmissionFence,
8549
+ observerAccountId
8550
+ );
8551
+ }
8552
+ }
8553
+ if (peek.kind === "attempt-owned") {
8554
+ let ownerActivityState = "unknown";
8555
+ if (inspectSessionAttemptActivity) {
8556
+ try {
8557
+ ownerActivityState = await inspectSessionAttemptActivity(peek.activityRef);
8558
+ } catch (error) {
8559
+ if (error instanceof CancelledFailure) throw error;
8560
+ if (currentActivityContext()?.cancellationSignal.aborted)
8561
+ throw new CancelledFailure("Control observation cancelled");
8562
+ }
8563
+ }
8564
+ if (currentActivityContext()?.cancellationSignal.aborted)
8565
+ throw new CancelledFailure("Control observation cancelled");
8566
+ const current = await peekSessionWorkFn(
8567
+ db,
8568
+ input.workspaceId,
8569
+ input.sessionId,
8570
+ input.includeAdmissionFence,
8571
+ observerAccountId
8572
+ );
8573
+ if (current.kind !== "attempt-owned" || current.turnId !== peek.turnId || current.attemptId !== peek.attemptId || current.executionGeneration !== peek.executionGeneration || current.activityRef.workflowId !== peek.activityRef.workflowId || current.activityRef.workflowRunId !== peek.activityRef.workflowRunId || current.activityRef.activityId !== peek.activityRef.activityId)
8574
+ return current;
8575
+ return { ...current, ownerActivityState };
8576
+ }
7096
8577
  await refreshQueuedTurnsGauge(db, observability, countQueuedTurnsFn, recordTurnsQueuedGaugeFn);
7097
8578
  return peek;
7098
8579
  }
@@ -7123,6 +8604,14 @@ function createSessionStateActivities(services, overrides = {}) {
7123
8604
  }
7124
8605
  return { action: result.action };
7125
8606
  }
8607
+ async function expireScheduledRunHumanWait(input) {
8608
+ const { db, bus } = await services();
8609
+ const result = await expireScheduledRunHumanWaitFn(db, input);
8610
+ if (result.events.length > 0) {
8611
+ await publishDurableSessionEventsFn(bus, input.workspaceId, input.sessionId, result.events);
8612
+ }
8613
+ return { action: result.action };
8614
+ }
7126
8615
  async function markSessionIdle(input) {
7127
8616
  const { db, bus, settings, observability, wakeSessionWorkflow } = await services();
7128
8617
  const settled = await settleSessionIdleWithParentOutboxFn(
@@ -7149,12 +8638,14 @@ function createSessionStateActivities(services, overrides = {}) {
7149
8638
  settleSessionInterruptions,
7150
8639
  persistSessionAttemptQuiescence,
7151
8640
  reconcileSessionAttemptQuiescence: reconcileSessionAttemptQuiescenceActivity,
8641
+ reconcileSettledSessionAttempt: reconcileSettledSessionAttemptActivity,
7152
8642
  recoverDispatch,
7153
8643
  recoverEscapedMcpTimeout,
7154
8644
  peekSessionWork,
7155
8645
  settleSessionInputWait,
7156
8646
  expireSessionHumanInput,
7157
8647
  expireSessionInteractionIntervention,
8648
+ expireScheduledRunHumanWait,
7158
8649
  markSessionIdle
7159
8650
  };
7160
8651
  }
@@ -7173,6 +8664,7 @@ import {
7173
8664
  getWorkspaceControlEvent
7174
8665
  } from "@opengeni/db";
7175
8666
  import { publishDurableWorkspaceControlEvent } from "@opengeni/events";
8667
+ import { recordWorkflowWakeReconciliation } from "@opengeni/observability";
7176
8668
  var BATCH_SIZE = 1e3;
7177
8669
  var MAX_DELIVERIES_PER_ACTIVITY = 1e4;
7178
8670
  var MAX_TITLE_FANOUT_PER_ACTIVITY = 1e3;
@@ -7258,23 +8750,7 @@ function createWorkflowWakeActivities(services) {
7258
8750
  pendingAdmissionBlockers,
7259
8751
  exhaustedBatchLimit
7260
8752
  };
7261
- if (claimed > 0) {
7262
- service.observability.info("session workflow wake delivery reconciled", {
7263
- claimed,
7264
- signaled,
7265
- delivered,
7266
- pendingAdmission,
7267
- unconfirmed,
7268
- failed,
7269
- exhaustedBatchLimit,
7270
- ...Object.fromEntries(
7271
- Object.entries(pendingAdmissionBlockers).map(([blocker, count]) => [
7272
- `pendingAdmissionBlockers.${blocker}`,
7273
- count
7274
- ])
7275
- )
7276
- });
7277
- }
8753
+ recordWorkflowWakeReconciliation(service.observability, result);
7278
8754
  return result;
7279
8755
  }
7280
8756
  };
@@ -7298,7 +8774,9 @@ function createControlActivitiesFromServices(services, resolveDocumentServices)
7298
8774
  ...createCodexCapacityActivities(services),
7299
8775
  ...createRigVerificationActivities(services),
7300
8776
  ...createBrowserStateArtifactMaintenanceActivities(services),
8777
+ ...createBrowserDeadlineCheckpointActivities(services),
7301
8778
  ...createFileUploadReaperActivities(services),
8779
+ ...createModelCallFactReconcilerActivities(services),
7302
8780
  ...createRetainedScreenshotMaintenanceActivities(services),
7303
8781
  ...createWorkflowWakeActivities(services),
7304
8782
  ...createSandboxLeaseActivities(services)
@@ -7319,4 +8797,4 @@ export {
7319
8797
  createControlActivities,
7320
8798
  createControlActivitiesFromServices
7321
8799
  };
7322
- //# sourceMappingURL=activities-control-HMFGSKC5.js.map
8800
+ //# sourceMappingURL=activities-control-BNPTRTOE.js.map