@opengeni/worker-bundle 1.0.0-canary.0 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (244) hide show
  1. package/dist/activities/agent-run-admission.d.ts +3 -1
  2. package/dist/activities/agent-turn/admission.d.ts +24 -3
  3. package/dist/activities/agent-turn/agent-build.d.ts +4 -0
  4. package/dist/activities/agent-turn/claude-usage-observer.d.ts +58 -0
  5. package/dist/activities/agent-turn/code-search.d.ts +60 -0
  6. package/dist/activities/agent-turn/codex-capacity.d.ts +13 -0
  7. package/dist/activities/agent-turn/codex-plan-entitlement.d.ts +105 -0
  8. package/dist/activities/agent-turn/codex.d.ts +3 -2
  9. package/dist/activities/agent-turn/compaction-prep.d.ts +21 -5
  10. package/dist/activities/agent-turn/credential-leases.d.ts +19 -33
  11. package/dist/activities/agent-turn/credential-restriction.d.ts +3 -0
  12. package/dist/activities/agent-turn/errors.d.ts +82 -11
  13. package/dist/activities/agent-turn/failure-settlement.d.ts +6 -3
  14. package/dist/activities/agent-turn/final-reply.d.ts +18 -0
  15. package/dist/activities/agent-turn/finalization-monitor.d.ts +17 -0
  16. package/dist/activities/agent-turn/finalization.d.ts +4 -2
  17. package/dist/activities/agent-turn/github-repository-bindings.d.ts +2 -0
  18. package/dist/activities/agent-turn/governance-model.d.ts +9 -2
  19. package/dist/activities/agent-turn/history-sink.d.ts +2 -0
  20. package/dist/activities/agent-turn/history.d.ts +2 -0
  21. package/dist/activities/agent-turn/input-wait-reply.d.ts +41 -0
  22. package/dist/activities/agent-turn/media-artifacts.d.ts +5 -3
  23. package/dist/activities/agent-turn/model-call-admission.d.ts +15 -0
  24. package/dist/activities/agent-turn/model-usage.d.ts +6 -0
  25. package/dist/activities/agent-turn/model-wire-dispatch.d.ts +15 -0
  26. package/dist/activities/agent-turn/optional-repositories.d.ts +59 -0
  27. package/dist/activities/agent-turn/provider-dispatch-barrier.d.ts +12 -1
  28. package/dist/activities/agent-turn/quiescence.d.ts +7 -7
  29. package/dist/activities/agent-turn/recovery-warning.d.ts +9 -0
  30. package/dist/activities/agent-turn/refresh-credentials.d.ts +6 -0
  31. package/dist/activities/agent-turn/run-credentials.d.ts +8 -3
  32. package/dist/activities/agent-turn/sandbox-establish.d.ts +2 -1
  33. package/dist/activities/agent-turn/sandbox-provision.d.ts +3 -1
  34. package/dist/activities/agent-turn/sandbox-route.d.ts +3 -20
  35. package/dist/activities/agent-turn/sandbox-runtime.d.ts +2 -2
  36. package/dist/activities/agent-turn/session-title.d.ts +34 -2
  37. package/dist/activities/agent-turn/skill-checkout.d.ts +25 -6
  38. package/dist/activities/agent-turn/skill-read.d.ts +70 -2
  39. package/dist/activities/agent-turn/skill-remove.d.ts +14 -0
  40. package/dist/activities/agent-turn/skill-search.d.ts +3 -1
  41. package/dist/activities/agent-turn/skill-tools.d.ts +15 -2
  42. package/dist/activities/agent-turn/skill-transfer.d.ts +18 -6
  43. package/dist/activities/agent-turn/stream-attempt.d.ts +7 -3
  44. package/dist/activities/agent-turn/subscription-lease.d.ts +38 -0
  45. package/dist/activities/agent-turn/tool-environment.d.ts +56 -6
  46. package/dist/activities/agent-turn/tool-policy.d.ts +43 -0
  47. package/dist/activities/agent-turn/turn-context.d.ts +29 -0
  48. package/dist/activities/agent-turn/turn-sandbox-access.d.ts +3 -0
  49. package/dist/activities/agent-turn/xai-capacity.d.ts +2 -1
  50. package/dist/activities/automations.d.ts +3 -1
  51. package/dist/activities/browser-deadline-checkpoint.d.ts +29 -0
  52. package/dist/activities/codemode-dispatcher.d.ts +9 -2
  53. package/dist/activities/codex-capacity.d.ts +18 -2
  54. package/dist/activities/codex-rotation.d.ts +7 -1
  55. package/dist/activities/context-compaction.d.ts +16 -2
  56. package/dist/activities/gmail-files.d.ts +32 -0
  57. package/dist/activities/goals.d.ts +8 -0
  58. package/dist/activities/knowledge-indexing.d.ts +19 -0
  59. package/dist/activities/mcp-account-routes.d.ts +3 -0
  60. package/dist/activities/mcp-credentials.d.ts +12 -13
  61. package/dist/activities/mcp-operation-authority.d.ts +0 -5
  62. package/dist/activities/model-call-fact-reconciler.d.ts +34 -0
  63. package/dist/activities/open-suffix-resume.d.ts +13 -0
  64. package/dist/activities/parent-wake.d.ts +6 -12
  65. package/dist/activities/retained-screenshots.d.ts +7 -2
  66. package/dist/activities/rig-verification.d.ts +2 -3
  67. package/dist/activities/run-credential-renewal.d.ts +1 -0
  68. package/dist/activities/run-credentials.d.ts +15 -3
  69. package/dist/activities/sandbox-images.d.ts +1 -1
  70. package/dist/activities/sandbox-lease.d.ts +47 -9
  71. package/dist/activities/scheduled-tasks.d.ts +4 -35
  72. package/dist/activities/session-state.d.ts +70 -3
  73. package/dist/activities/types.d.ts +55 -6
  74. package/dist/activities/workspace-capture.d.ts +2 -0
  75. package/dist/activities/workspace-credential-provider.d.ts +46 -0
  76. package/dist/{activities-control-HMFGSKC5.js → activities-control-BNPTRTOE.js} +1896 -418
  77. package/dist/activities-control-BNPTRTOE.js.map +1 -0
  78. package/dist/activities-control.d.ts +182 -4
  79. package/dist/{activities-turn-AP67JADS.js → activities-turn-5LGZOSVU.js} +4476 -1327
  80. package/dist/activities-turn-5LGZOSVU.js.map +1 -0
  81. package/dist/activities.d.ts +182 -4
  82. package/dist/artifact-materializer-entry.js +5 -2
  83. package/dist/artifact-materializer-entry.js.map +1 -1
  84. package/dist/artifact-outbox-entry.js +5 -2
  85. package/dist/artifact-outbox-entry.js.map +1 -1
  86. package/dist/browser-deadline-checkpoint-contract.d.ts +2 -0
  87. package/dist/{chunk-A4XQWVZK.js → chunk-3ZTRWDMN.js} +307 -4
  88. package/dist/chunk-3ZTRWDMN.js.map +1 -0
  89. package/dist/{chunk-RAMNYSAW.js → chunk-4ER3YXOT.js} +22 -4
  90. package/dist/chunk-4ER3YXOT.js.map +1 -0
  91. package/dist/{chunk-CYITU5UH.js → chunk-LJX7Z2KP.js} +1445 -343
  92. package/dist/chunk-LJX7Z2KP.js.map +1 -0
  93. package/dist/editable-artifact-materializer-service.d.ts +2 -2
  94. package/dist/editable-artifact-outbox-service.d.ts +2 -2
  95. package/dist/host-export-pump.d.ts +3 -1
  96. package/dist/host-export.d.ts +1 -1
  97. package/dist/host-export.js +1 -1
  98. package/dist/http.d.ts +1 -1
  99. package/dist/index.d.ts +95 -3
  100. package/dist/index.js +137 -43
  101. package/dist/index.js.map +1 -1
  102. package/dist/observability-metrics.d.ts +118 -8
  103. package/dist/retained-process-diagnostics.d.ts +4 -0
  104. package/dist/retained-process-retry.d.ts +7 -0
  105. package/dist/sandbox-resume.d.ts +55 -13
  106. package/dist/sandbox-routing.d.ts +5 -3
  107. package/dist/sandbox-snapshot-diagnostics.d.ts +20 -0
  108. package/dist/session-event-fanout.d.ts +6 -0
  109. package/dist/warm-capture-holder.d.ts +3 -0
  110. package/dist/worker-service-lifecycle.d.ts +12 -0
  111. package/dist/workflow-bundle.js +273 -41
  112. package/dist/workflows/activities.d.ts +1 -1
  113. package/dist/workflows/browser-deadline-checkpoint.d.ts +9 -0
  114. package/dist/workflows/scheduled-tasks.d.ts +4 -0
  115. package/dist/workflows/session.d.ts +36 -0
  116. package/dist/workflows.d.ts +1 -0
  117. package/package.json +21 -19
  118. package/src/activities/agent-run-admission.ts +17 -1
  119. package/src/activities/agent-turn/admission.ts +66 -3
  120. package/src/activities/agent-turn/agent-build.ts +84 -10
  121. package/src/activities/agent-turn/claim.ts +186 -43
  122. package/src/activities/agent-turn/claude-usage-observer.ts +221 -0
  123. package/src/activities/agent-turn/code-search.ts +275 -0
  124. package/src/activities/agent-turn/codex-capacity.ts +172 -8
  125. package/src/activities/agent-turn/codex-plan-entitlement.ts +254 -0
  126. package/src/activities/agent-turn/codex.ts +13 -5
  127. package/src/activities/agent-turn/compaction-prep.ts +170 -102
  128. package/src/activities/agent-turn/credential-leases.ts +122 -216
  129. package/src/activities/agent-turn/credential-restriction.ts +14 -0
  130. package/src/activities/agent-turn/errors.ts +647 -48
  131. package/src/activities/agent-turn/failure-settlement.ts +682 -130
  132. package/src/activities/agent-turn/final-reply.ts +46 -0
  133. package/src/activities/agent-turn/finalization-monitor.ts +120 -0
  134. package/src/activities/agent-turn/finalization.ts +146 -43
  135. package/src/activities/agent-turn/github-repository-bindings.ts +3 -1
  136. package/src/activities/agent-turn/governance-model.ts +69 -28
  137. package/src/activities/agent-turn/history-sink.ts +16 -2
  138. package/src/activities/agent-turn/history.ts +38 -11
  139. package/src/activities/agent-turn/input-wait-reply.ts +103 -0
  140. package/src/activities/agent-turn/knowledge-source-tools.ts +2 -0
  141. package/src/activities/agent-turn/media-artifacts.ts +2 -1
  142. package/src/activities/agent-turn/model-call-admission.ts +94 -0
  143. package/src/activities/agent-turn/model-usage.ts +64 -3
  144. package/src/activities/agent-turn/model-wire-dispatch.ts +27 -0
  145. package/src/activities/agent-turn/optional-repositories.ts +224 -0
  146. package/src/activities/agent-turn/provider-dispatch-barrier.ts +22 -0
  147. package/src/activities/agent-turn/quiescence.ts +10 -7
  148. package/src/activities/agent-turn/recovery-warning.ts +16 -0
  149. package/src/activities/agent-turn/refresh-credentials.ts +44 -0
  150. package/src/activities/agent-turn/run-credentials.ts +116 -55
  151. package/src/activities/agent-turn/run.ts +320 -93
  152. package/src/activities/agent-turn/sandbox-establish.ts +84 -8
  153. package/src/activities/agent-turn/sandbox-provision.ts +3 -1
  154. package/src/activities/agent-turn/sandbox-route.ts +4 -21
  155. package/src/activities/agent-turn/sandbox-runtime.ts +343 -18
  156. package/src/activities/agent-turn/session-title.ts +81 -5
  157. package/src/activities/agent-turn/skill-checkout.ts +111 -24
  158. package/src/activities/agent-turn/skill-install.ts +1 -1
  159. package/src/activities/agent-turn/skill-read.ts +374 -28
  160. package/src/activities/agent-turn/skill-remove.ts +46 -0
  161. package/src/activities/agent-turn/skill-save.ts +1 -1
  162. package/src/activities/agent-turn/skill-search.ts +10 -1
  163. package/src/activities/agent-turn/skill-selection.ts +4 -0
  164. package/src/activities/agent-turn/skill-tools.ts +112 -18
  165. package/src/activities/agent-turn/skill-transfer.ts +68 -23
  166. package/src/activities/agent-turn/stream-attempt.ts +313 -77
  167. package/src/activities/agent-turn/subscription-lease.ts +115 -0
  168. package/src/activities/agent-turn/tool-environment.ts +316 -167
  169. package/src/activities/agent-turn/tool-policy.ts +80 -1
  170. package/src/activities/agent-turn/turn-context.ts +35 -0
  171. package/src/activities/agent-turn/turn-sandbox-access.ts +7 -1
  172. package/src/activities/agent-turn/xai-capacity.ts +94 -51
  173. package/src/activities/automations.ts +76 -4
  174. package/src/activities/browser-deadline-checkpoint.ts +279 -0
  175. package/src/activities/capabilities.ts +108 -11
  176. package/src/activities/codemode-dispatcher.ts +36 -8
  177. package/src/activities/codex-capacity.ts +60 -25
  178. package/src/activities/codex-rotation.ts +39 -17
  179. package/src/activities/context-compaction.ts +75 -18
  180. package/src/activities/gmail-files.ts +88 -0
  181. package/src/activities/goals.ts +116 -121
  182. package/src/activities/google-drive-publication.ts +1 -3
  183. package/src/activities/image-generation-operation.ts +1 -1
  184. package/src/activities/knowledge-indexing.ts +269 -15
  185. package/src/activities/knowledge-source-sync.ts +10 -0
  186. package/src/activities/mcp-account-routes.ts +17 -0
  187. package/src/activities/mcp-credentials.ts +83 -194
  188. package/src/activities/mcp-operation-authority.ts +16 -41
  189. package/src/activities/model-call-fact-reconciler.ts +112 -0
  190. package/src/activities/open-suffix-resume.ts +15 -2
  191. package/src/activities/parent-wake.ts +46 -21
  192. package/src/activities/retained-screenshots.ts +82 -11
  193. package/src/activities/rig-verification.ts +38 -37
  194. package/src/activities/run-credential-renewal.ts +13 -1
  195. package/src/activities/run-credentials.ts +59 -17
  196. package/src/activities/run-input.ts +25 -16
  197. package/src/activities/sandbox-images.ts +0 -1
  198. package/src/activities/sandbox-lease.ts +798 -82
  199. package/src/activities/scheduled-tasks.ts +550 -178
  200. package/src/activities/session-state.ts +245 -6
  201. package/src/activities/site-auth-maintenance.ts +3 -0
  202. package/src/activities/types.ts +72 -5
  203. package/src/activities/workflow-wake.ts +2 -17
  204. package/src/activities/workspace-capture.ts +13 -2
  205. package/src/activities/workspace-credential-provider.ts +283 -0
  206. package/src/activities-control.ts +4 -0
  207. package/src/activities-turn.ts +5 -0
  208. package/src/activities.ts +13 -0
  209. package/src/activity-services.ts +16 -4
  210. package/src/browser-deadline-checkpoint-contract.ts +5 -0
  211. package/src/editable-artifact-hint-broker.ts +3 -0
  212. package/src/editable-artifact-materializer-service.ts +7 -3
  213. package/src/editable-artifact-outbox-service.ts +4 -4
  214. package/src/github-rest-mcp.ts +1 -2
  215. package/src/host-export-pump.ts +26 -4
  216. package/src/host-export.ts +3 -0
  217. package/src/http.ts +4 -2
  218. package/src/index.ts +99 -37
  219. package/src/observability-metrics.ts +510 -5
  220. package/src/personal-github-git-credentials.ts +2 -1
  221. package/src/pr-review-credentials.ts +1 -3
  222. package/src/retained-process-diagnostics.ts +50 -0
  223. package/src/retained-process-retry.ts +36 -0
  224. package/src/sandbox-resume.ts +388 -101
  225. package/src/sandbox-routing.ts +150 -25
  226. package/src/sandbox-snapshot-diagnostics.ts +112 -0
  227. package/src/session-event-fanout.ts +23 -0
  228. package/src/warm-capture-holder.ts +15 -0
  229. package/src/worker-service-lifecycle.ts +53 -0
  230. package/src/workflows/activities.ts +16 -1
  231. package/src/workflows/browser-deadline-checkpoint.ts +50 -0
  232. package/src/workflows/file-upload-reaper.ts +2 -0
  233. package/src/workflows/scheduled-tasks.ts +26 -13
  234. package/src/workflows/session.ts +242 -22
  235. package/src/workflows.ts +4 -0
  236. package/dist/activities/agent-turn/knowledge-sources.d.ts +0 -20
  237. package/dist/activities/packs.d.ts +0 -55
  238. package/dist/activities-control-HMFGSKC5.js.map +0 -1
  239. package/dist/activities-turn-AP67JADS.js.map +0 -1
  240. package/dist/chunk-A4XQWVZK.js.map +0 -1
  241. package/dist/chunk-CYITU5UH.js.map +0 -1
  242. package/dist/chunk-RAMNYSAW.js.map +0 -1
  243. package/src/activities/agent-turn/knowledge-sources.ts +0 -86
  244. package/src/activities/packs.ts +0 -166
@@ -1,15 +1,29 @@
1
1
  import {
2
2
  advanceWorkspaceGeneration,
3
3
  verifyWorkspaceMutationSettlement,
4
- heartbeatLeaseHolder,
4
+ heartbeatLeaseHolderStatus,
5
5
  readLease,
6
6
  accrueWarmSeconds,
7
7
  SandboxWorkspaceMutationFencedError,
8
+ retainWorkspaceMutationProcess,
9
+ retainedProviderCommandPersistence,
10
+ SandboxRetainedProcessPromotionFencedError,
8
11
  } from "@opengeni/db";
12
+ import { createProviderCommandRetainer } from "@opengeni/db/retained-provider-commands";
9
13
  import {
10
14
  RoutingMutationOutcomeUnknownError,
15
+ ProviderCommandStartOutcomeUnknownError,
16
+ ProviderCommandObservationUnavailableError,
17
+ isModalCommandStartOutcomeUnknownError,
18
+ withModalCommandStartSignal,
19
+ resolveModalCheckpointProviderBindingForSession,
20
+ runManagedCodemodeClientHook,
11
21
  type EstablishedSandboxSession,
22
+ type ProviderCommandSession,
23
+ type RoutingRetainedProcess,
12
24
  } from "@opengeni/runtime";
25
+ import { ModalRouterProviderCommand } from "@opengeni/contracts";
26
+ import { isDeepStrictEqual } from "node:util";
13
27
  import {
14
28
  sandboxLifecycleTransitionWaitMs,
15
29
  sandboxWarmRateMicrosPerSecond,
@@ -18,7 +32,10 @@ import {
18
32
  import { createRuntimeBatcher, currentActivityContext } from "../streaming";
19
33
  import type { TurnActivityServices as ActivityServices, RunAgentTurnInput } from "../types";
20
34
  import { maybePersistWarmWorkspaceSnapshot, type ResumedTurnSandbox } from "../../sandbox-resume";
21
- import { recordCreditMicros } from "../../observability-metrics";
35
+ import {
36
+ recordCreditMicros,
37
+ runtimeMetricsHooksForObservability,
38
+ } from "../../observability-metrics";
22
39
  import { ChannelAPartialMutationError } from "@opengeni/runtime/sandbox";
23
40
 
24
41
  import { safeErrorDiagnostic } from "./errors";
@@ -46,6 +63,114 @@ export type SandboxTurnRuntimeDeps = {
46
63
 
47
64
  export type SandboxTurnRuntime = ReturnType<typeof createSandboxTurnRuntime>;
48
65
 
66
+ const retainInternalProviderCommand = createProviderCommandRetainer(
67
+ retainWorkspaceMutationProcess,
68
+ (error) => (error instanceof SandboxRetainedProcessPromotionFencedError ? error.process : null),
69
+ );
70
+
71
+ /** Only the complete runtime producer boundary supplies a command descriptor.
72
+ * A bare SDK boundary can veto rejection, but its task/exec fields alone cannot
73
+ * establish its sandbox, PTY or complete original invocation. */
74
+ function internalMutationUnknownCommand(
75
+ error: unknown,
76
+ instanceId: string,
77
+ ): { unknown: boolean; command: ModalRouterProviderCommand | null } {
78
+ const pending = [{ value: error, depth: 0 }];
79
+ const seen = new Set<object>();
80
+ const commands: ModalRouterProviderCommand[] = [];
81
+ let unknown = false;
82
+ let incomplete = false;
83
+ for (let inspected = 0; pending.length && inspected < 128; inspected++) {
84
+ const { value, depth } = pending.shift()!;
85
+ if (!value || typeof value !== "object" || seen.has(value)) continue;
86
+ seen.add(value);
87
+ try {
88
+ if (
89
+ value instanceof ProviderCommandStartOutcomeUnknownError ||
90
+ value instanceof ProviderCommandObservationUnavailableError
91
+ ) {
92
+ unknown = true;
93
+ // Zod reads ordinary properties. First make a bounded own-data-only
94
+ // copy so neither validation nor equality executes descriptor getters.
95
+ // This contract has no arrays, functions or symbol-keyed fields.
96
+ let remaining = 64;
97
+ const copyCommandData = (candidate: unknown, descriptorDepth = 0): unknown => {
98
+ if (--remaining < 0 || descriptorDepth > 4)
99
+ throw new Error("Internal command descriptor exceeded inspection bounds");
100
+ if (
101
+ candidate === null ||
102
+ candidate === undefined ||
103
+ typeof candidate === "string" ||
104
+ typeof candidate === "number" ||
105
+ typeof candidate === "boolean"
106
+ )
107
+ return candidate;
108
+ if (typeof candidate !== "object" || Array.isArray(candidate))
109
+ throw new Error("Internal command descriptor is not a data object");
110
+ const keys = Reflect.ownKeys(candidate);
111
+ if (keys.length > 16)
112
+ throw new Error("Internal command descriptor exceeded field bounds");
113
+ const copy: Record<string, unknown> = Object.create(null);
114
+ for (const key of keys) {
115
+ const property = Object.getOwnPropertyDescriptor(candidate, key);
116
+ if (typeof key !== "string" || !property || !("value" in property))
117
+ throw new Error("Internal command descriptor contains a non-data field");
118
+ copy[key] = copyCommandData(property.value, descriptorDepth + 1);
119
+ }
120
+ return copy;
121
+ };
122
+ const parsed = ModalRouterProviderCommand.safeParse(
123
+ copyCommandData(Object.getOwnPropertyDescriptor(value, "command")?.value),
124
+ );
125
+ if (!parsed.success || parsed.data.sandboxId !== instanceId) incomplete = true;
126
+ else commands.push(parsed.data);
127
+ // This complete producer boundary owns its original SDK cause. Do not
128
+ // reinterpret that cause's partial locator as a second command.
129
+ continue;
130
+ }
131
+ const sdkUnknown = isModalCommandStartOutcomeUnknownError(value);
132
+ if (sdkUnknown || value instanceof RoutingMutationOutcomeUnknownError) unknown = true;
133
+ const nested: unknown[] = [];
134
+ for (const key of ["cause", "error", "errors"] as const) {
135
+ const property = Object.getOwnPropertyDescriptor(value, key);
136
+ if (!property) continue;
137
+ if (!("value" in property)) {
138
+ incomplete = true;
139
+ continue;
140
+ }
141
+ if (key === "errors" && Array.isArray(property.value)) {
142
+ const length = Object.getOwnPropertyDescriptor(property.value, "length")?.value;
143
+ if (typeof length !== "number" || !Number.isSafeInteger(length) || length < 0) {
144
+ incomplete = true;
145
+ continue;
146
+ }
147
+ if (length > 64) incomplete = true;
148
+ for (let index = 0; index < Math.min(length, 64); index++) {
149
+ const element = Object.getOwnPropertyDescriptor(property.value, String(index));
150
+ if (!element || !("value" in element)) incomplete = true;
151
+ else nested.push(element.value);
152
+ }
153
+ } else if (property.value !== undefined) nested.push(property.value);
154
+ }
155
+ if (
156
+ sdkUnknown &&
157
+ (nested.length === 0 ||
158
+ Object.getOwnPropertyDescriptor(value, "taskId") ||
159
+ Object.getOwnPropertyDescriptor(value, "execId"))
160
+ )
161
+ incomplete = true;
162
+ if (depth >= 12 && nested.length) incomplete = true;
163
+ else for (const child of nested) pending.push({ value: child, depth: depth + 1 });
164
+ } catch {
165
+ incomplete = true;
166
+ }
167
+ }
168
+ if (pending.length) incomplete = true;
169
+ const command = commands[0] ?? null;
170
+ if (commands.some((candidate) => !isDeepStrictEqual(candidate, command))) incomplete = true;
171
+ return { unknown: unknown || incomplete, command: incomplete ? null : command };
172
+ }
173
+
49
174
  /**
50
175
  * Provider-deadline rotation is a preemption boundary, not a cooperative
51
176
  * snapshot boundary. Snapshotting while this attempt still owns active tools
@@ -81,12 +206,62 @@ export function createSandboxTurnRuntime(deps: SandboxTurnRuntimeDeps) {
81
206
  eventing,
82
207
  attempt,
83
208
  } = deps;
209
+ const commandStartSignal = cancellationSignal
210
+ ? AbortSignal.any([cancellationSignal, sandboxRotationController.signal])
211
+ : sandboxRotationController.signal;
212
+
213
+ // The exact existing private release closure is the holder authority. Keep
214
+ // its ordinary eager release, but never pass a false writer-drain proof that
215
+ // would let releaseLeaseHolder settle an ambiguous null-outcome admission.
216
+ const releaseGuards = new WeakMap<
217
+ ResumedTurnSandbox["release"],
218
+ {
219
+ pending: Map<string, { operation: string; cause?: unknown }>;
220
+ release: ResumedTurnSandbox["release"];
221
+ }
222
+ >();
223
+ const guardSandboxRelease = (sandbox: ResumedTurnSandbox) => {
224
+ if (typeof sandbox.release !== "function") return null;
225
+ const original = sandbox.release;
226
+ let guard = releaseGuards.get(original);
227
+ if (!guard) {
228
+ const pending = new Map<string, { operation: string; cause?: unknown }>();
229
+ const release: ResumedTurnSandbox["release"] = async (options) => {
230
+ if (options?.workspaceWritersQuiesced && pending.size) {
231
+ await original();
232
+ throw new RoutingMutationOutcomeUnknownError(
233
+ "internalWorkspaceMutationDrain",
234
+ "Internal workspace command is still outcome-unknown; its admission remains fenced until exact retention or physical settlement",
235
+ { cause: new AggregateError([...pending.values()].map((entry) => entry.cause)) },
236
+ );
237
+ }
238
+ await original(options);
239
+ };
240
+ guard = { pending, release };
241
+ releaseGuards.set(original, guard);
242
+ releaseGuards.set(release, guard);
243
+ }
244
+ // Rebound preparation may use a shallow copy of the published sandbox.
245
+ // Protect every known copy of this SAME private holder release closure,
246
+ // never another lease or a newly fabricated holder.
247
+ for (const target of [
248
+ sandbox,
249
+ sandboxState.resolvedSandbox,
250
+ sandboxState.prefetchedManagedBoxResult,
251
+ ...(sandboxState.lateSandboxesAwaitingWriterDrain ?? []),
252
+ ]) {
253
+ if (target?.release === original || target?.release === guard.release)
254
+ target.release = guard.release;
255
+ }
256
+ return guard;
257
+ };
84
258
 
85
259
  // P1.2 ownership inversion: when sandboxOwnershipEnabled, the turn resolves
86
260
  // the one box by id from the group lease and injects it NON-OWNED into the
87
261
  // run. null when the flag is off (byte-for-byte the legacy build-and-discard
88
262
  // path) OR when the backend is "none". Released + dropped in `finally`.
89
263
  const releaseLateSandbox = async (sandbox: ResumedTurnSandbox): Promise<void> => {
264
+ guardSandboxRelease(sandbox);
90
265
  sandboxState.lateSandboxesAwaitingWriterDrain.add(sandbox);
91
266
  // Drop the holder/timer immediately, but keep its null-outcome admissions
92
267
  // fenced until the shared attempt writer drain completes. The staged
@@ -114,26 +289,48 @@ export function createSandboxTurnRuntime(deps: SandboxTurnRuntimeDeps) {
114
289
  // Platform setup (beforeAgentStart hooks + file materialization) execs against
115
290
  // THIS handle so a mid-turn sandbox_swap can never re-route those execs onto a
116
291
  // connected machine (the user's real computer).
117
- const finalizeTurnOpStreamOps = async (): Promise<void> => {
292
+ const finalizeTurnOpStreamOps = async (toolCallIds?: readonly string[]): Promise<void> => {
118
293
  await finalizeDurableTurnOpStreams(
119
294
  [sandboxState.lazyOwnedSandbox?.session, sandboxState.resolvedSandbox?.established.session],
120
295
  sandboxState.machinePrimarySession,
296
+ toolCallIds,
121
297
  );
122
298
  };
123
299
  // A same-target API repair can replace the home provider while this turn is
124
300
  // alive. Keep setup/snapshot persistence on the rebound raw session while
125
301
  // preserving the SDK-owned routing proxy for eager turns. Lazy turns hold the
126
302
  // proxy separately, so their worker-side handle may replace its raw session.
127
- const onHomeSandboxRebound = (rebound: {
303
+ const onHomeSandboxRebound = async (rebound: {
128
304
  established: EstablishedSandboxSession;
129
305
  leaseEpoch: number;
130
- }): void => {
306
+ }): Promise<void> => {
131
307
  const current = sandboxState.resolvedSandbox;
308
+ if (!current) throw new Error("Home sandbox rebound before turn sandbox admission");
309
+ if (
310
+ rebound.established.backendId !== "selfhosted" &&
311
+ rebound.established.instanceId !== current.established.instanceId
312
+ ) {
313
+ // An older archive may lack this worker release's selected client. Only
314
+ // prepare that client, never replay turn-start hooks/credentials/resources.
315
+ // Fence the replacement itself, not the stale turn-start handle. Both
316
+ // mutation admission and settlement precede raw-state/route publication.
317
+ const commandFence = eventing.toolCancellationFenceRef.current;
318
+ await runWorkspaceMutationForSandbox(
319
+ { ...current, established: rebound.established, leaseEpoch: rebound.leaseEpoch },
320
+ "homeSandboxClientPreparation",
321
+ () =>
322
+ runManagedCodemodeClientHook(rebound.established.session as never, {
323
+ environment: {},
324
+ ...(commandFence
325
+ ? { commandRunner: commandFence.runSandboxCommand.bind(commandFence) }
326
+ : {}),
327
+ }),
328
+ );
329
+ }
132
330
  const previousSession = current?.established.session;
133
331
  const preserveRoutingProxy =
134
332
  current !== null && previousSession !== sandboxState.setupBoxSession;
135
333
  sandboxState.setupBoxSession = rebound.established.session;
136
- if (!current) return;
137
334
  current.leaseEpoch = rebound.leaseEpoch;
138
335
  current.established = preserveRoutingProxy
139
336
  ? {
@@ -289,13 +486,116 @@ export function createSandboxTurnRuntime(deps: SandboxTurnRuntimeDeps) {
289
486
  }
290
487
  };
291
488
  let result: T;
489
+ const releaseGuard = guardSandboxRelease(sandbox);
490
+ releaseGuard?.pending.set(admission.id, { operation });
292
491
  const providerStartedAt = performance.now();
293
492
  let providerOutcome: "completed" | "failed" = "failed";
294
493
  try {
295
- result = await mutation();
494
+ result = await withModalCommandStartSignal(commandStartSignal, mutation);
296
495
  providerOutcome = "completed";
297
496
  } catch (providerError) {
298
497
  observeMutationPhase("provider", providerOutcome, performance.now() - providerStartedAt);
498
+ const ambiguity = internalMutationUnknownCommand(providerError, identity.expectedInstanceId);
499
+ if (ambiguity.unknown) {
500
+ releaseGuard?.pending.set(admission.id, { operation, cause: providerError });
501
+ if (!ambiguity.command) {
502
+ throw new RoutingMutationOutcomeUnknownError(
503
+ operation,
504
+ `Platform workspace mutation "${operation}" has no unique exact command descriptor; its admission remains fenced and it was not replayed`,
505
+ { cause: providerError },
506
+ );
507
+ }
508
+ const process: RoutingRetainedProcess = {
509
+ id: crypto.randomUUID(),
510
+ providerSessionId: admission.workspaceGeneration,
511
+ providerCommand: ambiguity.command,
512
+ };
513
+ let exactRetentionVerified = false;
514
+ try {
515
+ const providerBinding = await resolveModalCheckpointProviderBindingForSession(
516
+ settings,
517
+ sandbox.established.session,
518
+ );
519
+ try {
520
+ await retainInternalProviderCommand(db, {
521
+ accountId: input.accountId,
522
+ workspaceId: input.workspaceId,
523
+ sessionId: input.sessionId,
524
+ processId: process.id,
525
+ providerSessionId: process.providerSessionId,
526
+ providerCommand: ambiguity.command,
527
+ admissionId: admission.id,
528
+ admittedWorkspaceGeneration: admission.workspaceGeneration,
529
+ operation,
530
+ providerBinding,
531
+ owner: { kind: "turn", ...identity },
532
+ });
533
+ } catch (promotionError) {
534
+ if (!(promotionError instanceof SandboxRetainedProcessPromotionFencedError))
535
+ throw promotionError;
536
+ const durable = promotionError.process;
537
+ if (
538
+ durable.id !== process.id ||
539
+ durable.providerSessionId !== process.providerSessionId ||
540
+ durable.parentAdmissionId !== admission.id ||
541
+ durable.accountId !== identity.accountId ||
542
+ durable.workspaceId !== identity.workspaceId ||
543
+ durable.sessionId !== identity.sessionId ||
544
+ durable.leaseId !== admission.leaseId ||
545
+ durable.sandboxGroupId !== identity.sandboxGroupId ||
546
+ durable.ownerActorKind !== admission.actorKind ||
547
+ durable.ownerActorId !== admission.actorId ||
548
+ durable.ownerTurnId !== identity.turnId ||
549
+ durable.ownerAttemptId !== identity.attemptId ||
550
+ durable.ownerExecutionGeneration !== identity.executionGeneration ||
551
+ durable.leaseEpoch !== identity.expectedEpoch ||
552
+ durable.providerBackend !== "modal" ||
553
+ durable.providerInstanceId !== identity.expectedInstanceId ||
554
+ durable.providerBindingKey !== providerBinding.key ||
555
+ !isDeepStrictEqual(durable.providerBinding, providerBinding.binding) ||
556
+ durable.routeKind !== admission.routeKind ||
557
+ durable.routeTargetId !== admission.routeTargetId ||
558
+ durable.routeEpoch !== admission.routeEpoch ||
559
+ durable.state !== "active"
560
+ )
561
+ throw promotionError;
562
+ // The established retainer committed BOTH locator and holder before
563
+ // rejecting stale output. Do not undo that physical retention.
564
+ }
565
+ const persistence = retainedProviderCommandPersistence(db, {
566
+ accountId: input.accountId,
567
+ workspaceId: input.workspaceId,
568
+ sessionId: input.sessionId,
569
+ processId: process.id,
570
+ });
571
+ if (!isDeepStrictEqual(await persistence.load(), ambiguity.command))
572
+ throw new Error("Internal command retention lost its exact original locator", {
573
+ cause: providerError,
574
+ });
575
+ exactRetentionVerified = true;
576
+ releaseGuard?.pending.delete(admission.id);
577
+ (sandbox.established.session as ProviderCommandSession).bindProviderCommand?.(
578
+ process.providerSessionId,
579
+ ambiguity.command,
580
+ persistence,
581
+ );
582
+ } catch (retentionError) {
583
+ throw new RoutingMutationOutcomeUnknownError(
584
+ operation,
585
+ `Platform workspace mutation "${operation}" remains outcome-unknown and its exact retention or observer binding failed; it was not replayed`,
586
+ {
587
+ cause: new AggregateError([providerError, retentionError]),
588
+ ...(exactRetentionVerified ? { retainedProcess: process } : {}),
589
+ },
590
+ );
591
+ }
592
+ throw new RoutingMutationOutcomeUnknownError(
593
+ operation,
594
+ `Platform workspace mutation "${operation}" has an unknown Start outcome; its exact original invocation is retained and it was not replayed`,
595
+ { cause: providerError, retainedProcess: process },
596
+ );
597
+ }
598
+ releaseGuard?.pending.delete(admission.id);
299
599
  const partialMutation = providerError instanceof ChannelAPartialMutationError;
300
600
  try {
301
601
  await settleMutation(partialMutation ? "resolved" : "rejected");
@@ -317,6 +617,7 @@ export function createSandboxTurnRuntime(deps: SandboxTurnRuntimeDeps) {
317
617
  }
318
618
  throw providerError;
319
619
  }
620
+ releaseGuard?.pending.delete(admission.id);
320
621
  observeMutationPhase("provider", providerOutcome, performance.now() - providerStartedAt);
321
622
  try {
322
623
  await settleMutation("resolved");
@@ -501,12 +802,15 @@ export function createSandboxTurnRuntime(deps: SandboxTurnRuntimeDeps) {
501
802
  // home turn that degraded to the cloud group box (swap-away / flag-off), that
502
803
  // is the deployment default (modal), so the fallback box is warm-metered at
503
804
  // the cloud rate instead of selfhosted's rate-0 (which would under-bill).
504
- const warmRate = sandboxWarmRateMicrosPerSecond(
505
- settings,
506
- warmBackend ?? (sandbox.established.backendId as Settings["sandboxBackend"]),
507
- );
805
+ const warmRate =
806
+ settings.sandboxWarmBillingMode === "usage_only"
807
+ ? 0
808
+ : sandboxWarmRateMicrosPerSecond(
809
+ settings,
810
+ warmBackend ?? (sandbox.established.backendId as Settings["sandboxBackend"]),
811
+ );
508
812
  sandboxState.leaseHeartbeatTimer = setInterval(() => {
509
- void heartbeatLeaseHolder(db, {
813
+ void heartbeatLeaseHolderStatus(db, {
510
814
  accountId: input.accountId,
511
815
  workspaceId: input.workspaceId,
512
816
  sandboxGroupId: heartbeatGroupId,
@@ -514,9 +818,17 @@ export function createSandboxTurnRuntime(deps: SandboxTurnRuntimeDeps) {
514
818
  holderId: heartbeatHolderId,
515
819
  leaseTtlMs: settings.sandboxLeaseTtlMs,
516
820
  expectedEpoch: heartbeatEpoch,
821
+ billingMode: settings.sandboxWarmBillingMode,
517
822
  })
518
- .then(async (alive) => {
519
- if (alive) return;
823
+ .then(async (status) => {
824
+ if (status.fence === "funding") {
825
+ stopLeaseHeartbeat();
826
+ sandboxRotationController.abort(
827
+ new Error("Insufficient Opengeni credits to extend paid sandbox compute"),
828
+ );
829
+ return;
830
+ }
831
+ if (status.leaseExtended) return;
520
832
  const rotation = await beginRotationPreemption(sandbox, heartbeatEpoch, heartbeatGroupId);
521
833
  if (rotation === "not_rotating") {
522
834
  // The holder was reaped, the exact attempt closed, the epoch was
@@ -532,15 +844,22 @@ export function createSandboxTurnRuntime(deps: SandboxTurnRuntimeDeps) {
532
844
  sandboxGroupId: heartbeatGroupId,
533
845
  expectedEpoch: heartbeatEpoch,
534
846
  warmRateMicrosPerSecond: warmRate,
847
+ billingMode: settings.sandboxWarmBillingMode,
535
848
  subjectId: input.sessionId,
536
849
  })
537
- .then((result) => recordCreditMicros(observability, "usage", result.costMicros))
850
+ .then((result) => {
851
+ if (settings.sandboxWarmBillingMode === "credits") {
852
+ recordCreditMicros(observability, "usage", result.costMicros);
853
+ }
854
+ })
538
855
  .catch(() => undefined);
539
856
  // MID-SESSION snapshot (sandbox-file-persistence): while the turn holds
540
857
  // the box, fold a fresh /workspace snapshot onto the lease every
541
858
  // sandboxSnapshotIntervalMs, so a box death the reaper never sees
542
- // (Modal hard timeout mid-busy, OOM, infra) costs at most one interval
543
- // of work — a legit multi-day turn is otherwise completely unprotected
859
+ // (Modal hard timeout mid-busy, OOM, infra) has a recent recovery point
860
+ // when capture succeeds. This interval is not a maximum recovery-point
861
+ // age: live writers and failed captures can defer publication. A legit
862
+ // multi-day turn is otherwise completely unprotected
544
863
  // (the reaper only drain-persists IDLE leases). Uses the UN-proxied box
545
864
  // session (setupBoxSession): the routing veneer could swap mid-op and a
546
865
  // selfhosted target has no persistWorkspace anyway. Best-effort +
@@ -558,7 +877,13 @@ export function createSandboxTurnRuntime(deps: SandboxTurnRuntimeDeps) {
558
877
  })
559
878
  ) {
560
879
  const snapshot = maybePersistWarmWorkspaceSnapshot(
561
- { db, settings, objectStorage },
880
+ {
881
+ db,
882
+ settings,
883
+ objectStorage,
884
+ sandboxMetrics: runtimeMetricsHooksForObservability(observability),
885
+ observability,
886
+ },
562
887
  {
563
888
  accountId: input.accountId,
564
889
  workspaceId: input.workspaceId,
@@ -1,11 +1,18 @@
1
1
  import { hasPermission } from "@opengeni/core";
2
2
  import type { AttemptToolDefinition } from "@opengeni/codemode";
3
- import type { GeneratedSessionTitle } from "@opengeni/runtime";
3
+ import { isManagedOpenRouterFreeRoute, type ModelCapabilitiesV1 } from "@opengeni/config";
4
+ import type {
5
+ GeneratedSessionTitle,
6
+ GenerateSessionTitleOptions,
7
+ OpenGeniRuntime,
8
+ } from "@opengeni/runtime";
4
9
  import {
5
10
  AUTOMATIC_SESSION_TITLE_FALLBACK,
6
11
  DEFAULT_FIRST_PARTY_MCP_PERMISSIONS,
12
+ ReasoningEffort,
7
13
  type FirstPartyMcpToolName,
8
14
  type Permission,
15
+ type ResolvedAgentConfig,
9
16
  type ToolRef,
10
17
  } from "@opengeni/contracts";
11
18
 
@@ -17,6 +24,7 @@ const SESSION_TITLE_DESCRIPTION =
17
24
  export function shouldRequestMissingSessionTitle(input: {
18
25
  title: string | null;
19
26
  titleSource: "user" | "agent" | null;
27
+ agentConfig?: ResolvedAgentConfig | null;
20
28
  firstPartyMcpTools: readonly FirstPartyMcpToolName[];
21
29
  firstPartyMcpPermissions: readonly Permission[] | null;
22
30
  }): boolean {
@@ -24,16 +32,34 @@ export function shouldRequestMissingSessionTitle(input: {
24
32
  const needsSemanticTitle =
25
33
  input.titleSource !== "user" && (!title || title === AUTOMATIC_SESSION_TITLE_FALLBACK);
26
34
  if (!needsSemanticTitle) return false;
27
- if (!input.firstPartyMcpTools.includes("set_session_title")) return false;
35
+ // Configured sessions title through the runtime, not an opted-in capability.
36
+ // Legacy sessions keep their exact selected-tool and permission admission.
37
+ if (!input.agentConfig && !input.firstPartyMcpTools.includes("set_session_title")) return false;
28
38
  const permissions = input.firstPartyMcpPermissions ?? DEFAULT_FIRST_PARTY_MCP_PERMISSIONS;
29
39
  return hasPermission([...permissions], "sessions:control");
30
40
  }
31
41
 
42
+ /**
43
+ * Whether the turn's route can afford a model request spent only on a title.
44
+ * The managed OpenRouter free route draws on one deployment-wide per-minute
45
+ * and per-day request quota that users' turns need, so an untitled session on
46
+ * it gets no title sidecar and no title tool (whose call would cost a
47
+ * follow-up request). Clients keep showing the prompt preview, and a later
48
+ * turn on another route titles the session.
49
+ */
50
+ export function routeAllowsSessionTitleRequests(
51
+ resolvedModel: ReturnType<OpenGeniRuntime["resolveTurnModel"]>,
52
+ ): boolean {
53
+ return !resolvedModel || !isManagedOpenRouterFreeRoute(resolvedModel);
54
+ }
55
+
32
56
  export function sessionTitleToolPlan(input: {
33
57
  tools: readonly ToolRef[];
58
+ agentConfig?: ResolvedAgentConfig | null;
34
59
  selectedFirstPartyMcpTools: readonly FirstPartyMcpToolName[];
35
60
  shouldRequestTitle: boolean;
36
61
  parallelGenerationAvailable: boolean;
62
+ routeAllowsTitleRequests: boolean;
37
63
  }): {
38
64
  promoteTitleTool: boolean;
39
65
  generateTitleInParallel: boolean;
@@ -42,9 +68,11 @@ export function sessionTitleToolPlan(input: {
42
68
  } {
43
69
  const titleToolAvailable =
44
70
  input.shouldRequestTitle &&
45
- input.tools.some((tool) => tool.kind === "mcp" && tool.id === "opengeni");
46
- const generateTitleInParallel = titleToolAvailable && input.parallelGenerationAvailable;
47
- const promoteTitleTool = titleToolAvailable && !generateTitleInParallel;
71
+ (input.agentConfig != null ||
72
+ input.tools.some((tool) => tool.kind === "mcp" && tool.id === "opengeni"));
73
+ const titleRequestAllowed = titleToolAvailable && input.routeAllowsTitleRequests;
74
+ const generateTitleInParallel = titleRequestAllowed && input.parallelGenerationAvailable;
75
+ const promoteTitleTool = titleRequestAllowed && !generateTitleInParallel;
48
76
  return {
49
77
  promoteTitleTool,
50
78
  generateTitleInParallel,
@@ -57,6 +85,54 @@ export function sessionTitleToolPlan(input: {
57
85
 
58
86
  export const PARALLEL_SESSION_TITLE_TIMEOUT_MS = 15_000;
59
87
 
88
+ /**
89
+ * The lowest reasoning effort the resolved model can run, for the auxiliary
90
+ * title request only. A title needs no deliberation, and a provider default
91
+ * effort can use most of the output budget before any visible text. Returns
92
+ * undefined when the model declares no runnable reasoning control, so the
93
+ * request carries no reasoning parameter.
94
+ */
95
+ export function sessionTitleReasoningEffort(
96
+ capabilities: Pick<ModelCapabilitiesV1, "reasoning"> | undefined,
97
+ ): ReasoningEffort | undefined {
98
+ const reasoning = capabilities?.reasoning;
99
+ if (!reasoning?.runnable) return undefined;
100
+ const order = ReasoningEffort.options;
101
+ let lowest: ReasoningEffort | undefined;
102
+ for (const effort of reasoning.efforts) {
103
+ if (!lowest || order.indexOf(effort) < order.indexOf(lowest)) lowest = effort;
104
+ }
105
+ return lowest;
106
+ }
107
+
108
+ /**
109
+ * Options for the parallel title request. It uses the turn's resolved
110
+ * provider and credential authority, but its own lowest runnable reasoning
111
+ * effort rather than the turn's effort.
112
+ */
113
+ export function sessionTitleGenerationOptions(input: {
114
+ resolvedModel: ReturnType<OpenGeniRuntime["resolveTurnModel"]>;
115
+ modelName: string;
116
+ serviceTier: GenerateSessionTitleOptions["serviceTier"] | null | undefined;
117
+ signal: AbortSignal;
118
+ }): GenerateSessionTitleOptions {
119
+ const { resolvedModel, serviceTier } = input;
120
+ const reasoningEffort = sessionTitleReasoningEffort(resolvedModel?.configured.capabilities);
121
+ return {
122
+ ...(resolvedModel
123
+ ? {
124
+ client: resolvedModel.client,
125
+ provider: resolvedModel.provider,
126
+ model: resolvedModel.model,
127
+ }
128
+ : {}),
129
+ modelName: input.modelName,
130
+ ...(serviceTier ? { serviceTier } : {}),
131
+ ...(reasoningEffort ? { reasoningEffort } : {}),
132
+ signal: input.signal,
133
+ };
134
+ }
135
+
60
136
  export type ParallelSessionTitleGeneration = {
61
137
  finish: () => Promise<GeneratedSessionTitle | null>;
62
138
  cancel: () => Promise<void>;