@opengeni/worker-bundle 0.20.3 → 0.22.0-canary.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (184) hide show
  1. package/dist/activities/agent-turn/admission.d.ts +52 -0
  2. package/dist/activities/agent-turn/agent-build.d.ts +89 -0
  3. package/dist/activities/agent-turn/claim.d.ts +67 -0
  4. package/dist/activities/agent-turn/codex-capacity.d.ts +40 -0
  5. package/dist/activities/agent-turn/codex.d.ts +32 -0
  6. package/dist/activities/agent-turn/compaction-prep.d.ts +88 -0
  7. package/dist/activities/agent-turn/credential-leases.d.ts +56 -0
  8. package/dist/activities/agent-turn/errors.d.ts +253 -0
  9. package/dist/activities/agent-turn/failure-settlement.d.ts +33 -0
  10. package/dist/activities/agent-turn/file-resources.d.ts +37 -0
  11. package/dist/activities/agent-turn/finalization.d.ts +42 -0
  12. package/dist/activities/agent-turn/github-repository-bindings.d.ts +105 -0
  13. package/dist/activities/agent-turn/governance-model.d.ts +66 -0
  14. package/dist/activities/agent-turn/history-sink.d.ts +52 -0
  15. package/dist/activities/agent-turn/history.d.ts +63 -0
  16. package/dist/activities/agent-turn/media-artifacts.d.ts +145 -0
  17. package/dist/activities/agent-turn/model-usage.d.ts +218 -0
  18. package/dist/activities/agent-turn/quiescence.d.ts +135 -0
  19. package/dist/activities/agent-turn/run-credentials.d.ts +84 -0
  20. package/dist/activities/agent-turn/run.d.ts +3 -0
  21. package/dist/activities/agent-turn/sandbox-establish.d.ts +93 -0
  22. package/dist/activities/agent-turn/sandbox-provision.d.ts +89 -0
  23. package/dist/activities/agent-turn/sandbox-route.d.ts +151 -0
  24. package/dist/activities/agent-turn/sandbox-runtime.d.ts +44 -0
  25. package/dist/activities/agent-turn/stream-attempt.d.ts +107 -0
  26. package/dist/activities/agent-turn/tool-environment.d.ts +142 -0
  27. package/dist/activities/agent-turn/tool-policy.d.ts +127 -0
  28. package/dist/activities/agent-turn/tool-result-spill.d.ts +59 -0
  29. package/dist/activities/agent-turn/turn-context.d.ts +144 -0
  30. package/dist/activities/agent-turn/xai-capacity.d.ts +2 -0
  31. package/dist/activities/agent-turn.d.ts +12 -954
  32. package/dist/activities/automations.d.ts +16 -0
  33. package/dist/activities/common.d.ts +1 -1
  34. package/dist/activities/connector-attachments.d.ts +13 -0
  35. package/dist/activities/documents.d.ts +1 -0
  36. package/dist/activities/environment.d.ts +26 -15
  37. package/dist/activities/generated-images.d.ts +1 -1
  38. package/dist/activities/git-credential-renewal.d.ts +3 -1
  39. package/dist/activities/goals.d.ts +4 -1
  40. package/dist/activities/google-drive-provider.d.ts +29 -0
  41. package/dist/activities/google-drive-publication.d.ts +17 -4
  42. package/dist/activities/incident-telemetry-authority.d.ts +52 -0
  43. package/dist/activities/incident-telemetry-preflight.d.ts +71 -0
  44. package/dist/activities/knowledge-source-sync.d.ts +4 -0
  45. package/dist/activities/mcp-credentials.d.ts +9 -3
  46. package/dist/activities/open-suffix-resume.d.ts +77 -0
  47. package/dist/activities/packs.d.ts +3 -11
  48. package/dist/activities/parent-wake.d.ts +19 -0
  49. package/dist/activities/retained-screenshot-reaper.d.ts +1 -1
  50. package/dist/activities/retained-screenshots.d.ts +1 -1
  51. package/dist/activities/rig-verification.d.ts +1 -1
  52. package/dist/activities/run-input.d.ts +9 -10
  53. package/dist/activities/sandbox-images.d.ts +1 -14
  54. package/dist/activities/sandbox-lease.d.ts +21 -3
  55. package/dist/activities/scheduled-tasks.d.ts +47 -0
  56. package/dist/activities/types.d.ts +23 -2
  57. package/dist/activities/video-media-validation.d.ts +1 -1
  58. package/dist/activities/video-output-retention.d.ts +1 -2
  59. package/dist/activities/xai-auth.d.ts +3 -0
  60. package/dist/{activities-control-BJSDRSIP.js → activities-control-DKG22ONJ.js} +2646 -551
  61. package/dist/activities-control-DKG22ONJ.js.map +1 -0
  62. package/dist/activities-control.d.ts +6 -2
  63. package/dist/activities-turn-4IMC3MAA.js +21326 -0
  64. package/dist/activities-turn-4IMC3MAA.js.map +1 -0
  65. package/dist/activities.d.ts +7 -3
  66. package/dist/artifact-materializer-entry.js +41 -22
  67. package/dist/artifact-materializer-entry.js.map +1 -1
  68. package/dist/{chunk-N5TJIFCE.js → chunk-CE7TPKBS.js} +468 -1
  69. package/dist/chunk-CE7TPKBS.js.map +1 -0
  70. package/dist/chunk-CNYGVRKF.js +4698 -0
  71. package/dist/chunk-CNYGVRKF.js.map +1 -0
  72. package/dist/connection-provider-use.d.ts +18 -0
  73. package/dist/editable-artifact-materializer.d.ts +4 -3
  74. package/dist/github-rest-mcp.d.ts +26 -0
  75. package/dist/index.d.ts +10 -2
  76. package/dist/index.js +58 -14
  77. package/dist/index.js.map +1 -1
  78. package/dist/knowledge-source-sync-activity-policy.d.ts +2 -0
  79. package/dist/model-context-contributions.d.ts +8 -4
  80. package/dist/observability-metrics.d.ts +63 -4
  81. package/dist/opensandbox-kubernetes-inventory.d.ts +21 -0
  82. package/dist/personal-github-git-credentials.d.ts +9 -0
  83. package/dist/pr-review-credentials.d.ts +6 -0
  84. package/dist/sandbox-archive-storage.d.ts +45 -0
  85. package/dist/sandbox-resume.d.ts +65 -6
  86. package/dist/sandbox-routing.d.ts +38 -2
  87. package/dist/scheduled-alert-occurrence.d.ts +33 -0
  88. package/dist/workflow-bundle.js +87 -28
  89. package/dist/workflows/activities.d.ts +1 -0
  90. package/dist/workflows/automations.d.ts +6 -0
  91. package/dist/workflows/document-index.d.ts +1 -0
  92. package/dist/workflows.d.ts +1 -0
  93. package/package.json +17 -17
  94. package/src/activities/agent-turn/admission.ts +292 -0
  95. package/src/activities/agent-turn/agent-build.ts +802 -0
  96. package/src/activities/agent-turn/claim.ts +590 -0
  97. package/src/activities/agent-turn/codex-capacity.ts +720 -0
  98. package/src/activities/agent-turn/codex.ts +94 -0
  99. package/src/activities/agent-turn/compaction-prep.ts +722 -0
  100. package/src/activities/agent-turn/credential-leases.ts +240 -0
  101. package/src/activities/agent-turn/errors.ts +1068 -0
  102. package/src/activities/agent-turn/failure-settlement.ts +1436 -0
  103. package/src/activities/agent-turn/file-resources.ts +227 -0
  104. package/src/activities/agent-turn/finalization.ts +754 -0
  105. package/src/activities/agent-turn/github-repository-bindings.ts +300 -0
  106. package/src/activities/agent-turn/governance-model.ts +465 -0
  107. package/src/activities/agent-turn/history-sink.ts +139 -0
  108. package/src/activities/agent-turn/history.ts +200 -0
  109. package/src/activities/agent-turn/media-artifacts.ts +359 -0
  110. package/src/activities/agent-turn/model-usage.ts +823 -0
  111. package/src/activities/agent-turn/quiescence.ts +353 -0
  112. package/src/activities/agent-turn/run-credentials.ts +761 -0
  113. package/src/activities/agent-turn/run.ts +1530 -0
  114. package/src/activities/agent-turn/sandbox-establish.ts +1207 -0
  115. package/src/activities/agent-turn/sandbox-provision.ts +569 -0
  116. package/src/activities/agent-turn/sandbox-route.ts +358 -0
  117. package/src/activities/agent-turn/sandbox-runtime.ts +731 -0
  118. package/src/activities/agent-turn/stream-attempt.ts +1670 -0
  119. package/src/activities/agent-turn/tool-environment.ts +729 -0
  120. package/src/activities/agent-turn/tool-policy.ts +216 -0
  121. package/src/activities/agent-turn/tool-result-spill.ts +265 -0
  122. package/src/activities/agent-turn/turn-context.ts +281 -0
  123. package/src/activities/agent-turn/xai-capacity.ts +229 -0
  124. package/src/activities/agent-turn.ts +12 -12701
  125. package/src/activities/api-integrations.ts +3 -0
  126. package/src/activities/automations.ts +214 -0
  127. package/src/activities/codemode-dispatcher.ts +12 -1
  128. package/src/activities/common.ts +1 -5
  129. package/src/activities/connector-attachments.ts +168 -0
  130. package/src/activities/context-compaction.ts +18 -0
  131. package/src/activities/environment.ts +151 -55
  132. package/src/activities/generated-images.ts +26 -11
  133. package/src/activities/git-credential-renewal.ts +13 -3
  134. package/src/activities/goals.ts +85 -19
  135. package/src/activities/google-drive-provider.ts +178 -0
  136. package/src/activities/google-drive-publication.ts +147 -49
  137. package/src/activities/incident-telemetry-authority.ts +327 -0
  138. package/src/activities/incident-telemetry-preflight.ts +317 -0
  139. package/src/activities/knowledge-source-sync.ts +178 -37
  140. package/src/activities/mcp-credentials.ts +191 -17
  141. package/src/activities/open-suffix-resume.ts +422 -0
  142. package/src/activities/packs.ts +12 -60
  143. package/src/activities/parent-wake.ts +64 -14
  144. package/src/activities/retained-screenshot-reaper.ts +6 -3
  145. package/src/activities/retained-screenshots.ts +15 -47
  146. package/src/activities/rig-verification.ts +1 -1
  147. package/src/activities/run-input.ts +102 -142
  148. package/src/activities/sandbox-images.ts +7 -166
  149. package/src/activities/sandbox-lease.ts +406 -15
  150. package/src/activities/scheduled-tasks.ts +2103 -411
  151. package/src/activities/session-state.ts +10 -2
  152. package/src/activities/types.ts +32 -2
  153. package/src/activities/video-media-validation.ts +3 -3
  154. package/src/activities/video-output-retention.ts +2 -57
  155. package/src/activities/xai-auth.ts +8 -0
  156. package/src/activities-control.ts +2 -0
  157. package/src/activities.ts +3 -0
  158. package/src/activity-services.ts +13 -4
  159. package/src/connection-provider-use.ts +36 -0
  160. package/src/editable-artifact-materializer-subprocess.ts +25 -14
  161. package/src/editable-artifact-materializer.ts +29 -18
  162. package/src/github-rest-mcp.ts +713 -0
  163. package/src/http.ts +16 -6
  164. package/src/index.ts +40 -5
  165. package/src/knowledge-source-sync-activity-policy.ts +2 -0
  166. package/src/model-context-contributions.ts +33 -10
  167. package/src/observability-metrics.ts +357 -0
  168. package/src/opensandbox-kubernetes-inventory.ts +292 -0
  169. package/src/personal-github-git-credentials.ts +331 -0
  170. package/src/pr-review-credentials.ts +163 -0
  171. package/src/sandbox-archive-storage.ts +159 -0
  172. package/src/sandbox-resume.ts +449 -63
  173. package/src/sandbox-routing.ts +314 -16
  174. package/src/scheduled-alert-occurrence.ts +176 -0
  175. package/src/workflows/activities.ts +18 -2
  176. package/src/workflows/automations.ts +11 -0
  177. package/src/workflows/session.ts +13 -3
  178. package/src/workflows.ts +1 -0
  179. package/dist/activities-control-BJSDRSIP.js.map +0 -1
  180. package/dist/activities-turn-J5YO7XXX.js +0 -17243
  181. package/dist/activities-turn-J5YO7XXX.js.map +0 -1
  182. package/dist/chunk-N5TJIFCE.js.map +0 -1
  183. package/dist/chunk-P6Z2SLAA.js +0 -3092
  184. package/dist/chunk-P6Z2SLAA.js.map +0 -1
@@ -16,6 +16,7 @@
16
16
 
17
17
  import { createHash, randomUUID } from "node:crypto";
18
18
  import { Context } from "@temporalio/activity";
19
+ import { OpLostReason, OpState, type OpStatus } from "@opengeni/agent-proto";
19
20
  import {
20
21
  accrueWarmSeconds,
21
22
  adoptLegacyModalCheckpointArtifact,
@@ -61,6 +62,15 @@ import {
61
62
  type SandboxRetainedProcess,
62
63
  type LeaseSnapshot,
63
64
  } from "@opengeni/db";
65
+ import {
66
+ claimConnectedMachineSessionBackgroundCommands,
67
+ deferConnectedMachineBackgroundCommandReconciliation,
68
+ recordConnectedMachineBackgroundCommandProof,
69
+ settleClaimedConnectedMachineBackgroundCommand,
70
+ settleSessionBackgroundCommandForRetainedProcess,
71
+ type ConnectedMachineBackgroundCommandClaim,
72
+ type ConnectedMachineBackgroundCommandProof,
73
+ } from "@opengeni/db/session-background-commands";
64
74
  import { sandboxWarmRateMicrosPerSecond } from "@opengeni/config";
65
75
  import { sandboxLeaseTelemetryKey } from "@opengeni/observability";
66
76
  import {
@@ -70,6 +80,7 @@ import {
70
80
  // establishSandboxSessionFromEnvelope in strict resume-only mode so it can
71
81
  // reuse the runtime's provider-ready proof without ever cold-restoring.
72
82
  captureVerifiedWorkspaceArchive,
83
+ cancelSelfhostedOp,
73
84
  assertConsistentSandboxProviderIdentity,
74
85
  createSandboxClientForBackend,
75
86
  deleteModalCheckpointSnapshot,
@@ -78,18 +89,22 @@ import {
78
89
  inspectModalSandboxLifecycle,
79
90
  isExecSessionLostBanner,
80
91
  isProviderSandboxNotFoundError,
92
+ NatsControlRpc,
81
93
  parseExecBannerExitCode,
82
94
  prepareProviderForTeardownAfterCapture,
83
95
  providerWorkspaceCapturePolicy,
84
96
  resolveModalCheckpointProviderBindingForLiveSandbox,
85
97
  resolveModalCheckpointProviderBinding,
86
98
  resolveModalCheckpointProviderBindingForSession,
99
+ querySelfhostedOp,
87
100
  resumeExactSandboxSession,
88
101
  sandboxProviderContinuityForState,
89
102
  sandboxBackendForSdkBackendId,
90
103
  sandboxProviderInstanceIdFromEnvelope,
91
104
  sandboxCommandExitCode,
92
105
  sandboxCommandStillRunning,
106
+ SelfhostedControlError,
107
+ subjectFor,
93
108
  sweepModalOrphanSandboxes,
94
109
  terminateManagedSandboxSession,
95
110
  terminateModalSandboxById,
@@ -98,6 +113,8 @@ import {
98
113
  type ModalCheckpointProviderBinding,
99
114
  type ProviderWorkspaceCapturePolicy,
100
115
  type WorkspaceArchiveDescriptor,
116
+ type ControlRpc,
117
+ type NatsRequestConnection,
101
118
  } from "@opengeni/runtime/sandbox";
102
119
  import {
103
120
  SANDBOX_REAPER_ACTIVITY_HEARTBEAT_INTERVAL_MS,
@@ -113,6 +130,7 @@ import {
113
130
  recordCreditBalanceGauges,
114
131
  recordCreditMicros,
115
132
  recordExpiredDrainingSandboxLeaseGauges,
133
+ recordOpenSandboxKubernetesInventoryGauges,
116
134
  recordRetainedProcessInventoryGauges,
117
135
  recordRetainedProcessReconciliation,
118
136
  recordSandboxCheckpointArtifactGauges,
@@ -126,8 +144,19 @@ import {
126
144
  recordSandboxOrphansTerminated,
127
145
  recordSandboxRotationBacklogGauges,
128
146
  recordTurnsQueuedGauge,
147
+ runtimeMetricsHooksForObservability,
129
148
  } from "../observability-metrics";
130
- import { providerIdentityFromResumeState } from "../sandbox-routing";
149
+ import {
150
+ inspectOpenSandboxKubernetesInventory,
151
+ type OpenSandboxKubernetesInventory,
152
+ } from "../opensandbox-kubernetes-inventory";
153
+ import type { ObjectStorage } from "@opengeni/storage";
154
+ import {
155
+ collectWorkspaceArchiveObjectKeys,
156
+ deleteUnpublishedWorkspaceArchiveObject,
157
+ deleteWorkspaceArchiveObjectKeys,
158
+ putVersion1TarArchiveOrInline,
159
+ } from "../sandbox-archive-storage";
131
160
 
132
161
  export { sandboxLeaseTelemetryKey } from "@opengeni/observability";
133
162
 
@@ -218,6 +247,10 @@ export type SweepModalOrphansFn = (
218
247
  observability: ActivityServices["observability"],
219
248
  ) => Promise<number>;
220
249
 
250
+ export type InspectOpenSandboxKubernetesInventoryFn = (input: {
251
+ namespace: string;
252
+ }) => Promise<OpenSandboxKubernetesInventory>;
253
+
221
254
  export type SandboxLeaseActivityOptions = {
222
255
  /** Override the provider terminate (tests spy this; defaults to the real
223
256
  * resume-by-id + provider stop()). */
@@ -225,6 +258,8 @@ export type SandboxLeaseActivityOptions = {
225
258
  /** Override the provider-side Modal orphan sweep (tests spy this; defaults to
226
259
  * Modal list+tag comparison when Modal is configured). */
227
260
  sweepModalOrphans?: SweepModalOrphansFn;
261
+ /** Override the read-only bounded OpenSandbox Kubernetes projection. */
262
+ inspectOpenSandboxKubernetesInventory?: InspectOpenSandboxKubernetesInventoryFn;
228
263
  /** Override only the read-only provider process probe. The canonical DB
229
264
  * settlement remains real in tests and production. */
230
265
  probeRetainedProcess?: RetainedProcessProbeFn;
@@ -261,6 +296,7 @@ export type RetainedProcessProbeFn = (
261
296
  settings: ActivityServices["settings"],
262
297
  lease: LeaseSnapshot,
263
298
  process: SandboxRetainedProcess,
299
+ mode?: "observe" | "cancel",
264
300
  ) => Promise<RetainedProcessProbeResult>;
265
301
 
266
302
  export type HistoricalModalSandboxLifecycleProbeFn = typeof inspectModalSandboxLifecycle;
@@ -273,6 +309,8 @@ export type DrainableProviderProbeFn = (
273
309
  export const RETAINED_PROCESS_RECONCILIATION_LIMIT = 20;
274
310
  export const RETAINED_PROCESS_RECONCILIATION_CLAIM_TTL_MS = 5 * 60_000;
275
311
  export const RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS = 5_000;
312
+ export const CONNECTED_COMMAND_RECONCILIATION_LIMIT = 20;
313
+ export const CONNECTED_COMMAND_RECONCILIATION_CLAIM_TTL_MS = 5 * 60_000;
276
314
 
277
315
  export class SandboxProviderCaptureTimeoutError extends Error {
278
316
  readonly name = "SandboxProviderCaptureTimeoutError";
@@ -437,6 +475,8 @@ export function createSandboxLeaseActivities(
437
475
  ));
438
476
  const sweepModalOrphans: SweepModalOrphansFn =
439
477
  options.sweepModalOrphans ?? sweepModalOrphansForConfiguredBackend;
478
+ const inspectOpenSandboxInventory =
479
+ options.inspectOpenSandboxKubernetesInventory ?? inspectOpenSandboxKubernetesInventory;
440
480
  const probeRetainedProcess = options.probeRetainedProcess ?? probeRetainedProcessAtProvider;
441
481
  const probeDrainableProvider = options.probeDrainableProvider ?? probeDrainableProviderReadiness;
442
482
  async function prepareSandboxLeaseSweep(): Promise<SandboxLeaseSweepPlan> {
@@ -445,8 +485,19 @@ export function createSandboxLeaseActivities(
445
485
  const { db, settings, observability } = await services();
446
486
  const timing = sandboxDrainTiming(settings);
447
487
  if (!settings.sandboxOwnershipEnabled) {
488
+ // Turns skip leases when the flag is off, but Computer/Browser attach
489
+ // still acquire them. Leaving drainable rows untouched strands Desktop
490
+ // behind rotation_in_progress forever. Inventory and drain those rows;
491
+ // do not request NEW deadline rotations or meter warm time.
492
+ const drainableInventory = await reapStaleLeaseHoldersGlobal(db, {
493
+ viewerHolderTtlMs: settings.sandboxViewerHolderTtlMs,
494
+ turnHolderTtlMs: settings.sandboxLeaseTtlMs,
495
+ interactionHolderTtlMs: settings.sandboxInteractionHolderTtlMs,
496
+ idleGraceMs: settings.sandboxIdleGraceMs,
497
+ });
498
+ const drainable = drainableInventory.slice(0, SANDBOX_REAPER_CHILD_DISPATCH_LIMIT);
448
499
  return {
449
- drainable: [],
500
+ drainable,
450
501
  timeoutClass: timing.timeoutClass,
451
502
  snapshotTimeoutMs: timing.snapshotTimeoutMs,
452
503
  captureTimeoutMs: timing.captureTimeoutMs,
@@ -518,8 +569,7 @@ export function createSandboxLeaseActivities(
518
569
  instanceId: input.target.instanceId,
519
570
  });
520
571
  try {
521
- const { db, settings, observability } = await services();
522
- if (!settings.sandboxOwnershipEnabled) return { status: "skipped" };
572
+ const { db, settings, observability, objectStorage } = await services();
523
573
  assertSandboxDrainInputTiming(input);
524
574
  const drainSettings =
525
575
  settings.sandboxSnapshotTimeoutMs === input.snapshotTimeoutMs
@@ -542,6 +592,7 @@ export function createSandboxLeaseActivities(
542
592
  terminateBox,
543
593
  probeDrainableProvider,
544
594
  captureAttempt,
595
+ objectStorage,
545
596
  );
546
597
  return { status: drainedCold ? "terminated" : "skipped" };
547
598
  } catch (error) {
@@ -611,10 +662,21 @@ export function createSandboxLeaseActivities(
611
662
  let forceDrained = 0;
612
663
  const rotationsRequested = input.rotationsRequested;
613
664
  let modalOrphansTerminated = 0;
665
+
666
+ // Connected Machine commands are not managed-sandbox lease state. Their
667
+ // immutable launch locators must reconcile even when managed ownership is
668
+ // disabled, otherwise an exact runner exit/loss proof can remain stranded.
669
+ await reconcileConnectedMachineBackgroundCommands(db, settings, observability, service.bus);
670
+
614
671
  if (!settings.sandboxOwnershipEnabled) {
615
672
  // Inventory remains useful while provider ownership is intentionally
616
673
  // disabled, and this scheduled activity is its single projection owner.
617
- await refreshQueueLeaseAndCreditGauges(db, observability);
674
+ await refreshQueueLeaseAndCreditGauges(
675
+ db,
676
+ settings,
677
+ observability,
678
+ inspectOpenSandboxInventory,
679
+ );
618
680
  return {
619
681
  metered,
620
682
  forceDrained,
@@ -683,7 +745,12 @@ export function createSandboxLeaseActivities(
683
745
  },
684
746
  );
685
747
 
686
- await refreshQueueLeaseAndCreditGauges(db, observability);
748
+ await refreshQueueLeaseAndCreditGauges(
749
+ db,
750
+ settings,
751
+ observability,
752
+ inspectOpenSandboxInventory,
753
+ );
687
754
 
688
755
  if (
689
756
  input.examined > 0 ||
@@ -789,6 +856,30 @@ const CHECKPOINT_TOMBSTONE_RETENTION_MS = 30 * 24 * 60 * 60_000;
789
856
  const CHECKPOINT_TOMBSTONE_PRUNE_LIMIT = 500;
790
857
  const SANDBOX_MAINTENANCE_ITEM_CONCURRENCY = 8;
791
858
 
859
+ type ConnectedCommandReconciliationOutcome =
860
+ | "claim_failed"
861
+ | "provider_running"
862
+ | "provider_offline"
863
+ | "provider_error"
864
+ | "proof_exited"
865
+ | "proof_lost"
866
+ | "proof_checkpoint_failed"
867
+ | "settled_exited"
868
+ | "settled_lost"
869
+ | "settlement_failed"
870
+ | "defer_failed";
871
+
872
+ function recordConnectedCommandReconciliation(
873
+ observability: ActivityServices["observability"],
874
+ outcome: ConnectedCommandReconciliationOutcome,
875
+ ): void {
876
+ observability.incrementCounter({
877
+ name: "opengeni_connected_background_command_reconciliation_total",
878
+ help: "Exact-locator Connected Machine background-command reconciliation outcomes.",
879
+ labels: { outcome },
880
+ });
881
+ }
882
+
792
883
  async function forEachWithConcurrency<T>(
793
884
  values: readonly T[],
794
885
  concurrency: number,
@@ -934,6 +1025,199 @@ async function adoptLegacyModalCheckpointReceipts(
934
1025
  return adopted;
935
1026
  }
936
1027
 
1028
+ /** Reconcile commands only against the immutable physical locator copied at
1029
+ * adoption. A successor enrollment connection is intentionally irrelevant: no
1030
+ * query or cancellation is ever rebound to it. */
1031
+ async function reconcileConnectedMachineBackgroundCommands(
1032
+ db: ActivityServices["db"],
1033
+ settings: ActivityServices["settings"],
1034
+ observability: ActivityServices["observability"],
1035
+ bus: ActivityServices["bus"],
1036
+ ): Promise<void> {
1037
+ const claimId = crypto.randomUUID();
1038
+ let claims: ConnectedMachineBackgroundCommandClaim[];
1039
+ try {
1040
+ claims = await claimConnectedMachineSessionBackgroundCommands(db, {
1041
+ claimId,
1042
+ limit: CONNECTED_COMMAND_RECONCILIATION_LIMIT,
1043
+ claimTtlMs: CONNECTED_COMMAND_RECONCILIATION_CLAIM_TTL_MS,
1044
+ });
1045
+ } catch (error) {
1046
+ recordConnectedCommandReconciliation(observability, "claim_failed");
1047
+ observability.warn("sandbox reaper: connected-command claim failed", {
1048
+ error: error instanceof Error ? error.message : String(error),
1049
+ });
1050
+ return;
1051
+ }
1052
+
1053
+ const controlRpc = new NatsControlRpc(
1054
+ async (): Promise<NatsRequestConnection | null> => bus.getRequestConnection(),
1055
+ );
1056
+ await forEachWithConcurrency(claims, SANDBOX_MAINTENANCE_ITEM_CONCURRENCY, async (claim) => {
1057
+ let proof = claim.proof;
1058
+ if (!proof) {
1059
+ try {
1060
+ proof = await probeConnectedMachineBackgroundCommand(
1061
+ claim,
1062
+ controlRpc,
1063
+ Math.min(
1064
+ settings.sandboxSelfhostedControlTimeoutMs,
1065
+ RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS,
1066
+ ),
1067
+ );
1068
+ if (!proof) {
1069
+ await deferConnectedCommandClaim(db, settings, observability, claim, "provider_running");
1070
+ return;
1071
+ }
1072
+ } catch (error) {
1073
+ const offline =
1074
+ error instanceof SelfhostedControlError &&
1075
+ (error.agentOffline || error.reason === "agent_reconnecting" || error.draining);
1076
+ if (!offline) {
1077
+ observability.warn("sandbox reaper: connected-command provider probe failed", {
1078
+ commandId: claim.commandId,
1079
+ enrollmentId: claim.enrollmentId,
1080
+ connectionInstanceId: claim.connectionInstanceId,
1081
+ error: error instanceof Error ? error.message : String(error),
1082
+ });
1083
+ }
1084
+ await deferConnectedCommandClaim(
1085
+ db,
1086
+ settings,
1087
+ observability,
1088
+ claim,
1089
+ offline ? "provider_offline" : "provider_error",
1090
+ );
1091
+ return;
1092
+ }
1093
+
1094
+ try {
1095
+ await recordConnectedMachineBackgroundCommandProof(db, { claim, proof });
1096
+ recordConnectedCommandReconciliation(observability, `proof_${proof.outcome}`);
1097
+ } catch (error) {
1098
+ recordConnectedCommandReconciliation(observability, "proof_checkpoint_failed");
1099
+ observability.warn("sandbox reaper: connected-command proof checkpoint failed", {
1100
+ commandId: claim.commandId,
1101
+ error: error instanceof Error ? error.message : String(error),
1102
+ });
1103
+ return;
1104
+ }
1105
+ }
1106
+
1107
+ try {
1108
+ if (!(await settleClaimedConnectedMachineBackgroundCommand(db, { claim }))) {
1109
+ throw new Error("Connected command settlement lost its exact claim or proof");
1110
+ }
1111
+ recordConnectedCommandReconciliation(observability, `settled_${proof.outcome}`);
1112
+ } catch (error) {
1113
+ recordConnectedCommandReconciliation(observability, "settlement_failed");
1114
+ observability.warn("sandbox reaper: connected-command settlement failed", {
1115
+ commandId: claim.commandId,
1116
+ error: error instanceof Error ? error.message : String(error),
1117
+ });
1118
+ await deferConnectedCommandClaim(db, settings, observability, claim, "settlement_failed");
1119
+ }
1120
+ });
1121
+ }
1122
+
1123
+ export async function probeConnectedMachineBackgroundCommand(
1124
+ claim: ConnectedMachineBackgroundCommandClaim,
1125
+ controlRpc: ControlRpc,
1126
+ controlTimeoutMs: number,
1127
+ observedAt = new Date(),
1128
+ ): Promise<ConnectedMachineBackgroundCommandProof | null> {
1129
+ const input = {
1130
+ controlRpc,
1131
+ rpcSubject: subjectFor(
1132
+ claim.controlWorkspaceId,
1133
+ claim.enrollmentId,
1134
+ claim.connectionInstanceId,
1135
+ ),
1136
+ opId: claim.opId,
1137
+ controlTimeoutMs,
1138
+ };
1139
+ const status =
1140
+ claim.state === "stopping" ? await cancelSelfhostedOp(input) : await querySelfhostedOp(input);
1141
+ return connectedCommandProofFromStatus(status, observedAt);
1142
+ }
1143
+
1144
+ export function connectedCommandProofFromStatus(
1145
+ status: OpStatus,
1146
+ observedAt: Date,
1147
+ ): ConnectedMachineBackgroundCommandProof | null {
1148
+ switch (status.state) {
1149
+ case OpState.OP_STATE_ACCEPTED:
1150
+ case OpState.OP_STATE_RUNNING:
1151
+ return null;
1152
+ case OpState.OP_STATE_COMPLETE: {
1153
+ const exit = status.exit;
1154
+ if (!exit) {
1155
+ throw new Error("Connected command completed without an exit record");
1156
+ }
1157
+ const failureCode = exit.failureCode.replace(/[^A-Za-z0-9_-]/g, "_").slice(0, 128);
1158
+ const reason = exit.cancelled
1159
+ ? "op_cancelled"
1160
+ : exit.timedOut
1161
+ ? "op_timed_out"
1162
+ : failureCode
1163
+ ? `op_failure_${failureCode}`
1164
+ : "op_exit";
1165
+ return {
1166
+ outcome: "exited",
1167
+ exitCode: exit.exitCode,
1168
+ reason,
1169
+ observedAt,
1170
+ };
1171
+ }
1172
+ case OpState.OP_STATE_LOST:
1173
+ return {
1174
+ outcome: "lost",
1175
+ exitCode: null,
1176
+ reason:
1177
+ status.lostReason === OpLostReason.OP_LOST_REASON_EVICTED
1178
+ ? "op_lost_evicted"
1179
+ : status.lostReason === OpLostReason.OP_LOST_REASON_AGENT_RESTARTED
1180
+ ? "op_lost_agent_restarted"
1181
+ : "op_lost_unspecified",
1182
+ observedAt,
1183
+ };
1184
+ default:
1185
+ throw new Error(`Connected command returned invalid op state ${status.state}`);
1186
+ }
1187
+ }
1188
+
1189
+ async function deferConnectedCommandClaim(
1190
+ db: ActivityServices["db"],
1191
+ settings: ActivityServices["settings"],
1192
+ observability: ActivityServices["observability"],
1193
+ claim: ConnectedMachineBackgroundCommandClaim,
1194
+ outcome: "provider_running" | "provider_offline" | "provider_error" | "settlement_failed",
1195
+ ): Promise<void> {
1196
+ const exponent = Math.min(4, Math.max(0, claim.reconcileAttempts - 1));
1197
+ const retryAfterMs = Math.min(
1198
+ 5 * 60_000,
1199
+ Math.max(settings.sandboxLeaseReaperPeriodMs, 30_000) * 2 ** exponent,
1200
+ );
1201
+ try {
1202
+ if (
1203
+ !(await deferConnectedMachineBackgroundCommandReconciliation(db, {
1204
+ claim,
1205
+ outcome,
1206
+ retryAfterMs,
1207
+ }))
1208
+ ) {
1209
+ throw new Error("Connected command defer lost its exact claim");
1210
+ }
1211
+ recordConnectedCommandReconciliation(observability, outcome);
1212
+ } catch (error) {
1213
+ recordConnectedCommandReconciliation(observability, "defer_failed");
1214
+ observability.warn("sandbox reaper: connected-command defer failed", {
1215
+ commandId: claim.commandId,
1216
+ error: error instanceof Error ? error.message : String(error),
1217
+ });
1218
+ }
1219
+ }
1220
+
937
1221
  async function reconcileTerminalRetainedProcesses(
938
1222
  db: ActivityServices["db"],
939
1223
  settings: ActivityServices["settings"],
@@ -1070,7 +1354,12 @@ async function reconcileTerminalRetainedProcesses(
1070
1354
  }
1071
1355
  } else {
1072
1356
  try {
1073
- observation = await probe(settings, lease!, process);
1357
+ observation = await probe(
1358
+ settings,
1359
+ lease!,
1360
+ process,
1361
+ claim.ownerState === "background_stopping" ? "cancel" : "observe",
1362
+ );
1074
1363
  } catch (error) {
1075
1364
  observability.warn("sandbox reaper: retained-process provider probe failed", {
1076
1365
  processId: process.id,
@@ -1150,6 +1439,15 @@ async function reconcileTerminalRetainedProcesses(
1150
1439
  reason: proof.reason,
1151
1440
  idleGraceMs: settings.sandboxIdleGraceMs,
1152
1441
  });
1442
+ await settleSessionBackgroundCommandForRetainedProcess(db, {
1443
+ accountId: process.accountId,
1444
+ workspaceId: process.workspaceId,
1445
+ sessionId: process.sessionId,
1446
+ retainedProcessId: process.id,
1447
+ outcome: proof.outcome,
1448
+ exitCode: proof.exitCode,
1449
+ reason: proof.reason,
1450
+ });
1153
1451
  recordRetainedProcessReconciliation(observability, `settled_${proof.outcome}`);
1154
1452
  } catch (error) {
1155
1453
  recordRetainedProcessReconciliation(observability, "settlement_failed");
@@ -1244,6 +1542,7 @@ export async function probeRetainedProcessAtProvider(
1244
1542
  settings: ActivityServices["settings"],
1245
1543
  lease: LeaseSnapshot,
1246
1544
  process: SandboxRetainedProcess,
1545
+ mode: "observe" | "cancel" = "observe",
1247
1546
  ): Promise<RetainedProcessProbeResult> {
1248
1547
  if (
1249
1548
  lease.id !== process.leaseId ||
@@ -1263,7 +1562,7 @@ export async function probeRetainedProcessAtProvider(
1263
1562
  if (envelopeBackend !== undefined && envelopeBackend !== process.providerBackend) {
1264
1563
  return { status: "deferred", reason: "identity_mismatch" };
1265
1564
  }
1266
- if (providerIdentityFromResumeState(lease.resumeState) !== process.providerInstanceId) {
1565
+ if (sandboxProviderInstanceIdFromEnvelope(lease.resumeState) !== process.providerInstanceId) {
1267
1566
  return { status: "deferred", reason: "identity_mismatch" };
1268
1567
  }
1269
1568
 
@@ -1378,7 +1677,7 @@ export async function probeRetainedProcessAtProvider(
1378
1677
  result = await withRetainedProcessProbeTimeout(
1379
1678
  session.writeStdin({
1380
1679
  sessionId: process.providerSessionId,
1381
- chars: "",
1680
+ chars: mode === "cancel" ? "\u0003" : "",
1382
1681
  yieldTimeMs: 1_000,
1383
1682
  maxOutputTokens: 2_000,
1384
1683
  }),
@@ -1533,7 +1832,9 @@ async function accrueWarmTick(
1533
1832
 
1534
1833
  async function refreshQueueLeaseAndCreditGauges(
1535
1834
  db: ActivityServices["db"],
1835
+ settings: ActivityServices["settings"],
1536
1836
  observability: ActivityServices["observability"],
1837
+ inspectOpenSandboxInventory: InspectOpenSandboxKubernetesInventoryFn,
1537
1838
  ): Promise<void> {
1538
1839
  await Promise.all([
1539
1840
  (async () => {
@@ -1584,6 +1885,24 @@ async function refreshQueueLeaseAndCreditGauges(
1584
1885
  await countExpiredDrainingSandboxLeases(db),
1585
1886
  );
1586
1887
  }),
1888
+ ...(settings.sandboxBackend === "opensandbox" &&
1889
+ settings.openSandboxKubernetesInventoryNamespace
1890
+ ? [
1891
+ refreshSandboxInventoryGauge(
1892
+ observability,
1893
+ "opensandbox_kubernetes",
1894
+ "OpenSandbox Kubernetes",
1895
+ async () => {
1896
+ recordOpenSandboxKubernetesInventoryGauges(
1897
+ observability,
1898
+ await inspectOpenSandboxInventory({
1899
+ namespace: settings.openSandboxKubernetesInventoryNamespace!,
1900
+ }),
1901
+ );
1902
+ },
1903
+ ),
1904
+ ]
1905
+ : []),
1587
1906
  (async () => {
1588
1907
  try {
1589
1908
  recordCreditBalanceGauges(observability, await listCreditBalancesByAccount(db));
@@ -1813,6 +2132,24 @@ async function probeDrainableProviderReadiness(
1813
2132
  }
1814
2133
  }
1815
2134
 
2135
+ export type SandboxDrainReason =
2136
+ | "idle"
2137
+ | "provider_deadline"
2138
+ | "operator"
2139
+ | "teardown_claim"
2140
+ | "rotation_requested";
2141
+
2142
+ /** The bounded drain reason carried on `sandbox.box.terminated`: the lease's
2143
+ * rotation reason when a rotation was requested, otherwise the ordinary idle
2144
+ * drain. A requested rotation without a recorded reason (legacy rows) is
2145
+ * reported as `rotation_requested` rather than guessed. */
2146
+ export function sandboxDrainReason(
2147
+ lease: Pick<LeaseSnapshot, "rotationRequestedAt" | "rotationReason">,
2148
+ ): SandboxDrainReason {
2149
+ if (lease.rotationRequestedAt === null) return "idle";
2150
+ return lease.rotationReason ?? "rotation_requested";
2151
+ }
2152
+
1816
2153
  /**
1817
2154
  * Terminate one drainable box by id, then CAS its lease draining->cold under the
1818
2155
  * epoch fence. Returns true when the lease went cold (the box is ours to stop
@@ -1837,6 +2174,7 @@ async function terminateDrainableBox(
1837
2174
  terminateBox: TerminateBoxFn,
1838
2175
  probeDrainableProvider: DrainableProviderProbeFn,
1839
2176
  attempt: SandboxDrainCaptureAttempt,
2177
+ objectStorage: ObjectStorage | null,
1840
2178
  ): Promise<boolean> {
1841
2179
  // Resolve the account for the RLS-scoped confirmDrainCold (the global sweep
1842
2180
  // returns no account_id; the workspace->account map is the bootstrap read).
@@ -2092,6 +2430,7 @@ async function terminateDrainableBox(
2092
2430
  if (!lease.instanceId || !captureClaim) {
2093
2431
  return { wrote: false, archiveRevision: null };
2094
2432
  }
2433
+ const archiveMetrics = runtimeMetricsHooksForObservability(observability);
2095
2434
  let checkpointArtifactId: string | null = null;
2096
2435
  if (
2097
2436
  archiveBase64 &&
@@ -2138,12 +2477,59 @@ async function terminateDrainableBox(
2138
2477
  if (!archiveMetadata) {
2139
2478
  throw new Error("Sandbox snapshot publication requires a verified archive descriptor");
2140
2479
  }
2141
- result = await persistDrainSnapshot(db, {
2142
- ...baseInput,
2143
- workspaceArchive: archiveBase64,
2144
- workspaceArchiveMeta: archiveMetadata,
2145
- ...(checkpointArtifactId ? { checkpointArtifactId } : {}),
2146
- });
2480
+ const priorKeys = collectWorkspaceArchiveObjectKeys(
2481
+ (lease.resumeState as Record<string, unknown> | null | undefined) ?? null,
2482
+ );
2483
+ let workspaceArchiveRef:
2484
+ | Awaited<ReturnType<typeof putVersion1TarArchiveOrInline>>["workspaceArchiveRef"]
2485
+ | undefined;
2486
+ try {
2487
+ const published = await putVersion1TarArchiveOrInline({
2488
+ backend: lease.backend,
2489
+ objectStorage,
2490
+ accountId,
2491
+ workspaceId: row.workspaceId,
2492
+ sandboxGroupId: row.sandboxGroupId,
2493
+ archive: {
2494
+ bytes: Buffer.from(archiveBase64, "base64"),
2495
+ descriptor: archiveMetadata,
2496
+ base64: archiveBase64,
2497
+ },
2498
+ metrics: archiveMetrics,
2499
+ });
2500
+ workspaceArchiveRef = published.workspaceArchiveRef;
2501
+ result = await persistDrainSnapshot(db, {
2502
+ ...baseInput,
2503
+ workspaceArchiveMeta: archiveMetadata,
2504
+ ...published,
2505
+ ...(checkpointArtifactId ? { checkpointArtifactId } : {}),
2506
+ });
2507
+ if (published.workspaceArchiveRef && objectStorage) {
2508
+ if (!result.wrote) {
2509
+ await deleteUnpublishedWorkspaceArchiveObject(
2510
+ objectStorage,
2511
+ published.workspaceArchiveRef,
2512
+ archiveMetrics,
2513
+ );
2514
+ } else {
2515
+ const afterLease = await readLease(db, row.workspaceId, row.sandboxGroupId);
2516
+ const afterKeys = collectWorkspaceArchiveObjectKeys(
2517
+ (afterLease?.resumeState as Record<string, unknown> | null | undefined) ?? null,
2518
+ );
2519
+ await deleteWorkspaceArchiveObjectKeys(
2520
+ objectStorage,
2521
+ [...priorKeys].filter((key) => !afterKeys.has(key)),
2522
+ ).catch(() => undefined);
2523
+ }
2524
+ }
2525
+ } catch (error) {
2526
+ await deleteUnpublishedWorkspaceArchiveObject(
2527
+ objectStorage,
2528
+ workspaceArchiveRef,
2529
+ archiveMetrics,
2530
+ );
2531
+ throw error;
2532
+ }
2147
2533
  }
2148
2534
  if (!result.wrote && checkpointArtifactId) {
2149
2535
  await markSandboxCheckpointArtifactDeletePending(db, {
@@ -2214,6 +2600,11 @@ async function terminateDrainableBox(
2214
2600
  archiveRevision,
2215
2601
  instanceId: lease.instanceId,
2216
2602
  providerMissingBeforeCapture: providerMissing,
2603
+ // Why the box drained: the lease's bounded rotation reason when one
2604
+ // was requested (provider deadline, operator/image conflict, teardown
2605
+ // claim), otherwise the ordinary idle drain. Additive diagnostic
2606
+ // fact; no identifiers beyond the ones already carried above.
2607
+ drainReason: sandboxDrainReason(lease),
2217
2608
  },
2218
2609
  });
2219
2610
  } catch (eventError) {