@opengeni/worker-bundle 0.12.7 → 0.13.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. package/dist/activities/agent-turn.d.ts +778 -0
  2. package/dist/activities/capabilities.d.ts +18 -0
  3. package/dist/activities/codex-auth.d.ts +1 -0
  4. package/dist/activities/codex-capacity.d.ts +31 -0
  5. package/dist/activities/codex-fleet-shadow.d.ts +74 -0
  6. package/dist/activities/codex-rotation.d.ts +261 -0
  7. package/dist/activities/common.d.ts +17 -0
  8. package/dist/activities/context-compaction.d.ts +81 -0
  9. package/dist/activities/documents.d.ts +39 -0
  10. package/dist/activities/environment.d.ts +97 -0
  11. package/dist/activities/expiring-material-renewal.d.ts +38 -0
  12. package/dist/activities/file-upload-reaper.d.ts +25 -0
  13. package/dist/activities/git-credential-renewal.d.ts +41 -0
  14. package/dist/activities/goals.d.ts +16 -0
  15. package/dist/activities/mcp-credentials.d.ts +14 -0
  16. package/dist/activities/packs.d.ts +77 -0
  17. package/dist/activities/parent-wake.d.ts +54 -0
  18. package/dist/activities/recording.d.ts +90 -0
  19. package/dist/activities/redaction.d.ts +1 -0
  20. package/dist/activities/rig-verification.d.ts +131 -0
  21. package/dist/activities/run-credential-renewal.d.ts +28 -0
  22. package/dist/activities/run-credentials.d.ts +36 -0
  23. package/dist/activities/run-input.d.ts +106 -0
  24. package/dist/activities/sandbox-lease.d.ts +128 -0
  25. package/dist/activities/scheduled-tasks.d.ts +4 -0
  26. package/dist/activities/session-state.d.ts +39 -0
  27. package/dist/activities/streaming.d.ts +24 -0
  28. package/dist/activities/toolspace-token-renewal.d.ts +23 -0
  29. package/dist/activities/turn-attempt-fenced.d.ts +11 -0
  30. package/dist/activities/types.d.ts +240 -0
  31. package/dist/activities/workflow-wake.d.ts +15 -0
  32. package/dist/activities/workspace-capture.d.ts +105 -0
  33. package/dist/activities.d.ts +312 -0
  34. package/dist/concurrency.d.ts +19 -0
  35. package/dist/host-export-pump.d.ts +66 -0
  36. package/dist/host-export.d.ts +6 -67
  37. package/dist/http.d.ts +32 -0
  38. package/dist/index.d.ts +27 -388
  39. package/dist/index.js +2730 -1211
  40. package/dist/index.js.map +1 -1
  41. package/dist/observability-metrics.d.ts +200 -0
  42. package/dist/op-journal.d.ts +10 -0
  43. package/dist/sandbox-resume.d.ts +152 -0
  44. package/dist/sandbox-routing.d.ts +159 -0
  45. package/dist/worker-service-lifecycle.d.ts +20 -0
  46. package/dist/workflow-bundle.js +88 -20
  47. package/dist/workflows/activities.d.ts +27 -0
  48. package/dist/workflows/document-index.d.ts +41 -0
  49. package/dist/workflows/file-upload-reaper.d.ts +2 -0
  50. package/dist/workflows/rig-verification.d.ts +3 -0
  51. package/dist/workflows/sandbox-reaper.d.ts +1 -0
  52. package/dist/workflows/scheduled-tasks.d.ts +17 -0
  53. package/dist/workflows/session.d.ts +58 -0
  54. package/dist/workflows/workflow-wake.d.ts +3 -0
  55. package/dist/workflows.d.ts +7 -0
  56. package/package.json +13 -13
  57. package/src/activities/agent-turn.ts +874 -134
  58. package/src/activities/context-compaction.ts +265 -76
  59. package/src/activities/environment.ts +1 -0
  60. package/src/activities/goals.ts +14 -0
  61. package/src/activities/rig-verification.ts +2 -0
  62. package/src/activities/run-input.ts +18 -4
  63. package/src/activities/sandbox-lease.ts +787 -133
  64. package/src/activities/session-state.ts +56 -0
  65. package/src/activities/types.ts +21 -0
  66. package/src/activities/workspace-capture.ts +95 -4
  67. package/src/activities.ts +3 -0
  68. package/src/observability-metrics.ts +94 -0
  69. package/src/sandbox-resume.ts +383 -90
  70. package/src/sandbox-routing.ts +32 -10
  71. package/src/workflows/activities.ts +1 -0
  72. package/src/workflows/session.ts +70 -0
@@ -1,6 +1,7 @@
1
1
  import {
2
2
  settleSessionAttemptInterruptions,
3
3
  applySessionTurnSettlement,
4
+ requestSessionTurnRecovery,
4
5
  recoverSessionDispatch,
5
6
  reconcileSessionAttemptQuiescence,
6
7
  peekSessionWork as peekSessionWorkDb,
@@ -29,11 +30,14 @@ import type {
29
30
  ReconcileSessionAttemptQuiescenceResult,
30
31
  RecoverDispatchInput,
31
32
  RecoverDispatchResult,
33
+ RecoverEscapedMcpTimeoutInput,
34
+ RecoverEscapedMcpTimeoutResult,
32
35
  } from "./types";
33
36
 
34
37
  export type SessionStateActivityOverrides = Partial<{
35
38
  settleSessionAttemptInterruptions: typeof settleSessionAttemptInterruptions;
36
39
  applySessionTurnSettlement: typeof applySessionTurnSettlement;
40
+ requestSessionTurnRecovery: typeof requestSessionTurnRecovery;
37
41
  recoverSessionDispatch: typeof recoverSessionDispatch;
38
42
  reconcileSessionAttemptQuiescence: typeof reconcileSessionAttemptQuiescence;
39
43
  peekSessionWork: typeof peekSessionWorkDb;
@@ -65,6 +69,8 @@ export function createSessionStateActivities(
65
69
  overrides.settleSessionAttemptInterruptions ?? settleSessionAttemptInterruptions;
66
70
  const applySessionTurnSettlementFn =
67
71
  overrides.applySessionTurnSettlement ?? applySessionTurnSettlement;
72
+ const requestSessionTurnRecoveryFn =
73
+ overrides.requestSessionTurnRecovery ?? requestSessionTurnRecovery;
68
74
  const recoverSessionDispatchFn = overrides.recoverSessionDispatch ?? recoverSessionDispatch;
69
75
  const reconcileSessionAttemptQuiescenceFn =
70
76
  overrides.reconcileSessionAttemptQuiescence ?? reconcileSessionAttemptQuiescence;
@@ -271,6 +277,55 @@ export function createSessionStateActivities(
271
277
  };
272
278
  }
273
279
 
280
+ /**
281
+ * Finish a retryable MCP timeout checkpoint that escaped a recovered turn's
282
+ * activity before any model request. Temporal carries the immutable turn
283
+ * identity in ApplicationFailure details; this activity re-reads and fences
284
+ * every field before mutating the exact still-owned attempt. A checkpoint
285
+ * that committed before event fanout failed is already recovering and
286
+ * therefore returns stale without another event or child callback.
287
+ */
288
+ async function recoverEscapedMcpTimeout(
289
+ input: RecoverEscapedMcpTimeoutInput,
290
+ ): Promise<RecoverEscapedMcpTimeoutResult> {
291
+ const { db, bus, observability } = await services();
292
+ const turn = await getSessionTurnForAttemptFn(
293
+ db,
294
+ input.workspaceId,
295
+ input.sessionId,
296
+ input.attemptId,
297
+ );
298
+ if (!turn) return { action: "stale" };
299
+ if (
300
+ turn.id !== input.turnId ||
301
+ turn.triggerEventId !== input.triggerEventId ||
302
+ turn.executionGeneration !== input.executionGeneration ||
303
+ input.executionGeneration <= 1
304
+ ) {
305
+ return { action: "ineligible" };
306
+ }
307
+ const recovery = await requestSessionTurnRecoveryFn(db, input.workspaceId, {
308
+ sessionId: input.sessionId,
309
+ turnId: input.turnId,
310
+ triggerEventId: input.triggerEventId,
311
+ attemptId: input.attemptId,
312
+ reason: "mcp_transport_timeout",
313
+ detail: {
314
+ code: "mcp_transport_timeout",
315
+ retryable: true,
316
+ continueDelayMs: 60_000,
317
+ recoverySource: "workflow_activity_failure",
318
+ },
319
+ fromStatuses: ["running"],
320
+ });
321
+ if (recovery.action !== "recovering") {
322
+ return { action: "stale" };
323
+ }
324
+ await publishDurableSessionEventsFn(bus, input.workspaceId, input.sessionId, recovery.events);
325
+ await refreshQueuedTurnsGauge(db, observability, countQueuedTurnsFn, recordTurnsQueuedGaugeFn);
326
+ return { action: "recovering" };
327
+ }
328
+
274
329
  async function peekSessionWork(input: PeekSessionWorkInput) {
275
330
  const { db, observability } = await services();
276
331
  const peek = await peekSessionWorkFn(db, input.workspaceId, input.sessionId);
@@ -325,6 +380,7 @@ export function createSessionStateActivities(
325
380
  persistSessionAttemptQuiescence,
326
381
  reconcileSessionAttemptQuiescence: reconcileSessionAttemptQuiescenceActivity,
327
382
  recoverDispatch,
383
+ recoverEscapedMcpTimeout,
328
384
  peekSessionWork,
329
385
  expireSessionHumanInput,
330
386
  markSessionIdle,
@@ -202,6 +202,27 @@ export type RecoverDispatchResult =
202
202
  // ceiling), so the failed attempt was worker death number redispatches + 1.
203
203
  | { action: "exceeded"; turnId: string; redispatches: number };
204
204
 
205
+ export const ESCAPED_MCP_TIMEOUT_RECOVERY_FAILURE_TYPE = "EscapedMcpTimeoutRecoveryFailure";
206
+ export const ESCAPED_MCP_TIMEOUT_RECOVERY_FAILURE_MESSAGE =
207
+ "MCP request timeout recovery checkpoint failed before model request";
208
+
209
+ export type EscapedMcpTimeoutRecoveryDetail = {
210
+ turnId: string;
211
+ triggerEventId: string;
212
+ executionGeneration: number;
213
+ };
214
+
215
+ export type RecoverEscapedMcpTimeoutInput = EscapedMcpTimeoutRecoveryDetail & {
216
+ accountId: string;
217
+ workspaceId: string;
218
+ sessionId: string;
219
+ attemptId: string;
220
+ };
221
+
222
+ export type RecoverEscapedMcpTimeoutResult = {
223
+ action: "recovering" | "stale" | "ineligible";
224
+ };
225
+
205
226
  export type PeekSessionWorkInput = {
206
227
  workspaceId: string;
207
228
  sessionId: string;
@@ -14,7 +14,7 @@
14
14
  // • This module NEVER calls close()/terminate()/kill() on the session handle.
15
15
  // session.close() == terminate() on Modal and would kill the user's box.
16
16
  // Only the reaper terminates. We drop references, nothing more.
17
- // • The whole capture is time-capped (60s) and best-effort: ANY failure logs
17
+ // • The whole capture is time-capped (120s) and best-effort: ANY failure logs
18
18
  // "workspace capture failed — turn outcome unaffected" and returns; nothing
19
19
  // throws past captureWorkspaceRevision.
20
20
  // • The DB write is fenced on the exact attempt, accepted control requests,
@@ -62,7 +62,12 @@ import type {
62
62
  } from "@opengeni/contracts";
63
63
 
64
64
  // ─── Guards & constants for pathological cases; configurable ─────
65
- export const CAPTURE_TIMEOUT_MS = 60_000;
65
+ // Production Modal captures of the acceptance workspace normally take ~30s:
66
+ // repository diff framing deliberately trades provider round-trips for bounded
67
+ // output and symlink-safe reads. Sixty seconds gave that healthy path only a 2x
68
+ // margin and repeatedly timed it out under ordinary provider variance. The turn
69
+ // holder keeps the exact lease pinned for this whole window.
70
+ export const CAPTURE_TIMEOUT_MS = 120_000;
66
71
  export const PER_FILE_CONTENT_GUARD_BYTES = 5 * 1024 * 1024; // after-image; over → tooLarge marker
67
72
  export const PER_FILE_DIFF_GUARD_BYTES = 10 * 1024 * 1024; // per-file diff; over → truncated marker
68
73
  export const WHOLE_CAPTURE_GUARD_BYTES = 200 * 1024 * 1024; // total → skip, fall back to live
@@ -185,6 +190,14 @@ export type CaptureWorkspaceRevisionInput = {
185
190
  publish: CaptureEventPublisher | null;
186
191
  /** The un-proxied setupBoxSession (NOT the routing veneer). */
187
192
  session: ChannelASession;
193
+ /**
194
+ * Reopen the exact live provider instance as a fresh, non-owning read handle.
195
+ * Modal sessions keep mutable exec-process bookkeeping; reusing the handle
196
+ * that just drove model tools can strand a later capture command behind stale
197
+ * provider state. The lease still owns the box. Capture drops this handle
198
+ * without calling close(), which would terminate Modal's sandbox.
199
+ */
200
+ openReadSession?: () => Promise<ChannelASession>;
188
201
  leaseEpoch: number;
189
202
  sandboxGroupId: string;
190
203
  accountId: string;
@@ -211,6 +224,68 @@ type CaptureRepositoryReadResult =
211
224
  }
212
225
  | { complete: false; degradedReason: "repository_read_unavailable" };
213
226
 
227
+ type ResumableCaptureClient = {
228
+ resume?: (state: unknown) => Promise<unknown>;
229
+ };
230
+
231
+ type StatefulCaptureSession = ChannelASession & {
232
+ state?: unknown;
233
+ };
234
+
235
+ function captureSessionInstanceId(session: unknown): string | null {
236
+ const state =
237
+ session && typeof session === "object"
238
+ ? (session as { state?: Record<string, unknown> }).state
239
+ : undefined;
240
+ if (!state || typeof state !== "object") return null;
241
+ for (const value of [
242
+ state.sandboxId,
243
+ state.instanceId,
244
+ state.id,
245
+ state.hostId,
246
+ state.containerId,
247
+ state.workspaceRootPath,
248
+ ]) {
249
+ if (typeof value === "string" && value.length > 0) return value;
250
+ }
251
+ return null;
252
+ }
253
+
254
+ /**
255
+ * Reattach a clean SDK handle to the exact already-live Modal sandbox. Modal's
256
+ * resume contract is a pure from-id reattach. Other providers keep the original
257
+ * session because their generic resume contract may restore or replace a dead
258
+ * instance, which capture does not own.
259
+ */
260
+ export async function openFreshWorkspaceCaptureSession(input: {
261
+ backendId: string;
262
+ client: unknown;
263
+ session: ChannelASession;
264
+ expectedInstanceId: string;
265
+ }): Promise<ChannelASession> {
266
+ if (input.backendId !== "modal") {
267
+ return input.session;
268
+ }
269
+ const client = input.client as ResumableCaptureClient;
270
+ const state = (input.session as StatefulCaptureSession).state;
271
+ if (typeof client.resume !== "function" || state === undefined) {
272
+ return input.session;
273
+ }
274
+ // The lease—not this temporary reader—owns provider teardown. Preserve the
275
+ // exact state while explicitly removing close/terminate authority from
276
+ // providers (Modal) that encode it in the resumed state.
277
+ const readState =
278
+ state !== null && typeof state === "object" ? { ...state, ownsSandbox: false } : state;
279
+ const reopened = await client.resume(readState);
280
+ const actualInstanceId = captureSessionInstanceId(reopened);
281
+ if (input.expectedInstanceId.length > 0 && actualInstanceId !== input.expectedInstanceId) {
282
+ throw new Error(
283
+ `workspace capture reopened provider instance ${actualInstanceId ?? "unknown"}, expected ${input.expectedInstanceId}`,
284
+ );
285
+ }
286
+ return reopened as ChannelASession;
287
+ }
288
+
214
289
  /** One discovered repository is all-or-nothing capture authority. A provider
215
290
  * transport failure must never be represented as an authoritative empty diff. */
216
291
  export async function readCaptureRepository(
@@ -250,7 +325,7 @@ let loggedStorageNullOnce = false;
250
325
 
251
326
  /**
252
327
  * Turn-end entrypoint. Gates on the flag + configured object storage, races the
253
- * whole capture against a 60s cap, and swallows every failure ("turn outcome
328
+ * whole capture against a 120s cap, and swallows every failure ("turn outcome
254
329
  * unaffected"). Returns void — the caller awaits it (it self-caps) and moves on
255
330
  * to release() regardless.
256
331
  */
@@ -258,6 +333,7 @@ export async function captureWorkspaceRevision(
258
333
  input: CaptureWorkspaceRevisionInput,
259
334
  ): Promise<void> {
260
335
  const { observability } = input;
336
+ const stage = { value: "preflight" };
261
337
  if (input.signal?.aborted) {
262
338
  observability.incrementCounter({
263
339
  name: "opengeni_workspace_capture_total",
@@ -305,6 +381,7 @@ export async function captureWorkspaceRevision(
305
381
  { ...input, objectStorage: input.objectStorage },
306
382
  startedAt,
307
383
  controller.signal,
384
+ stage,
308
385
  ).finally(() => {
309
386
  observability.incrementGauge({
310
387
  name: "opengeni_workspace_captures_inflight",
@@ -339,6 +416,7 @@ export async function captureWorkspaceRevision(
339
416
  "opengeni.turn_id": input.turnId ?? "",
340
417
  "error.message": error instanceof Error ? error.message : String(error),
341
418
  "workspace_capture.duration_ms": Date.now() - startedAt,
419
+ "workspace_capture.stage": stage.value,
342
420
  });
343
421
  observability.incrementCounter({
344
422
  name: "opengeni_workspace_capture_total",
@@ -357,22 +435,28 @@ async function runCapture(
357
435
  ctx: CaptureWorkspaceRevisionInput & { objectStorage: ObjectStorage },
358
436
  startedAt: number,
359
437
  signal: AbortSignal,
438
+ stage: { value: string },
360
439
  ): Promise<void> {
361
440
  const { observability } = input;
362
441
  const storage = ctx.objectStorage;
363
442
  const keepN = input.keepLatest ?? KEEP_LATEST_REVISIONS;
443
+ stage.value = "open_read_session";
444
+ const readSession = input.openReadSession ? await input.openReadSession() : input.session;
445
+ throwIfCaptureAborted(signal);
364
446
  // Reads only — no emitter (we publish the announce ourselves after commit).
365
- const svc = new SandboxChannelAService({ session: input.session, leaseEpoch: input.leaseEpoch });
447
+ const svc = new SandboxChannelAService({ session: readSession, leaseEpoch: input.leaseEpoch });
366
448
 
367
449
  // Resolve the previous revision before discovery so even a failed discovery
368
450
  // can commit a monotonic, explicit degraded marker. That marker becomes the
369
451
  // newest read result and prevents an older successful capture from being
370
452
  // mistaken for the current turn's authoritative workspace state.
453
+ stage.value = "load_previous_capture";
371
454
  const prev = await latestWorkspaceCapture(input.db, input.workspaceId, input.sessionId);
372
455
  throwIfCaptureAborted(signal);
373
456
  const revision = (prev?.revision ?? -1) + 1;
374
457
 
375
458
  // ── 1. per-repo status + diff, union the touched set ──────────────────────
459
+ stage.value = "repository_discovery";
376
460
  const discovery = await svc.detectReposDetailed();
377
461
  throwIfCaptureAborted(signal);
378
462
  if (process.env.OPENGENI_TEST_SCENARIO === "sandbox") {
@@ -400,6 +484,7 @@ async function runCapture(
400
484
  let deletions = 0;
401
485
 
402
486
  for (const root of repoRoots) {
487
+ stage.value = "repository_read";
403
488
  throwIfCaptureAborted(signal);
404
489
  // Repository discovery defines the authoritative set. Once a root is in
405
490
  // that set, status/diff failure must degrade the entire revision rather than
@@ -475,6 +560,7 @@ async function runCapture(
475
560
  let binaryCount = 0;
476
561
 
477
562
  for (const [wsPath, info] of touched) {
563
+ stage.value = "file_read";
478
564
  throwIfCaptureAborted(signal);
479
565
  if (info.deleted) {
480
566
  files.push({
@@ -580,6 +666,7 @@ async function runCapture(
580
666
  }
581
667
 
582
668
  // ── 5. tree index (one bounded listing; residue dirs pruned at source) ─────
669
+ stage.value = "tree_index";
583
670
  const tree = await buildTreeIndex(svc, startedAt, signal);
584
671
  throwIfCaptureAborted(signal);
585
672
 
@@ -602,6 +689,7 @@ async function runCapture(
602
689
  let refreshedFiles = 0;
603
690
  const omittedFiles = new Set<string>();
604
691
  for (const file of files) {
692
+ stage.value = "content_upload";
605
693
  throwIfCaptureAborted(signal);
606
694
  if (file.deleted || file.tooLarge || !file.contentRef || !file.hash) {
607
695
  continue;
@@ -730,10 +818,12 @@ async function runCapture(
730
818
  entryCount: tree.entryCount,
731
819
  }),
732
820
  );
821
+ stage.value = "tree_upload";
733
822
  await storage.putObject({ key: treeKey, contentType: "application/json", body: treeBytes });
734
823
  throwIfCaptureAborted(signal);
735
824
  stats.durationMs = Date.now() - startedAt;
736
825
  const manifestBytes = utf8(JSON.stringify(manifest));
826
+ stage.value = "manifest_upload";
737
827
  await storage.putObject({
738
828
  key: manifestKey,
739
829
  contentType: "application/json",
@@ -743,6 +833,7 @@ async function runCapture(
743
833
  const sizeBytes = totalBytes + treeBytes.byteLength + manifestBytes.byteLength;
744
834
 
745
835
  // ── 8. epoch-fenced insert (superseded lease → zero rows) ─────────────────
836
+ stage.value = "database_commit";
746
837
  const inserted = await insertWorkspaceCapture(input.db, {
747
838
  accountId: input.accountId,
748
839
  workspaceId: input.workspaceId,
package/src/activities.ts CHANGED
@@ -36,6 +36,8 @@ export type {
36
36
  ReconcileCodexCapacityWaitResult,
37
37
  RecoverDispatchInput,
38
38
  RecoverDispatchResult,
39
+ RecoverEscapedMcpTimeoutInput,
40
+ RecoverEscapedMcpTimeoutResult,
39
41
  PersistSessionAttemptQuiescenceInput,
40
42
  ReconcileSessionAttemptQuiescenceInput,
41
43
  ReconcileSessionAttemptQuiescenceResult,
@@ -167,6 +169,7 @@ export const persistSessionAttemptQuiescence =
167
169
  export const reconcileSessionAttemptQuiescence =
168
170
  defaultControlActivities.reconcileSessionAttemptQuiescence;
169
171
  export const recoverDispatch = defaultControlActivities.recoverDispatch;
172
+ export const recoverEscapedMcpTimeout = defaultControlActivities.recoverEscapedMcpTimeout;
170
173
  export const peekSessionWork = defaultControlActivities.peekSessionWork;
171
174
  export const expireSessionHumanInput = defaultControlActivities.expireSessionHumanInput;
172
175
  export const markSessionIdle = defaultControlActivities.markSessionIdle;
@@ -384,6 +384,97 @@ export function recordSandboxOrphansTerminated(observability: Observability, cou
384
384
  });
385
385
  }
386
386
 
387
+ export type SandboxCheckpointArtifactMetricState =
388
+ | "candidate"
389
+ | "current"
390
+ | "previous"
391
+ | "delete_pending"
392
+ | "deleting"
393
+ | "delete_failed"
394
+ | "deleted";
395
+
396
+ export function recordSandboxCheckpointArtifactGauges(
397
+ observability: Observability,
398
+ counts: Record<SandboxCheckpointArtifactMetricState, number>,
399
+ ): void {
400
+ for (const state of [
401
+ "candidate",
402
+ "current",
403
+ "previous",
404
+ "delete_pending",
405
+ "deleting",
406
+ "delete_failed",
407
+ "deleted",
408
+ ] as const) {
409
+ observability.setGauge({
410
+ name: "opengeni_sandbox_checkpoint_artifacts",
411
+ help: "Current durable provider checkpoint artifacts by lifecycle state.",
412
+ labels: { state },
413
+ value: counts[state],
414
+ });
415
+ }
416
+ }
417
+
418
+ export type SandboxCheckpointArtifactOutcome =
419
+ | "legacy_adopted"
420
+ | "claimed"
421
+ | "deleted"
422
+ | "delete_failed"
423
+ | "tombstone_pruned";
424
+
425
+ export function recordSandboxCheckpointArtifactOutcome(
426
+ observability: Observability,
427
+ outcome: SandboxCheckpointArtifactOutcome,
428
+ count = 1,
429
+ ): void {
430
+ if (count <= 0) return;
431
+ observability.incrementCounter({
432
+ name: "opengeni_sandbox_checkpoint_artifact_operations_total",
433
+ help: "Durable provider checkpoint artifact operations by fixed outcome.",
434
+ labels: { outcome },
435
+ amount: count,
436
+ });
437
+ }
438
+
439
+ export function recordSandboxDeadlineRotationsRequested(
440
+ observability: Observability,
441
+ count: number,
442
+ ): void {
443
+ if (count <= 0) return;
444
+ observability.incrementCounter({
445
+ name: "opengeni_sandbox_deadline_rotations_requested_total",
446
+ help: "Total finite-lifetime sandbox rotations requested before provider deadline.",
447
+ amount: count,
448
+ });
449
+ }
450
+
451
+ export function recordSandboxRotationBacklogGauges(
452
+ observability: Observability,
453
+ backlog: {
454
+ requested: number;
455
+ overdue: number;
456
+ turnBlocked: number;
457
+ directBlocked: number;
458
+ processBlocked: number;
459
+ },
460
+ ): void {
461
+ const values = {
462
+ requested: backlog.requested,
463
+ overdue: backlog.overdue,
464
+ turn_blocked: backlog.turnBlocked,
465
+ direct_blocked: backlog.directBlocked,
466
+ process_blocked: backlog.processBlocked,
467
+ } as const;
468
+ for (const [kind, value] of Object.entries(values)) {
469
+ observability.setGauge({
470
+ name: "opengeni_sandbox_rotation_backlog",
471
+ help: "Current finite-lifetime sandbox rotation backlog by fixed condition.",
472
+ labels: { kind },
473
+ value,
474
+ });
475
+ }
476
+ }
477
+
387
478
  const RETAINED_PROCESS_OWNER_STATES = [
388
479
  "direct",
389
480
  "queued",
@@ -503,6 +594,9 @@ export const RETAINED_PROCESS_RECONCILIATION_OUTCOMES = [
503
594
  "provider_unknown",
504
595
  "provider_timeout",
505
596
  "provider_error",
597
+ "provider_binding_missing",
598
+ "provider_binding_mismatch",
599
+ "provider_binding_adopted",
506
600
  "defer_failed",
507
601
  ] as const;
508
602