@opengeni/worker-bundle 0.11.2 → 0.11.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,8 +1,7 @@
1
1
  import {
2
2
  environmentsEncryptionKeyBytes,
3
- parseModelProvidersJson,
4
- type RegistryProvider,
5
3
  type Settings,
4
+ withCodexCatalogProvider,
6
5
  } from "@opengeni/config";
7
6
  import { settingsWithMcpCapabilityServers } from "@opengeni/core";
8
7
  import {
@@ -10,14 +9,6 @@ import {
10
9
  CODEX_APPS_MCP_SERVER_NAME,
11
10
  CODEX_APPS_MCP_URL,
12
11
  CODEX_APPS_STARTUP_TIMEOUT_MS,
13
- CODEX_FALLBACK_MODEL_SLUGS,
14
- CODEX_MODEL_CONTEXT_WINDOW_TOKENS,
15
- CODEX_MODEL_EFFECTIVE_CONTEXT_WINDOW_TOKENS,
16
- CODEX_MODEL_AUTO_COMPACT_TOKEN_LIMIT,
17
- CODEX_MODEL_TOOL_OUTPUT_TRUNCATION_TOKENS,
18
- CODEX_MODEL_ID_PREFIX,
19
- CODEX_PROVIDER_BASE_URL,
20
- CODEX_PROVIDER_ID,
21
12
  } from "@opengeni/codex";
22
13
  import {
23
14
  listEnabledMcpCapabilityServers,
@@ -43,6 +34,9 @@ export async function settingsWithSessionMcpServersForRun(
43
34
  sessionId: string,
44
35
  attemptId: string,
45
36
  settings: Settings,
37
+ options?: {
38
+ onResolvedServers?: (servers: readonly SessionMcpServerForRun[]) => void;
39
+ },
46
40
  ): Promise<Settings> {
47
41
  const encryptionKey = environmentsEncryptionKeyBytes(settings);
48
42
  if (!encryptionKey) {
@@ -56,10 +50,18 @@ export async function settingsWithSessionMcpServersForRun(
56
50
  );
57
51
  }
58
52
  }
59
- return settingsWithSessionMcpServers(
60
- settings,
61
- await listSessionMcpServersForRun(db, workspaceId, sessionId, attemptId, encryptionKey ?? null),
53
+ const servers = await listSessionMcpServersForRun(
54
+ db,
55
+ workspaceId,
56
+ sessionId,
57
+ attemptId,
58
+ encryptionKey ?? null,
62
59
  );
60
+ // Keep credential provenance coupled to the exact decrypted rows that are
61
+ // overlaid into settings. A session projection read earlier in the turn can
62
+ // be stale after a concurrent mcpCredentialUpdates renewal.
63
+ options?.onResolvedServers?.(servers);
64
+ return settingsWithSessionMcpServers(settings, servers);
63
65
  }
64
66
 
65
67
  export function settingsWithSessionMcpServers(
@@ -115,6 +117,9 @@ export async function settingsWithCodexCredential(
115
117
  return settings; // disabled / not connected / needs_relogin / error -> leave settings unchanged
116
118
  }
117
119
  const withProvider = withCodexProvider(settings);
120
+ if (!settings.codexConnectedAppsEnabled) {
121
+ return withProvider;
122
+ }
118
123
  // Additive: append the synthetic codex_apps connectors MCP server for ANY
119
124
  // active credential. Connector access is gated SERVER-SIDE per ChatGPT account
120
125
  // (via chatgpt-account-id), NOT by token scopes — confirmed live: a `pro` token
@@ -162,28 +167,5 @@ export function withCodexAppsMcpServer(settings: Settings): Settings {
162
167
 
163
168
  /** Pure: append the synthetic codex-subscription provider, idempotently. */
164
169
  export function withCodexProvider(settings: Settings): Settings {
165
- const providers = parseModelProvidersJson(settings.modelProvidersJson);
166
- if (providers.some((provider) => provider.id === CODEX_PROVIDER_ID)) {
167
- return settings; // already injected
168
- }
169
- const codexProvider: RegistryProvider = {
170
- kind: "codex-subscription",
171
- id: CODEX_PROVIDER_ID,
172
- label: "Codex (ChatGPT subscription)",
173
- api: "responses",
174
- baseUrl: CODEX_PROVIDER_BASE_URL,
175
- models: CODEX_FALLBACK_MODEL_SLUGS.map((slug) => ({
176
- id: `${CODEX_MODEL_ID_PREFIX}${slug}`,
177
- upstreamModelId: slug,
178
- label: slug,
179
- reasoningEffort: true,
180
- // These three values are the live Codex CLI catalog policy, kept distinct
181
- // so proactive compaction and the effective input guard match Codex core.
182
- contextWindowTokens: CODEX_MODEL_CONTEXT_WINDOW_TOKENS,
183
- effectiveContextWindowTokens: CODEX_MODEL_EFFECTIVE_CONTEXT_WINDOW_TOKENS,
184
- autoCompactTokenLimit: CODEX_MODEL_AUTO_COMPACT_TOKEN_LIMIT,
185
- toolOutputTruncationTokens: CODEX_MODEL_TOOL_OUTPUT_TRUNCATION_TOKENS,
186
- })),
187
- };
188
- return { ...settings, modelProvidersJson: JSON.stringify([...providers, codexProvider]) };
170
+ return withCodexCatalogProvider(settings);
189
171
  }
@@ -200,7 +200,7 @@ export function createCodexCapacityActivities(services: () => Promise<ActivitySe
200
200
  }
201
201
  }
202
202
  if (result.action === "resumed") {
203
- return { action: "resumed", updateId: result.update.id };
203
+ return { action: "resumed" };
204
204
  }
205
205
  if (result.action === "waiting") {
206
206
  return {
@@ -1,56 +1,9 @@
1
- const MIN_REDACTABLE_VALUE_LENGTH = 6;
2
-
3
- export type SecretForRedaction = {
4
- name: string;
5
- value: string;
6
- };
7
-
8
- /**
9
- * Defense-in-depth against an agent echoing injected workspace environment
10
- * values (for example via `env`) into session events. Deep-walks payloads and
11
- * replaces every exact occurrence of each secret value with
12
- * `[redacted:<NAME>]`. Values shorter than six characters are skipped to avoid
13
- * a false-positive flood; secrets are applied longest-value-first so
14
- * overlapping values redact deterministically.
15
- */
16
- export function createSecretRedactor(secrets: SecretForRedaction[]): (payload: unknown) => unknown {
17
- const usable = secrets
18
- .filter((secret) => secret.value.length >= MIN_REDACTABLE_VALUE_LENGTH)
19
- .sort((a, b) => b.value.length - a.value.length);
20
- if (usable.length === 0) {
21
- return identityRedactor;
22
- }
23
- const redact = (payload: unknown): unknown => {
24
- if (typeof payload === "string") {
25
- let out = payload;
26
- for (const secret of usable) {
27
- if (out.includes(secret.value)) {
28
- out = out.split(secret.value).join(`[redacted:${secret.name}]`);
29
- }
30
- }
31
- return out;
32
- }
33
- if (Array.isArray(payload)) {
34
- return payload.map(redact);
35
- }
36
- if (isPlainObject(payload)) {
37
- return Object.fromEntries(
38
- Object.entries(payload).map(([key, value]) => [key, redact(value)]),
39
- );
40
- }
41
- return payload;
42
- };
43
- return redact;
44
- }
45
-
46
- export function identityRedactor(payload: unknown): unknown {
47
- return payload;
48
- }
49
-
50
- function isPlainObject(value: unknown): value is Record<string, unknown> {
51
- if (!value || typeof value !== "object") {
52
- return false;
53
- }
54
- const proto = Object.getPrototypeOf(value);
55
- return proto === Object.prototype || proto === null;
56
- }
1
+ export {
2
+ createSecretRedactor,
3
+ identityRedactor,
4
+ isCredentialHeaderName,
5
+ redactSensitiveData,
6
+ redactSensitiveText,
7
+ redactSerializedJson,
8
+ type SecretForRedaction,
9
+ } from "@opengeni/contracts";
@@ -34,21 +34,32 @@ import {
34
34
  accrueWarmSeconds,
35
35
  confirmDrainCold,
36
36
  appendSessionEventToSandboxGroup,
37
+ claimTerminalRetainedProcesses,
38
+ countActiveRetainedProcessesByOwnerState,
39
+ countExpiredDrainingSandboxLeases,
37
40
  countQueuedTurns,
38
41
  countSandboxLeasesByLiveness,
42
+ deferRetainedProcessReconciliation,
39
43
  forceDrainOverLimitViewerOnlyBoxes,
40
44
  getBillingBalance,
41
45
  listCreditBalancesByAccount,
42
46
  listLiveModalSandboxLeaseAttributions,
43
47
  listMeterableWarmLeases,
44
48
  persistDrainSnapshot,
49
+ recordRetainedProcessReconciliationProof,
45
50
  readWorkspaceArchiveCapturePreflight,
46
51
  readLease,
47
52
  reapExpiredSessionListSnapshots,
48
53
  reapStaleLeaseHoldersGlobal,
54
+ retainedProcessReconciliationProof,
55
+ retainedProcessSettlementIdentity,
49
56
  rlsContextForWorkspace,
57
+ settleRetainedProcess,
50
58
  type MeterableWarmLease,
51
59
  type ReapDrainable,
60
+ type RetainedProcessProviderProof,
61
+ type SandboxRetainedProcess,
62
+ type LeaseSnapshot,
52
63
  } from "@opengeni/db";
53
64
  import { sandboxWarmRateMicrosPerSecond } from "@opengeni/config";
54
65
  import {
@@ -62,7 +73,9 @@ import {
62
73
  createSandboxClientForBackend,
63
74
  deletePriorPersistedSnapshot,
64
75
  deserializeSandboxSessionStateEnvelope,
76
+ isExecSessionLostBanner,
65
77
  isProviderSandboxNotFoundError,
78
+ parseExecBannerExitCode,
66
79
  sweepModalOrphanSandboxes,
67
80
  terminateModalSandboxById,
68
81
  type ModalOrphanSweepTermination,
@@ -73,10 +86,15 @@ import { reconcilePendingParentSystemUpdates } from "./parent-wake";
73
86
  import {
74
87
  recordCreditBalanceGauges,
75
88
  recordCreditMicros,
89
+ recordExpiredDrainingSandboxLeaseGauges,
90
+ recordRetainedProcessInventoryGauges,
91
+ recordRetainedProcessReconciliation,
92
+ type RetainedProcessReconciliationOutcome,
76
93
  recordSandboxLeaseGauges,
77
94
  recordSandboxOrphansTerminated,
78
95
  recordTurnsQueuedGauge,
79
96
  } from "../observability-metrics";
97
+ import { providerIdentityFromResumeState } from "../sandbox-routing";
80
98
 
81
99
  export type ReapSandboxLeasesResult = {
82
100
  /** Stale viewer holders + warming-death rows the sweep touched is folded into
@@ -179,8 +197,35 @@ export type SandboxLeaseActivityOptions = {
179
197
  /** Override the provider-side Modal orphan sweep (tests spy this; defaults to
180
198
  * Modal list+tag comparison when Modal is configured). */
181
199
  sweepModalOrphans?: SweepModalOrphansFn;
200
+ /** Override only the read-only provider process probe. The canonical DB
201
+ * settlement remains real in tests and production. */
202
+ probeRetainedProcess?: RetainedProcessProbeFn;
182
203
  };
183
204
 
205
+ export type RetainedProcessProbeResult =
206
+ | { status: "proved"; proof: RetainedProcessProviderProof }
207
+ | {
208
+ status: "deferred";
209
+ reason:
210
+ | "identity_mismatch"
211
+ | "resume_state_missing"
212
+ | "backend_unsupported"
213
+ | "provider_running"
214
+ | "provider_unknown"
215
+ | "provider_timeout"
216
+ | "provider_error";
217
+ };
218
+
219
+ export type RetainedProcessProbeFn = (
220
+ settings: ActivityServices["settings"],
221
+ lease: LeaseSnapshot,
222
+ process: SandboxRetainedProcess,
223
+ ) => Promise<RetainedProcessProbeResult>;
224
+
225
+ export const RETAINED_PROCESS_RECONCILIATION_LIMIT = 20;
226
+ export const RETAINED_PROCESS_RECONCILIATION_CLAIM_TTL_MS = 5 * 60_000;
227
+ export const RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS = 5_000;
228
+
184
229
  export function createSandboxLeaseActivities(
185
230
  services: () => Promise<ActivityServices>,
186
231
  options: SandboxLeaseActivityOptions = {},
@@ -188,6 +233,7 @@ export function createSandboxLeaseActivities(
188
233
  const terminateBox: TerminateBoxFn = options.terminateBox ?? terminateProviderBox;
189
234
  const sweepModalOrphans: SweepModalOrphansFn =
190
235
  options.sweepModalOrphans ?? sweepModalOrphansForConfiguredBackend;
236
+ const probeRetainedProcess = options.probeRetainedProcess ?? probeRetainedProcessAtProvider;
191
237
  /**
192
238
  * The one global reaper sweep. Idempotent; concurrency-safe with itself.
193
239
  * Gated by the caller (the Schedule is only registered when
@@ -248,6 +294,12 @@ export function createSandboxLeaseActivities(
248
294
  observability,
249
295
  );
250
296
 
297
+ // (0c) Terminal-owner retained-process reconciliation. Owner state selects
298
+ // only a bounded inspection batch; exact provider exit/loss proof is
299
+ // checkpointed before canonical settlement. Ambiguous, running, timed-out,
300
+ // unsupported, and transient provider states preserve every durable row.
301
+ await reconcileTerminalRetainedProcesses(db, settings, observability, probeRetainedProcess);
302
+
251
303
  // (1) The DB-only cross-workspace sweep. Returns the drainable rows.
252
304
  const drainable: ReapDrainable[] = await reapStaleLeaseHoldersGlobal(db, {
253
305
  viewerHolderTtlMs: settings.sandboxViewerHolderTtlMs,
@@ -336,6 +388,345 @@ export function createSandboxLeaseActivities(
336
388
  return { reapSandboxLeases };
337
389
  }
338
390
 
391
+ async function reconcileTerminalRetainedProcesses(
392
+ db: ActivityServices["db"],
393
+ settings: ActivityServices["settings"],
394
+ observability: ActivityServices["observability"],
395
+ probe: RetainedProcessProbeFn,
396
+ ): Promise<void> {
397
+ const claimId = crypto.randomUUID();
398
+ let claims: Awaited<ReturnType<typeof claimTerminalRetainedProcesses>>;
399
+ try {
400
+ claims = await claimTerminalRetainedProcesses(db, {
401
+ claimId,
402
+ limit: RETAINED_PROCESS_RECONCILIATION_LIMIT,
403
+ claimTtlMs: RETAINED_PROCESS_RECONCILIATION_CLAIM_TTL_MS,
404
+ });
405
+ } catch (error) {
406
+ recordRetainedProcessReconciliation(observability, "claim_failed");
407
+ observability.warn("sandbox reaper: retained-process claim failed", {
408
+ error: error instanceof Error ? error.message : String(error),
409
+ });
410
+ return;
411
+ }
412
+
413
+ for (const claim of claims) {
414
+ let process = claim.process;
415
+ const expected = retainedProcessSettlementIdentity(process);
416
+ let proof = retainedProcessReconciliationProof(process);
417
+ if (!proof) {
418
+ const lease = await readLease(db, process.workspaceId, process.sandboxGroupId).catch(
419
+ () => null,
420
+ );
421
+ if (
422
+ !lease ||
423
+ lease.id !== process.leaseId ||
424
+ lease.leaseEpoch !== process.leaseEpoch ||
425
+ lease.backend !== process.providerBackend ||
426
+ lease.instanceId !== process.providerInstanceId ||
427
+ process.routeTargetId !== null
428
+ ) {
429
+ await deferRetainedProcessClaim(
430
+ db,
431
+ settings,
432
+ observability,
433
+ process,
434
+ expected,
435
+ claim.claimId,
436
+ "identity_mismatch",
437
+ );
438
+ continue;
439
+ }
440
+
441
+ let observation: RetainedProcessProbeResult;
442
+ try {
443
+ observation = await probe(settings, lease, process);
444
+ } catch (error) {
445
+ observability.warn("sandbox reaper: retained-process provider probe failed", {
446
+ processId: process.id,
447
+ providerBackend: process.providerBackend,
448
+ error: error instanceof Error ? error.message : String(error),
449
+ });
450
+ observation = { status: "deferred", reason: "provider_error" };
451
+ }
452
+ if (observation.status === "deferred") {
453
+ await deferRetainedProcessClaim(
454
+ db,
455
+ settings,
456
+ observability,
457
+ process,
458
+ expected,
459
+ claim.claimId,
460
+ observation.reason,
461
+ );
462
+ continue;
463
+ }
464
+ try {
465
+ process = await recordRetainedProcessReconciliationProof(db, {
466
+ accountId: process.accountId,
467
+ workspaceId: process.workspaceId,
468
+ sessionId: process.sessionId,
469
+ processId: process.id,
470
+ expected,
471
+ claimId: claim.claimId,
472
+ proof: observation.proof,
473
+ });
474
+ proof = observation.proof;
475
+ recordRetainedProcessReconciliation(observability, `proof_${proof.outcome}`);
476
+ } catch (error) {
477
+ recordRetainedProcessReconciliation(observability, "proof_checkpoint_failed");
478
+ observability.warn("sandbox reaper: retained-process proof checkpoint failed", {
479
+ processId: process.id,
480
+ error: error instanceof Error ? error.message : String(error),
481
+ });
482
+ continue;
483
+ }
484
+ }
485
+
486
+ try {
487
+ await settleRetainedProcess(db, {
488
+ accountId: process.accountId,
489
+ workspaceId: process.workspaceId,
490
+ sessionId: process.sessionId,
491
+ processId: process.id,
492
+ expected,
493
+ reconciliationClaimId: claim.claimId,
494
+ outcome: proof.outcome,
495
+ exitCode: proof.exitCode,
496
+ reason: proof.reason,
497
+ idleGraceMs: settings.sandboxIdleGraceMs,
498
+ });
499
+ recordRetainedProcessReconciliation(observability, `settled_${proof.outcome}`);
500
+ } catch (error) {
501
+ recordRetainedProcessReconciliation(observability, "settlement_failed");
502
+ observability.warn("sandbox reaper: retained-process settlement failed", {
503
+ processId: process.id,
504
+ error: error instanceof Error ? error.message : String(error),
505
+ });
506
+ await deferRetainedProcessClaim(
507
+ db,
508
+ settings,
509
+ observability,
510
+ process,
511
+ expected,
512
+ claim.claimId,
513
+ "settlement_failed",
514
+ );
515
+ }
516
+ }
517
+ }
518
+
519
+ async function deferRetainedProcessClaim(
520
+ db: ActivityServices["db"],
521
+ settings: ActivityServices["settings"],
522
+ observability: ActivityServices["observability"],
523
+ process: SandboxRetainedProcess,
524
+ expected: ReturnType<typeof retainedProcessSettlementIdentity>,
525
+ claimId: string,
526
+ outcome: RetainedProcessReconciliationOutcome,
527
+ ): Promise<void> {
528
+ const exponent = Math.min(4, Math.max(0, process.reconcileAttempts - 1));
529
+ const retryAfterMs = Math.min(
530
+ 5 * 60_000,
531
+ Math.max(settings.sandboxLeaseReaperPeriodMs, 30_000) * 2 ** exponent,
532
+ );
533
+ try {
534
+ await deferRetainedProcessReconciliation(db, {
535
+ accountId: process.accountId,
536
+ workspaceId: process.workspaceId,
537
+ sessionId: process.sessionId,
538
+ processId: process.id,
539
+ expected,
540
+ claimId,
541
+ outcome,
542
+ retryAfterMs,
543
+ });
544
+ recordRetainedProcessReconciliation(observability, outcome);
545
+ } catch (error) {
546
+ recordRetainedProcessReconciliation(observability, "defer_failed");
547
+ observability.warn("sandbox reaper: retained-process defer failed", {
548
+ processId: process.id,
549
+ error: error instanceof Error ? error.message : String(error),
550
+ });
551
+ }
552
+ }
553
+
554
+ type RetainedProcessProbeClient = {
555
+ backendId: string;
556
+ resume?: (state: unknown) => Promise<unknown>;
557
+ deserializeSessionState?: (state: Record<string, unknown>) => Promise<unknown>;
558
+ };
559
+
560
+ type RetainedProcessProbeSession = {
561
+ writeStdin?: (args: {
562
+ sessionId: number;
563
+ chars: string;
564
+ yieldTimeMs: number;
565
+ maxOutputTokens: number;
566
+ }) => Promise<unknown>;
567
+ };
568
+
569
+ const RETAINED_PROCESS_PROBE_TIMEOUT = Symbol("retained-process-provider-probe-timeout");
570
+
571
+ async function withRetainedProcessProbeTimeout<T>(promise: Promise<T>): Promise<T> {
572
+ let timer: ReturnType<typeof setTimeout> | undefined;
573
+ try {
574
+ return await Promise.race([
575
+ promise,
576
+ new Promise<never>((_resolve, reject) => {
577
+ timer = setTimeout(
578
+ () => reject(RETAINED_PROCESS_PROBE_TIMEOUT),
579
+ RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS,
580
+ );
581
+ timer.unref?.();
582
+ }),
583
+ ]);
584
+ } finally {
585
+ if (timer) clearTimeout(timer);
586
+ }
587
+ }
588
+
589
+ export async function probeRetainedProcessAtProvider(
590
+ settings: ActivityServices["settings"],
591
+ lease: LeaseSnapshot,
592
+ process: SandboxRetainedProcess,
593
+ ): Promise<RetainedProcessProbeResult> {
594
+ if (
595
+ lease.id !== process.leaseId ||
596
+ lease.sandboxGroupId !== process.sandboxGroupId ||
597
+ lease.leaseEpoch !== process.leaseEpoch ||
598
+ lease.backend !== process.providerBackend ||
599
+ lease.instanceId !== process.providerInstanceId ||
600
+ lease.resumeBackendId !== process.providerBackend ||
601
+ process.routeTargetId !== null
602
+ ) {
603
+ return { status: "deferred", reason: "identity_mismatch" };
604
+ }
605
+ if (!lease.resumeState) {
606
+ return { status: "deferred", reason: "resume_state_missing" };
607
+ }
608
+ const envelopeBackend = (lease.resumeState as { backendId?: unknown }).backendId;
609
+ if (envelopeBackend !== undefined && envelopeBackend !== process.providerBackend) {
610
+ return { status: "deferred", reason: "identity_mismatch" };
611
+ }
612
+ if (providerIdentityFromResumeState(lease.resumeState) !== process.providerInstanceId) {
613
+ return { status: "deferred", reason: "identity_mismatch" };
614
+ }
615
+
616
+ let client: RetainedProcessProbeClient | undefined;
617
+ try {
618
+ client = createSandboxClientForBackend(process.providerBackend as never, settings) as
619
+ | RetainedProcessProbeClient
620
+ | undefined;
621
+ } catch {
622
+ return { status: "deferred", reason: "provider_error" };
623
+ }
624
+ if (!client || client.backendId !== process.providerBackend || !client.resume) {
625
+ return { status: "deferred", reason: "backend_unsupported" };
626
+ }
627
+ const envelopeSessionState =
628
+ (lease.resumeState as { sessionState?: unknown }).sessionState ?? lease.resumeState;
629
+ let resumedState: unknown;
630
+ try {
631
+ resumedState = await deserializeSandboxSessionStateEnvelope(
632
+ client as never,
633
+ envelopeSessionState,
634
+ );
635
+ } catch {
636
+ return { status: "deferred", reason: "provider_error" };
637
+ }
638
+ if (resumedState === undefined) {
639
+ return { status: "deferred", reason: "resume_state_missing" };
640
+ }
641
+
642
+ let session: RetainedProcessProbeSession;
643
+ try {
644
+ session = (await withRetainedProcessProbeTimeout(
645
+ client.resume(resumedState),
646
+ )) as RetainedProcessProbeSession;
647
+ } catch (error) {
648
+ if (error === RETAINED_PROCESS_PROBE_TIMEOUT) {
649
+ return { status: "deferred", reason: "provider_timeout" };
650
+ }
651
+ if (isProviderSandboxNotFoundError(client.backendId, error)) {
652
+ return {
653
+ status: "proved",
654
+ proof: {
655
+ outcome: "lost",
656
+ exitCode: null,
657
+ reason: "provider_instance_not_found",
658
+ },
659
+ };
660
+ }
661
+ return { status: "deferred", reason: "provider_error" };
662
+ }
663
+ if (typeof session.writeStdin !== "function") {
664
+ return { status: "deferred", reason: "backend_unsupported" };
665
+ }
666
+
667
+ let result: unknown;
668
+ try {
669
+ result = await withRetainedProcessProbeTimeout(
670
+ session.writeStdin({
671
+ sessionId: process.providerSessionId,
672
+ chars: "",
673
+ yieldTimeMs: 1_000,
674
+ maxOutputTokens: 2_000,
675
+ }),
676
+ );
677
+ } catch (error) {
678
+ if (error === RETAINED_PROCESS_PROBE_TIMEOUT) {
679
+ return { status: "deferred", reason: "provider_timeout" };
680
+ }
681
+ if (isProviderSandboxNotFoundError(client.backendId, error)) {
682
+ return {
683
+ status: "proved",
684
+ proof: {
685
+ outcome: "lost",
686
+ exitCode: null,
687
+ reason: "provider_instance_not_found",
688
+ },
689
+ };
690
+ }
691
+ return { status: "deferred", reason: "provider_error" };
692
+ }
693
+ return classifyRetainedProcessPollResult(result, process.providerSessionId);
694
+ }
695
+
696
+ /** Classify only exact SDK control banners. Arbitrary output, malformed text,
697
+ * and a running marker are observations to retry; none is physical exit proof. */
698
+ export function classifyRetainedProcessPollResult(
699
+ result: unknown,
700
+ providerSessionId: number,
701
+ ): RetainedProcessProbeResult {
702
+ if (typeof result !== "string") {
703
+ return { status: "deferred", reason: "provider_unknown" };
704
+ }
705
+ if (isExecSessionLostBanner(result, providerSessionId)) {
706
+ return {
707
+ status: "proved",
708
+ proof: {
709
+ outcome: "lost",
710
+ exitCode: null,
711
+ reason: "provider_session_lost_banner",
712
+ },
713
+ };
714
+ }
715
+ const exitCode = parseExecBannerExitCode(result);
716
+ if (exitCode !== null) {
717
+ return {
718
+ status: "proved",
719
+ proof: { outcome: "exited", exitCode, reason: "provider_exit_banner" },
720
+ };
721
+ }
722
+ return {
723
+ status: "deferred",
724
+ reason: result.includes("Process running with session ID")
725
+ ? "provider_running"
726
+ : "provider_unknown",
727
+ };
728
+ }
729
+
339
730
  /**
340
731
  * The reaper-tick warm-meter pass (P2.1). Accrues warm-seconds for every WARM
341
732
  * viewer-only lease cross-workspace (the list fn excludes turn-held boxes — those
@@ -405,6 +796,26 @@ async function refreshQueueLeaseAndCreditGauges(
405
796
  error: error instanceof Error ? error.message : String(error),
406
797
  });
407
798
  }
799
+ try {
800
+ recordRetainedProcessInventoryGauges(
801
+ observability,
802
+ await countActiveRetainedProcessesByOwnerState(db),
803
+ );
804
+ } catch (error) {
805
+ observability.warn("sandbox reaper: retained-process gauge refresh failed", {
806
+ error: error instanceof Error ? error.message : String(error),
807
+ });
808
+ }
809
+ try {
810
+ recordExpiredDrainingSandboxLeaseGauges(
811
+ observability,
812
+ await countExpiredDrainingSandboxLeases(db),
813
+ );
814
+ } catch (error) {
815
+ observability.warn("sandbox reaper: expired-draining gauge refresh failed", {
816
+ error: error instanceof Error ? error.message : String(error),
817
+ });
818
+ }
408
819
  try {
409
820
  recordCreditBalanceGauges(observability, await listCreditBalancesByAccount(db));
410
821
  } catch (error) {
@@ -120,8 +120,7 @@ export type ReconcileCodexCapacityWaitInput = {
120
120
 
121
121
  export type ReconcileCodexCapacityWaitResult =
122
122
  | ({ action: "waiting" } & CodexCapacityWaitRef)
123
- | { action: "resumed"; updateId: string }
124
- | { action: "superseded" | "stale" };
123
+ | { action: "resumed" | "paused" | "superseded" | "stale" };
125
124
 
126
125
  export type ActivityDependencies = Partial<ActivityServices>;
127
126
 
@@ -241,7 +240,7 @@ export type IndexDocumentInput = {
241
240
  type ClaimedRunAgentTurnResult = {
242
241
  // "recovering": this attempt ended after durably preserving the same current
243
242
  // inference for a new attempt. Recovery is not prompt queue work.
244
- status: "idle" | "requires_action" | "failed" | "cancelled" | "recovering";
243
+ status: "idle" | "requires_action" | "failed" | "cancelled" | "recovering" | "waiting_capacity";
245
244
  turnId: string;
246
245
  attemptId: string;
247
246
  // Provider backpressure pacing: when set on an idle or recovering result, the
@@ -253,9 +252,10 @@ type ClaimedRunAgentTurnResult = {
253
252
  // "continue now" (invariant 4: NO THRASH). Distinct from a normal continueDelayMs:0
254
253
  // which legitimately means "a rotation candidate is ready, re-dispatch immediately".
255
254
  idleUntilReset?: boolean;
256
- // Durable native zero-pool wait. Unlike continueDelayMs, this reference is
257
- // persisted in Postgres and reconstructed after workflow/worker restart.
258
- // The workflow must not call maybeContinueGoal while this waiter is active.
255
+ // Durable native zero-pool wait for this same nonterminal logical turn.
256
+ // Unlike continueDelayMs, this reference is persisted in Postgres and
257
+ // reconstructed after workflow/worker restart. The workflow must not call
258
+ // maybeContinueGoal or manufacture queue/input work while it is active.
259
259
  capacityWait?: CodexCapacityWaitRef;
260
260
  // This execution reached a durable terminal-for-now boundary (for example,
261
261
  // maintenance could not run or same-turn context recovery failed). End this