@opengeni/worker-bundle 0.26.0-canary.0 → 0.26.0-canary.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. package/dist/activities/agent-turn/errors.d.ts +5 -4
  2. package/dist/activities/agent-turn/history-prefix.d.ts +8 -0
  3. package/dist/activities/agent-turn/history-sink.d.ts +6 -2
  4. package/dist/activities/agent-turn/quiescence.d.ts +4 -2
  5. package/dist/activities/sandbox-lease.d.ts +7 -2
  6. package/dist/activities/xai-quota.d.ts +3 -0
  7. package/dist/{activities-control-CTEMLTHG.js → activities-control-3SKLDTXQ.js} +205 -86
  8. package/dist/activities-control-3SKLDTXQ.js.map +1 -0
  9. package/dist/{activities-turn-S5IU347A.js → activities-turn-GHSFZB64.js} +176 -1077
  10. package/dist/activities-turn-GHSFZB64.js.map +1 -0
  11. package/dist/{chunk-TAUE3GYT.js → chunk-O2LQWJIK.js} +3115 -1948
  12. package/dist/chunk-O2LQWJIK.js.map +1 -0
  13. package/dist/index.js +1 -1
  14. package/dist/sandbox-routing.d.ts +6 -0
  15. package/dist/workflow-bundle.js +24 -2
  16. package/package.json +17 -17
  17. package/src/activities/agent-turn/agent-build.ts +1 -0
  18. package/src/activities/agent-turn/errors.ts +44 -11
  19. package/src/activities/agent-turn/failure-settlement.ts +21 -8
  20. package/src/activities/agent-turn/history-prefix.ts +51 -0
  21. package/src/activities/agent-turn/history-sink.ts +30 -6
  22. package/src/activities/agent-turn/quiescence.ts +8 -2
  23. package/src/activities/agent-turn/run.ts +4 -0
  24. package/src/activities/agent-turn/stream-attempt.ts +20 -7
  25. package/src/activities/agent-turn/tool-environment.ts +12 -0
  26. package/src/activities/agent-turn/xai-capacity.ts +11 -0
  27. package/src/activities/codex-capacity.ts +18 -0
  28. package/src/activities/run-input.ts +6 -7
  29. package/src/activities/sandbox-lease.ts +229 -87
  30. package/src/activities/session-state.ts +8 -7
  31. package/src/activities/xai-quota.ts +35 -0
  32. package/src/sandbox-routing.ts +115 -0
  33. package/src/workflows/session.ts +29 -1
  34. package/dist/activities-control-CTEMLTHG.js.map +0 -1
  35. package/dist/activities-turn-S5IU347A.js.map +0 -1
  36. package/dist/chunk-TAUE3GYT.js.map +0 -1
@@ -67,10 +67,11 @@ export declare function escapedMcpTimeoutRecoveryFailure(input: {
67
67
  * Convert the atomic claim transaction's failure into a small, stable
68
68
  * Temporal wire contract. The original error remains in activity diagnostics,
69
69
  * but SQL text, parameters, and arbitrary invariant messages never enter
70
- * workflow history. Contention and operational database unavailability are
71
- * safe to re-read after backoff because the claim transaction contains no
72
- * model/tool effects; permanent database and state failures require terminal
73
- * settlement.
70
+ * workflow history. Every database failure is safe to re-read after backoff
71
+ * because the claim transaction contains no model/tool effects. Database
72
+ * constraints and authorization guards can also be repaired by a deployment or
73
+ * authority refresh, so they must preserve pending work. Only a non-database
74
+ * claim invariant is terminal.
74
75
  */
75
76
  export declare function preClaimAdmissionFailure(error: unknown): ApplicationFailure;
76
77
  /**
@@ -0,0 +1,8 @@
1
+ /** A position is an append cursor only while the items before it stay identical. */
2
+ export declare class HistoryPrefixGuard {
3
+ private persisted;
4
+ seed(items: Array<Record<string, unknown>>, count: number, truncationTokens?: number): void;
5
+ verify(items: Array<Record<string, unknown>>, truncationTokens?: number): string[];
6
+ acknowledge(keys: string[], count: number): void;
7
+ private keys;
8
+ }
@@ -19,8 +19,8 @@ export type TurnHistorySinkDeps = {
19
19
  * Dual-write of conversation truth (issue #35): completed items are reconciled
20
20
  * into session_history_items after every model response and at every turn-end
21
21
  * path (idempotent on position), and the sandbox recovery envelope is upserted
22
- * alongside. Best-effort by design: persistence problems must never fail the
23
- * run.
22
+ * alongside. Provider dispatch and settlement require durable persistence;
23
+ * the prefix guard rejects SDK history removal/reordering before slicing.
24
24
  *
25
25
  * Orphaned-tool-output guard: `stream.state.history` is NOT a plain
26
26
  * append-only array — it is a computed getter
@@ -40,6 +40,10 @@ export type TurnHistorySinkDeps = {
40
40
  */
41
41
  export declare class TurnHistorySink {
42
42
  private readonly deps;
43
+ private readonly prefix;
44
+ seedHistory(input: string | readonly unknown[] | {
45
+ readonly history: unknown[];
46
+ }, count: number): void;
43
47
  persistedHistoryCount: number;
44
48
  nextHistoryPosition: number;
45
49
  providerArtifactCandidates: Awaited<ReturnType<typeof turnInput>>["providerArtifactCandidates"];
@@ -142,8 +142,10 @@ export declare function assertSuccessfulAgentStreamCompletion(input: {
142
142
  /** Call after any more-specific empty-stream recovery has had priority. */
143
143
  export declare function assertAgentStreamNotCancelled(cancelled: boolean): void;
144
144
  /** Interruption streams do not produce a normal final output. Call this only
145
- * after the interruption branch has settled any required action. */
146
- export declare function requireAgentStreamFinalOutput(finalOutput: unknown | undefined): unknown;
145
+ * after the interruption branch has settled any required action. A successful
146
+ * runtime-owned input wait is a normal terminal result without assistant text;
147
+ * its authority comes from prepared tools, never the provider's output. */
148
+ export declare function requireAgentStreamFinalOutput(finalOutput: unknown | undefined, inputWaitYielded?: boolean): unknown;
147
149
  /**
148
150
  * Terminal settlement closes the active attempt before turn finalization. The
149
151
  * ordinary workspace admission therefore correctly returns `attempt_fenced`
@@ -94,7 +94,7 @@ export type RetainedProcessProbeResult = {
94
94
  status: "deferred";
95
95
  reason: "identity_mismatch" | "resume_state_missing" | "backend_unsupported" | "provider_running" | "provider_unknown" | "provider_timeout" | "provider_error" | "provider_binding_missing" | "provider_binding_mismatch";
96
96
  };
97
- export type RetainedProcessProbeFn = (settings: ActivityServices["settings"], lease: LeaseSnapshot, process: SandboxRetainedProcess, mode?: "observe" | "cancel") => Promise<RetainedProcessProbeResult>;
97
+ export type RetainedProcessProbeFn = (settings: ActivityServices["settings"], lease: LeaseSnapshot, process: SandboxRetainedProcess, mode?: "observe" | "cancel", captureOutput?: (result: unknown, chunkId: string) => Promise<void>) => Promise<RetainedProcessProbeResult>;
98
98
  export type HistoricalModalSandboxLifecycleProbeFn = typeof inspectModalSandboxLifecycle;
99
99
  export type DrainableProviderProbeFn = (settings: ActivityServices["settings"], lease: LeaseSnapshot) => Promise<"ready" | "missing">;
100
100
  export declare const RETAINED_PROCESS_RECONCILIATION_LIMIT = 20;
@@ -141,6 +141,10 @@ export declare function createSandboxLeaseActivities(services: () => Promise<Act
141
141
  maintainSandboxLeaseSweep: (input: SandboxLeaseSweepMaintenanceInput) => Promise<SandboxLeaseSweepMaintenanceResult>;
142
142
  reapSandboxLeases: () => Promise<ReapSandboxLeasesResult>;
143
143
  };
144
+ /** Reconcile commands only against the immutable physical locator copied at
145
+ * adoption. A successor enrollment connection is intentionally irrelevant: no
146
+ * query or cancellation is ever rebound to it. */
147
+ export declare function reconcileConnectedMachineBackgroundCommands(db: ActivityServices["db"], settings: ActivityServices["settings"], observability: ActivityServices["observability"], bus: ActivityServices["bus"], controlRpcOverride?: ControlRpc): Promise<void>;
144
148
  export declare function probeConnectedMachineBackgroundCommand(claim: ConnectedMachineBackgroundCommandClaim, controlRpc: ControlRpc, controlTimeoutMs: number, observedAt?: Date): Promise<ConnectedMachineBackgroundCommandProof | null>;
145
149
  export declare function connectedCommandProofFromStatus(status: OpStatus, observedAt: Date): ConnectedMachineBackgroundCommandProof | null;
146
150
  type RetainedProcessDeferralOutcome = Extract<RetainedProcessProbeResult, {
@@ -152,7 +156,8 @@ export declare function retainedProcessReconciliationDeferral(settings: Pick<Act
152
156
  retryAfterMs: number;
153
157
  quarantined: boolean;
154
158
  };
155
- export declare function probeRetainedProcessAtProvider(settings: ActivityServices["settings"], lease: LeaseSnapshot, process: SandboxRetainedProcess, mode?: "observe" | "cancel"): Promise<RetainedProcessProbeResult>;
159
+ export declare function captureRetainedProbeOutput(processId: string, result: unknown, capture: (result: unknown, chunkId: string) => Promise<void>): Promise<void>;
160
+ export declare function probeRetainedProcessAtProvider(settings: ActivityServices["settings"], lease: LeaseSnapshot, process: SandboxRetainedProcess, mode?: "observe" | "cancel", captureOutput?: (result: unknown, chunkId: string) => Promise<void>): Promise<RetainedProcessProbeResult>;
156
161
  /** Classify only exact SDK control banners. Arbitrary output, malformed text,
157
162
  * and a running marker are observations to retry; none is physical exit proof. */
158
163
  export declare function classifyRetainedProcessPollResult(result: unknown, providerSessionId: number): RetainedProcessProbeResult;
@@ -0,0 +1,3 @@
1
+ import { buildXaiTurnRequestAuthorization } from "./xai-auth.js";
2
+ /** Refresh billing, never inference, before treating cached exhaustion as current. */
3
+ export declare function refreshExhaustedXaiQuota(input: Omit<Parameters<typeof buildXaiTurnRequestAuthorization>[0], "credentialId">): Promise<void>;
@@ -9,6 +9,7 @@ import {
9
9
  INCIDENT_TELEMETRY_AUTHORITY_FENCE_LINEAGE_KEY,
10
10
  MAX_AUTOMATIC_PROVIDER_RECOVERIES,
11
11
  assertReusableSessionRevivable,
12
+ captureConnectedCommandOutput,
12
13
  collectWorkspaceArchiveObjectKeys,
13
14
  createCodexCapacityActivities,
14
15
  createGoalActivities,
@@ -32,7 +33,7 @@ import {
32
33
  scheduledUserMessagePayload,
33
34
  withFirstPartyTools,
34
35
  workflowIdForSession
35
- } from "./chunk-TAUE3GYT.js";
36
+ } from "./chunk-O2LQWJIK.js";
36
37
  import {
37
38
  inspectOpenSandboxKubernetesInventory,
38
39
  recordCreditBalanceGauges,
@@ -5236,6 +5237,7 @@ import {
5236
5237
  recordConnectedMachineBackgroundCommandProof
5237
5238
  } from "@opengeni/db/session-background-commands";
5238
5239
  import { sandboxWarmRateMicrosPerSecond } from "@opengeni/config";
5240
+ import { appendSessionCommandOutput } from "@opengeni/db/session-command-output";
5239
5241
  import { sandboxLeaseTelemetryKey } from "@opengeni/observability";
5240
5242
  import {
5241
5243
  captureVerifiedWorkspaceArchive,
@@ -5249,6 +5251,10 @@ import {
5249
5251
  isExecSessionLostBanner,
5250
5252
  isProviderSandboxNotFoundError,
5251
5253
  NatsControlRpc,
5254
+ NatsOpStreamTransport,
5255
+ OpStreamExecClient,
5256
+ defaultSelfhostedRetryClock,
5257
+ stripExecBanner,
5252
5258
  parseExecBannerExitCode,
5253
5259
  prepareProviderForTeardownAfterCapture,
5254
5260
  providerWorkspaceCapturePolicy,
@@ -5809,97 +5815,172 @@ async function adoptLegacyModalCheckpointReceipts(db, settings, observability) {
5809
5815
  });
5810
5816
  return adopted;
5811
5817
  }
5812
- async function reconcileConnectedMachineBackgroundCommands(db, settings, observability, bus) {
5813
- const claimId = crypto.randomUUID();
5814
- let claims;
5815
- try {
5816
- claims = await claimConnectedMachineSessionBackgroundCommands(db, {
5817
- claimId,
5818
- limit: CONNECTED_COMMAND_RECONCILIATION_LIMIT,
5819
- claimTtlMs: CONNECTED_COMMAND_RECONCILIATION_CLAIM_TTL_MS
5820
- });
5821
- } catch (error) {
5822
- recordConnectedCommandReconciliation(observability, "claim_failed");
5823
- observability.warn("sandbox reaper: connected-command claim failed", {
5824
- error: error instanceof Error ? error.message : String(error)
5825
- });
5826
- return;
5827
- }
5828
- const controlRpc = new NatsControlRpc(
5829
- async () => bus.getRequestConnection()
5830
- );
5831
- await forEachWithConcurrency(claims, SANDBOX_MAINTENANCE_ITEM_CONCURRENCY, async (claim) => {
5832
- let proof = claim.proof;
5833
- if (!proof) {
5834
- try {
5835
- proof = await probeConnectedMachineBackgroundCommand(
5836
- claim,
5837
- controlRpc,
5838
- Math.min(
5839
- settings.sandboxSelfhostedControlTimeoutMs,
5840
- RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS
5841
- )
5842
- );
5843
- if (!proof) {
5844
- await deferConnectedCommandClaim(db, settings, observability, claim, "provider_running");
5818
+ async function reconcileConnectedMachineBackgroundCommands(db, settings, observability, bus, controlRpcOverride) {
5819
+ const dueBefore = /* @__PURE__ */ new Date();
5820
+ const deadline = Date.now() + settings.sandboxLeaseReaperPeriodMs;
5821
+ const offlineConnections = /* @__PURE__ */ new Set();
5822
+ const firstConnectionProbes = /* @__PURE__ */ new Map();
5823
+ do {
5824
+ const claimId = crypto.randomUUID();
5825
+ let claims;
5826
+ try {
5827
+ claims = await claimConnectedMachineSessionBackgroundCommands(db, {
5828
+ claimId,
5829
+ dueBefore,
5830
+ limit: CONNECTED_COMMAND_RECONCILIATION_LIMIT,
5831
+ claimTtlMs: CONNECTED_COMMAND_RECONCILIATION_CLAIM_TTL_MS
5832
+ });
5833
+ } catch (error) {
5834
+ recordConnectedCommandReconciliation(observability, "claim_failed");
5835
+ observability.warn("sandbox reaper: connected-command claim failed", {
5836
+ error: error instanceof Error ? error.message : String(error)
5837
+ });
5838
+ return;
5839
+ }
5840
+ const controlRpc = controlRpcOverride ?? new NatsControlRpc(
5841
+ async () => bus.getRequestConnection()
5842
+ );
5843
+ await forEachWithConcurrency(claims, SANDBOX_MAINTENANCE_ITEM_CONCURRENCY, async (claim) => {
5844
+ let proof = claim.proof;
5845
+ const connectionKey = JSON.stringify([
5846
+ claim.controlWorkspaceId,
5847
+ claim.enrollmentId,
5848
+ claim.connectionInstanceId
5849
+ ]);
5850
+ if (!proof) {
5851
+ const firstProbe = firstConnectionProbes.get(connectionKey);
5852
+ if (firstProbe) await firstProbe;
5853
+ if (offlineConnections.has(connectionKey)) {
5854
+ await deferConnectedCommandClaim(db, settings, observability, claim, "provider_offline");
5845
5855
  return;
5846
5856
  }
5847
- } catch (error) {
5848
- const offline = error instanceof SelfhostedControlError && (error.agentOffline || error.reason === "agent_reconnecting" || error.draining);
5849
- if (!offline) {
5850
- observability.warn("sandbox reaper: connected-command provider probe failed", {
5857
+ let finishFirstProbe;
5858
+ if (!firstProbe) {
5859
+ firstConnectionProbes.set(
5860
+ connectionKey,
5861
+ new Promise((resolve) => {
5862
+ finishFirstProbe = resolve;
5863
+ })
5864
+ );
5865
+ }
5866
+ try {
5867
+ proof = await probeConnectedMachineBackgroundCommand(
5868
+ claim,
5869
+ controlRpc,
5870
+ Math.min(
5871
+ settings.sandboxSelfhostedControlTimeoutMs,
5872
+ RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS
5873
+ )
5874
+ );
5875
+ if (proof?.outcome !== "lost") {
5876
+ if (!bus.getOpStreamConnection)
5877
+ throw new Error("Connected command output transport unavailable");
5878
+ const client = new OpStreamExecClient({
5879
+ workspaceId: claim.controlWorkspaceId,
5880
+ agentId: claim.enrollmentId,
5881
+ connectionInstanceId: claim.connectionInstanceId,
5882
+ epoch: 0,
5883
+ controlRpc,
5884
+ rpcSubject: subjectFor(
5885
+ claim.controlWorkspaceId,
5886
+ claim.enrollmentId,
5887
+ claim.connectionInstanceId
5888
+ ),
5889
+ transport: new NatsOpStreamTransport(
5890
+ async () => bus.getOpStreamConnection?.() ?? null
5891
+ ),
5892
+ controlTimeoutMs: Math.min(
5893
+ settings.sandboxSelfhostedControlTimeoutMs,
5894
+ RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS
5895
+ ),
5896
+ retryClock: defaultSelfhostedRetryClock,
5897
+ journal: { attachGeneration: () => String(Date.now()), persistSettled: () => {
5898
+ } }
5899
+ });
5900
+ const replay = await client.readExisting(
5901
+ claim.opId,
5902
+ proof ? 5e3 : 250,
5903
+ async (frames) => {
5904
+ await captureConnectedCommandOutput(db, claim, bus)(claim.commandId, frames);
5905
+ }
5906
+ );
5907
+ if (proof && replay.status !== "completed" && !replay.terminal) {
5908
+ throw new Error(
5909
+ "Connected command terminal output replay has not reached its exit frontier"
5910
+ );
5911
+ }
5912
+ }
5913
+ if (!proof) {
5914
+ await deferConnectedCommandClaim(
5915
+ db,
5916
+ settings,
5917
+ observability,
5918
+ claim,
5919
+ "provider_running"
5920
+ );
5921
+ return;
5922
+ }
5923
+ } catch (error) {
5924
+ const offline = error instanceof SelfhostedControlError && (error.agentOffline || error.reason === "agent_reconnecting" || error.draining);
5925
+ if (offline) offlineConnections.add(connectionKey);
5926
+ if (!offline) {
5927
+ observability.warn("sandbox reaper: connected-command provider probe failed", {
5928
+ commandId: claim.commandId,
5929
+ enrollmentId: claim.enrollmentId,
5930
+ connectionInstanceId: claim.connectionInstanceId,
5931
+ error: error instanceof Error ? error.message : String(error)
5932
+ });
5933
+ }
5934
+ await deferConnectedCommandClaim(
5935
+ db,
5936
+ settings,
5937
+ observability,
5938
+ claim,
5939
+ offline ? "provider_offline" : "provider_error"
5940
+ );
5941
+ return;
5942
+ } finally {
5943
+ finishFirstProbe?.();
5944
+ }
5945
+ try {
5946
+ await recordConnectedMachineBackgroundCommandProof(db, { claim, proof });
5947
+ recordConnectedCommandReconciliation(observability, `proof_${proof.outcome}`);
5948
+ } catch (error) {
5949
+ recordConnectedCommandReconciliation(observability, "proof_checkpoint_failed");
5950
+ observability.warn("sandbox reaper: connected-command proof checkpoint failed", {
5851
5951
  commandId: claim.commandId,
5852
- enrollmentId: claim.enrollmentId,
5853
- connectionInstanceId: claim.connectionInstanceId,
5854
5952
  error: error instanceof Error ? error.message : String(error)
5855
5953
  });
5954
+ return;
5856
5955
  }
5857
- await deferConnectedCommandClaim(
5858
- db,
5859
- settings,
5860
- observability,
5861
- claim,
5862
- offline ? "provider_offline" : "provider_error"
5863
- );
5864
- return;
5865
5956
  }
5866
5957
  try {
5867
- await recordConnectedMachineBackgroundCommandProof(db, { claim, proof });
5868
- recordConnectedCommandReconciliation(observability, `proof_${proof.outcome}`);
5958
+ const settlement = await settleClaimedConnectedMachineBackgroundCommand(db, { claim });
5959
+ if (!settlement.settled) {
5960
+ throw new Error("Connected command settlement lost its exact claim or proof");
5961
+ }
5962
+ if (settlement.events.length > 0) {
5963
+ try {
5964
+ await bus.publish(claim.workspaceId, claim.sessionId, settlement.events);
5965
+ } catch (error) {
5966
+ observability.warn("sandbox reaper: connected-command event fanout failed", {
5967
+ commandId: claim.commandId,
5968
+ error: error instanceof Error ? error.message : String(error)
5969
+ });
5970
+ }
5971
+ }
5972
+ recordConnectedCommandReconciliation(observability, `settled_${proof.outcome}`);
5869
5973
  } catch (error) {
5870
- recordConnectedCommandReconciliation(observability, "proof_checkpoint_failed");
5871
- observability.warn("sandbox reaper: connected-command proof checkpoint failed", {
5974
+ recordConnectedCommandReconciliation(observability, "settlement_failed");
5975
+ observability.warn("sandbox reaper: connected-command settlement failed", {
5872
5976
  commandId: claim.commandId,
5873
5977
  error: error instanceof Error ? error.message : String(error)
5874
5978
  });
5875
- return;
5876
- }
5877
- }
5878
- try {
5879
- const settlement = await settleClaimedConnectedMachineBackgroundCommand(db, { claim });
5880
- if (!settlement.settled) {
5881
- throw new Error("Connected command settlement lost its exact claim or proof");
5882
- }
5883
- if (settlement.events.length > 0) {
5884
- try {
5885
- await bus.publish(claim.workspaceId, claim.sessionId, settlement.events);
5886
- } catch (error) {
5887
- observability.warn("sandbox reaper: connected-command event fanout failed", {
5888
- commandId: claim.commandId,
5889
- error: error instanceof Error ? error.message : String(error)
5890
- });
5891
- }
5979
+ await deferConnectedCommandClaim(db, settings, observability, claim, "settlement_failed");
5892
5980
  }
5893
- recordConnectedCommandReconciliation(observability, `settled_${proof.outcome}`);
5894
- } catch (error) {
5895
- recordConnectedCommandReconciliation(observability, "settlement_failed");
5896
- observability.warn("sandbox reaper: connected-command settlement failed", {
5897
- commandId: claim.commandId,
5898
- error: error instanceof Error ? error.message : String(error)
5899
- });
5900
- await deferConnectedCommandClaim(db, settings, observability, claim, "settlement_failed");
5901
- }
5902
- });
5981
+ });
5982
+ if (claims.length < CONNECTED_COMMAND_RECONCILIATION_LIMIT) break;
5983
+ } while (Date.now() < deadline);
5903
5984
  }
5904
5985
  async function probeConnectedMachineBackgroundCommand(claim, controlRpc, controlTimeoutMs, observedAt = /* @__PURE__ */ new Date()) {
5905
5986
  const input = {
@@ -5926,11 +6007,14 @@ function connectedCommandProofFromStatus(status, observedAt) {
5926
6007
  throw new Error("Connected command completed without an exit record");
5927
6008
  }
5928
6009
  const failureCode = exit.failureCode.replace(/[^A-Za-z0-9_-]/g, "_").slice(0, 128);
5929
- const reason = exit.cancelled ? "op_cancelled" : exit.timedOut ? "op_timed_out" : failureCode ? `op_failure_${failureCode}` : "op_exit";
6010
+ const reason = failureCode ? `op_failure_${failureCode}` : exit.cancelled ? "op_cancelled" : exit.timedOut ? "op_timed_out" : "op_exit";
5930
6011
  return {
5931
6012
  outcome: "exited",
5932
6013
  exitCode: exit.exitCode,
5933
6014
  reason,
6015
+ ...failureCode ? {
6016
+ failure: { code: failureCode, detail: exit.failureDetail, retryable: false }
6017
+ } : {},
5934
6018
  observedAt
5935
6019
  };
5936
6020
  }
@@ -6075,7 +6159,23 @@ async function reconcileTerminalRetainedProcesses(db, settings, observability, p
6075
6159
  settings,
6076
6160
  lease,
6077
6161
  process,
6078
- claim.ownerState === "background_stopping" ? "cancel" : "observe"
6162
+ claim.ownerState === "background_stopping" ? "cancel" : "observe",
6163
+ async (result, chunkId) => {
6164
+ if (typeof result !== "string" || isExecSessionLostBanner(result, process.providerSessionId))
6165
+ return;
6166
+ const events = await appendSessionCommandOutput(db, {
6167
+ accountId: process.accountId,
6168
+ workspaceId: process.workspaceId,
6169
+ sessionId: process.sessionId,
6170
+ commandId: process.id,
6171
+ chunkId,
6172
+ stream: "stdout",
6173
+ streamFidelity: "merged",
6174
+ chunk: stripExecBanner(result)
6175
+ });
6176
+ if (events.length && bus)
6177
+ await bus.publish(process.workspaceId, process.sessionId, events).catch(() => void 0);
6178
+ }
6079
6179
  );
6080
6180
  } catch (error) {
6081
6181
  observability.warn("sandbox reaper: retained-process provider probe failed", {
@@ -6253,13 +6353,29 @@ async function withRetainedProcessProbeTimeout(promise) {
6253
6353
  if (timer) clearTimeout(timer);
6254
6354
  }
6255
6355
  }
6256
- async function probeRetainedProcessAtProvider(settings, lease, process, mode = "observe") {
6356
+ var pendingRetainedProbeOutput = /* @__PURE__ */ new Map();
6357
+ async function captureRetainedProbeOutput(processId, result, capture) {
6358
+ const pending = pendingRetainedProbeOutput.get(processId) ?? {
6359
+ result,
6360
+ chunkId: crypto.randomUUID()
6361
+ };
6362
+ pendingRetainedProbeOutput.set(processId, pending);
6363
+ await capture(pending.result, pending.chunkId);
6364
+ pendingRetainedProbeOutput.delete(processId);
6365
+ }
6366
+ async function probeRetainedProcessAtProvider(settings, lease, process, mode = "observe", captureOutput) {
6257
6367
  if (lease.id !== process.leaseId || lease.sandboxGroupId !== process.sandboxGroupId || lease.leaseEpoch !== process.leaseEpoch || lease.backend !== process.providerBackend || lease.instanceId !== process.providerInstanceId || lease.resumeBackendId !== process.providerBackend || process.routeTargetId !== null) {
6258
6368
  return { status: "deferred", reason: "identity_mismatch" };
6259
6369
  }
6260
6370
  if (!lease.resumeState) {
6261
6371
  return { status: "deferred", reason: "resume_state_missing" };
6262
6372
  }
6373
+ const pending = pendingRetainedProbeOutput.get(process.id);
6374
+ if (pending) {
6375
+ if (!captureOutput) throw new Error("Pending command output requires its persistence callback");
6376
+ await captureRetainedProbeOutput(process.id, pending.result, captureOutput);
6377
+ return classifyRetainedProcessPollResult(pending.result, process.providerSessionId);
6378
+ }
6263
6379
  const envelopeBackend = lease.resumeState.backendId;
6264
6380
  if (envelopeBackend !== void 0 && envelopeBackend !== process.providerBackend) {
6265
6381
  return { status: "deferred", reason: "identity_mismatch" };
@@ -6391,6 +6507,7 @@ async function probeRetainedProcessAtProvider(settings, lease, process, mode = "
6391
6507
  }
6392
6508
  return { status: "deferred", reason: "provider_error" };
6393
6509
  }
6510
+ if (captureOutput) await captureRetainedProbeOutput(process.id, result, captureOutput);
6394
6511
  const observation = classifyRetainedProcessPollResult(result, process.providerSessionId);
6395
6512
  if (observation.status === "deferred" && observation.reason === "provider_running" && lease.rotationRequestedAt !== null) {
6396
6513
  try {
@@ -6402,6 +6519,7 @@ async function probeRetainedProcessAtProvider(settings, lease, process, mode = "
6402
6519
  maxOutputTokens: 2e3
6403
6520
  })
6404
6521
  );
6522
+ if (captureOutput) await captureRetainedProbeOutput(process.id, interrupted, captureOutput);
6405
6523
  return classifyRetainedProcessPollResult(interrupted, process.providerSessionId);
6406
6524
  } catch (error) {
6407
6525
  if (isProviderSandboxNotFoundError(client.backendId, error)) {
@@ -9674,7 +9792,8 @@ function createSessionStateActivities(services, overrides = {}) {
9674
9792
  sessionId: input.sessionId,
9675
9793
  workflowId,
9676
9794
  trigger: input.trigger,
9677
- error: input.error ?? "Agent turn admission failed before attempt claim."
9795
+ error: input.error ?? "Agent turn admission failed before attempt claim.",
9796
+ ...input.preClaimFailure ? { admissionFailure: input.preClaimFailure } : {}
9678
9797
  });
9679
9798
  if (failed.action === "terminal") return { action: "terminal" };
9680
9799
  if (failed.action === "stale") return { action: "stale" };
@@ -9715,7 +9834,7 @@ function createSessionStateActivities(services, overrides = {}) {
9715
9834
  if (providerRecoveryCount !== void 0 && (!Number.isSafeInteger(providerRecoveryCount) || providerRecoveryCount <= 0 || providerRecoveryCount !== providerRecoveryCountFromMetadata(turn.metadata ?? {}) + 1 || providerRecoveryCount > MAX_AUTOMATIC_PROVIDER_RECOVERIES || typeof providerFailureCode !== "string" || !/^[a-z][a-z0-9_]{0,63}$/.test(providerFailureCode))) {
9716
9835
  return { action: "stale" };
9717
9836
  }
9718
- const recoveredClaimCode = postClaimIdentityMatches ? postClaimRecovery.code : input.preClaimFailure?.disposition === "retryable" && input.preClaimFailure.code !== "claim_invariant" ? input.preClaimFailure.code : null;
9837
+ const recoveredClaimCode = postClaimIdentityMatches ? postClaimRecovery.code : input.preClaimFailure?.disposition === "retryable" && input.preClaimFailure.code !== "claim_invariant" ? input.preClaimFailure.code : input.preClaimFailureDisposition === "retryable" ? "legacy_retryable_preclaim_database_failure" : null;
9719
9838
  if (recoveredClaimCode) {
9720
9839
  const recovery = await requestSessionTurnRecoveryFn(db, input.workspaceId, {
9721
9840
  sessionId: input.sessionId,
@@ -9971,7 +10090,7 @@ function createSessionStateActivities(services, overrides = {}) {
9971
10090
  await publishDurableSessionEventsFn(bus, input.workspaceId, input.sessionId, settled.events);
9972
10091
  }
9973
10092
  await refreshQueuedTurnsGauge(db, observability, countQueuedTurnsFn, recordTurnsQueuedGaugeFn);
9974
- if (settled.action === "stale") {
10093
+ if (settled.action === "stale" || !settled.notifyParent) {
9975
10094
  return;
9976
10095
  }
9977
10096
  await notifyParentOfChildIdleFn(
@@ -10081,4 +10200,4 @@ export {
10081
10200
  createControlActivities,
10082
10201
  createControlActivitiesFromServices
10083
10202
  };
10084
- //# sourceMappingURL=activities-control-CTEMLTHG.js.map
10203
+ //# sourceMappingURL=activities-control-3SKLDTXQ.js.map