@opengeni/worker-bundle 0.24.2-canary.0 → 0.26.0-canary.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. package/dist/activities/agent-turn/agent-build.d.ts +3 -10
  2. package/dist/activities/agent-turn/claim.d.ts +7 -0
  3. package/dist/activities/agent-turn/credential-leases.d.ts +8 -0
  4. package/dist/activities/agent-turn/errors.d.ts +6 -5
  5. package/dist/activities/agent-turn/failure-settlement.d.ts +43 -1
  6. package/dist/activities/agent-turn/history-prefix.d.ts +8 -0
  7. package/dist/activities/agent-turn/history-sink.d.ts +6 -2
  8. package/dist/activities/agent-turn/quiescence.d.ts +13 -8
  9. package/dist/activities/agent-turn/tool-environment.d.ts +2 -13
  10. package/dist/activities/agent-turn/tool-policy.d.ts +4 -0
  11. package/dist/activities/agent-turn/turn-context.d.ts +9 -1
  12. package/dist/activities/codemode-dispatcher.d.ts +10 -1
  13. package/dist/activities/codex-capacity.d.ts +32 -3
  14. package/dist/activities/codex-fleet-shadow.d.ts +1 -1
  15. package/dist/activities/codex-image-generation.d.ts +1 -1
  16. package/dist/activities/codex-rotation.d.ts +8 -67
  17. package/dist/activities/goals.d.ts +1 -1
  18. package/dist/activities/google-drive-publication.d.ts +2 -13
  19. package/dist/activities/image-generation-operation.d.ts +13 -4
  20. package/dist/activities/sandbox-lease.d.ts +7 -2
  21. package/dist/activities/session-state.d.ts +4 -2
  22. package/dist/activities/types.d.ts +11 -1
  23. package/dist/activities/xai-quota.d.ts +3 -0
  24. package/dist/{activities-control-ZWH7J4KZ.js → activities-control-KS3ZPQVY.js} +233 -88
  25. package/dist/activities-control-KS3ZPQVY.js.map +1 -0
  26. package/dist/activities-control.d.ts +2 -0
  27. package/dist/{activities-turn-I4OPLQLS.js → activities-turn-GHSFZB64.js} +1109 -1865
  28. package/dist/activities-turn-GHSFZB64.js.map +1 -0
  29. package/dist/activities.d.ts +2 -0
  30. package/dist/{chunk-65JEWP2Q.js → chunk-D4ZEFOYH.js} +1 -1
  31. package/dist/chunk-D4ZEFOYH.js.map +1 -0
  32. package/dist/{chunk-RQNH6CGR.js → chunk-O2LQWJIK.js} +3250 -2061
  33. package/dist/chunk-O2LQWJIK.js.map +1 -0
  34. package/dist/github-rest-mcp.d.ts +0 -2
  35. package/dist/index.d.ts +1 -0
  36. package/dist/index.js +2 -2
  37. package/dist/index.js.map +1 -1
  38. package/dist/observability-metrics.d.ts +1 -1
  39. package/dist/sandbox-routing.d.ts +6 -0
  40. package/dist/workflow-bundle.js +94 -9
  41. package/dist/workflows/activities.d.ts +1 -1
  42. package/dist/workflows/session.d.ts +3 -0
  43. package/package.json +17 -17
  44. package/src/activities/agent-turn/agent-build.ts +21 -26
  45. package/src/activities/agent-turn/claim.ts +26 -1
  46. package/src/activities/agent-turn/codex-capacity.ts +311 -234
  47. package/src/activities/agent-turn/credential-leases.ts +36 -0
  48. package/src/activities/agent-turn/errors.ts +73 -12
  49. package/src/activities/agent-turn/failure-settlement.ts +472 -331
  50. package/src/activities/agent-turn/history-prefix.ts +51 -0
  51. package/src/activities/agent-turn/history-sink.ts +30 -6
  52. package/src/activities/agent-turn/quiescence.ts +29 -17
  53. package/src/activities/agent-turn/run.ts +22 -5
  54. package/src/activities/agent-turn/stream-attempt.ts +35 -8
  55. package/src/activities/agent-turn/tool-environment.ts +42 -32
  56. package/src/activities/agent-turn/tool-policy.ts +14 -8
  57. package/src/activities/agent-turn/turn-context.ts +13 -0
  58. package/src/activities/agent-turn/xai-capacity.ts +11 -0
  59. package/src/activities/codemode-dispatcher.ts +235 -59
  60. package/src/activities/codex-capacity.ts +107 -10
  61. package/src/activities/codex-fleet-shadow.ts +8 -1
  62. package/src/activities/codex-image-generation.ts +19 -2
  63. package/src/activities/codex-rotation.ts +45 -142
  64. package/src/activities/goals.ts +9 -9
  65. package/src/activities/google-drive-publication.ts +4 -81
  66. package/src/activities/image-generation-operation.ts +28 -3
  67. package/src/activities/run-input.ts +6 -7
  68. package/src/activities/sandbox-lease.ts +246 -87
  69. package/src/activities/session-state.ts +20 -1
  70. package/src/activities/types.ts +16 -8
  71. package/src/activities/xai-quota.ts +35 -0
  72. package/src/activities.ts +1 -0
  73. package/src/github-rest-mcp.ts +0 -82
  74. package/src/observability-metrics.ts +4 -0
  75. package/src/sandbox-routing.ts +186 -16
  76. package/src/workflows/activities.ts +1 -0
  77. package/src/workflows/session.ts +114 -8
  78. package/dist/activities/api-integrations.d.ts +0 -21
  79. package/dist/activities-control-ZWH7J4KZ.js.map +0 -1
  80. package/dist/activities-turn-I4OPLQLS.js.map +0 -1
  81. package/dist/chunk-65JEWP2Q.js.map +0 -1
  82. package/dist/chunk-RQNH6CGR.js.map +0 -1
  83. package/src/activities/api-integrations.ts +0 -151
@@ -1,8 +1,8 @@
1
- import { settleSessionAttemptInterruptions, applySessionTurnSettlement, enqueueSessionWorkflowWake, failSessionWorkBeforeAttemptClaim, requestSessionTurnRecovery, recoverSessionDispatch, reconcileSessionAttemptQuiescence, peekSessionWork as peekSessionWorkDb, countQueuedTurns, getSessionAttemptActivityRef, getSessionEvent, getSessionTurnForAttempt, expireSessionInteractionIntervention as expireSessionInteractionInterventionDb, expireSessionHumanInputRequest, markSessionAttemptQuiesced, requireSession, settleSessionIdleWithParentOutbox } from "@opengeni/db";
1
+ import { settleSessionAttemptInterruptions, applySessionTurnSettlement, enqueueSessionWorkflowWake, failSessionWorkBeforeAttemptClaim, requestSessionTurnRecovery, recoverSessionDispatch, reconcileSessionAttemptQuiescence, peekSessionWork as peekSessionWorkDb, settleSessionInputWait as settleSessionInputWaitDb, countQueuedTurns, getSessionAttemptActivityRef, getSessionEvent, getSessionTurnForAttempt, expireSessionInteractionIntervention as expireSessionInteractionInterventionDb, expireSessionHumanInputRequest, markSessionAttemptQuiesced, requireSession, settleSessionIdleWithParentOutbox } from "@opengeni/db";
2
2
  import { publishDurableSessionEvents } from "@opengeni/events";
3
3
  import { deliverFailedChildTurnToParent, notifyParentOfChildIdle } from "./parent-wake.js";
4
4
  import { recordTurnsQueuedGauge, recordWorkerDeathRecoveryMetrics } from "../observability-metrics.js";
5
- import type { ControlActivityServices, ExpireSessionHumanInputInput, ExpireSessionHumanInputResult, ExpireSessionInteractionInterventionInput, ExpireSessionInteractionInterventionResult, PeekSessionWorkInput, FailSessionAttemptInput, FailSessionAttemptResult, SettleSessionInterruptionsInput, MarkSessionIdleInput, PersistSessionAttemptQuiescenceInput, ReconcileSessionAttemptQuiescenceInput, ReconcileSessionAttemptQuiescenceResult, RecoverDispatchInput, RecoverDispatchResult, RecoverEscapedMcpTimeoutInput, RecoverEscapedMcpTimeoutResult } from "./types.js";
5
+ import type { ControlActivityServices, ExpireSessionHumanInputInput, ExpireSessionHumanInputResult, ExpireSessionInteractionInterventionInput, ExpireSessionInteractionInterventionResult, PeekSessionWorkInput, FailSessionAttemptInput, FailSessionAttemptResult, SettleSessionInterruptionsInput, MarkSessionIdleInput, PersistSessionAttemptQuiescenceInput, ReconcileSessionAttemptQuiescenceInput, ReconcileSessionAttemptQuiescenceResult, RecoverDispatchInput, RecoverDispatchResult, RecoverEscapedMcpTimeoutInput, RecoverEscapedMcpTimeoutResult, SettleSessionInputWaitInput, SettleSessionInputWaitResult } from "./types.js";
6
6
  export type SessionStateActivityOverrides = Partial<{
7
7
  settleSessionAttemptInterruptions: typeof settleSessionAttemptInterruptions;
8
8
  applySessionTurnSettlement: typeof applySessionTurnSettlement;
@@ -12,6 +12,7 @@ export type SessionStateActivityOverrides = Partial<{
12
12
  recoverSessionDispatch: typeof recoverSessionDispatch;
13
13
  reconcileSessionAttemptQuiescence: typeof reconcileSessionAttemptQuiescence;
14
14
  peekSessionWork: typeof peekSessionWorkDb;
15
+ settleSessionInputWait: typeof settleSessionInputWaitDb;
15
16
  countQueuedTurns: typeof countQueuedTurns;
16
17
  getSessionAttemptActivityRef: typeof getSessionAttemptActivityRef;
17
18
  getSessionEvent: typeof getSessionEvent;
@@ -38,6 +39,7 @@ export declare function createSessionStateActivities(services: () => Promise<Con
38
39
  recoverDispatch: (input: RecoverDispatchInput) => Promise<RecoverDispatchResult>;
39
40
  recoverEscapedMcpTimeout: (input: RecoverEscapedMcpTimeoutInput) => Promise<RecoverEscapedMcpTimeoutResult>;
40
41
  peekSessionWork: (input: PeekSessionWorkInput) => Promise<import("@opengeni/db").SessionWorkPeek>;
42
+ settleSessionInputWait: (input: SettleSessionInputWaitInput) => Promise<SettleSessionInputWaitResult>;
41
43
  expireSessionHumanInput: (input: ExpireSessionHumanInputInput) => Promise<ExpireSessionHumanInputResult>;
42
44
  expireSessionInteractionIntervention: (input: ExpireSessionInteractionInterventionInput) => Promise<ExpireSessionInteractionInterventionResult>;
43
45
  markSessionIdle: (input: MarkSessionIdleInput) => Promise<void>;
@@ -272,6 +272,16 @@ export type PeekSessionWorkInput = {
272
272
  workspaceId: string;
273
273
  sessionId: string;
274
274
  };
275
+ export type SettleSessionInputWaitInput = {
276
+ accountId: string;
277
+ workspaceId: string;
278
+ sessionId: string;
279
+ waitTurnId: string;
280
+ disposition: "held" | "timeout" | "superseded";
281
+ };
282
+ export type SettleSessionInputWaitResult = {
283
+ action: "held" | "timeout" | "superseded" | "stale";
284
+ };
275
285
  export type ExpireSessionHumanInputInput = {
276
286
  accountId: string;
277
287
  workspaceId: string;
@@ -301,7 +311,7 @@ export type MaybeContinueGoalInput = {
301
311
  workflowId: string;
302
312
  };
303
313
  export type MaybeContinueGoalResult = {
304
- action: "none" | "queue" | "continue" | "paused" | "held" | "deferred";
314
+ action: "none" | "queue" | "continue" | "paused" | "deferred";
305
315
  };
306
316
  export type DispatchScheduledTaskRunInput = {
307
317
  workspaceId: string;
@@ -0,0 +1,3 @@
1
+ import { buildXaiTurnRequestAuthorization } from "./xai-auth.js";
2
+ /** Refresh billing, never inference, before treating cached exhaustion as current. */
3
+ export declare function refreshExhaustedXaiQuota(input: Omit<Parameters<typeof buildXaiTurnRequestAuthorization>[0], "credentialId">): Promise<void>;
@@ -9,6 +9,7 @@ import {
9
9
  INCIDENT_TELEMETRY_AUTHORITY_FENCE_LINEAGE_KEY,
10
10
  MAX_AUTOMATIC_PROVIDER_RECOVERIES,
11
11
  assertReusableSessionRevivable,
12
+ captureConnectedCommandOutput,
12
13
  collectWorkspaceArchiveObjectKeys,
13
14
  createCodexCapacityActivities,
14
15
  createGoalActivities,
@@ -32,7 +33,7 @@ import {
32
33
  scheduledUserMessagePayload,
33
34
  withFirstPartyTools,
34
35
  workflowIdForSession
35
- } from "./chunk-RQNH6CGR.js";
36
+ } from "./chunk-O2LQWJIK.js";
36
37
  import {
37
38
  inspectOpenSandboxKubernetesInventory,
38
39
  recordCreditBalanceGauges,
@@ -52,7 +53,7 @@ import {
52
53
  recordTurnsQueuedGauge,
53
54
  recordWorkerDeathRecoveryMetrics,
54
55
  runtimeMetricsHooksForObservability
55
- } from "./chunk-65JEWP2Q.js";
56
+ } from "./chunk-D4ZEFOYH.js";
56
57
 
57
58
  // src/activities/browser-state-artifact-reaper.ts
58
59
  import {
@@ -5225,6 +5226,7 @@ import {
5225
5226
  workspaceArchiveCaptureDeadlineElapsed,
5226
5227
  retainedProcessReconciliationProof,
5227
5228
  retainedProcessSettlementIdentity,
5229
+ settleClaimedConnectedMachineBackgroundCommand,
5228
5230
  settleSandboxCheckpointArtifactGc,
5229
5231
  rlsContextForWorkspace as rlsContextForWorkspace2,
5230
5232
  settleRetainedProcess
@@ -5232,11 +5234,10 @@ import {
5232
5234
  import {
5233
5235
  claimConnectedMachineSessionBackgroundCommands,
5234
5236
  deferConnectedMachineBackgroundCommandReconciliation,
5235
- recordConnectedMachineBackgroundCommandProof,
5236
- settleClaimedConnectedMachineBackgroundCommand,
5237
- settleSessionBackgroundCommandForRetainedProcess
5237
+ recordConnectedMachineBackgroundCommandProof
5238
5238
  } from "@opengeni/db/session-background-commands";
5239
5239
  import { sandboxWarmRateMicrosPerSecond } from "@opengeni/config";
5240
+ import { appendSessionCommandOutput } from "@opengeni/db/session-command-output";
5240
5241
  import { sandboxLeaseTelemetryKey } from "@opengeni/observability";
5241
5242
  import {
5242
5243
  captureVerifiedWorkspaceArchive,
@@ -5250,6 +5251,10 @@ import {
5250
5251
  isExecSessionLostBanner,
5251
5252
  isProviderSandboxNotFoundError,
5252
5253
  NatsControlRpc,
5254
+ NatsOpStreamTransport,
5255
+ OpStreamExecClient,
5256
+ defaultSelfhostedRetryClock,
5257
+ stripExecBanner,
5253
5258
  parseExecBannerExitCode,
5254
5259
  prepareProviderForTeardownAfterCapture,
5255
5260
  providerWorkspaceCapturePolicy,
@@ -5570,7 +5575,8 @@ function createSandboxLeaseActivities(services, options = {}) {
5570
5575
  settings,
5571
5576
  observability,
5572
5577
  probeRetainedProcess,
5573
- options.inspectHistoricalModalSandbox ?? inspectModalSandboxLifecycle
5578
+ options.inspectHistoricalModalSandbox ?? inspectModalSandboxLifecycle,
5579
+ service.bus
5574
5580
  );
5575
5581
  try {
5576
5582
  modalOrphansTerminated = await sweepModalOrphans(settings, db, observability);
@@ -5809,86 +5815,172 @@ async function adoptLegacyModalCheckpointReceipts(db, settings, observability) {
5809
5815
  });
5810
5816
  return adopted;
5811
5817
  }
5812
- async function reconcileConnectedMachineBackgroundCommands(db, settings, observability, bus) {
5813
- const claimId = crypto.randomUUID();
5814
- let claims;
5815
- try {
5816
- claims = await claimConnectedMachineSessionBackgroundCommands(db, {
5817
- claimId,
5818
- limit: CONNECTED_COMMAND_RECONCILIATION_LIMIT,
5819
- claimTtlMs: CONNECTED_COMMAND_RECONCILIATION_CLAIM_TTL_MS
5820
- });
5821
- } catch (error) {
5822
- recordConnectedCommandReconciliation(observability, "claim_failed");
5823
- observability.warn("sandbox reaper: connected-command claim failed", {
5824
- error: error instanceof Error ? error.message : String(error)
5825
- });
5826
- return;
5827
- }
5828
- const controlRpc = new NatsControlRpc(
5829
- async () => bus.getRequestConnection()
5830
- );
5831
- await forEachWithConcurrency(claims, SANDBOX_MAINTENANCE_ITEM_CONCURRENCY, async (claim) => {
5832
- let proof = claim.proof;
5833
- if (!proof) {
5834
- try {
5835
- proof = await probeConnectedMachineBackgroundCommand(
5836
- claim,
5837
- controlRpc,
5838
- Math.min(
5839
- settings.sandboxSelfhostedControlTimeoutMs,
5840
- RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS
5841
- )
5842
- );
5843
- if (!proof) {
5844
- await deferConnectedCommandClaim(db, settings, observability, claim, "provider_running");
5818
+ async function reconcileConnectedMachineBackgroundCommands(db, settings, observability, bus, controlRpcOverride) {
5819
+ const dueBefore = /* @__PURE__ */ new Date();
5820
+ const deadline = Date.now() + settings.sandboxLeaseReaperPeriodMs;
5821
+ const offlineConnections = /* @__PURE__ */ new Set();
5822
+ const firstConnectionProbes = /* @__PURE__ */ new Map();
5823
+ do {
5824
+ const claimId = crypto.randomUUID();
5825
+ let claims;
5826
+ try {
5827
+ claims = await claimConnectedMachineSessionBackgroundCommands(db, {
5828
+ claimId,
5829
+ dueBefore,
5830
+ limit: CONNECTED_COMMAND_RECONCILIATION_LIMIT,
5831
+ claimTtlMs: CONNECTED_COMMAND_RECONCILIATION_CLAIM_TTL_MS
5832
+ });
5833
+ } catch (error) {
5834
+ recordConnectedCommandReconciliation(observability, "claim_failed");
5835
+ observability.warn("sandbox reaper: connected-command claim failed", {
5836
+ error: error instanceof Error ? error.message : String(error)
5837
+ });
5838
+ return;
5839
+ }
5840
+ const controlRpc = controlRpcOverride ?? new NatsControlRpc(
5841
+ async () => bus.getRequestConnection()
5842
+ );
5843
+ await forEachWithConcurrency(claims, SANDBOX_MAINTENANCE_ITEM_CONCURRENCY, async (claim) => {
5844
+ let proof = claim.proof;
5845
+ const connectionKey = JSON.stringify([
5846
+ claim.controlWorkspaceId,
5847
+ claim.enrollmentId,
5848
+ claim.connectionInstanceId
5849
+ ]);
5850
+ if (!proof) {
5851
+ const firstProbe = firstConnectionProbes.get(connectionKey);
5852
+ if (firstProbe) await firstProbe;
5853
+ if (offlineConnections.has(connectionKey)) {
5854
+ await deferConnectedCommandClaim(db, settings, observability, claim, "provider_offline");
5845
5855
  return;
5846
5856
  }
5847
- } catch (error) {
5848
- const offline = error instanceof SelfhostedControlError && (error.agentOffline || error.reason === "agent_reconnecting" || error.draining);
5849
- if (!offline) {
5850
- observability.warn("sandbox reaper: connected-command provider probe failed", {
5857
+ let finishFirstProbe;
5858
+ if (!firstProbe) {
5859
+ firstConnectionProbes.set(
5860
+ connectionKey,
5861
+ new Promise((resolve) => {
5862
+ finishFirstProbe = resolve;
5863
+ })
5864
+ );
5865
+ }
5866
+ try {
5867
+ proof = await probeConnectedMachineBackgroundCommand(
5868
+ claim,
5869
+ controlRpc,
5870
+ Math.min(
5871
+ settings.sandboxSelfhostedControlTimeoutMs,
5872
+ RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS
5873
+ )
5874
+ );
5875
+ if (proof?.outcome !== "lost") {
5876
+ if (!bus.getOpStreamConnection)
5877
+ throw new Error("Connected command output transport unavailable");
5878
+ const client = new OpStreamExecClient({
5879
+ workspaceId: claim.controlWorkspaceId,
5880
+ agentId: claim.enrollmentId,
5881
+ connectionInstanceId: claim.connectionInstanceId,
5882
+ epoch: 0,
5883
+ controlRpc,
5884
+ rpcSubject: subjectFor(
5885
+ claim.controlWorkspaceId,
5886
+ claim.enrollmentId,
5887
+ claim.connectionInstanceId
5888
+ ),
5889
+ transport: new NatsOpStreamTransport(
5890
+ async () => bus.getOpStreamConnection?.() ?? null
5891
+ ),
5892
+ controlTimeoutMs: Math.min(
5893
+ settings.sandboxSelfhostedControlTimeoutMs,
5894
+ RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS
5895
+ ),
5896
+ retryClock: defaultSelfhostedRetryClock,
5897
+ journal: { attachGeneration: () => String(Date.now()), persistSettled: () => {
5898
+ } }
5899
+ });
5900
+ const replay = await client.readExisting(
5901
+ claim.opId,
5902
+ proof ? 5e3 : 250,
5903
+ async (frames) => {
5904
+ await captureConnectedCommandOutput(db, claim, bus)(claim.commandId, frames);
5905
+ }
5906
+ );
5907
+ if (proof && replay.status !== "completed" && !replay.terminal) {
5908
+ throw new Error(
5909
+ "Connected command terminal output replay has not reached its exit frontier"
5910
+ );
5911
+ }
5912
+ }
5913
+ if (!proof) {
5914
+ await deferConnectedCommandClaim(
5915
+ db,
5916
+ settings,
5917
+ observability,
5918
+ claim,
5919
+ "provider_running"
5920
+ );
5921
+ return;
5922
+ }
5923
+ } catch (error) {
5924
+ const offline = error instanceof SelfhostedControlError && (error.agentOffline || error.reason === "agent_reconnecting" || error.draining);
5925
+ if (offline) offlineConnections.add(connectionKey);
5926
+ if (!offline) {
5927
+ observability.warn("sandbox reaper: connected-command provider probe failed", {
5928
+ commandId: claim.commandId,
5929
+ enrollmentId: claim.enrollmentId,
5930
+ connectionInstanceId: claim.connectionInstanceId,
5931
+ error: error instanceof Error ? error.message : String(error)
5932
+ });
5933
+ }
5934
+ await deferConnectedCommandClaim(
5935
+ db,
5936
+ settings,
5937
+ observability,
5938
+ claim,
5939
+ offline ? "provider_offline" : "provider_error"
5940
+ );
5941
+ return;
5942
+ } finally {
5943
+ finishFirstProbe?.();
5944
+ }
5945
+ try {
5946
+ await recordConnectedMachineBackgroundCommandProof(db, { claim, proof });
5947
+ recordConnectedCommandReconciliation(observability, `proof_${proof.outcome}`);
5948
+ } catch (error) {
5949
+ recordConnectedCommandReconciliation(observability, "proof_checkpoint_failed");
5950
+ observability.warn("sandbox reaper: connected-command proof checkpoint failed", {
5851
5951
  commandId: claim.commandId,
5852
- enrollmentId: claim.enrollmentId,
5853
- connectionInstanceId: claim.connectionInstanceId,
5854
5952
  error: error instanceof Error ? error.message : String(error)
5855
5953
  });
5954
+ return;
5856
5955
  }
5857
- await deferConnectedCommandClaim(
5858
- db,
5859
- settings,
5860
- observability,
5861
- claim,
5862
- offline ? "provider_offline" : "provider_error"
5863
- );
5864
- return;
5865
5956
  }
5866
5957
  try {
5867
- await recordConnectedMachineBackgroundCommandProof(db, { claim, proof });
5868
- recordConnectedCommandReconciliation(observability, `proof_${proof.outcome}`);
5958
+ const settlement = await settleClaimedConnectedMachineBackgroundCommand(db, { claim });
5959
+ if (!settlement.settled) {
5960
+ throw new Error("Connected command settlement lost its exact claim or proof");
5961
+ }
5962
+ if (settlement.events.length > 0) {
5963
+ try {
5964
+ await bus.publish(claim.workspaceId, claim.sessionId, settlement.events);
5965
+ } catch (error) {
5966
+ observability.warn("sandbox reaper: connected-command event fanout failed", {
5967
+ commandId: claim.commandId,
5968
+ error: error instanceof Error ? error.message : String(error)
5969
+ });
5970
+ }
5971
+ }
5972
+ recordConnectedCommandReconciliation(observability, `settled_${proof.outcome}`);
5869
5973
  } catch (error) {
5870
- recordConnectedCommandReconciliation(observability, "proof_checkpoint_failed");
5871
- observability.warn("sandbox reaper: connected-command proof checkpoint failed", {
5974
+ recordConnectedCommandReconciliation(observability, "settlement_failed");
5975
+ observability.warn("sandbox reaper: connected-command settlement failed", {
5872
5976
  commandId: claim.commandId,
5873
5977
  error: error instanceof Error ? error.message : String(error)
5874
5978
  });
5875
- return;
5979
+ await deferConnectedCommandClaim(db, settings, observability, claim, "settlement_failed");
5876
5980
  }
5877
- }
5878
- try {
5879
- if (!await settleClaimedConnectedMachineBackgroundCommand(db, { claim })) {
5880
- throw new Error("Connected command settlement lost its exact claim or proof");
5881
- }
5882
- recordConnectedCommandReconciliation(observability, `settled_${proof.outcome}`);
5883
- } catch (error) {
5884
- recordConnectedCommandReconciliation(observability, "settlement_failed");
5885
- observability.warn("sandbox reaper: connected-command settlement failed", {
5886
- commandId: claim.commandId,
5887
- error: error instanceof Error ? error.message : String(error)
5888
- });
5889
- await deferConnectedCommandClaim(db, settings, observability, claim, "settlement_failed");
5890
- }
5891
- });
5981
+ });
5982
+ if (claims.length < CONNECTED_COMMAND_RECONCILIATION_LIMIT) break;
5983
+ } while (Date.now() < deadline);
5892
5984
  }
5893
5985
  async function probeConnectedMachineBackgroundCommand(claim, controlRpc, controlTimeoutMs, observedAt = /* @__PURE__ */ new Date()) {
5894
5986
  const input = {
@@ -5915,11 +6007,14 @@ function connectedCommandProofFromStatus(status, observedAt) {
5915
6007
  throw new Error("Connected command completed without an exit record");
5916
6008
  }
5917
6009
  const failureCode = exit.failureCode.replace(/[^A-Za-z0-9_-]/g, "_").slice(0, 128);
5918
- const reason = exit.cancelled ? "op_cancelled" : exit.timedOut ? "op_timed_out" : failureCode ? `op_failure_${failureCode}` : "op_exit";
6010
+ const reason = failureCode ? `op_failure_${failureCode}` : exit.cancelled ? "op_cancelled" : exit.timedOut ? "op_timed_out" : "op_exit";
5919
6011
  return {
5920
6012
  outcome: "exited",
5921
6013
  exitCode: exit.exitCode,
5922
6014
  reason,
6015
+ ...failureCode ? {
6016
+ failure: { code: failureCode, detail: exit.failureDetail, retryable: false }
6017
+ } : {},
5923
6018
  observedAt
5924
6019
  };
5925
6020
  }
@@ -5957,7 +6052,7 @@ async function deferConnectedCommandClaim(db, settings, observability, claim, ou
5957
6052
  });
5958
6053
  }
5959
6054
  }
5960
- async function reconcileTerminalRetainedProcesses(db, settings, observability, probe, inspectHistoricalModalSandbox) {
6055
+ async function reconcileTerminalRetainedProcesses(db, settings, observability, probe, inspectHistoricalModalSandbox, bus) {
5961
6056
  const claimId = crypto.randomUUID();
5962
6057
  let claims;
5963
6058
  try {
@@ -6064,7 +6159,23 @@ async function reconcileTerminalRetainedProcesses(db, settings, observability, p
6064
6159
  settings,
6065
6160
  lease,
6066
6161
  process,
6067
- claim.ownerState === "background_stopping" ? "cancel" : "observe"
6162
+ claim.ownerState === "background_stopping" ? "cancel" : "observe",
6163
+ async (result, chunkId) => {
6164
+ if (typeof result !== "string" || isExecSessionLostBanner(result, process.providerSessionId))
6165
+ return;
6166
+ const events = await appendSessionCommandOutput(db, {
6167
+ accountId: process.accountId,
6168
+ workspaceId: process.workspaceId,
6169
+ sessionId: process.sessionId,
6170
+ commandId: process.id,
6171
+ chunkId,
6172
+ stream: "stdout",
6173
+ streamFidelity: "merged",
6174
+ chunk: stripExecBanner(result)
6175
+ });
6176
+ if (events.length && bus)
6177
+ await bus.publish(process.workspaceId, process.sessionId, events).catch(() => void 0);
6178
+ }
6068
6179
  );
6069
6180
  } catch (error) {
6070
6181
  observability.warn("sandbox reaper: retained-process provider probe failed", {
@@ -6147,15 +6258,20 @@ async function reconcileTerminalRetainedProcesses(db, settings, observability, p
6147
6258
  if (settlement.process.state === "active") {
6148
6259
  throw new Error("Retained-process reconciliation returned an active durable process");
6149
6260
  }
6150
- await settleSessionBackgroundCommandForRetainedProcess(db, {
6151
- accountId: process.accountId,
6152
- workspaceId: process.workspaceId,
6153
- sessionId: process.sessionId,
6154
- retainedProcessId: process.id,
6155
- outcome: settlement.process.state,
6156
- exitCode: settlement.process.exitCode,
6157
- reason: settlement.process.settlementReason ?? proof.reason
6158
- });
6261
+ if (settlement.backgroundCommandEvents.length > 0) {
6262
+ try {
6263
+ await bus.publish(
6264
+ process.workspaceId,
6265
+ process.sessionId,
6266
+ settlement.backgroundCommandEvents
6267
+ );
6268
+ } catch (error) {
6269
+ observability.warn("sandbox reaper: retained-command event fanout failed", {
6270
+ processId: process.id,
6271
+ error: error instanceof Error ? error.message : String(error)
6272
+ });
6273
+ }
6274
+ }
6159
6275
  recordRetainedProcessReconciliation(observability, `settled_${proof.outcome}`);
6160
6276
  } catch (error) {
6161
6277
  recordRetainedProcessReconciliation(observability, "settlement_failed");
@@ -6237,13 +6353,29 @@ async function withRetainedProcessProbeTimeout(promise) {
6237
6353
  if (timer) clearTimeout(timer);
6238
6354
  }
6239
6355
  }
6240
- async function probeRetainedProcessAtProvider(settings, lease, process, mode = "observe") {
6356
+ var pendingRetainedProbeOutput = /* @__PURE__ */ new Map();
6357
+ async function captureRetainedProbeOutput(processId, result, capture) {
6358
+ const pending = pendingRetainedProbeOutput.get(processId) ?? {
6359
+ result,
6360
+ chunkId: crypto.randomUUID()
6361
+ };
6362
+ pendingRetainedProbeOutput.set(processId, pending);
6363
+ await capture(pending.result, pending.chunkId);
6364
+ pendingRetainedProbeOutput.delete(processId);
6365
+ }
6366
+ async function probeRetainedProcessAtProvider(settings, lease, process, mode = "observe", captureOutput) {
6241
6367
  if (lease.id !== process.leaseId || lease.sandboxGroupId !== process.sandboxGroupId || lease.leaseEpoch !== process.leaseEpoch || lease.backend !== process.providerBackend || lease.instanceId !== process.providerInstanceId || lease.resumeBackendId !== process.providerBackend || process.routeTargetId !== null) {
6242
6368
  return { status: "deferred", reason: "identity_mismatch" };
6243
6369
  }
6244
6370
  if (!lease.resumeState) {
6245
6371
  return { status: "deferred", reason: "resume_state_missing" };
6246
6372
  }
6373
+ const pending = pendingRetainedProbeOutput.get(process.id);
6374
+ if (pending) {
6375
+ if (!captureOutput) throw new Error("Pending command output requires its persistence callback");
6376
+ await captureRetainedProbeOutput(process.id, pending.result, captureOutput);
6377
+ return classifyRetainedProcessPollResult(pending.result, process.providerSessionId);
6378
+ }
6247
6379
  const envelopeBackend = lease.resumeState.backendId;
6248
6380
  if (envelopeBackend !== void 0 && envelopeBackend !== process.providerBackend) {
6249
6381
  return { status: "deferred", reason: "identity_mismatch" };
@@ -6375,6 +6507,7 @@ async function probeRetainedProcessAtProvider(settings, lease, process, mode = "
6375
6507
  }
6376
6508
  return { status: "deferred", reason: "provider_error" };
6377
6509
  }
6510
+ if (captureOutput) await captureRetainedProbeOutput(process.id, result, captureOutput);
6378
6511
  const observation = classifyRetainedProcessPollResult(result, process.providerSessionId);
6379
6512
  if (observation.status === "deferred" && observation.reason === "provider_running" && lease.rotationRequestedAt !== null) {
6380
6513
  try {
@@ -6386,6 +6519,7 @@ async function probeRetainedProcessAtProvider(settings, lease, process, mode = "
6386
6519
  maxOutputTokens: 2e3
6387
6520
  })
6388
6521
  );
6522
+ if (captureOutput) await captureRetainedProbeOutput(process.id, interrupted, captureOutput);
6389
6523
  return classifyRetainedProcessPollResult(interrupted, process.providerSessionId);
6390
6524
  } catch (error) {
6391
6525
  if (isProviderSandboxNotFoundError(client.backendId, error)) {
@@ -9592,6 +9726,7 @@ import {
9592
9726
  recoverSessionDispatch,
9593
9727
  reconcileSessionAttemptQuiescence,
9594
9728
  peekSessionWork as peekSessionWorkDb,
9729
+ settleSessionInputWait as settleSessionInputWaitDb,
9595
9730
  countQueuedTurns as countQueuedTurns2,
9596
9731
  getSessionAttemptActivityRef,
9597
9732
  getSessionEvent,
@@ -9613,6 +9748,7 @@ function createSessionStateActivities(services, overrides = {}) {
9613
9748
  const recoverSessionDispatchFn = overrides.recoverSessionDispatch ?? recoverSessionDispatch;
9614
9749
  const reconcileSessionAttemptQuiescenceFn = overrides.reconcileSessionAttemptQuiescence ?? reconcileSessionAttemptQuiescence;
9615
9750
  const peekSessionWorkFn = overrides.peekSessionWork ?? peekSessionWorkDb;
9751
+ const settleSessionInputWaitFn = overrides.settleSessionInputWait ?? settleSessionInputWaitDb;
9616
9752
  const countQueuedTurnsFn = overrides.countQueuedTurns ?? countQueuedTurns2;
9617
9753
  const getSessionAttemptActivityRefFn = overrides.getSessionAttemptActivityRef ?? getSessionAttemptActivityRef;
9618
9754
  const getSessionEventFn = overrides.getSessionEvent ?? getSessionEvent;
@@ -9697,7 +9833,7 @@ function createSessionStateActivities(services, overrides = {}) {
9697
9833
  if (providerRecoveryCount !== void 0 && (!Number.isSafeInteger(providerRecoveryCount) || providerRecoveryCount <= 0 || providerRecoveryCount !== providerRecoveryCountFromMetadata(turn.metadata ?? {}) + 1 || providerRecoveryCount > MAX_AUTOMATIC_PROVIDER_RECOVERIES || typeof providerFailureCode !== "string" || !/^[a-z][a-z0-9_]{0,63}$/.test(providerFailureCode))) {
9698
9834
  return { action: "stale" };
9699
9835
  }
9700
- const recoveredClaimCode = postClaimIdentityMatches ? postClaimRecovery.code : input.preClaimFailure?.disposition === "retryable" && input.preClaimFailure.code !== "claim_invariant" ? input.preClaimFailure.code : null;
9836
+ const recoveredClaimCode = postClaimIdentityMatches ? postClaimRecovery.code : input.preClaimFailure?.disposition === "retryable" && input.preClaimFailure.code !== "claim_invariant" ? input.preClaimFailure.code : input.preClaimFailureDisposition === "retryable" ? "legacy_retryable_preclaim_database_failure" : null;
9701
9837
  if (recoveredClaimCode) {
9702
9838
  const recovery = await requestSessionTurnRecoveryFn(db, input.workspaceId, {
9703
9839
  sessionId: input.sessionId,
@@ -9915,6 +10051,14 @@ function createSessionStateActivities(services, overrides = {}) {
9915
10051
  await refreshQueuedTurnsGauge(db, observability, countQueuedTurnsFn, recordTurnsQueuedGaugeFn);
9916
10052
  return peek;
9917
10053
  }
10054
+ async function settleSessionInputWait(input) {
10055
+ const { db, bus } = await services();
10056
+ const result = await settleSessionInputWaitFn(db, input);
10057
+ if (result.events.length > 0) {
10058
+ await publishDurableSessionEventsFn(bus, input.workspaceId, input.sessionId, result.events);
10059
+ }
10060
+ return { action: result.action };
10061
+ }
9918
10062
  async function expireSessionHumanInput(input) {
9919
10063
  const { db, bus } = await services();
9920
10064
  const result = await expireSessionHumanInputRequestFn(db, input);
@@ -9963,6 +10107,7 @@ function createSessionStateActivities(services, overrides = {}) {
9963
10107
  recoverDispatch,
9964
10108
  recoverEscapedMcpTimeout,
9965
10109
  peekSessionWork,
10110
+ settleSessionInputWait,
9966
10111
  expireSessionHumanInput,
9967
10112
  expireSessionInteractionIntervention,
9968
10113
  markSessionIdle
@@ -10054,4 +10199,4 @@ export {
10054
10199
  createControlActivities,
10055
10200
  createControlActivitiesFromServices
10056
10201
  };
10057
- //# sourceMappingURL=activities-control-ZWH7J4KZ.js.map
10202
+ //# sourceMappingURL=activities-control-KS3ZPQVY.js.map