@opengeni/worker-bundle 0.24.2-canary.0 → 0.26.0-canary.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. package/dist/activities/agent-turn/agent-build.d.ts +3 -10
  2. package/dist/activities/agent-turn/claim.d.ts +7 -0
  3. package/dist/activities/agent-turn/credential-leases.d.ts +8 -0
  4. package/dist/activities/agent-turn/errors.d.ts +6 -5
  5. package/dist/activities/agent-turn/failure-settlement.d.ts +43 -1
  6. package/dist/activities/agent-turn/history-prefix.d.ts +8 -0
  7. package/dist/activities/agent-turn/history-sink.d.ts +6 -2
  8. package/dist/activities/agent-turn/quiescence.d.ts +13 -8
  9. package/dist/activities/agent-turn/tool-environment.d.ts +2 -13
  10. package/dist/activities/agent-turn/tool-policy.d.ts +4 -0
  11. package/dist/activities/agent-turn/turn-context.d.ts +9 -1
  12. package/dist/activities/codemode-dispatcher.d.ts +10 -1
  13. package/dist/activities/codex-capacity.d.ts +32 -3
  14. package/dist/activities/codex-fleet-shadow.d.ts +1 -1
  15. package/dist/activities/codex-image-generation.d.ts +1 -1
  16. package/dist/activities/codex-rotation.d.ts +8 -67
  17. package/dist/activities/goals.d.ts +1 -1
  18. package/dist/activities/google-drive-publication.d.ts +2 -13
  19. package/dist/activities/image-generation-operation.d.ts +13 -4
  20. package/dist/activities/sandbox-lease.d.ts +7 -2
  21. package/dist/activities/session-state.d.ts +4 -2
  22. package/dist/activities/types.d.ts +11 -1
  23. package/dist/activities/xai-quota.d.ts +3 -0
  24. package/dist/{activities-control-ZWH7J4KZ.js → activities-control-KS3ZPQVY.js} +233 -88
  25. package/dist/activities-control-KS3ZPQVY.js.map +1 -0
  26. package/dist/activities-control.d.ts +2 -0
  27. package/dist/{activities-turn-I4OPLQLS.js → activities-turn-GHSFZB64.js} +1109 -1865
  28. package/dist/activities-turn-GHSFZB64.js.map +1 -0
  29. package/dist/activities.d.ts +2 -0
  30. package/dist/{chunk-65JEWP2Q.js → chunk-D4ZEFOYH.js} +1 -1
  31. package/dist/chunk-D4ZEFOYH.js.map +1 -0
  32. package/dist/{chunk-RQNH6CGR.js → chunk-O2LQWJIK.js} +3250 -2061
  33. package/dist/chunk-O2LQWJIK.js.map +1 -0
  34. package/dist/github-rest-mcp.d.ts +0 -2
  35. package/dist/index.d.ts +1 -0
  36. package/dist/index.js +2 -2
  37. package/dist/index.js.map +1 -1
  38. package/dist/observability-metrics.d.ts +1 -1
  39. package/dist/sandbox-routing.d.ts +6 -0
  40. package/dist/workflow-bundle.js +94 -9
  41. package/dist/workflows/activities.d.ts +1 -1
  42. package/dist/workflows/session.d.ts +3 -0
  43. package/package.json +17 -17
  44. package/src/activities/agent-turn/agent-build.ts +21 -26
  45. package/src/activities/agent-turn/claim.ts +26 -1
  46. package/src/activities/agent-turn/codex-capacity.ts +311 -234
  47. package/src/activities/agent-turn/credential-leases.ts +36 -0
  48. package/src/activities/agent-turn/errors.ts +73 -12
  49. package/src/activities/agent-turn/failure-settlement.ts +472 -331
  50. package/src/activities/agent-turn/history-prefix.ts +51 -0
  51. package/src/activities/agent-turn/history-sink.ts +30 -6
  52. package/src/activities/agent-turn/quiescence.ts +29 -17
  53. package/src/activities/agent-turn/run.ts +22 -5
  54. package/src/activities/agent-turn/stream-attempt.ts +35 -8
  55. package/src/activities/agent-turn/tool-environment.ts +42 -32
  56. package/src/activities/agent-turn/tool-policy.ts +14 -8
  57. package/src/activities/agent-turn/turn-context.ts +13 -0
  58. package/src/activities/agent-turn/xai-capacity.ts +11 -0
  59. package/src/activities/codemode-dispatcher.ts +235 -59
  60. package/src/activities/codex-capacity.ts +107 -10
  61. package/src/activities/codex-fleet-shadow.ts +8 -1
  62. package/src/activities/codex-image-generation.ts +19 -2
  63. package/src/activities/codex-rotation.ts +45 -142
  64. package/src/activities/goals.ts +9 -9
  65. package/src/activities/google-drive-publication.ts +4 -81
  66. package/src/activities/image-generation-operation.ts +28 -3
  67. package/src/activities/run-input.ts +6 -7
  68. package/src/activities/sandbox-lease.ts +246 -87
  69. package/src/activities/session-state.ts +20 -1
  70. package/src/activities/types.ts +16 -8
  71. package/src/activities/xai-quota.ts +35 -0
  72. package/src/activities.ts +1 -0
  73. package/src/github-rest-mcp.ts +0 -82
  74. package/src/observability-metrics.ts +4 -0
  75. package/src/sandbox-routing.ts +186 -16
  76. package/src/workflows/activities.ts +1 -0
  77. package/src/workflows/session.ts +114 -8
  78. package/dist/activities/api-integrations.d.ts +0 -21
  79. package/dist/activities-control-ZWH7J4KZ.js.map +0 -1
  80. package/dist/activities-turn-I4OPLQLS.js.map +0 -1
  81. package/dist/chunk-65JEWP2Q.js.map +0 -1
  82. package/dist/chunk-RQNH6CGR.js.map +0 -1
  83. package/src/activities/api-integrations.ts +0 -151
@@ -53,6 +53,7 @@ import {
53
53
  workspaceArchiveCaptureDeadlineElapsed,
54
54
  retainedProcessReconciliationProof,
55
55
  retainedProcessSettlementIdentity,
56
+ settleClaimedConnectedMachineBackgroundCommand,
56
57
  settleSandboxCheckpointArtifactGc,
57
58
  rlsContextForWorkspace,
58
59
  settleRetainedProcess,
@@ -66,12 +67,12 @@ import {
66
67
  claimConnectedMachineSessionBackgroundCommands,
67
68
  deferConnectedMachineBackgroundCommandReconciliation,
68
69
  recordConnectedMachineBackgroundCommandProof,
69
- settleClaimedConnectedMachineBackgroundCommand,
70
- settleSessionBackgroundCommandForRetainedProcess,
71
70
  type ConnectedMachineBackgroundCommandClaim,
72
71
  type ConnectedMachineBackgroundCommandProof,
73
72
  } from "@opengeni/db/session-background-commands";
74
73
  import { sandboxWarmRateMicrosPerSecond } from "@opengeni/config";
74
+ import { appendSessionCommandOutput } from "@opengeni/db/session-command-output";
75
+ import { captureConnectedCommandOutput } from "../sandbox-routing";
75
76
  import { sandboxLeaseTelemetryKey } from "@opengeni/observability";
76
77
  import {
77
78
  // Normal drain teardown builds the client and resumes the envelope directly:
@@ -90,6 +91,10 @@ import {
90
91
  isExecSessionLostBanner,
91
92
  isProviderSandboxNotFoundError,
92
93
  NatsControlRpc,
94
+ NatsOpStreamTransport,
95
+ OpStreamExecClient,
96
+ defaultSelfhostedRetryClock,
97
+ stripExecBanner,
93
98
  parseExecBannerExitCode,
94
99
  prepareProviderForTeardownAfterCapture,
95
100
  providerWorkspaceCapturePolicy,
@@ -297,6 +302,7 @@ export type RetainedProcessProbeFn = (
297
302
  lease: LeaseSnapshot,
298
303
  process: SandboxRetainedProcess,
299
304
  mode?: "observe" | "cancel",
305
+ captureOutput?: (result: unknown, chunkId: string) => Promise<void>,
300
306
  ) => Promise<RetainedProcessProbeResult>;
301
307
 
302
308
  export type HistoricalModalSandboxLifecycleProbeFn = typeof inspectModalSandboxLifecycle;
@@ -727,6 +733,7 @@ export function createSandboxLeaseActivities(
727
733
  observability,
728
734
  probeRetainedProcess,
729
735
  options.inspectHistoricalModalSandbox ?? inspectModalSandboxLifecycle,
736
+ service.bus,
730
737
  );
731
738
 
732
739
  try {
@@ -1030,96 +1037,189 @@ async function adoptLegacyModalCheckpointReceipts(
1030
1037
  /** Reconcile commands only against the immutable physical locator copied at
1031
1038
  * adoption. A successor enrollment connection is intentionally irrelevant: no
1032
1039
  * query or cancellation is ever rebound to it. */
1033
- async function reconcileConnectedMachineBackgroundCommands(
1040
+ export async function reconcileConnectedMachineBackgroundCommands(
1034
1041
  db: ActivityServices["db"],
1035
1042
  settings: ActivityServices["settings"],
1036
1043
  observability: ActivityServices["observability"],
1037
1044
  bus: ActivityServices["bus"],
1045
+ controlRpcOverride?: ControlRpc,
1038
1046
  ): Promise<void> {
1039
- const claimId = crypto.randomUUID();
1040
- let claims: ConnectedMachineBackgroundCommandClaim[];
1041
- try {
1042
- claims = await claimConnectedMachineSessionBackgroundCommands(db, {
1043
- claimId,
1044
- limit: CONNECTED_COMMAND_RECONCILIATION_LIMIT,
1045
- claimTtlMs: CONNECTED_COMMAND_RECONCILIATION_CLAIM_TTL_MS,
1046
- });
1047
- } catch (error) {
1048
- recordConnectedCommandReconciliation(observability, "claim_failed");
1049
- observability.warn("sandbox reaper: connected-command claim failed", {
1050
- error: error instanceof Error ? error.message : String(error),
1051
- });
1052
- return;
1053
- }
1047
+ const dueBefore = new Date();
1048
+ const deadline = Date.now() + settings.sandboxLeaseReaperPeriodMs;
1049
+ const offlineConnections = new Set<string>();
1050
+ const firstConnectionProbes = new Map<string, Promise<void>>();
1051
+ // Drain the fixed due frontier with the existing concurrency. Yield to the
1052
+ // remaining maintenance work each period; retries belong to a later sweep.
1053
+ do {
1054
+ const claimId = crypto.randomUUID();
1055
+ let claims: ConnectedMachineBackgroundCommandClaim[];
1056
+ try {
1057
+ claims = await claimConnectedMachineSessionBackgroundCommands(db, {
1058
+ claimId,
1059
+ dueBefore,
1060
+ limit: CONNECTED_COMMAND_RECONCILIATION_LIMIT,
1061
+ claimTtlMs: CONNECTED_COMMAND_RECONCILIATION_CLAIM_TTL_MS,
1062
+ });
1063
+ } catch (error) {
1064
+ recordConnectedCommandReconciliation(observability, "claim_failed");
1065
+ observability.warn("sandbox reaper: connected-command claim failed", {
1066
+ error: error instanceof Error ? error.message : String(error),
1067
+ });
1068
+ return;
1069
+ }
1054
1070
 
1055
- const controlRpc = new NatsControlRpc(
1056
- async (): Promise<NatsRequestConnection | null> => bus.getRequestConnection(),
1057
- );
1058
- await forEachWithConcurrency(claims, SANDBOX_MAINTENANCE_ITEM_CONCURRENCY, async (claim) => {
1059
- let proof = claim.proof;
1060
- if (!proof) {
1061
- try {
1062
- proof = await probeConnectedMachineBackgroundCommand(
1063
- claim,
1064
- controlRpc,
1065
- Math.min(
1066
- settings.sandboxSelfhostedControlTimeoutMs,
1067
- RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS,
1068
- ),
1069
- );
1070
- if (!proof) {
1071
- await deferConnectedCommandClaim(db, settings, observability, claim, "provider_running");
1071
+ const controlRpc =
1072
+ controlRpcOverride ??
1073
+ new NatsControlRpc(
1074
+ async (): Promise<NatsRequestConnection | null> => bus.getRequestConnection(),
1075
+ );
1076
+ await forEachWithConcurrency(claims, SANDBOX_MAINTENANCE_ITEM_CONCURRENCY, async (claim) => {
1077
+ let proof = claim.proof;
1078
+ const connectionKey = JSON.stringify([
1079
+ claim.controlWorkspaceId,
1080
+ claim.enrollmentId,
1081
+ claim.connectionInstanceId,
1082
+ ]);
1083
+ if (!proof) {
1084
+ const firstProbe = firstConnectionProbes.get(connectionKey);
1085
+ if (firstProbe) await firstProbe;
1086
+ if (offlineConnections.has(connectionKey)) {
1087
+ await deferConnectedCommandClaim(db, settings, observability, claim, "provider_offline");
1072
1088
  return;
1073
1089
  }
1074
- } catch (error) {
1075
- const offline =
1076
- error instanceof SelfhostedControlError &&
1077
- (error.agentOffline || error.reason === "agent_reconnecting" || error.draining);
1078
- if (!offline) {
1079
- observability.warn("sandbox reaper: connected-command provider probe failed", {
1090
+ // Share only the initial connection observation, never another command's result.
1091
+ let finishFirstProbe: (() => void) | undefined;
1092
+ if (!firstProbe) {
1093
+ firstConnectionProbes.set(
1094
+ connectionKey,
1095
+ new Promise<void>((resolve) => {
1096
+ finishFirstProbe = resolve;
1097
+ }),
1098
+ );
1099
+ }
1100
+ try {
1101
+ proof = await probeConnectedMachineBackgroundCommand(
1102
+ claim,
1103
+ controlRpc,
1104
+ Math.min(
1105
+ settings.sandboxSelfhostedControlTimeoutMs,
1106
+ RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS,
1107
+ ),
1108
+ );
1109
+ // Replay currently retained frames for running as well as completed
1110
+ // commands. This is attach-only and leaves final ACK/observation alone.
1111
+ if (proof?.outcome !== "lost") {
1112
+ if (!bus.getOpStreamConnection)
1113
+ throw new Error("Connected command output transport unavailable");
1114
+ const client = new OpStreamExecClient({
1115
+ workspaceId: claim.controlWorkspaceId,
1116
+ agentId: claim.enrollmentId,
1117
+ connectionInstanceId: claim.connectionInstanceId,
1118
+ epoch: 0,
1119
+ controlRpc,
1120
+ rpcSubject: subjectFor(
1121
+ claim.controlWorkspaceId,
1122
+ claim.enrollmentId,
1123
+ claim.connectionInstanceId,
1124
+ ),
1125
+ transport: new NatsOpStreamTransport(
1126
+ async () => bus.getOpStreamConnection?.() ?? null,
1127
+ ),
1128
+ controlTimeoutMs: Math.min(
1129
+ settings.sandboxSelfhostedControlTimeoutMs,
1130
+ RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS,
1131
+ ),
1132
+ retryClock: defaultSelfhostedRetryClock,
1133
+ journal: { attachGeneration: () => String(Date.now()), persistSettled: () => {} },
1134
+ });
1135
+ const replay = await client.readExisting(
1136
+ claim.opId,
1137
+ proof ? 5_000 : 250,
1138
+ async (frames) => {
1139
+ await captureConnectedCommandOutput(db, claim, bus)(claim.commandId, frames);
1140
+ },
1141
+ );
1142
+ if (proof && replay.status !== "completed" && !replay.terminal) {
1143
+ throw new Error(
1144
+ "Connected command terminal output replay has not reached its exit frontier",
1145
+ );
1146
+ }
1147
+ }
1148
+ if (!proof) {
1149
+ await deferConnectedCommandClaim(
1150
+ db,
1151
+ settings,
1152
+ observability,
1153
+ claim,
1154
+ "provider_running",
1155
+ );
1156
+ return;
1157
+ }
1158
+ } catch (error) {
1159
+ const offline =
1160
+ error instanceof SelfhostedControlError &&
1161
+ (error.agentOffline || error.reason === "agent_reconnecting" || error.draining);
1162
+ if (offline) offlineConnections.add(connectionKey);
1163
+ if (!offline) {
1164
+ observability.warn("sandbox reaper: connected-command provider probe failed", {
1165
+ commandId: claim.commandId,
1166
+ enrollmentId: claim.enrollmentId,
1167
+ connectionInstanceId: claim.connectionInstanceId,
1168
+ error: error instanceof Error ? error.message : String(error),
1169
+ });
1170
+ }
1171
+ await deferConnectedCommandClaim(
1172
+ db,
1173
+ settings,
1174
+ observability,
1175
+ claim,
1176
+ offline ? "provider_offline" : "provider_error",
1177
+ );
1178
+ return;
1179
+ } finally {
1180
+ finishFirstProbe?.();
1181
+ }
1182
+
1183
+ try {
1184
+ await recordConnectedMachineBackgroundCommandProof(db, { claim, proof });
1185
+ recordConnectedCommandReconciliation(observability, `proof_${proof.outcome}`);
1186
+ } catch (error) {
1187
+ recordConnectedCommandReconciliation(observability, "proof_checkpoint_failed");
1188
+ observability.warn("sandbox reaper: connected-command proof checkpoint failed", {
1080
1189
  commandId: claim.commandId,
1081
- enrollmentId: claim.enrollmentId,
1082
- connectionInstanceId: claim.connectionInstanceId,
1083
1190
  error: error instanceof Error ? error.message : String(error),
1084
1191
  });
1192
+ return;
1085
1193
  }
1086
- await deferConnectedCommandClaim(
1087
- db,
1088
- settings,
1089
- observability,
1090
- claim,
1091
- offline ? "provider_offline" : "provider_error",
1092
- );
1093
- return;
1094
1194
  }
1095
1195
 
1096
1196
  try {
1097
- await recordConnectedMachineBackgroundCommandProof(db, { claim, proof });
1098
- recordConnectedCommandReconciliation(observability, `proof_${proof.outcome}`);
1197
+ const settlement = await settleClaimedConnectedMachineBackgroundCommand(db, { claim });
1198
+ if (!settlement.settled) {
1199
+ throw new Error("Connected command settlement lost its exact claim or proof");
1200
+ }
1201
+ if (settlement.events.length > 0) {
1202
+ try {
1203
+ await bus.publish(claim.workspaceId, claim.sessionId, settlement.events);
1204
+ } catch (error) {
1205
+ observability.warn("sandbox reaper: connected-command event fanout failed", {
1206
+ commandId: claim.commandId,
1207
+ error: error instanceof Error ? error.message : String(error),
1208
+ });
1209
+ }
1210
+ }
1211
+ recordConnectedCommandReconciliation(observability, `settled_${proof.outcome}`);
1099
1212
  } catch (error) {
1100
- recordConnectedCommandReconciliation(observability, "proof_checkpoint_failed");
1101
- observability.warn("sandbox reaper: connected-command proof checkpoint failed", {
1213
+ recordConnectedCommandReconciliation(observability, "settlement_failed");
1214
+ observability.warn("sandbox reaper: connected-command settlement failed", {
1102
1215
  commandId: claim.commandId,
1103
1216
  error: error instanceof Error ? error.message : String(error),
1104
1217
  });
1105
- return;
1106
- }
1107
- }
1108
-
1109
- try {
1110
- if (!(await settleClaimedConnectedMachineBackgroundCommand(db, { claim }))) {
1111
- throw new Error("Connected command settlement lost its exact claim or proof");
1218
+ await deferConnectedCommandClaim(db, settings, observability, claim, "settlement_failed");
1112
1219
  }
1113
- recordConnectedCommandReconciliation(observability, `settled_${proof.outcome}`);
1114
- } catch (error) {
1115
- recordConnectedCommandReconciliation(observability, "settlement_failed");
1116
- observability.warn("sandbox reaper: connected-command settlement failed", {
1117
- commandId: claim.commandId,
1118
- error: error instanceof Error ? error.message : String(error),
1119
- });
1120
- await deferConnectedCommandClaim(db, settings, observability, claim, "settlement_failed");
1121
- }
1122
- });
1220
+ });
1221
+ if (claims.length < CONNECTED_COMMAND_RECONCILIATION_LIMIT) break;
1222
+ } while (Date.now() < deadline);
1123
1223
  }
1124
1224
 
1125
1225
  export async function probeConnectedMachineBackgroundCommand(
@@ -1157,17 +1257,22 @@ export function connectedCommandProofFromStatus(
1157
1257
  throw new Error("Connected command completed without an exit record");
1158
1258
  }
1159
1259
  const failureCode = exit.failureCode.replace(/[^A-Za-z0-9_-]/g, "_").slice(0, 128);
1160
- const reason = exit.cancelled
1161
- ? "op_cancelled"
1162
- : exit.timedOut
1163
- ? "op_timed_out"
1164
- : failureCode
1165
- ? `op_failure_${failureCode}`
1260
+ const reason = failureCode
1261
+ ? `op_failure_${failureCode}`
1262
+ : exit.cancelled
1263
+ ? "op_cancelled"
1264
+ : exit.timedOut
1265
+ ? "op_timed_out"
1166
1266
  : "op_exit";
1167
1267
  return {
1168
1268
  outcome: "exited",
1169
1269
  exitCode: exit.exitCode,
1170
1270
  reason,
1271
+ ...(failureCode
1272
+ ? {
1273
+ failure: { code: failureCode, detail: exit.failureDetail, retryable: false as const },
1274
+ }
1275
+ : {}),
1171
1276
  observedAt,
1172
1277
  };
1173
1278
  }
@@ -1226,6 +1331,7 @@ async function reconcileTerminalRetainedProcesses(
1226
1331
  observability: ActivityServices["observability"],
1227
1332
  probe: RetainedProcessProbeFn,
1228
1333
  inspectHistoricalModalSandbox: HistoricalModalSandboxLifecycleProbeFn,
1334
+ bus: ActivityServices["bus"],
1229
1335
  ): Promise<void> {
1230
1336
  const claimId = crypto.randomUUID();
1231
1337
  let claims: Awaited<ReturnType<typeof claimTerminalRetainedProcesses>>;
@@ -1361,6 +1467,27 @@ async function reconcileTerminalRetainedProcesses(
1361
1467
  lease!,
1362
1468
  process,
1363
1469
  claim.ownerState === "background_stopping" ? "cancel" : "observe",
1470
+ async (result, chunkId) => {
1471
+ if (
1472
+ typeof result !== "string" ||
1473
+ isExecSessionLostBanner(result, process.providerSessionId)
1474
+ )
1475
+ return;
1476
+ const events = await appendSessionCommandOutput(db, {
1477
+ accountId: process.accountId,
1478
+ workspaceId: process.workspaceId,
1479
+ sessionId: process.sessionId,
1480
+ commandId: process.id,
1481
+ chunkId,
1482
+ stream: "stdout",
1483
+ streamFidelity: "merged",
1484
+ chunk: stripExecBanner(result),
1485
+ });
1486
+ if (events.length && bus)
1487
+ await bus
1488
+ .publish(process.workspaceId, process.sessionId, events)
1489
+ .catch(() => undefined);
1490
+ },
1364
1491
  );
1365
1492
  } catch (error) {
1366
1493
  observability.warn("sandbox reaper: retained-process provider probe failed", {
@@ -1444,15 +1571,20 @@ async function reconcileTerminalRetainedProcesses(
1444
1571
  if (settlement.process.state === "active") {
1445
1572
  throw new Error("Retained-process reconciliation returned an active durable process");
1446
1573
  }
1447
- await settleSessionBackgroundCommandForRetainedProcess(db, {
1448
- accountId: process.accountId,
1449
- workspaceId: process.workspaceId,
1450
- sessionId: process.sessionId,
1451
- retainedProcessId: process.id,
1452
- outcome: settlement.process.state,
1453
- exitCode: settlement.process.exitCode,
1454
- reason: settlement.process.settlementReason ?? proof.reason,
1455
- });
1574
+ if (settlement.backgroundCommandEvents.length > 0) {
1575
+ try {
1576
+ await bus.publish(
1577
+ process.workspaceId,
1578
+ process.sessionId,
1579
+ settlement.backgroundCommandEvents,
1580
+ );
1581
+ } catch (error) {
1582
+ observability.warn("sandbox reaper: retained-command event fanout failed", {
1583
+ processId: process.id,
1584
+ error: error instanceof Error ? error.message : String(error),
1585
+ });
1586
+ }
1587
+ }
1456
1588
  recordRetainedProcessReconciliation(observability, `settled_${proof.outcome}`);
1457
1589
  } catch (error) {
1458
1590
  recordRetainedProcessReconciliation(observability, "settlement_failed");
@@ -1580,11 +1712,30 @@ async function withRetainedProcessProbeTimeout<T>(promise: Promise<T>): Promise<
1580
1712
  }
1581
1713
  }
1582
1714
 
1715
+ /** Consumed SDK output cannot be polled again. Retain the receipt across
1716
+ * persistence retries in this worker; a worker crash remains unreplayable. */
1717
+ const pendingRetainedProbeOutput = new Map<string, { result: unknown; chunkId: string }>();
1718
+
1719
+ export async function captureRetainedProbeOutput(
1720
+ processId: string,
1721
+ result: unknown,
1722
+ capture: (result: unknown, chunkId: string) => Promise<void>,
1723
+ ): Promise<void> {
1724
+ const pending = pendingRetainedProbeOutput.get(processId) ?? {
1725
+ result,
1726
+ chunkId: crypto.randomUUID(),
1727
+ };
1728
+ pendingRetainedProbeOutput.set(processId, pending);
1729
+ await capture(pending.result, pending.chunkId);
1730
+ pendingRetainedProbeOutput.delete(processId);
1731
+ }
1732
+
1583
1733
  export async function probeRetainedProcessAtProvider(
1584
1734
  settings: ActivityServices["settings"],
1585
1735
  lease: LeaseSnapshot,
1586
1736
  process: SandboxRetainedProcess,
1587
1737
  mode: "observe" | "cancel" = "observe",
1738
+ captureOutput?: (result: unknown, chunkId: string) => Promise<void>,
1588
1739
  ): Promise<RetainedProcessProbeResult> {
1589
1740
  if (
1590
1741
  lease.id !== process.leaseId ||
@@ -1600,6 +1751,12 @@ export async function probeRetainedProcessAtProvider(
1600
1751
  if (!lease.resumeState) {
1601
1752
  return { status: "deferred", reason: "resume_state_missing" };
1602
1753
  }
1754
+ const pending = pendingRetainedProbeOutput.get(process.id);
1755
+ if (pending) {
1756
+ if (!captureOutput) throw new Error("Pending command output requires its persistence callback");
1757
+ await captureRetainedProbeOutput(process.id, pending.result, captureOutput);
1758
+ return classifyRetainedProcessPollResult(pending.result, process.providerSessionId);
1759
+ }
1603
1760
  const envelopeBackend = (lease.resumeState as { backendId?: unknown }).backendId;
1604
1761
  if (envelopeBackend !== undefined && envelopeBackend !== process.providerBackend) {
1605
1762
  return { status: "deferred", reason: "identity_mismatch" };
@@ -1743,6 +1900,7 @@ export async function probeRetainedProcessAtProvider(
1743
1900
  }
1744
1901
  return { status: "deferred", reason: "provider_error" };
1745
1902
  }
1903
+ if (captureOutput) await captureRetainedProbeOutput(process.id, result, captureOutput);
1746
1904
  const observation = classifyRetainedProcessPollResult(result, process.providerSessionId);
1747
1905
  if (
1748
1906
  observation.status === "deferred" &&
@@ -1761,6 +1919,7 @@ export async function probeRetainedProcessAtProvider(
1761
1919
  maxOutputTokens: 2_000,
1762
1920
  }),
1763
1921
  );
1922
+ if (captureOutput) await captureRetainedProbeOutput(process.id, interrupted, captureOutput);
1764
1923
  return classifyRetainedProcessPollResult(interrupted, process.providerSessionId);
1765
1924
  } catch (error) {
1766
1925
  if (isProviderSandboxNotFoundError(client.backendId, error)) {
@@ -7,6 +7,7 @@ import {
7
7
  recoverSessionDispatch,
8
8
  reconcileSessionAttemptQuiescence,
9
9
  peekSessionWork as peekSessionWorkDb,
10
+ settleSessionInputWait as settleSessionInputWaitDb,
10
11
  countQueuedTurns,
11
12
  getSessionAttemptActivityRef,
12
13
  getSessionEvent,
@@ -42,6 +43,8 @@ import type {
42
43
  RecoverDispatchResult,
43
44
  RecoverEscapedMcpTimeoutInput,
44
45
  RecoverEscapedMcpTimeoutResult,
46
+ SettleSessionInputWaitInput,
47
+ SettleSessionInputWaitResult,
45
48
  } from "./types";
46
49
 
47
50
  export type SessionStateActivityOverrides = Partial<{
@@ -53,6 +56,7 @@ export type SessionStateActivityOverrides = Partial<{
53
56
  recoverSessionDispatch: typeof recoverSessionDispatch;
54
57
  reconcileSessionAttemptQuiescence: typeof reconcileSessionAttemptQuiescence;
55
58
  peekSessionWork: typeof peekSessionWorkDb;
59
+ settleSessionInputWait: typeof settleSessionInputWaitDb;
56
60
  countQueuedTurns: typeof countQueuedTurns;
57
61
  getSessionAttemptActivityRef: typeof getSessionAttemptActivityRef;
58
62
  getSessionEvent: typeof getSessionEvent;
@@ -93,6 +97,7 @@ export function createSessionStateActivities(
93
97
  const reconcileSessionAttemptQuiescenceFn =
94
98
  overrides.reconcileSessionAttemptQuiescence ?? reconcileSessionAttemptQuiescence;
95
99
  const peekSessionWorkFn = overrides.peekSessionWork ?? peekSessionWorkDb;
100
+ const settleSessionInputWaitFn = overrides.settleSessionInputWait ?? settleSessionInputWaitDb;
96
101
  const countQueuedTurnsFn = overrides.countQueuedTurns ?? countQueuedTurns;
97
102
  const getSessionAttemptActivityRefFn =
98
103
  overrides.getSessionAttemptActivityRef ?? getSessionAttemptActivityRef;
@@ -219,7 +224,9 @@ export function createSessionStateActivities(
219
224
  : input.preClaimFailure?.disposition === "retryable" &&
220
225
  input.preClaimFailure.code !== "claim_invariant"
221
226
  ? input.preClaimFailure.code
222
- : null;
227
+ : input.preClaimFailureDisposition === "retryable"
228
+ ? "legacy_retryable_preclaim_database_failure"
229
+ : null;
223
230
  if (recoveredClaimCode) {
224
231
  const recovery = await requestSessionTurnRecoveryFn(db, input.workspaceId, {
225
232
  sessionId: input.sessionId,
@@ -486,6 +493,17 @@ export function createSessionStateActivities(
486
493
  return peek;
487
494
  }
488
495
 
496
+ async function settleSessionInputWait(
497
+ input: SettleSessionInputWaitInput,
498
+ ): Promise<SettleSessionInputWaitResult> {
499
+ const { db, bus } = await services();
500
+ const result = await settleSessionInputWaitFn(db, input);
501
+ if (result.events.length > 0) {
502
+ await publishDurableSessionEventsFn(bus, input.workspaceId, input.sessionId, result.events);
503
+ }
504
+ return { action: result.action };
505
+ }
506
+
489
507
  async function expireSessionHumanInput(
490
508
  input: ExpireSessionHumanInputInput,
491
509
  ): Promise<ExpireSessionHumanInputResult> {
@@ -546,6 +564,7 @@ export function createSessionStateActivities(
546
564
  recoverDispatch,
547
565
  recoverEscapedMcpTimeout,
548
566
  peekSessionWork,
567
+ settleSessionInputWait,
549
568
  expireSessionHumanInput,
550
569
  expireSessionInteractionIntervention,
551
570
  markSessionIdle,
@@ -343,6 +343,18 @@ export type PeekSessionWorkInput = {
343
343
  sessionId: string;
344
344
  };
345
345
 
346
+ export type SettleSessionInputWaitInput = {
347
+ accountId: string;
348
+ workspaceId: string;
349
+ sessionId: string;
350
+ waitTurnId: string;
351
+ disposition: "held" | "timeout" | "superseded";
352
+ };
353
+
354
+ export type SettleSessionInputWaitResult = {
355
+ action: "held" | "timeout" | "superseded" | "stale";
356
+ };
357
+
346
358
  export type ExpireSessionHumanInputInput = {
347
359
  accountId: string;
348
360
  workspaceId: string;
@@ -378,14 +390,10 @@ export type MaybeContinueGoalInput = {
378
390
  };
379
391
 
380
392
  export type MaybeContinueGoalResult = {
381
- // `held`: an agent-declared `goal_wait` hold is current. No continuation was
382
- // materialized and the goal obligation stays armed; the workflow closes like
383
- // `none` and the wake outbox (deadline) or any producer's signalWithStart
384
- // restarts it.
385
- // `deferred`: idle backoff between consecutive no-input continuations. Same
386
- // shape as `held`: nothing materialized, obligation armed, a delayed outbox
387
- // wake at the pacing deadline (pulled to now by any new input) restarts it.
388
- action: "none" | "queue" | "continue" | "paused" | "held" | "deferred";
393
+ // `deferred`: idle backoff between consecutive no-input continuations.
394
+ // Nothing was materialized; the obligation remains armed and a delayed
395
+ // outbox wake at the pacing deadline restarts it.
396
+ action: "none" | "queue" | "continue" | "paused" | "deferred";
389
397
  };
390
398
 
391
399
  export type DispatchScheduledTaskRunInput = {
@@ -0,0 +1,35 @@
1
+ import { listXaiSubscriptionAccountsMetadata, updateXaiQuotaMetadata } from "@opengeni/db";
2
+ import { fetchXaiSubscriptionQuota } from "@opengeni/xai-subscription";
3
+ import { buildXaiTurnRequestAuthorization } from "./xai-auth";
4
+
5
+ /** Refresh billing, never inference, before treating cached exhaustion as current. */
6
+ export async function refreshExhaustedXaiQuota(
7
+ input: Omit<Parameters<typeof buildXaiTurnRequestAuthorization>[0], "credentialId">,
8
+ ) {
9
+ const accounts = await listXaiSubscriptionAccountsMetadata(input.db, input);
10
+ for (const account of accounts) {
11
+ if (account.status !== "active" || !account.allocatorEnabled || !account.exhaustedUntil)
12
+ continue;
13
+ if (account.quotaCheckedAt && Date.now() - account.quotaCheckedAt.getTime() < 30_000) continue;
14
+ try {
15
+ const auth = await buildXaiTurnRequestAuthorization({ ...input, credentialId: account.id });
16
+ const quota = await fetchXaiSubscriptionQuota({ context: auth.context });
17
+ // Unknown billing is not evidence that a refusal has cleared.
18
+ if (quota.usedPercent === null) continue;
19
+ await updateXaiQuotaMetadata(input.db, {
20
+ workspaceId: input.workspaceId,
21
+ subjectId: input.subjectId,
22
+ credentialId: account.id,
23
+ quotaUsedPercent: quota.usedPercent,
24
+ quotaResetAt: quota.period?.end ?? null,
25
+ quotaCheckedAt: quota.checkedAt,
26
+ expectedExhaustedUntil: account.exhaustedUntil,
27
+ expectedQuotaCheckedAt: account.quotaCheckedAt,
28
+ exhaustedUntil:
29
+ quota.usedPercent < 100 ? null : (quota.period?.end ?? account.exhaustedUntil),
30
+ });
31
+ } catch {
32
+ // Preserve provider refusal on refresh failure; the durable timer retries.
33
+ }
34
+ }
35
+ }
package/src/activities.ts CHANGED
@@ -101,6 +101,7 @@ export const reconcileSessionAttemptQuiescence =
101
101
  export const recoverDispatch = defaultControlActivities.recoverDispatch;
102
102
  export const recoverEscapedMcpTimeout = defaultControlActivities.recoverEscapedMcpTimeout;
103
103
  export const peekSessionWork = defaultControlActivities.peekSessionWork;
104
+ export const settleSessionInputWait = defaultControlActivities.settleSessionInputWait;
104
105
  export const expireSessionHumanInput = defaultControlActivities.expireSessionHumanInput;
105
106
  export const expireSessionInteractionIntervention =
106
107
  defaultControlActivities.expireSessionInteractionIntervention;