@opengeni/worker-bundle 0.26.0-canary.0 → 0.26.0-canary.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. package/dist/activities/agent-turn/errors.d.ts +5 -4
  2. package/dist/activities/agent-turn/history-prefix.d.ts +8 -0
  3. package/dist/activities/agent-turn/history-sink.d.ts +6 -2
  4. package/dist/activities/agent-turn/quiescence.d.ts +4 -2
  5. package/dist/activities/sandbox-lease.d.ts +7 -2
  6. package/dist/activities/xai-quota.d.ts +3 -0
  7. package/dist/{activities-control-CTEMLTHG.js → activities-control-3SKLDTXQ.js} +205 -86
  8. package/dist/activities-control-3SKLDTXQ.js.map +1 -0
  9. package/dist/{activities-turn-S5IU347A.js → activities-turn-GHSFZB64.js} +176 -1077
  10. package/dist/activities-turn-GHSFZB64.js.map +1 -0
  11. package/dist/{chunk-TAUE3GYT.js → chunk-O2LQWJIK.js} +3115 -1948
  12. package/dist/chunk-O2LQWJIK.js.map +1 -0
  13. package/dist/index.js +1 -1
  14. package/dist/sandbox-routing.d.ts +6 -0
  15. package/dist/workflow-bundle.js +24 -2
  16. package/package.json +17 -17
  17. package/src/activities/agent-turn/agent-build.ts +1 -0
  18. package/src/activities/agent-turn/errors.ts +44 -11
  19. package/src/activities/agent-turn/failure-settlement.ts +21 -8
  20. package/src/activities/agent-turn/history-prefix.ts +51 -0
  21. package/src/activities/agent-turn/history-sink.ts +30 -6
  22. package/src/activities/agent-turn/quiescence.ts +8 -2
  23. package/src/activities/agent-turn/run.ts +4 -0
  24. package/src/activities/agent-turn/stream-attempt.ts +20 -7
  25. package/src/activities/agent-turn/tool-environment.ts +12 -0
  26. package/src/activities/agent-turn/xai-capacity.ts +11 -0
  27. package/src/activities/codex-capacity.ts +18 -0
  28. package/src/activities/run-input.ts +6 -7
  29. package/src/activities/sandbox-lease.ts +229 -87
  30. package/src/activities/session-state.ts +8 -7
  31. package/src/activities/xai-quota.ts +35 -0
  32. package/src/sandbox-routing.ts +115 -0
  33. package/src/workflows/session.ts +29 -1
  34. package/dist/activities-control-CTEMLTHG.js.map +0 -1
  35. package/dist/activities-turn-S5IU347A.js.map +0 -1
  36. package/dist/chunk-TAUE3GYT.js.map +0 -1
@@ -71,6 +71,8 @@ import {
71
71
  type ConnectedMachineBackgroundCommandProof,
72
72
  } from "@opengeni/db/session-background-commands";
73
73
  import { sandboxWarmRateMicrosPerSecond } from "@opengeni/config";
74
+ import { appendSessionCommandOutput } from "@opengeni/db/session-command-output";
75
+ import { captureConnectedCommandOutput } from "../sandbox-routing";
74
76
  import { sandboxLeaseTelemetryKey } from "@opengeni/observability";
75
77
  import {
76
78
  // Normal drain teardown builds the client and resumes the envelope directly:
@@ -89,6 +91,10 @@ import {
89
91
  isExecSessionLostBanner,
90
92
  isProviderSandboxNotFoundError,
91
93
  NatsControlRpc,
94
+ NatsOpStreamTransport,
95
+ OpStreamExecClient,
96
+ defaultSelfhostedRetryClock,
97
+ stripExecBanner,
92
98
  parseExecBannerExitCode,
93
99
  prepareProviderForTeardownAfterCapture,
94
100
  providerWorkspaceCapturePolicy,
@@ -296,6 +302,7 @@ export type RetainedProcessProbeFn = (
296
302
  lease: LeaseSnapshot,
297
303
  process: SandboxRetainedProcess,
298
304
  mode?: "observe" | "cancel",
305
+ captureOutput?: (result: unknown, chunkId: string) => Promise<void>,
299
306
  ) => Promise<RetainedProcessProbeResult>;
300
307
 
301
308
  export type HistoricalModalSandboxLifecycleProbeFn = typeof inspectModalSandboxLifecycle;
@@ -1030,107 +1037,189 @@ async function adoptLegacyModalCheckpointReceipts(
1030
1037
  /** Reconcile commands only against the immutable physical locator copied at
1031
1038
  * adoption. A successor enrollment connection is intentionally irrelevant: no
1032
1039
  * query or cancellation is ever rebound to it. */
1033
- async function reconcileConnectedMachineBackgroundCommands(
1040
+ export async function reconcileConnectedMachineBackgroundCommands(
1034
1041
  db: ActivityServices["db"],
1035
1042
  settings: ActivityServices["settings"],
1036
1043
  observability: ActivityServices["observability"],
1037
1044
  bus: ActivityServices["bus"],
1045
+ controlRpcOverride?: ControlRpc,
1038
1046
  ): Promise<void> {
1039
- const claimId = crypto.randomUUID();
1040
- let claims: ConnectedMachineBackgroundCommandClaim[];
1041
- try {
1042
- claims = await claimConnectedMachineSessionBackgroundCommands(db, {
1043
- claimId,
1044
- limit: CONNECTED_COMMAND_RECONCILIATION_LIMIT,
1045
- claimTtlMs: CONNECTED_COMMAND_RECONCILIATION_CLAIM_TTL_MS,
1046
- });
1047
- } catch (error) {
1048
- recordConnectedCommandReconciliation(observability, "claim_failed");
1049
- observability.warn("sandbox reaper: connected-command claim failed", {
1050
- error: error instanceof Error ? error.message : String(error),
1051
- });
1052
- return;
1053
- }
1047
+ const dueBefore = new Date();
1048
+ const deadline = Date.now() + settings.sandboxLeaseReaperPeriodMs;
1049
+ const offlineConnections = new Set<string>();
1050
+ const firstConnectionProbes = new Map<string, Promise<void>>();
1051
+ // Drain the fixed due frontier with the existing concurrency. Yield to the
1052
+ // remaining maintenance work each period; retries belong to a later sweep.
1053
+ do {
1054
+ const claimId = crypto.randomUUID();
1055
+ let claims: ConnectedMachineBackgroundCommandClaim[];
1056
+ try {
1057
+ claims = await claimConnectedMachineSessionBackgroundCommands(db, {
1058
+ claimId,
1059
+ dueBefore,
1060
+ limit: CONNECTED_COMMAND_RECONCILIATION_LIMIT,
1061
+ claimTtlMs: CONNECTED_COMMAND_RECONCILIATION_CLAIM_TTL_MS,
1062
+ });
1063
+ } catch (error) {
1064
+ recordConnectedCommandReconciliation(observability, "claim_failed");
1065
+ observability.warn("sandbox reaper: connected-command claim failed", {
1066
+ error: error instanceof Error ? error.message : String(error),
1067
+ });
1068
+ return;
1069
+ }
1054
1070
 
1055
- const controlRpc = new NatsControlRpc(
1056
- async (): Promise<NatsRequestConnection | null> => bus.getRequestConnection(),
1057
- );
1058
- await forEachWithConcurrency(claims, SANDBOX_MAINTENANCE_ITEM_CONCURRENCY, async (claim) => {
1059
- let proof = claim.proof;
1060
- if (!proof) {
1061
- try {
1062
- proof = await probeConnectedMachineBackgroundCommand(
1063
- claim,
1064
- controlRpc,
1065
- Math.min(
1066
- settings.sandboxSelfhostedControlTimeoutMs,
1067
- RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS,
1068
- ),
1069
- );
1070
- if (!proof) {
1071
- await deferConnectedCommandClaim(db, settings, observability, claim, "provider_running");
1071
+ const controlRpc =
1072
+ controlRpcOverride ??
1073
+ new NatsControlRpc(
1074
+ async (): Promise<NatsRequestConnection | null> => bus.getRequestConnection(),
1075
+ );
1076
+ await forEachWithConcurrency(claims, SANDBOX_MAINTENANCE_ITEM_CONCURRENCY, async (claim) => {
1077
+ let proof = claim.proof;
1078
+ const connectionKey = JSON.stringify([
1079
+ claim.controlWorkspaceId,
1080
+ claim.enrollmentId,
1081
+ claim.connectionInstanceId,
1082
+ ]);
1083
+ if (!proof) {
1084
+ const firstProbe = firstConnectionProbes.get(connectionKey);
1085
+ if (firstProbe) await firstProbe;
1086
+ if (offlineConnections.has(connectionKey)) {
1087
+ await deferConnectedCommandClaim(db, settings, observability, claim, "provider_offline");
1072
1088
  return;
1073
1089
  }
1074
- } catch (error) {
1075
- const offline =
1076
- error instanceof SelfhostedControlError &&
1077
- (error.agentOffline || error.reason === "agent_reconnecting" || error.draining);
1078
- if (!offline) {
1079
- observability.warn("sandbox reaper: connected-command provider probe failed", {
1090
+ // Share only the initial connection observation, never another command's result.
1091
+ let finishFirstProbe: (() => void) | undefined;
1092
+ if (!firstProbe) {
1093
+ firstConnectionProbes.set(
1094
+ connectionKey,
1095
+ new Promise<void>((resolve) => {
1096
+ finishFirstProbe = resolve;
1097
+ }),
1098
+ );
1099
+ }
1100
+ try {
1101
+ proof = await probeConnectedMachineBackgroundCommand(
1102
+ claim,
1103
+ controlRpc,
1104
+ Math.min(
1105
+ settings.sandboxSelfhostedControlTimeoutMs,
1106
+ RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS,
1107
+ ),
1108
+ );
1109
+ // Replay currently retained frames for running as well as completed
1110
+ // commands. This is attach-only and leaves final ACK/observation alone.
1111
+ if (proof?.outcome !== "lost") {
1112
+ if (!bus.getOpStreamConnection)
1113
+ throw new Error("Connected command output transport unavailable");
1114
+ const client = new OpStreamExecClient({
1115
+ workspaceId: claim.controlWorkspaceId,
1116
+ agentId: claim.enrollmentId,
1117
+ connectionInstanceId: claim.connectionInstanceId,
1118
+ epoch: 0,
1119
+ controlRpc,
1120
+ rpcSubject: subjectFor(
1121
+ claim.controlWorkspaceId,
1122
+ claim.enrollmentId,
1123
+ claim.connectionInstanceId,
1124
+ ),
1125
+ transport: new NatsOpStreamTransport(
1126
+ async () => bus.getOpStreamConnection?.() ?? null,
1127
+ ),
1128
+ controlTimeoutMs: Math.min(
1129
+ settings.sandboxSelfhostedControlTimeoutMs,
1130
+ RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS,
1131
+ ),
1132
+ retryClock: defaultSelfhostedRetryClock,
1133
+ journal: { attachGeneration: () => String(Date.now()), persistSettled: () => {} },
1134
+ });
1135
+ const replay = await client.readExisting(
1136
+ claim.opId,
1137
+ proof ? 5_000 : 250,
1138
+ async (frames) => {
1139
+ await captureConnectedCommandOutput(db, claim, bus)(claim.commandId, frames);
1140
+ },
1141
+ );
1142
+ if (proof && replay.status !== "completed" && !replay.terminal) {
1143
+ throw new Error(
1144
+ "Connected command terminal output replay has not reached its exit frontier",
1145
+ );
1146
+ }
1147
+ }
1148
+ if (!proof) {
1149
+ await deferConnectedCommandClaim(
1150
+ db,
1151
+ settings,
1152
+ observability,
1153
+ claim,
1154
+ "provider_running",
1155
+ );
1156
+ return;
1157
+ }
1158
+ } catch (error) {
1159
+ const offline =
1160
+ error instanceof SelfhostedControlError &&
1161
+ (error.agentOffline || error.reason === "agent_reconnecting" || error.draining);
1162
+ if (offline) offlineConnections.add(connectionKey);
1163
+ if (!offline) {
1164
+ observability.warn("sandbox reaper: connected-command provider probe failed", {
1165
+ commandId: claim.commandId,
1166
+ enrollmentId: claim.enrollmentId,
1167
+ connectionInstanceId: claim.connectionInstanceId,
1168
+ error: error instanceof Error ? error.message : String(error),
1169
+ });
1170
+ }
1171
+ await deferConnectedCommandClaim(
1172
+ db,
1173
+ settings,
1174
+ observability,
1175
+ claim,
1176
+ offline ? "provider_offline" : "provider_error",
1177
+ );
1178
+ return;
1179
+ } finally {
1180
+ finishFirstProbe?.();
1181
+ }
1182
+
1183
+ try {
1184
+ await recordConnectedMachineBackgroundCommandProof(db, { claim, proof });
1185
+ recordConnectedCommandReconciliation(observability, `proof_${proof.outcome}`);
1186
+ } catch (error) {
1187
+ recordConnectedCommandReconciliation(observability, "proof_checkpoint_failed");
1188
+ observability.warn("sandbox reaper: connected-command proof checkpoint failed", {
1080
1189
  commandId: claim.commandId,
1081
- enrollmentId: claim.enrollmentId,
1082
- connectionInstanceId: claim.connectionInstanceId,
1083
1190
  error: error instanceof Error ? error.message : String(error),
1084
1191
  });
1192
+ return;
1085
1193
  }
1086
- await deferConnectedCommandClaim(
1087
- db,
1088
- settings,
1089
- observability,
1090
- claim,
1091
- offline ? "provider_offline" : "provider_error",
1092
- );
1093
- return;
1094
1194
  }
1095
1195
 
1096
1196
  try {
1097
- await recordConnectedMachineBackgroundCommandProof(db, { claim, proof });
1098
- recordConnectedCommandReconciliation(observability, `proof_${proof.outcome}`);
1197
+ const settlement = await settleClaimedConnectedMachineBackgroundCommand(db, { claim });
1198
+ if (!settlement.settled) {
1199
+ throw new Error("Connected command settlement lost its exact claim or proof");
1200
+ }
1201
+ if (settlement.events.length > 0) {
1202
+ try {
1203
+ await bus.publish(claim.workspaceId, claim.sessionId, settlement.events);
1204
+ } catch (error) {
1205
+ observability.warn("sandbox reaper: connected-command event fanout failed", {
1206
+ commandId: claim.commandId,
1207
+ error: error instanceof Error ? error.message : String(error),
1208
+ });
1209
+ }
1210
+ }
1211
+ recordConnectedCommandReconciliation(observability, `settled_${proof.outcome}`);
1099
1212
  } catch (error) {
1100
- recordConnectedCommandReconciliation(observability, "proof_checkpoint_failed");
1101
- observability.warn("sandbox reaper: connected-command proof checkpoint failed", {
1213
+ recordConnectedCommandReconciliation(observability, "settlement_failed");
1214
+ observability.warn("sandbox reaper: connected-command settlement failed", {
1102
1215
  commandId: claim.commandId,
1103
1216
  error: error instanceof Error ? error.message : String(error),
1104
1217
  });
1105
- return;
1218
+ await deferConnectedCommandClaim(db, settings, observability, claim, "settlement_failed");
1106
1219
  }
1107
- }
1108
-
1109
- try {
1110
- const settlement = await settleClaimedConnectedMachineBackgroundCommand(db, { claim });
1111
- if (!settlement.settled) {
1112
- throw new Error("Connected command settlement lost its exact claim or proof");
1113
- }
1114
- if (settlement.events.length > 0) {
1115
- try {
1116
- await bus.publish(claim.workspaceId, claim.sessionId, settlement.events);
1117
- } catch (error) {
1118
- observability.warn("sandbox reaper: connected-command event fanout failed", {
1119
- commandId: claim.commandId,
1120
- error: error instanceof Error ? error.message : String(error),
1121
- });
1122
- }
1123
- }
1124
- recordConnectedCommandReconciliation(observability, `settled_${proof.outcome}`);
1125
- } catch (error) {
1126
- recordConnectedCommandReconciliation(observability, "settlement_failed");
1127
- observability.warn("sandbox reaper: connected-command settlement failed", {
1128
- commandId: claim.commandId,
1129
- error: error instanceof Error ? error.message : String(error),
1130
- });
1131
- await deferConnectedCommandClaim(db, settings, observability, claim, "settlement_failed");
1132
- }
1133
- });
1220
+ });
1221
+ if (claims.length < CONNECTED_COMMAND_RECONCILIATION_LIMIT) break;
1222
+ } while (Date.now() < deadline);
1134
1223
  }
1135
1224
 
1136
1225
  export async function probeConnectedMachineBackgroundCommand(
@@ -1168,17 +1257,22 @@ export function connectedCommandProofFromStatus(
1168
1257
  throw new Error("Connected command completed without an exit record");
1169
1258
  }
1170
1259
  const failureCode = exit.failureCode.replace(/[^A-Za-z0-9_-]/g, "_").slice(0, 128);
1171
- const reason = exit.cancelled
1172
- ? "op_cancelled"
1173
- : exit.timedOut
1174
- ? "op_timed_out"
1175
- : failureCode
1176
- ? `op_failure_${failureCode}`
1260
+ const reason = failureCode
1261
+ ? `op_failure_${failureCode}`
1262
+ : exit.cancelled
1263
+ ? "op_cancelled"
1264
+ : exit.timedOut
1265
+ ? "op_timed_out"
1177
1266
  : "op_exit";
1178
1267
  return {
1179
1268
  outcome: "exited",
1180
1269
  exitCode: exit.exitCode,
1181
1270
  reason,
1271
+ ...(failureCode
1272
+ ? {
1273
+ failure: { code: failureCode, detail: exit.failureDetail, retryable: false as const },
1274
+ }
1275
+ : {}),
1182
1276
  observedAt,
1183
1277
  };
1184
1278
  }
@@ -1373,6 +1467,27 @@ async function reconcileTerminalRetainedProcesses(
1373
1467
  lease!,
1374
1468
  process,
1375
1469
  claim.ownerState === "background_stopping" ? "cancel" : "observe",
1470
+ async (result, chunkId) => {
1471
+ if (
1472
+ typeof result !== "string" ||
1473
+ isExecSessionLostBanner(result, process.providerSessionId)
1474
+ )
1475
+ return;
1476
+ const events = await appendSessionCommandOutput(db, {
1477
+ accountId: process.accountId,
1478
+ workspaceId: process.workspaceId,
1479
+ sessionId: process.sessionId,
1480
+ commandId: process.id,
1481
+ chunkId,
1482
+ stream: "stdout",
1483
+ streamFidelity: "merged",
1484
+ chunk: stripExecBanner(result),
1485
+ });
1486
+ if (events.length && bus)
1487
+ await bus
1488
+ .publish(process.workspaceId, process.sessionId, events)
1489
+ .catch(() => undefined);
1490
+ },
1376
1491
  );
1377
1492
  } catch (error) {
1378
1493
  observability.warn("sandbox reaper: retained-process provider probe failed", {
@@ -1597,11 +1712,30 @@ async function withRetainedProcessProbeTimeout<T>(promise: Promise<T>): Promise<
1597
1712
  }
1598
1713
  }
1599
1714
 
1715
+ /** Consumed SDK output cannot be polled again. Retain the receipt across
1716
+ * persistence retries in this worker; a worker crash remains unreplayable. */
1717
+ const pendingRetainedProbeOutput = new Map<string, { result: unknown; chunkId: string }>();
1718
+
1719
+ export async function captureRetainedProbeOutput(
1720
+ processId: string,
1721
+ result: unknown,
1722
+ capture: (result: unknown, chunkId: string) => Promise<void>,
1723
+ ): Promise<void> {
1724
+ const pending = pendingRetainedProbeOutput.get(processId) ?? {
1725
+ result,
1726
+ chunkId: crypto.randomUUID(),
1727
+ };
1728
+ pendingRetainedProbeOutput.set(processId, pending);
1729
+ await capture(pending.result, pending.chunkId);
1730
+ pendingRetainedProbeOutput.delete(processId);
1731
+ }
1732
+
1600
1733
  export async function probeRetainedProcessAtProvider(
1601
1734
  settings: ActivityServices["settings"],
1602
1735
  lease: LeaseSnapshot,
1603
1736
  process: SandboxRetainedProcess,
1604
1737
  mode: "observe" | "cancel" = "observe",
1738
+ captureOutput?: (result: unknown, chunkId: string) => Promise<void>,
1605
1739
  ): Promise<RetainedProcessProbeResult> {
1606
1740
  if (
1607
1741
  lease.id !== process.leaseId ||
@@ -1617,6 +1751,12 @@ export async function probeRetainedProcessAtProvider(
1617
1751
  if (!lease.resumeState) {
1618
1752
  return { status: "deferred", reason: "resume_state_missing" };
1619
1753
  }
1754
+ const pending = pendingRetainedProbeOutput.get(process.id);
1755
+ if (pending) {
1756
+ if (!captureOutput) throw new Error("Pending command output requires its persistence callback");
1757
+ await captureRetainedProbeOutput(process.id, pending.result, captureOutput);
1758
+ return classifyRetainedProcessPollResult(pending.result, process.providerSessionId);
1759
+ }
1620
1760
  const envelopeBackend = (lease.resumeState as { backendId?: unknown }).backendId;
1621
1761
  if (envelopeBackend !== undefined && envelopeBackend !== process.providerBackend) {
1622
1762
  return { status: "deferred", reason: "identity_mismatch" };
@@ -1760,6 +1900,7 @@ export async function probeRetainedProcessAtProvider(
1760
1900
  }
1761
1901
  return { status: "deferred", reason: "provider_error" };
1762
1902
  }
1903
+ if (captureOutput) await captureRetainedProbeOutput(process.id, result, captureOutput);
1763
1904
  const observation = classifyRetainedProcessPollResult(result, process.providerSessionId);
1764
1905
  if (
1765
1906
  observation.status === "deferred" &&
@@ -1778,6 +1919,7 @@ export async function probeRetainedProcessAtProvider(
1778
1919
  maxOutputTokens: 2_000,
1779
1920
  }),
1780
1921
  );
1922
+ if (captureOutput) await captureRetainedProbeOutput(process.id, interrupted, captureOutput);
1781
1923
  return classifyRetainedProcessPollResult(interrupted, process.providerSessionId);
1782
1924
  } catch (error) {
1783
1925
  if (isProviderSandboxNotFoundError(client.backendId, error)) {
@@ -160,6 +160,7 @@ export function createSessionStateActivities(
160
160
  workflowId,
161
161
  trigger: input.trigger,
162
162
  error: input.error ?? "Agent turn admission failed before attempt claim.",
163
+ ...(input.preClaimFailure ? { admissionFailure: input.preClaimFailure } : {}),
163
164
  });
164
165
  if (failed.action === "terminal") return { action: "terminal" };
165
166
  if (failed.action === "stale") return { action: "stale" };
@@ -224,7 +225,9 @@ export function createSessionStateActivities(
224
225
  : input.preClaimFailure?.disposition === "retryable" &&
225
226
  input.preClaimFailure.code !== "claim_invariant"
226
227
  ? input.preClaimFailure.code
227
- : null;
228
+ : input.preClaimFailureDisposition === "retryable"
229
+ ? "legacy_retryable_preclaim_database_failure"
230
+ : null;
228
231
  if (recoveredClaimCode) {
229
232
  const recovery = await requestSessionTurnRecoveryFn(db, input.workspaceId, {
230
233
  sessionId: input.sessionId,
@@ -538,14 +541,12 @@ export function createSessionStateActivities(
538
541
  await publishDurableSessionEventsFn(bus, input.workspaceId, input.sessionId, settled.events);
539
542
  }
540
543
  await refreshQueuedTurnsGauge(db, observability, countQueuedTurnsFn, recordTurnsQueuedGaugeFn);
541
- if (settled.action === "stale") {
544
+ if (settled.action === "stale" || !settled.notifyParent) {
542
545
  return;
543
546
  }
544
- // The workflow reaches markSessionIdle exactly when it has decided to stop
545
- // for now (no queued turn, no goal continuation): the terminal-for-now
546
- // point for a spawned worker, whatever the cause (goal completed, agent or
547
- // system paused goal, goalless work finished, idle control settlement). Wake
548
- // the parent here, deduped per idle episode so the manager is nudged once.
547
+ // The idle transaction distinguishes parked wait/goal obligations from
548
+ // completed work. Only a terminal idle boundary may notify the parent;
549
+ // workflow closure while waiting retains its durable wake without a result.
549
550
  await notifyParentOfChildIdleFn(
550
551
  { db, bus, settings, observability, wakeSessionWorkflow },
551
552
  input.workspaceId,
@@ -0,0 +1,35 @@
1
+ import { listXaiSubscriptionAccountsMetadata, updateXaiQuotaMetadata } from "@opengeni/db";
2
+ import { fetchXaiSubscriptionQuota } from "@opengeni/xai-subscription";
3
+ import { buildXaiTurnRequestAuthorization } from "./xai-auth";
4
+
5
+ /** Refresh billing, never inference, before treating cached exhaustion as current. */
6
+ export async function refreshExhaustedXaiQuota(
7
+ input: Omit<Parameters<typeof buildXaiTurnRequestAuthorization>[0], "credentialId">,
8
+ ) {
9
+ const accounts = await listXaiSubscriptionAccountsMetadata(input.db, input);
10
+ for (const account of accounts) {
11
+ if (account.status !== "active" || !account.allocatorEnabled || !account.exhaustedUntil)
12
+ continue;
13
+ if (account.quotaCheckedAt && Date.now() - account.quotaCheckedAt.getTime() < 30_000) continue;
14
+ try {
15
+ const auth = await buildXaiTurnRequestAuthorization({ ...input, credentialId: account.id });
16
+ const quota = await fetchXaiSubscriptionQuota({ context: auth.context });
17
+ // Unknown billing is not evidence that a refusal has cleared.
18
+ if (quota.usedPercent === null) continue;
19
+ await updateXaiQuotaMetadata(input.db, {
20
+ workspaceId: input.workspaceId,
21
+ subjectId: input.subjectId,
22
+ credentialId: account.id,
23
+ quotaUsedPercent: quota.usedPercent,
24
+ quotaResetAt: quota.period?.end ?? null,
25
+ quotaCheckedAt: quota.checkedAt,
26
+ expectedExhaustedUntil: account.exhaustedUntil,
27
+ expectedQuotaCheckedAt: account.quotaCheckedAt,
28
+ exhaustedUntil:
29
+ quota.usedPercent < 100 ? null : (quota.period?.end ?? account.exhaustedUntil),
30
+ });
31
+ } catch {
32
+ // Preserve provider refusal on refresh failure; the durable timer retries.
33
+ }
34
+ }
35
+ }
@@ -38,6 +38,9 @@ import {
38
38
  type EnrollmentRecord,
39
39
  type SandboxWorkspaceMutationAdmission,
40
40
  } from "@opengeni/db";
41
+ import { observeSessionBackgroundCommandCompletion } from "@opengeni/db/session-background-commands";
42
+ import { appendSessionCommandOutput } from "@opengeni/db/session-command-output";
43
+ import type { OpStreamOutputFrame } from "@opengeni/runtime/sandbox";
41
44
  import type { EventBus } from "@opengeni/events";
42
45
  import {
43
46
  buildSelfhostedBackendSession,
@@ -636,6 +639,76 @@ function afterRetainedProcessMutation(
636
639
  };
637
640
  }
638
641
 
642
+ export function captureConnectedCommandOutput(
643
+ db: Database,
644
+ ids: { accountId: string; workspaceId: string; sessionId: string },
645
+ bus?: EventBus,
646
+ ) {
647
+ return async (commandId: string, frames: OpStreamOutputFrame[]) => {
648
+ for (const frame of frames) {
649
+ const events = await appendSessionCommandOutput(db, {
650
+ ...ids,
651
+ commandId,
652
+ chunkId: `op-frame:${frame.sequence}`,
653
+ stream: frame.stream,
654
+ chunk: frame.chunk,
655
+ });
656
+ if (events.length && bus)
657
+ await bus.publish(ids.workspaceId, ids.sessionId, events).catch(() => undefined);
658
+ }
659
+ };
660
+ }
661
+
662
+ function captureRetainedProcessOutputForTurn(
663
+ services: RoutingWiringServices,
664
+ ids: RoutingWiringIds,
665
+ ) {
666
+ const accountId = ids.workspaceMutationFence?.accountId;
667
+ if (!accountId) return undefined;
668
+ return async ({
669
+ process,
670
+ chunkId,
671
+ chunk,
672
+ stream,
673
+ streamFidelity,
674
+ }: {
675
+ process: RoutingRetainedProcess;
676
+ chunkId: string;
677
+ chunk: string;
678
+ stream: "stdout" | "stderr";
679
+ streamFidelity: "separate" | "merged";
680
+ }) => {
681
+ const events = await appendSessionCommandOutput(services.db, {
682
+ accountId,
683
+ workspaceId: ids.workspaceId,
684
+ sessionId: ids.sessionId,
685
+ commandId: process.id,
686
+ chunkId,
687
+ chunk,
688
+ stream,
689
+ streamFidelity,
690
+ });
691
+ if (events.length && services.bus)
692
+ await services.bus.publish(ids.workspaceId, ids.sessionId, events).catch(() => undefined);
693
+ };
694
+ }
695
+
696
+ function observeRetainedProcessTerminalForTurn(
697
+ services: RoutingWiringServices,
698
+ ids: RoutingWiringIds,
699
+ ) {
700
+ const fence = ids.workspaceMutationFence;
701
+ if (!fence) return undefined;
702
+ return async ({ process }: { process: RoutingRetainedProcess }) => {
703
+ await observeSessionBackgroundCommandCompletion(services.db, {
704
+ accountId: fence.accountId,
705
+ workspaceId: ids.workspaceId,
706
+ sessionId: ids.sessionId,
707
+ commandId: process.id,
708
+ });
709
+ };
710
+ }
711
+
639
712
  function settleRetainedProcessForTurn(
640
713
  services: RoutingWiringServices,
641
714
  ids: RoutingWiringIds,
@@ -840,6 +913,7 @@ export function wrapTurnBoxWithRouting(
840
913
  outcome: command.outcome,
841
914
  exitCode: command.exitCode,
842
915
  reason: command.reason,
916
+ ...(command.failure ? { failure: command.failure } : {}),
843
917
  });
844
918
  if (settlement && bus) {
845
919
  await bus
@@ -847,6 +921,15 @@ export function wrapTurnBoxWithRouting(
847
921
  .catch(() => undefined);
848
922
  }
849
923
  },
924
+ selfhostedCaptureBackgroundCommandOutput: captureConnectedCommandOutput(
925
+ db,
926
+ {
927
+ accountId: backgroundCommandAccountId,
928
+ workspaceId: ids.workspaceId,
929
+ sessionId: ids.sessionId,
930
+ },
931
+ bus,
932
+ ),
850
933
  }
851
934
  : {}),
852
935
  // The turn's declared environment → a selfhosted swap target's manifest, so the
@@ -931,6 +1014,12 @@ export function wrapTurnBoxWithRouting(
931
1014
  ...(beforeProcessMutation ? { beforeProcessMutation } : {}),
932
1015
  ...(afterProcessMutation ? { afterProcessMutation } : {}),
933
1016
  ...(settleProcess ? { settleProcess } : {}),
1017
+ ...(ids.workspaceMutationFence
1018
+ ? { observeProcessTerminal: observeRetainedProcessTerminalForTurn(services, ids)! }
1019
+ : {}),
1020
+ ...(ids.workspaceMutationFence
1021
+ ? { captureProcessOutput: captureRetainedProcessOutputForTurn(services, ids)! }
1022
+ : {}),
934
1023
  ...(adoptProcessAsBackgroundCommand ? { adoptProcessAsBackgroundCommand } : {}),
935
1024
  ...(ids.homeLease
936
1025
  ? {
@@ -1094,6 +1183,7 @@ export function wrapLazyTurnBoxWithRouting(
1094
1183
  outcome: command.outcome,
1095
1184
  exitCode: command.exitCode,
1096
1185
  reason: command.reason,
1186
+ ...(command.failure ? { failure: command.failure } : {}),
1097
1187
  });
1098
1188
  if (settlement && bus) {
1099
1189
  await bus
@@ -1101,6 +1191,15 @@ export function wrapLazyTurnBoxWithRouting(
1101
1191
  .catch(() => undefined);
1102
1192
  }
1103
1193
  },
1194
+ selfhostedCaptureBackgroundCommandOutput: captureConnectedCommandOutput(
1195
+ db,
1196
+ {
1197
+ accountId: backgroundCommandAccountId,
1198
+ workspaceId: ids.workspaceId,
1199
+ sessionId: ids.sessionId,
1200
+ },
1201
+ bus,
1202
+ ),
1104
1203
  }
1105
1204
  : {}),
1106
1205
  ...(ids.environment !== undefined ? { environment: ids.environment } : {}),
@@ -1154,6 +1253,12 @@ export function wrapLazyTurnBoxWithRouting(
1154
1253
  ...(beforeProcessMutation ? { beforeProcessMutation } : {}),
1155
1254
  ...(afterProcessMutation ? { afterProcessMutation } : {}),
1156
1255
  ...(settleProcess ? { settleProcess } : {}),
1256
+ ...(ids.workspaceMutationFence
1257
+ ? { observeProcessTerminal: observeRetainedProcessTerminalForTurn(services, ids)! }
1258
+ : {}),
1259
+ ...(ids.workspaceMutationFence
1260
+ ? { captureProcessOutput: captureRetainedProcessOutputForTurn(services, ids)! }
1261
+ : {}),
1157
1262
  ...(adoptProcessAsBackgroundCommand ? { adoptProcessAsBackgroundCommand } : {}),
1158
1263
  ...(args.homeLeaseIdentity
1159
1264
  ? {
@@ -1432,6 +1537,15 @@ export async function establishSelfhostedTurnSession(
1432
1537
  },
1433
1538
  }
1434
1539
  : {}),
1540
+ captureBackgroundCommandOutput: captureConnectedCommandOutput(
1541
+ db,
1542
+ {
1543
+ accountId: args.accountId,
1544
+ workspaceId: args.workspaceId,
1545
+ sessionId: args.sessionId,
1546
+ },
1547
+ bus,
1548
+ ),
1435
1549
  settleBackgroundCommand: async (command) => {
1436
1550
  const settlement = await settleConnectedMachineSessionBackgroundCommand(db, {
1437
1551
  accountId: args.accountId,
@@ -1445,6 +1559,7 @@ export async function establishSelfhostedTurnSession(
1445
1559
  outcome: command.outcome,
1446
1560
  exitCode: command.exitCode,
1447
1561
  reason: command.reason,
1562
+ ...(command.failure ? { failure: command.failure } : {}),
1448
1563
  });
1449
1564
  if (settlement && bus) {
1450
1565
  await bus