@opengeni/worker-bundle 0.26.0-canary.0 → 0.26.0-canary.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn/errors.d.ts +5 -4
- package/dist/activities/agent-turn/history-prefix.d.ts +8 -0
- package/dist/activities/agent-turn/history-sink.d.ts +6 -2
- package/dist/activities/agent-turn/quiescence.d.ts +4 -2
- package/dist/activities/sandbox-lease.d.ts +7 -2
- package/dist/activities/xai-quota.d.ts +3 -0
- package/dist/{activities-control-CTEMLTHG.js → activities-control-KS3ZPQVY.js} +202 -84
- package/dist/activities-control-KS3ZPQVY.js.map +1 -0
- package/dist/{activities-turn-S5IU347A.js → activities-turn-GHSFZB64.js} +176 -1077
- package/dist/activities-turn-GHSFZB64.js.map +1 -0
- package/dist/{chunk-TAUE3GYT.js → chunk-O2LQWJIK.js} +3115 -1948
- package/dist/chunk-O2LQWJIK.js.map +1 -0
- package/dist/index.js +1 -1
- package/dist/sandbox-routing.d.ts +6 -0
- package/dist/workflow-bundle.js +24 -2
- package/package.json +17 -17
- package/src/activities/agent-turn/agent-build.ts +1 -0
- package/src/activities/agent-turn/errors.ts +44 -11
- package/src/activities/agent-turn/failure-settlement.ts +21 -8
- package/src/activities/agent-turn/history-prefix.ts +51 -0
- package/src/activities/agent-turn/history-sink.ts +30 -6
- package/src/activities/agent-turn/quiescence.ts +8 -2
- package/src/activities/agent-turn/run.ts +4 -0
- package/src/activities/agent-turn/stream-attempt.ts +20 -7
- package/src/activities/agent-turn/tool-environment.ts +12 -0
- package/src/activities/agent-turn/xai-capacity.ts +11 -0
- package/src/activities/codex-capacity.ts +18 -0
- package/src/activities/run-input.ts +6 -7
- package/src/activities/sandbox-lease.ts +229 -87
- package/src/activities/session-state.ts +3 -1
- package/src/activities/xai-quota.ts +35 -0
- package/src/sandbox-routing.ts +115 -0
- package/src/workflows/session.ts +29 -1
- package/dist/activities-control-CTEMLTHG.js.map +0 -1
- package/dist/activities-turn-S5IU347A.js.map +0 -1
- package/dist/chunk-TAUE3GYT.js.map +0 -1
|
@@ -71,6 +71,8 @@ import {
|
|
|
71
71
|
type ConnectedMachineBackgroundCommandProof,
|
|
72
72
|
} from "@opengeni/db/session-background-commands";
|
|
73
73
|
import { sandboxWarmRateMicrosPerSecond } from "@opengeni/config";
|
|
74
|
+
import { appendSessionCommandOutput } from "@opengeni/db/session-command-output";
|
|
75
|
+
import { captureConnectedCommandOutput } from "../sandbox-routing";
|
|
74
76
|
import { sandboxLeaseTelemetryKey } from "@opengeni/observability";
|
|
75
77
|
import {
|
|
76
78
|
// Normal drain teardown builds the client and resumes the envelope directly:
|
|
@@ -89,6 +91,10 @@ import {
|
|
|
89
91
|
isExecSessionLostBanner,
|
|
90
92
|
isProviderSandboxNotFoundError,
|
|
91
93
|
NatsControlRpc,
|
|
94
|
+
NatsOpStreamTransport,
|
|
95
|
+
OpStreamExecClient,
|
|
96
|
+
defaultSelfhostedRetryClock,
|
|
97
|
+
stripExecBanner,
|
|
92
98
|
parseExecBannerExitCode,
|
|
93
99
|
prepareProviderForTeardownAfterCapture,
|
|
94
100
|
providerWorkspaceCapturePolicy,
|
|
@@ -296,6 +302,7 @@ export type RetainedProcessProbeFn = (
|
|
|
296
302
|
lease: LeaseSnapshot,
|
|
297
303
|
process: SandboxRetainedProcess,
|
|
298
304
|
mode?: "observe" | "cancel",
|
|
305
|
+
captureOutput?: (result: unknown, chunkId: string) => Promise<void>,
|
|
299
306
|
) => Promise<RetainedProcessProbeResult>;
|
|
300
307
|
|
|
301
308
|
export type HistoricalModalSandboxLifecycleProbeFn = typeof inspectModalSandboxLifecycle;
|
|
@@ -1030,107 +1037,189 @@ async function adoptLegacyModalCheckpointReceipts(
|
|
|
1030
1037
|
/** Reconcile commands only against the immutable physical locator copied at
|
|
1031
1038
|
* adoption. A successor enrollment connection is intentionally irrelevant: no
|
|
1032
1039
|
* query or cancellation is ever rebound to it. */
|
|
1033
|
-
async function reconcileConnectedMachineBackgroundCommands(
|
|
1040
|
+
export async function reconcileConnectedMachineBackgroundCommands(
|
|
1034
1041
|
db: ActivityServices["db"],
|
|
1035
1042
|
settings: ActivityServices["settings"],
|
|
1036
1043
|
observability: ActivityServices["observability"],
|
|
1037
1044
|
bus: ActivityServices["bus"],
|
|
1045
|
+
controlRpcOverride?: ControlRpc,
|
|
1038
1046
|
): Promise<void> {
|
|
1039
|
-
const
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1047
|
+
const dueBefore = new Date();
|
|
1048
|
+
const deadline = Date.now() + settings.sandboxLeaseReaperPeriodMs;
|
|
1049
|
+
const offlineConnections = new Set<string>();
|
|
1050
|
+
const firstConnectionProbes = new Map<string, Promise<void>>();
|
|
1051
|
+
// Drain the fixed due frontier with the existing concurrency. Yield to the
|
|
1052
|
+
// remaining maintenance work each period; retries belong to a later sweep.
|
|
1053
|
+
do {
|
|
1054
|
+
const claimId = crypto.randomUUID();
|
|
1055
|
+
let claims: ConnectedMachineBackgroundCommandClaim[];
|
|
1056
|
+
try {
|
|
1057
|
+
claims = await claimConnectedMachineSessionBackgroundCommands(db, {
|
|
1058
|
+
claimId,
|
|
1059
|
+
dueBefore,
|
|
1060
|
+
limit: CONNECTED_COMMAND_RECONCILIATION_LIMIT,
|
|
1061
|
+
claimTtlMs: CONNECTED_COMMAND_RECONCILIATION_CLAIM_TTL_MS,
|
|
1062
|
+
});
|
|
1063
|
+
} catch (error) {
|
|
1064
|
+
recordConnectedCommandReconciliation(observability, "claim_failed");
|
|
1065
|
+
observability.warn("sandbox reaper: connected-command claim failed", {
|
|
1066
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1067
|
+
});
|
|
1068
|
+
return;
|
|
1069
|
+
}
|
|
1054
1070
|
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
);
|
|
1070
|
-
if (
|
|
1071
|
-
await deferConnectedCommandClaim(db, settings, observability, claim, "
|
|
1071
|
+
const controlRpc =
|
|
1072
|
+
controlRpcOverride ??
|
|
1073
|
+
new NatsControlRpc(
|
|
1074
|
+
async (): Promise<NatsRequestConnection | null> => bus.getRequestConnection(),
|
|
1075
|
+
);
|
|
1076
|
+
await forEachWithConcurrency(claims, SANDBOX_MAINTENANCE_ITEM_CONCURRENCY, async (claim) => {
|
|
1077
|
+
let proof = claim.proof;
|
|
1078
|
+
const connectionKey = JSON.stringify([
|
|
1079
|
+
claim.controlWorkspaceId,
|
|
1080
|
+
claim.enrollmentId,
|
|
1081
|
+
claim.connectionInstanceId,
|
|
1082
|
+
]);
|
|
1083
|
+
if (!proof) {
|
|
1084
|
+
const firstProbe = firstConnectionProbes.get(connectionKey);
|
|
1085
|
+
if (firstProbe) await firstProbe;
|
|
1086
|
+
if (offlineConnections.has(connectionKey)) {
|
|
1087
|
+
await deferConnectedCommandClaim(db, settings, observability, claim, "provider_offline");
|
|
1072
1088
|
return;
|
|
1073
1089
|
}
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
(
|
|
1078
|
-
|
|
1079
|
-
|
|
1090
|
+
// Share only the initial connection observation, never another command's result.
|
|
1091
|
+
let finishFirstProbe: (() => void) | undefined;
|
|
1092
|
+
if (!firstProbe) {
|
|
1093
|
+
firstConnectionProbes.set(
|
|
1094
|
+
connectionKey,
|
|
1095
|
+
new Promise<void>((resolve) => {
|
|
1096
|
+
finishFirstProbe = resolve;
|
|
1097
|
+
}),
|
|
1098
|
+
);
|
|
1099
|
+
}
|
|
1100
|
+
try {
|
|
1101
|
+
proof = await probeConnectedMachineBackgroundCommand(
|
|
1102
|
+
claim,
|
|
1103
|
+
controlRpc,
|
|
1104
|
+
Math.min(
|
|
1105
|
+
settings.sandboxSelfhostedControlTimeoutMs,
|
|
1106
|
+
RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS,
|
|
1107
|
+
),
|
|
1108
|
+
);
|
|
1109
|
+
// Replay currently retained frames for running as well as completed
|
|
1110
|
+
// commands. This is attach-only and leaves final ACK/observation alone.
|
|
1111
|
+
if (proof?.outcome !== "lost") {
|
|
1112
|
+
if (!bus.getOpStreamConnection)
|
|
1113
|
+
throw new Error("Connected command output transport unavailable");
|
|
1114
|
+
const client = new OpStreamExecClient({
|
|
1115
|
+
workspaceId: claim.controlWorkspaceId,
|
|
1116
|
+
agentId: claim.enrollmentId,
|
|
1117
|
+
connectionInstanceId: claim.connectionInstanceId,
|
|
1118
|
+
epoch: 0,
|
|
1119
|
+
controlRpc,
|
|
1120
|
+
rpcSubject: subjectFor(
|
|
1121
|
+
claim.controlWorkspaceId,
|
|
1122
|
+
claim.enrollmentId,
|
|
1123
|
+
claim.connectionInstanceId,
|
|
1124
|
+
),
|
|
1125
|
+
transport: new NatsOpStreamTransport(
|
|
1126
|
+
async () => bus.getOpStreamConnection?.() ?? null,
|
|
1127
|
+
),
|
|
1128
|
+
controlTimeoutMs: Math.min(
|
|
1129
|
+
settings.sandboxSelfhostedControlTimeoutMs,
|
|
1130
|
+
RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS,
|
|
1131
|
+
),
|
|
1132
|
+
retryClock: defaultSelfhostedRetryClock,
|
|
1133
|
+
journal: { attachGeneration: () => String(Date.now()), persistSettled: () => {} },
|
|
1134
|
+
});
|
|
1135
|
+
const replay = await client.readExisting(
|
|
1136
|
+
claim.opId,
|
|
1137
|
+
proof ? 5_000 : 250,
|
|
1138
|
+
async (frames) => {
|
|
1139
|
+
await captureConnectedCommandOutput(db, claim, bus)(claim.commandId, frames);
|
|
1140
|
+
},
|
|
1141
|
+
);
|
|
1142
|
+
if (proof && replay.status !== "completed" && !replay.terminal) {
|
|
1143
|
+
throw new Error(
|
|
1144
|
+
"Connected command terminal output replay has not reached its exit frontier",
|
|
1145
|
+
);
|
|
1146
|
+
}
|
|
1147
|
+
}
|
|
1148
|
+
if (!proof) {
|
|
1149
|
+
await deferConnectedCommandClaim(
|
|
1150
|
+
db,
|
|
1151
|
+
settings,
|
|
1152
|
+
observability,
|
|
1153
|
+
claim,
|
|
1154
|
+
"provider_running",
|
|
1155
|
+
);
|
|
1156
|
+
return;
|
|
1157
|
+
}
|
|
1158
|
+
} catch (error) {
|
|
1159
|
+
const offline =
|
|
1160
|
+
error instanceof SelfhostedControlError &&
|
|
1161
|
+
(error.agentOffline || error.reason === "agent_reconnecting" || error.draining);
|
|
1162
|
+
if (offline) offlineConnections.add(connectionKey);
|
|
1163
|
+
if (!offline) {
|
|
1164
|
+
observability.warn("sandbox reaper: connected-command provider probe failed", {
|
|
1165
|
+
commandId: claim.commandId,
|
|
1166
|
+
enrollmentId: claim.enrollmentId,
|
|
1167
|
+
connectionInstanceId: claim.connectionInstanceId,
|
|
1168
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1169
|
+
});
|
|
1170
|
+
}
|
|
1171
|
+
await deferConnectedCommandClaim(
|
|
1172
|
+
db,
|
|
1173
|
+
settings,
|
|
1174
|
+
observability,
|
|
1175
|
+
claim,
|
|
1176
|
+
offline ? "provider_offline" : "provider_error",
|
|
1177
|
+
);
|
|
1178
|
+
return;
|
|
1179
|
+
} finally {
|
|
1180
|
+
finishFirstProbe?.();
|
|
1181
|
+
}
|
|
1182
|
+
|
|
1183
|
+
try {
|
|
1184
|
+
await recordConnectedMachineBackgroundCommandProof(db, { claim, proof });
|
|
1185
|
+
recordConnectedCommandReconciliation(observability, `proof_${proof.outcome}`);
|
|
1186
|
+
} catch (error) {
|
|
1187
|
+
recordConnectedCommandReconciliation(observability, "proof_checkpoint_failed");
|
|
1188
|
+
observability.warn("sandbox reaper: connected-command proof checkpoint failed", {
|
|
1080
1189
|
commandId: claim.commandId,
|
|
1081
|
-
enrollmentId: claim.enrollmentId,
|
|
1082
|
-
connectionInstanceId: claim.connectionInstanceId,
|
|
1083
1190
|
error: error instanceof Error ? error.message : String(error),
|
|
1084
1191
|
});
|
|
1192
|
+
return;
|
|
1085
1193
|
}
|
|
1086
|
-
await deferConnectedCommandClaim(
|
|
1087
|
-
db,
|
|
1088
|
-
settings,
|
|
1089
|
-
observability,
|
|
1090
|
-
claim,
|
|
1091
|
-
offline ? "provider_offline" : "provider_error",
|
|
1092
|
-
);
|
|
1093
|
-
return;
|
|
1094
1194
|
}
|
|
1095
1195
|
|
|
1096
1196
|
try {
|
|
1097
|
-
await
|
|
1098
|
-
|
|
1197
|
+
const settlement = await settleClaimedConnectedMachineBackgroundCommand(db, { claim });
|
|
1198
|
+
if (!settlement.settled) {
|
|
1199
|
+
throw new Error("Connected command settlement lost its exact claim or proof");
|
|
1200
|
+
}
|
|
1201
|
+
if (settlement.events.length > 0) {
|
|
1202
|
+
try {
|
|
1203
|
+
await bus.publish(claim.workspaceId, claim.sessionId, settlement.events);
|
|
1204
|
+
} catch (error) {
|
|
1205
|
+
observability.warn("sandbox reaper: connected-command event fanout failed", {
|
|
1206
|
+
commandId: claim.commandId,
|
|
1207
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1208
|
+
});
|
|
1209
|
+
}
|
|
1210
|
+
}
|
|
1211
|
+
recordConnectedCommandReconciliation(observability, `settled_${proof.outcome}`);
|
|
1099
1212
|
} catch (error) {
|
|
1100
|
-
recordConnectedCommandReconciliation(observability, "
|
|
1101
|
-
observability.warn("sandbox reaper: connected-command
|
|
1213
|
+
recordConnectedCommandReconciliation(observability, "settlement_failed");
|
|
1214
|
+
observability.warn("sandbox reaper: connected-command settlement failed", {
|
|
1102
1215
|
commandId: claim.commandId,
|
|
1103
1216
|
error: error instanceof Error ? error.message : String(error),
|
|
1104
1217
|
});
|
|
1105
|
-
|
|
1218
|
+
await deferConnectedCommandClaim(db, settings, observability, claim, "settlement_failed");
|
|
1106
1219
|
}
|
|
1107
|
-
}
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
const settlement = await settleClaimedConnectedMachineBackgroundCommand(db, { claim });
|
|
1111
|
-
if (!settlement.settled) {
|
|
1112
|
-
throw new Error("Connected command settlement lost its exact claim or proof");
|
|
1113
|
-
}
|
|
1114
|
-
if (settlement.events.length > 0) {
|
|
1115
|
-
try {
|
|
1116
|
-
await bus.publish(claim.workspaceId, claim.sessionId, settlement.events);
|
|
1117
|
-
} catch (error) {
|
|
1118
|
-
observability.warn("sandbox reaper: connected-command event fanout failed", {
|
|
1119
|
-
commandId: claim.commandId,
|
|
1120
|
-
error: error instanceof Error ? error.message : String(error),
|
|
1121
|
-
});
|
|
1122
|
-
}
|
|
1123
|
-
}
|
|
1124
|
-
recordConnectedCommandReconciliation(observability, `settled_${proof.outcome}`);
|
|
1125
|
-
} catch (error) {
|
|
1126
|
-
recordConnectedCommandReconciliation(observability, "settlement_failed");
|
|
1127
|
-
observability.warn("sandbox reaper: connected-command settlement failed", {
|
|
1128
|
-
commandId: claim.commandId,
|
|
1129
|
-
error: error instanceof Error ? error.message : String(error),
|
|
1130
|
-
});
|
|
1131
|
-
await deferConnectedCommandClaim(db, settings, observability, claim, "settlement_failed");
|
|
1132
|
-
}
|
|
1133
|
-
});
|
|
1220
|
+
});
|
|
1221
|
+
if (claims.length < CONNECTED_COMMAND_RECONCILIATION_LIMIT) break;
|
|
1222
|
+
} while (Date.now() < deadline);
|
|
1134
1223
|
}
|
|
1135
1224
|
|
|
1136
1225
|
export async function probeConnectedMachineBackgroundCommand(
|
|
@@ -1168,17 +1257,22 @@ export function connectedCommandProofFromStatus(
|
|
|
1168
1257
|
throw new Error("Connected command completed without an exit record");
|
|
1169
1258
|
}
|
|
1170
1259
|
const failureCode = exit.failureCode.replace(/[^A-Za-z0-9_-]/g, "_").slice(0, 128);
|
|
1171
|
-
const reason =
|
|
1172
|
-
?
|
|
1173
|
-
: exit.
|
|
1174
|
-
? "
|
|
1175
|
-
:
|
|
1176
|
-
?
|
|
1260
|
+
const reason = failureCode
|
|
1261
|
+
? `op_failure_${failureCode}`
|
|
1262
|
+
: exit.cancelled
|
|
1263
|
+
? "op_cancelled"
|
|
1264
|
+
: exit.timedOut
|
|
1265
|
+
? "op_timed_out"
|
|
1177
1266
|
: "op_exit";
|
|
1178
1267
|
return {
|
|
1179
1268
|
outcome: "exited",
|
|
1180
1269
|
exitCode: exit.exitCode,
|
|
1181
1270
|
reason,
|
|
1271
|
+
...(failureCode
|
|
1272
|
+
? {
|
|
1273
|
+
failure: { code: failureCode, detail: exit.failureDetail, retryable: false as const },
|
|
1274
|
+
}
|
|
1275
|
+
: {}),
|
|
1182
1276
|
observedAt,
|
|
1183
1277
|
};
|
|
1184
1278
|
}
|
|
@@ -1373,6 +1467,27 @@ async function reconcileTerminalRetainedProcesses(
|
|
|
1373
1467
|
lease!,
|
|
1374
1468
|
process,
|
|
1375
1469
|
claim.ownerState === "background_stopping" ? "cancel" : "observe",
|
|
1470
|
+
async (result, chunkId) => {
|
|
1471
|
+
if (
|
|
1472
|
+
typeof result !== "string" ||
|
|
1473
|
+
isExecSessionLostBanner(result, process.providerSessionId)
|
|
1474
|
+
)
|
|
1475
|
+
return;
|
|
1476
|
+
const events = await appendSessionCommandOutput(db, {
|
|
1477
|
+
accountId: process.accountId,
|
|
1478
|
+
workspaceId: process.workspaceId,
|
|
1479
|
+
sessionId: process.sessionId,
|
|
1480
|
+
commandId: process.id,
|
|
1481
|
+
chunkId,
|
|
1482
|
+
stream: "stdout",
|
|
1483
|
+
streamFidelity: "merged",
|
|
1484
|
+
chunk: stripExecBanner(result),
|
|
1485
|
+
});
|
|
1486
|
+
if (events.length && bus)
|
|
1487
|
+
await bus
|
|
1488
|
+
.publish(process.workspaceId, process.sessionId, events)
|
|
1489
|
+
.catch(() => undefined);
|
|
1490
|
+
},
|
|
1376
1491
|
);
|
|
1377
1492
|
} catch (error) {
|
|
1378
1493
|
observability.warn("sandbox reaper: retained-process provider probe failed", {
|
|
@@ -1597,11 +1712,30 @@ async function withRetainedProcessProbeTimeout<T>(promise: Promise<T>): Promise<
|
|
|
1597
1712
|
}
|
|
1598
1713
|
}
|
|
1599
1714
|
|
|
1715
|
+
/** Consumed SDK output cannot be polled again. Retain the receipt across
|
|
1716
|
+
* persistence retries in this worker; a worker crash remains unreplayable. */
|
|
1717
|
+
const pendingRetainedProbeOutput = new Map<string, { result: unknown; chunkId: string }>();
|
|
1718
|
+
|
|
1719
|
+
export async function captureRetainedProbeOutput(
|
|
1720
|
+
processId: string,
|
|
1721
|
+
result: unknown,
|
|
1722
|
+
capture: (result: unknown, chunkId: string) => Promise<void>,
|
|
1723
|
+
): Promise<void> {
|
|
1724
|
+
const pending = pendingRetainedProbeOutput.get(processId) ?? {
|
|
1725
|
+
result,
|
|
1726
|
+
chunkId: crypto.randomUUID(),
|
|
1727
|
+
};
|
|
1728
|
+
pendingRetainedProbeOutput.set(processId, pending);
|
|
1729
|
+
await capture(pending.result, pending.chunkId);
|
|
1730
|
+
pendingRetainedProbeOutput.delete(processId);
|
|
1731
|
+
}
|
|
1732
|
+
|
|
1600
1733
|
export async function probeRetainedProcessAtProvider(
|
|
1601
1734
|
settings: ActivityServices["settings"],
|
|
1602
1735
|
lease: LeaseSnapshot,
|
|
1603
1736
|
process: SandboxRetainedProcess,
|
|
1604
1737
|
mode: "observe" | "cancel" = "observe",
|
|
1738
|
+
captureOutput?: (result: unknown, chunkId: string) => Promise<void>,
|
|
1605
1739
|
): Promise<RetainedProcessProbeResult> {
|
|
1606
1740
|
if (
|
|
1607
1741
|
lease.id !== process.leaseId ||
|
|
@@ -1617,6 +1751,12 @@ export async function probeRetainedProcessAtProvider(
|
|
|
1617
1751
|
if (!lease.resumeState) {
|
|
1618
1752
|
return { status: "deferred", reason: "resume_state_missing" };
|
|
1619
1753
|
}
|
|
1754
|
+
const pending = pendingRetainedProbeOutput.get(process.id);
|
|
1755
|
+
if (pending) {
|
|
1756
|
+
if (!captureOutput) throw new Error("Pending command output requires its persistence callback");
|
|
1757
|
+
await captureRetainedProbeOutput(process.id, pending.result, captureOutput);
|
|
1758
|
+
return classifyRetainedProcessPollResult(pending.result, process.providerSessionId);
|
|
1759
|
+
}
|
|
1620
1760
|
const envelopeBackend = (lease.resumeState as { backendId?: unknown }).backendId;
|
|
1621
1761
|
if (envelopeBackend !== undefined && envelopeBackend !== process.providerBackend) {
|
|
1622
1762
|
return { status: "deferred", reason: "identity_mismatch" };
|
|
@@ -1760,6 +1900,7 @@ export async function probeRetainedProcessAtProvider(
|
|
|
1760
1900
|
}
|
|
1761
1901
|
return { status: "deferred", reason: "provider_error" };
|
|
1762
1902
|
}
|
|
1903
|
+
if (captureOutput) await captureRetainedProbeOutput(process.id, result, captureOutput);
|
|
1763
1904
|
const observation = classifyRetainedProcessPollResult(result, process.providerSessionId);
|
|
1764
1905
|
if (
|
|
1765
1906
|
observation.status === "deferred" &&
|
|
@@ -1778,6 +1919,7 @@ export async function probeRetainedProcessAtProvider(
|
|
|
1778
1919
|
maxOutputTokens: 2_000,
|
|
1779
1920
|
}),
|
|
1780
1921
|
);
|
|
1922
|
+
if (captureOutput) await captureRetainedProbeOutput(process.id, interrupted, captureOutput);
|
|
1781
1923
|
return classifyRetainedProcessPollResult(interrupted, process.providerSessionId);
|
|
1782
1924
|
} catch (error) {
|
|
1783
1925
|
if (isProviderSandboxNotFoundError(client.backendId, error)) {
|
|
@@ -224,7 +224,9 @@ export function createSessionStateActivities(
|
|
|
224
224
|
: input.preClaimFailure?.disposition === "retryable" &&
|
|
225
225
|
input.preClaimFailure.code !== "claim_invariant"
|
|
226
226
|
? input.preClaimFailure.code
|
|
227
|
-
:
|
|
227
|
+
: input.preClaimFailureDisposition === "retryable"
|
|
228
|
+
? "legacy_retryable_preclaim_database_failure"
|
|
229
|
+
: null;
|
|
228
230
|
if (recoveredClaimCode) {
|
|
229
231
|
const recovery = await requestSessionTurnRecoveryFn(db, input.workspaceId, {
|
|
230
232
|
sessionId: input.sessionId,
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import { listXaiSubscriptionAccountsMetadata, updateXaiQuotaMetadata } from "@opengeni/db";
|
|
2
|
+
import { fetchXaiSubscriptionQuota } from "@opengeni/xai-subscription";
|
|
3
|
+
import { buildXaiTurnRequestAuthorization } from "./xai-auth";
|
|
4
|
+
|
|
5
|
+
/** Refresh billing, never inference, before treating cached exhaustion as current. */
|
|
6
|
+
export async function refreshExhaustedXaiQuota(
|
|
7
|
+
input: Omit<Parameters<typeof buildXaiTurnRequestAuthorization>[0], "credentialId">,
|
|
8
|
+
) {
|
|
9
|
+
const accounts = await listXaiSubscriptionAccountsMetadata(input.db, input);
|
|
10
|
+
for (const account of accounts) {
|
|
11
|
+
if (account.status !== "active" || !account.allocatorEnabled || !account.exhaustedUntil)
|
|
12
|
+
continue;
|
|
13
|
+
if (account.quotaCheckedAt && Date.now() - account.quotaCheckedAt.getTime() < 30_000) continue;
|
|
14
|
+
try {
|
|
15
|
+
const auth = await buildXaiTurnRequestAuthorization({ ...input, credentialId: account.id });
|
|
16
|
+
const quota = await fetchXaiSubscriptionQuota({ context: auth.context });
|
|
17
|
+
// Unknown billing is not evidence that a refusal has cleared.
|
|
18
|
+
if (quota.usedPercent === null) continue;
|
|
19
|
+
await updateXaiQuotaMetadata(input.db, {
|
|
20
|
+
workspaceId: input.workspaceId,
|
|
21
|
+
subjectId: input.subjectId,
|
|
22
|
+
credentialId: account.id,
|
|
23
|
+
quotaUsedPercent: quota.usedPercent,
|
|
24
|
+
quotaResetAt: quota.period?.end ?? null,
|
|
25
|
+
quotaCheckedAt: quota.checkedAt,
|
|
26
|
+
expectedExhaustedUntil: account.exhaustedUntil,
|
|
27
|
+
expectedQuotaCheckedAt: account.quotaCheckedAt,
|
|
28
|
+
exhaustedUntil:
|
|
29
|
+
quota.usedPercent < 100 ? null : (quota.period?.end ?? account.exhaustedUntil),
|
|
30
|
+
});
|
|
31
|
+
} catch {
|
|
32
|
+
// Preserve provider refusal on refresh failure; the durable timer retries.
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
}
|
package/src/sandbox-routing.ts
CHANGED
|
@@ -38,6 +38,9 @@ import {
|
|
|
38
38
|
type EnrollmentRecord,
|
|
39
39
|
type SandboxWorkspaceMutationAdmission,
|
|
40
40
|
} from "@opengeni/db";
|
|
41
|
+
import { observeSessionBackgroundCommandCompletion } from "@opengeni/db/session-background-commands";
|
|
42
|
+
import { appendSessionCommandOutput } from "@opengeni/db/session-command-output";
|
|
43
|
+
import type { OpStreamOutputFrame } from "@opengeni/runtime/sandbox";
|
|
41
44
|
import type { EventBus } from "@opengeni/events";
|
|
42
45
|
import {
|
|
43
46
|
buildSelfhostedBackendSession,
|
|
@@ -636,6 +639,76 @@ function afterRetainedProcessMutation(
|
|
|
636
639
|
};
|
|
637
640
|
}
|
|
638
641
|
|
|
642
|
+
export function captureConnectedCommandOutput(
|
|
643
|
+
db: Database,
|
|
644
|
+
ids: { accountId: string; workspaceId: string; sessionId: string },
|
|
645
|
+
bus?: EventBus,
|
|
646
|
+
) {
|
|
647
|
+
return async (commandId: string, frames: OpStreamOutputFrame[]) => {
|
|
648
|
+
for (const frame of frames) {
|
|
649
|
+
const events = await appendSessionCommandOutput(db, {
|
|
650
|
+
...ids,
|
|
651
|
+
commandId,
|
|
652
|
+
chunkId: `op-frame:${frame.sequence}`,
|
|
653
|
+
stream: frame.stream,
|
|
654
|
+
chunk: frame.chunk,
|
|
655
|
+
});
|
|
656
|
+
if (events.length && bus)
|
|
657
|
+
await bus.publish(ids.workspaceId, ids.sessionId, events).catch(() => undefined);
|
|
658
|
+
}
|
|
659
|
+
};
|
|
660
|
+
}
|
|
661
|
+
|
|
662
|
+
function captureRetainedProcessOutputForTurn(
|
|
663
|
+
services: RoutingWiringServices,
|
|
664
|
+
ids: RoutingWiringIds,
|
|
665
|
+
) {
|
|
666
|
+
const accountId = ids.workspaceMutationFence?.accountId;
|
|
667
|
+
if (!accountId) return undefined;
|
|
668
|
+
return async ({
|
|
669
|
+
process,
|
|
670
|
+
chunkId,
|
|
671
|
+
chunk,
|
|
672
|
+
stream,
|
|
673
|
+
streamFidelity,
|
|
674
|
+
}: {
|
|
675
|
+
process: RoutingRetainedProcess;
|
|
676
|
+
chunkId: string;
|
|
677
|
+
chunk: string;
|
|
678
|
+
stream: "stdout" | "stderr";
|
|
679
|
+
streamFidelity: "separate" | "merged";
|
|
680
|
+
}) => {
|
|
681
|
+
const events = await appendSessionCommandOutput(services.db, {
|
|
682
|
+
accountId,
|
|
683
|
+
workspaceId: ids.workspaceId,
|
|
684
|
+
sessionId: ids.sessionId,
|
|
685
|
+
commandId: process.id,
|
|
686
|
+
chunkId,
|
|
687
|
+
chunk,
|
|
688
|
+
stream,
|
|
689
|
+
streamFidelity,
|
|
690
|
+
});
|
|
691
|
+
if (events.length && services.bus)
|
|
692
|
+
await services.bus.publish(ids.workspaceId, ids.sessionId, events).catch(() => undefined);
|
|
693
|
+
};
|
|
694
|
+
}
|
|
695
|
+
|
|
696
|
+
function observeRetainedProcessTerminalForTurn(
|
|
697
|
+
services: RoutingWiringServices,
|
|
698
|
+
ids: RoutingWiringIds,
|
|
699
|
+
) {
|
|
700
|
+
const fence = ids.workspaceMutationFence;
|
|
701
|
+
if (!fence) return undefined;
|
|
702
|
+
return async ({ process }: { process: RoutingRetainedProcess }) => {
|
|
703
|
+
await observeSessionBackgroundCommandCompletion(services.db, {
|
|
704
|
+
accountId: fence.accountId,
|
|
705
|
+
workspaceId: ids.workspaceId,
|
|
706
|
+
sessionId: ids.sessionId,
|
|
707
|
+
commandId: process.id,
|
|
708
|
+
});
|
|
709
|
+
};
|
|
710
|
+
}
|
|
711
|
+
|
|
639
712
|
function settleRetainedProcessForTurn(
|
|
640
713
|
services: RoutingWiringServices,
|
|
641
714
|
ids: RoutingWiringIds,
|
|
@@ -840,6 +913,7 @@ export function wrapTurnBoxWithRouting(
|
|
|
840
913
|
outcome: command.outcome,
|
|
841
914
|
exitCode: command.exitCode,
|
|
842
915
|
reason: command.reason,
|
|
916
|
+
...(command.failure ? { failure: command.failure } : {}),
|
|
843
917
|
});
|
|
844
918
|
if (settlement && bus) {
|
|
845
919
|
await bus
|
|
@@ -847,6 +921,15 @@ export function wrapTurnBoxWithRouting(
|
|
|
847
921
|
.catch(() => undefined);
|
|
848
922
|
}
|
|
849
923
|
},
|
|
924
|
+
selfhostedCaptureBackgroundCommandOutput: captureConnectedCommandOutput(
|
|
925
|
+
db,
|
|
926
|
+
{
|
|
927
|
+
accountId: backgroundCommandAccountId,
|
|
928
|
+
workspaceId: ids.workspaceId,
|
|
929
|
+
sessionId: ids.sessionId,
|
|
930
|
+
},
|
|
931
|
+
bus,
|
|
932
|
+
),
|
|
850
933
|
}
|
|
851
934
|
: {}),
|
|
852
935
|
// The turn's declared environment → a selfhosted swap target's manifest, so the
|
|
@@ -931,6 +1014,12 @@ export function wrapTurnBoxWithRouting(
|
|
|
931
1014
|
...(beforeProcessMutation ? { beforeProcessMutation } : {}),
|
|
932
1015
|
...(afterProcessMutation ? { afterProcessMutation } : {}),
|
|
933
1016
|
...(settleProcess ? { settleProcess } : {}),
|
|
1017
|
+
...(ids.workspaceMutationFence
|
|
1018
|
+
? { observeProcessTerminal: observeRetainedProcessTerminalForTurn(services, ids)! }
|
|
1019
|
+
: {}),
|
|
1020
|
+
...(ids.workspaceMutationFence
|
|
1021
|
+
? { captureProcessOutput: captureRetainedProcessOutputForTurn(services, ids)! }
|
|
1022
|
+
: {}),
|
|
934
1023
|
...(adoptProcessAsBackgroundCommand ? { adoptProcessAsBackgroundCommand } : {}),
|
|
935
1024
|
...(ids.homeLease
|
|
936
1025
|
? {
|
|
@@ -1094,6 +1183,7 @@ export function wrapLazyTurnBoxWithRouting(
|
|
|
1094
1183
|
outcome: command.outcome,
|
|
1095
1184
|
exitCode: command.exitCode,
|
|
1096
1185
|
reason: command.reason,
|
|
1186
|
+
...(command.failure ? { failure: command.failure } : {}),
|
|
1097
1187
|
});
|
|
1098
1188
|
if (settlement && bus) {
|
|
1099
1189
|
await bus
|
|
@@ -1101,6 +1191,15 @@ export function wrapLazyTurnBoxWithRouting(
|
|
|
1101
1191
|
.catch(() => undefined);
|
|
1102
1192
|
}
|
|
1103
1193
|
},
|
|
1194
|
+
selfhostedCaptureBackgroundCommandOutput: captureConnectedCommandOutput(
|
|
1195
|
+
db,
|
|
1196
|
+
{
|
|
1197
|
+
accountId: backgroundCommandAccountId,
|
|
1198
|
+
workspaceId: ids.workspaceId,
|
|
1199
|
+
sessionId: ids.sessionId,
|
|
1200
|
+
},
|
|
1201
|
+
bus,
|
|
1202
|
+
),
|
|
1104
1203
|
}
|
|
1105
1204
|
: {}),
|
|
1106
1205
|
...(ids.environment !== undefined ? { environment: ids.environment } : {}),
|
|
@@ -1154,6 +1253,12 @@ export function wrapLazyTurnBoxWithRouting(
|
|
|
1154
1253
|
...(beforeProcessMutation ? { beforeProcessMutation } : {}),
|
|
1155
1254
|
...(afterProcessMutation ? { afterProcessMutation } : {}),
|
|
1156
1255
|
...(settleProcess ? { settleProcess } : {}),
|
|
1256
|
+
...(ids.workspaceMutationFence
|
|
1257
|
+
? { observeProcessTerminal: observeRetainedProcessTerminalForTurn(services, ids)! }
|
|
1258
|
+
: {}),
|
|
1259
|
+
...(ids.workspaceMutationFence
|
|
1260
|
+
? { captureProcessOutput: captureRetainedProcessOutputForTurn(services, ids)! }
|
|
1261
|
+
: {}),
|
|
1157
1262
|
...(adoptProcessAsBackgroundCommand ? { adoptProcessAsBackgroundCommand } : {}),
|
|
1158
1263
|
...(args.homeLeaseIdentity
|
|
1159
1264
|
? {
|
|
@@ -1432,6 +1537,15 @@ export async function establishSelfhostedTurnSession(
|
|
|
1432
1537
|
},
|
|
1433
1538
|
}
|
|
1434
1539
|
: {}),
|
|
1540
|
+
captureBackgroundCommandOutput: captureConnectedCommandOutput(
|
|
1541
|
+
db,
|
|
1542
|
+
{
|
|
1543
|
+
accountId: args.accountId,
|
|
1544
|
+
workspaceId: args.workspaceId,
|
|
1545
|
+
sessionId: args.sessionId,
|
|
1546
|
+
},
|
|
1547
|
+
bus,
|
|
1548
|
+
),
|
|
1435
1549
|
settleBackgroundCommand: async (command) => {
|
|
1436
1550
|
const settlement = await settleConnectedMachineSessionBackgroundCommand(db, {
|
|
1437
1551
|
accountId: args.accountId,
|
|
@@ -1445,6 +1559,7 @@ export async function establishSelfhostedTurnSession(
|
|
|
1445
1559
|
outcome: command.outcome,
|
|
1446
1560
|
exitCode: command.exitCode,
|
|
1447
1561
|
reason: command.reason,
|
|
1562
|
+
...(command.failure ? { failure: command.failure } : {}),
|
|
1448
1563
|
});
|
|
1449
1564
|
if (settlement && bus) {
|
|
1450
1565
|
await bus
|