@opengeni/worker-bundle 0.24.2-canary.0 → 0.26.0-canary.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn/agent-build.d.ts +3 -10
- package/dist/activities/agent-turn/claim.d.ts +7 -0
- package/dist/activities/agent-turn/credential-leases.d.ts +8 -0
- package/dist/activities/agent-turn/errors.d.ts +6 -5
- package/dist/activities/agent-turn/failure-settlement.d.ts +43 -1
- package/dist/activities/agent-turn/history-prefix.d.ts +8 -0
- package/dist/activities/agent-turn/history-sink.d.ts +6 -2
- package/dist/activities/agent-turn/quiescence.d.ts +13 -8
- package/dist/activities/agent-turn/tool-environment.d.ts +2 -13
- package/dist/activities/agent-turn/tool-policy.d.ts +4 -0
- package/dist/activities/agent-turn/turn-context.d.ts +9 -1
- package/dist/activities/codemode-dispatcher.d.ts +10 -1
- package/dist/activities/codex-capacity.d.ts +32 -3
- package/dist/activities/codex-fleet-shadow.d.ts +1 -1
- package/dist/activities/codex-image-generation.d.ts +1 -1
- package/dist/activities/codex-rotation.d.ts +8 -67
- package/dist/activities/goals.d.ts +1 -1
- package/dist/activities/google-drive-publication.d.ts +2 -13
- package/dist/activities/image-generation-operation.d.ts +13 -4
- package/dist/activities/sandbox-lease.d.ts +7 -2
- package/dist/activities/session-state.d.ts +4 -2
- package/dist/activities/types.d.ts +11 -1
- package/dist/activities/xai-quota.d.ts +3 -0
- package/dist/{activities-control-ZWH7J4KZ.js → activities-control-KS3ZPQVY.js} +233 -88
- package/dist/activities-control-KS3ZPQVY.js.map +1 -0
- package/dist/activities-control.d.ts +2 -0
- package/dist/{activities-turn-I4OPLQLS.js → activities-turn-GHSFZB64.js} +1109 -1865
- package/dist/activities-turn-GHSFZB64.js.map +1 -0
- package/dist/activities.d.ts +2 -0
- package/dist/{chunk-65JEWP2Q.js → chunk-D4ZEFOYH.js} +1 -1
- package/dist/chunk-D4ZEFOYH.js.map +1 -0
- package/dist/{chunk-RQNH6CGR.js → chunk-O2LQWJIK.js} +3250 -2061
- package/dist/chunk-O2LQWJIK.js.map +1 -0
- package/dist/github-rest-mcp.d.ts +0 -2
- package/dist/index.d.ts +1 -0
- package/dist/index.js +2 -2
- package/dist/index.js.map +1 -1
- package/dist/observability-metrics.d.ts +1 -1
- package/dist/sandbox-routing.d.ts +6 -0
- package/dist/workflow-bundle.js +94 -9
- package/dist/workflows/activities.d.ts +1 -1
- package/dist/workflows/session.d.ts +3 -0
- package/package.json +17 -17
- package/src/activities/agent-turn/agent-build.ts +21 -26
- package/src/activities/agent-turn/claim.ts +26 -1
- package/src/activities/agent-turn/codex-capacity.ts +311 -234
- package/src/activities/agent-turn/credential-leases.ts +36 -0
- package/src/activities/agent-turn/errors.ts +73 -12
- package/src/activities/agent-turn/failure-settlement.ts +472 -331
- package/src/activities/agent-turn/history-prefix.ts +51 -0
- package/src/activities/agent-turn/history-sink.ts +30 -6
- package/src/activities/agent-turn/quiescence.ts +29 -17
- package/src/activities/agent-turn/run.ts +22 -5
- package/src/activities/agent-turn/stream-attempt.ts +35 -8
- package/src/activities/agent-turn/tool-environment.ts +42 -32
- package/src/activities/agent-turn/tool-policy.ts +14 -8
- package/src/activities/agent-turn/turn-context.ts +13 -0
- package/src/activities/agent-turn/xai-capacity.ts +11 -0
- package/src/activities/codemode-dispatcher.ts +235 -59
- package/src/activities/codex-capacity.ts +107 -10
- package/src/activities/codex-fleet-shadow.ts +8 -1
- package/src/activities/codex-image-generation.ts +19 -2
- package/src/activities/codex-rotation.ts +45 -142
- package/src/activities/goals.ts +9 -9
- package/src/activities/google-drive-publication.ts +4 -81
- package/src/activities/image-generation-operation.ts +28 -3
- package/src/activities/run-input.ts +6 -7
- package/src/activities/sandbox-lease.ts +246 -87
- package/src/activities/session-state.ts +20 -1
- package/src/activities/types.ts +16 -8
- package/src/activities/xai-quota.ts +35 -0
- package/src/activities.ts +1 -0
- package/src/github-rest-mcp.ts +0 -82
- package/src/observability-metrics.ts +4 -0
- package/src/sandbox-routing.ts +186 -16
- package/src/workflows/activities.ts +1 -0
- package/src/workflows/session.ts +114 -8
- package/dist/activities/api-integrations.d.ts +0 -21
- package/dist/activities-control-ZWH7J4KZ.js.map +0 -1
- package/dist/activities-turn-I4OPLQLS.js.map +0 -1
- package/dist/chunk-65JEWP2Q.js.map +0 -1
- package/dist/chunk-RQNH6CGR.js.map +0 -1
- package/src/activities/api-integrations.ts +0 -151
|
@@ -53,6 +53,7 @@ import {
|
|
|
53
53
|
workspaceArchiveCaptureDeadlineElapsed,
|
|
54
54
|
retainedProcessReconciliationProof,
|
|
55
55
|
retainedProcessSettlementIdentity,
|
|
56
|
+
settleClaimedConnectedMachineBackgroundCommand,
|
|
56
57
|
settleSandboxCheckpointArtifactGc,
|
|
57
58
|
rlsContextForWorkspace,
|
|
58
59
|
settleRetainedProcess,
|
|
@@ -66,12 +67,12 @@ import {
|
|
|
66
67
|
claimConnectedMachineSessionBackgroundCommands,
|
|
67
68
|
deferConnectedMachineBackgroundCommandReconciliation,
|
|
68
69
|
recordConnectedMachineBackgroundCommandProof,
|
|
69
|
-
settleClaimedConnectedMachineBackgroundCommand,
|
|
70
|
-
settleSessionBackgroundCommandForRetainedProcess,
|
|
71
70
|
type ConnectedMachineBackgroundCommandClaim,
|
|
72
71
|
type ConnectedMachineBackgroundCommandProof,
|
|
73
72
|
} from "@opengeni/db/session-background-commands";
|
|
74
73
|
import { sandboxWarmRateMicrosPerSecond } from "@opengeni/config";
|
|
74
|
+
import { appendSessionCommandOutput } from "@opengeni/db/session-command-output";
|
|
75
|
+
import { captureConnectedCommandOutput } from "../sandbox-routing";
|
|
75
76
|
import { sandboxLeaseTelemetryKey } from "@opengeni/observability";
|
|
76
77
|
import {
|
|
77
78
|
// Normal drain teardown builds the client and resumes the envelope directly:
|
|
@@ -90,6 +91,10 @@ import {
|
|
|
90
91
|
isExecSessionLostBanner,
|
|
91
92
|
isProviderSandboxNotFoundError,
|
|
92
93
|
NatsControlRpc,
|
|
94
|
+
NatsOpStreamTransport,
|
|
95
|
+
OpStreamExecClient,
|
|
96
|
+
defaultSelfhostedRetryClock,
|
|
97
|
+
stripExecBanner,
|
|
93
98
|
parseExecBannerExitCode,
|
|
94
99
|
prepareProviderForTeardownAfterCapture,
|
|
95
100
|
providerWorkspaceCapturePolicy,
|
|
@@ -297,6 +302,7 @@ export type RetainedProcessProbeFn = (
|
|
|
297
302
|
lease: LeaseSnapshot,
|
|
298
303
|
process: SandboxRetainedProcess,
|
|
299
304
|
mode?: "observe" | "cancel",
|
|
305
|
+
captureOutput?: (result: unknown, chunkId: string) => Promise<void>,
|
|
300
306
|
) => Promise<RetainedProcessProbeResult>;
|
|
301
307
|
|
|
302
308
|
export type HistoricalModalSandboxLifecycleProbeFn = typeof inspectModalSandboxLifecycle;
|
|
@@ -727,6 +733,7 @@ export function createSandboxLeaseActivities(
|
|
|
727
733
|
observability,
|
|
728
734
|
probeRetainedProcess,
|
|
729
735
|
options.inspectHistoricalModalSandbox ?? inspectModalSandboxLifecycle,
|
|
736
|
+
service.bus,
|
|
730
737
|
);
|
|
731
738
|
|
|
732
739
|
try {
|
|
@@ -1030,96 +1037,189 @@ async function adoptLegacyModalCheckpointReceipts(
|
|
|
1030
1037
|
/** Reconcile commands only against the immutable physical locator copied at
|
|
1031
1038
|
* adoption. A successor enrollment connection is intentionally irrelevant: no
|
|
1032
1039
|
* query or cancellation is ever rebound to it. */
|
|
1033
|
-
async function reconcileConnectedMachineBackgroundCommands(
|
|
1040
|
+
export async function reconcileConnectedMachineBackgroundCommands(
|
|
1034
1041
|
db: ActivityServices["db"],
|
|
1035
1042
|
settings: ActivityServices["settings"],
|
|
1036
1043
|
observability: ActivityServices["observability"],
|
|
1037
1044
|
bus: ActivityServices["bus"],
|
|
1045
|
+
controlRpcOverride?: ControlRpc,
|
|
1038
1046
|
): Promise<void> {
|
|
1039
|
-
const
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1047
|
+
const dueBefore = new Date();
|
|
1048
|
+
const deadline = Date.now() + settings.sandboxLeaseReaperPeriodMs;
|
|
1049
|
+
const offlineConnections = new Set<string>();
|
|
1050
|
+
const firstConnectionProbes = new Map<string, Promise<void>>();
|
|
1051
|
+
// Drain the fixed due frontier with the existing concurrency. Yield to the
|
|
1052
|
+
// remaining maintenance work each period; retries belong to a later sweep.
|
|
1053
|
+
do {
|
|
1054
|
+
const claimId = crypto.randomUUID();
|
|
1055
|
+
let claims: ConnectedMachineBackgroundCommandClaim[];
|
|
1056
|
+
try {
|
|
1057
|
+
claims = await claimConnectedMachineSessionBackgroundCommands(db, {
|
|
1058
|
+
claimId,
|
|
1059
|
+
dueBefore,
|
|
1060
|
+
limit: CONNECTED_COMMAND_RECONCILIATION_LIMIT,
|
|
1061
|
+
claimTtlMs: CONNECTED_COMMAND_RECONCILIATION_CLAIM_TTL_MS,
|
|
1062
|
+
});
|
|
1063
|
+
} catch (error) {
|
|
1064
|
+
recordConnectedCommandReconciliation(observability, "claim_failed");
|
|
1065
|
+
observability.warn("sandbox reaper: connected-command claim failed", {
|
|
1066
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1067
|
+
});
|
|
1068
|
+
return;
|
|
1069
|
+
}
|
|
1054
1070
|
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
);
|
|
1070
|
-
if (
|
|
1071
|
-
await deferConnectedCommandClaim(db, settings, observability, claim, "
|
|
1071
|
+
const controlRpc =
|
|
1072
|
+
controlRpcOverride ??
|
|
1073
|
+
new NatsControlRpc(
|
|
1074
|
+
async (): Promise<NatsRequestConnection | null> => bus.getRequestConnection(),
|
|
1075
|
+
);
|
|
1076
|
+
await forEachWithConcurrency(claims, SANDBOX_MAINTENANCE_ITEM_CONCURRENCY, async (claim) => {
|
|
1077
|
+
let proof = claim.proof;
|
|
1078
|
+
const connectionKey = JSON.stringify([
|
|
1079
|
+
claim.controlWorkspaceId,
|
|
1080
|
+
claim.enrollmentId,
|
|
1081
|
+
claim.connectionInstanceId,
|
|
1082
|
+
]);
|
|
1083
|
+
if (!proof) {
|
|
1084
|
+
const firstProbe = firstConnectionProbes.get(connectionKey);
|
|
1085
|
+
if (firstProbe) await firstProbe;
|
|
1086
|
+
if (offlineConnections.has(connectionKey)) {
|
|
1087
|
+
await deferConnectedCommandClaim(db, settings, observability, claim, "provider_offline");
|
|
1072
1088
|
return;
|
|
1073
1089
|
}
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
(
|
|
1078
|
-
|
|
1079
|
-
|
|
1090
|
+
// Share only the initial connection observation, never another command's result.
|
|
1091
|
+
let finishFirstProbe: (() => void) | undefined;
|
|
1092
|
+
if (!firstProbe) {
|
|
1093
|
+
firstConnectionProbes.set(
|
|
1094
|
+
connectionKey,
|
|
1095
|
+
new Promise<void>((resolve) => {
|
|
1096
|
+
finishFirstProbe = resolve;
|
|
1097
|
+
}),
|
|
1098
|
+
);
|
|
1099
|
+
}
|
|
1100
|
+
try {
|
|
1101
|
+
proof = await probeConnectedMachineBackgroundCommand(
|
|
1102
|
+
claim,
|
|
1103
|
+
controlRpc,
|
|
1104
|
+
Math.min(
|
|
1105
|
+
settings.sandboxSelfhostedControlTimeoutMs,
|
|
1106
|
+
RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS,
|
|
1107
|
+
),
|
|
1108
|
+
);
|
|
1109
|
+
// Replay currently retained frames for running as well as completed
|
|
1110
|
+
// commands. This is attach-only and leaves final ACK/observation alone.
|
|
1111
|
+
if (proof?.outcome !== "lost") {
|
|
1112
|
+
if (!bus.getOpStreamConnection)
|
|
1113
|
+
throw new Error("Connected command output transport unavailable");
|
|
1114
|
+
const client = new OpStreamExecClient({
|
|
1115
|
+
workspaceId: claim.controlWorkspaceId,
|
|
1116
|
+
agentId: claim.enrollmentId,
|
|
1117
|
+
connectionInstanceId: claim.connectionInstanceId,
|
|
1118
|
+
epoch: 0,
|
|
1119
|
+
controlRpc,
|
|
1120
|
+
rpcSubject: subjectFor(
|
|
1121
|
+
claim.controlWorkspaceId,
|
|
1122
|
+
claim.enrollmentId,
|
|
1123
|
+
claim.connectionInstanceId,
|
|
1124
|
+
),
|
|
1125
|
+
transport: new NatsOpStreamTransport(
|
|
1126
|
+
async () => bus.getOpStreamConnection?.() ?? null,
|
|
1127
|
+
),
|
|
1128
|
+
controlTimeoutMs: Math.min(
|
|
1129
|
+
settings.sandboxSelfhostedControlTimeoutMs,
|
|
1130
|
+
RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS,
|
|
1131
|
+
),
|
|
1132
|
+
retryClock: defaultSelfhostedRetryClock,
|
|
1133
|
+
journal: { attachGeneration: () => String(Date.now()), persistSettled: () => {} },
|
|
1134
|
+
});
|
|
1135
|
+
const replay = await client.readExisting(
|
|
1136
|
+
claim.opId,
|
|
1137
|
+
proof ? 5_000 : 250,
|
|
1138
|
+
async (frames) => {
|
|
1139
|
+
await captureConnectedCommandOutput(db, claim, bus)(claim.commandId, frames);
|
|
1140
|
+
},
|
|
1141
|
+
);
|
|
1142
|
+
if (proof && replay.status !== "completed" && !replay.terminal) {
|
|
1143
|
+
throw new Error(
|
|
1144
|
+
"Connected command terminal output replay has not reached its exit frontier",
|
|
1145
|
+
);
|
|
1146
|
+
}
|
|
1147
|
+
}
|
|
1148
|
+
if (!proof) {
|
|
1149
|
+
await deferConnectedCommandClaim(
|
|
1150
|
+
db,
|
|
1151
|
+
settings,
|
|
1152
|
+
observability,
|
|
1153
|
+
claim,
|
|
1154
|
+
"provider_running",
|
|
1155
|
+
);
|
|
1156
|
+
return;
|
|
1157
|
+
}
|
|
1158
|
+
} catch (error) {
|
|
1159
|
+
const offline =
|
|
1160
|
+
error instanceof SelfhostedControlError &&
|
|
1161
|
+
(error.agentOffline || error.reason === "agent_reconnecting" || error.draining);
|
|
1162
|
+
if (offline) offlineConnections.add(connectionKey);
|
|
1163
|
+
if (!offline) {
|
|
1164
|
+
observability.warn("sandbox reaper: connected-command provider probe failed", {
|
|
1165
|
+
commandId: claim.commandId,
|
|
1166
|
+
enrollmentId: claim.enrollmentId,
|
|
1167
|
+
connectionInstanceId: claim.connectionInstanceId,
|
|
1168
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1169
|
+
});
|
|
1170
|
+
}
|
|
1171
|
+
await deferConnectedCommandClaim(
|
|
1172
|
+
db,
|
|
1173
|
+
settings,
|
|
1174
|
+
observability,
|
|
1175
|
+
claim,
|
|
1176
|
+
offline ? "provider_offline" : "provider_error",
|
|
1177
|
+
);
|
|
1178
|
+
return;
|
|
1179
|
+
} finally {
|
|
1180
|
+
finishFirstProbe?.();
|
|
1181
|
+
}
|
|
1182
|
+
|
|
1183
|
+
try {
|
|
1184
|
+
await recordConnectedMachineBackgroundCommandProof(db, { claim, proof });
|
|
1185
|
+
recordConnectedCommandReconciliation(observability, `proof_${proof.outcome}`);
|
|
1186
|
+
} catch (error) {
|
|
1187
|
+
recordConnectedCommandReconciliation(observability, "proof_checkpoint_failed");
|
|
1188
|
+
observability.warn("sandbox reaper: connected-command proof checkpoint failed", {
|
|
1080
1189
|
commandId: claim.commandId,
|
|
1081
|
-
enrollmentId: claim.enrollmentId,
|
|
1082
|
-
connectionInstanceId: claim.connectionInstanceId,
|
|
1083
1190
|
error: error instanceof Error ? error.message : String(error),
|
|
1084
1191
|
});
|
|
1192
|
+
return;
|
|
1085
1193
|
}
|
|
1086
|
-
await deferConnectedCommandClaim(
|
|
1087
|
-
db,
|
|
1088
|
-
settings,
|
|
1089
|
-
observability,
|
|
1090
|
-
claim,
|
|
1091
|
-
offline ? "provider_offline" : "provider_error",
|
|
1092
|
-
);
|
|
1093
|
-
return;
|
|
1094
1194
|
}
|
|
1095
1195
|
|
|
1096
1196
|
try {
|
|
1097
|
-
await
|
|
1098
|
-
|
|
1197
|
+
const settlement = await settleClaimedConnectedMachineBackgroundCommand(db, { claim });
|
|
1198
|
+
if (!settlement.settled) {
|
|
1199
|
+
throw new Error("Connected command settlement lost its exact claim or proof");
|
|
1200
|
+
}
|
|
1201
|
+
if (settlement.events.length > 0) {
|
|
1202
|
+
try {
|
|
1203
|
+
await bus.publish(claim.workspaceId, claim.sessionId, settlement.events);
|
|
1204
|
+
} catch (error) {
|
|
1205
|
+
observability.warn("sandbox reaper: connected-command event fanout failed", {
|
|
1206
|
+
commandId: claim.commandId,
|
|
1207
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1208
|
+
});
|
|
1209
|
+
}
|
|
1210
|
+
}
|
|
1211
|
+
recordConnectedCommandReconciliation(observability, `settled_${proof.outcome}`);
|
|
1099
1212
|
} catch (error) {
|
|
1100
|
-
recordConnectedCommandReconciliation(observability, "
|
|
1101
|
-
observability.warn("sandbox reaper: connected-command
|
|
1213
|
+
recordConnectedCommandReconciliation(observability, "settlement_failed");
|
|
1214
|
+
observability.warn("sandbox reaper: connected-command settlement failed", {
|
|
1102
1215
|
commandId: claim.commandId,
|
|
1103
1216
|
error: error instanceof Error ? error.message : String(error),
|
|
1104
1217
|
});
|
|
1105
|
-
|
|
1106
|
-
}
|
|
1107
|
-
}
|
|
1108
|
-
|
|
1109
|
-
try {
|
|
1110
|
-
if (!(await settleClaimedConnectedMachineBackgroundCommand(db, { claim }))) {
|
|
1111
|
-
throw new Error("Connected command settlement lost its exact claim or proof");
|
|
1218
|
+
await deferConnectedCommandClaim(db, settings, observability, claim, "settlement_failed");
|
|
1112
1219
|
}
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
observability.warn("sandbox reaper: connected-command settlement failed", {
|
|
1117
|
-
commandId: claim.commandId,
|
|
1118
|
-
error: error instanceof Error ? error.message : String(error),
|
|
1119
|
-
});
|
|
1120
|
-
await deferConnectedCommandClaim(db, settings, observability, claim, "settlement_failed");
|
|
1121
|
-
}
|
|
1122
|
-
});
|
|
1220
|
+
});
|
|
1221
|
+
if (claims.length < CONNECTED_COMMAND_RECONCILIATION_LIMIT) break;
|
|
1222
|
+
} while (Date.now() < deadline);
|
|
1123
1223
|
}
|
|
1124
1224
|
|
|
1125
1225
|
export async function probeConnectedMachineBackgroundCommand(
|
|
@@ -1157,17 +1257,22 @@ export function connectedCommandProofFromStatus(
|
|
|
1157
1257
|
throw new Error("Connected command completed without an exit record");
|
|
1158
1258
|
}
|
|
1159
1259
|
const failureCode = exit.failureCode.replace(/[^A-Za-z0-9_-]/g, "_").slice(0, 128);
|
|
1160
|
-
const reason =
|
|
1161
|
-
?
|
|
1162
|
-
: exit.
|
|
1163
|
-
? "
|
|
1164
|
-
:
|
|
1165
|
-
?
|
|
1260
|
+
const reason = failureCode
|
|
1261
|
+
? `op_failure_${failureCode}`
|
|
1262
|
+
: exit.cancelled
|
|
1263
|
+
? "op_cancelled"
|
|
1264
|
+
: exit.timedOut
|
|
1265
|
+
? "op_timed_out"
|
|
1166
1266
|
: "op_exit";
|
|
1167
1267
|
return {
|
|
1168
1268
|
outcome: "exited",
|
|
1169
1269
|
exitCode: exit.exitCode,
|
|
1170
1270
|
reason,
|
|
1271
|
+
...(failureCode
|
|
1272
|
+
? {
|
|
1273
|
+
failure: { code: failureCode, detail: exit.failureDetail, retryable: false as const },
|
|
1274
|
+
}
|
|
1275
|
+
: {}),
|
|
1171
1276
|
observedAt,
|
|
1172
1277
|
};
|
|
1173
1278
|
}
|
|
@@ -1226,6 +1331,7 @@ async function reconcileTerminalRetainedProcesses(
|
|
|
1226
1331
|
observability: ActivityServices["observability"],
|
|
1227
1332
|
probe: RetainedProcessProbeFn,
|
|
1228
1333
|
inspectHistoricalModalSandbox: HistoricalModalSandboxLifecycleProbeFn,
|
|
1334
|
+
bus: ActivityServices["bus"],
|
|
1229
1335
|
): Promise<void> {
|
|
1230
1336
|
const claimId = crypto.randomUUID();
|
|
1231
1337
|
let claims: Awaited<ReturnType<typeof claimTerminalRetainedProcesses>>;
|
|
@@ -1361,6 +1467,27 @@ async function reconcileTerminalRetainedProcesses(
|
|
|
1361
1467
|
lease!,
|
|
1362
1468
|
process,
|
|
1363
1469
|
claim.ownerState === "background_stopping" ? "cancel" : "observe",
|
|
1470
|
+
async (result, chunkId) => {
|
|
1471
|
+
if (
|
|
1472
|
+
typeof result !== "string" ||
|
|
1473
|
+
isExecSessionLostBanner(result, process.providerSessionId)
|
|
1474
|
+
)
|
|
1475
|
+
return;
|
|
1476
|
+
const events = await appendSessionCommandOutput(db, {
|
|
1477
|
+
accountId: process.accountId,
|
|
1478
|
+
workspaceId: process.workspaceId,
|
|
1479
|
+
sessionId: process.sessionId,
|
|
1480
|
+
commandId: process.id,
|
|
1481
|
+
chunkId,
|
|
1482
|
+
stream: "stdout",
|
|
1483
|
+
streamFidelity: "merged",
|
|
1484
|
+
chunk: stripExecBanner(result),
|
|
1485
|
+
});
|
|
1486
|
+
if (events.length && bus)
|
|
1487
|
+
await bus
|
|
1488
|
+
.publish(process.workspaceId, process.sessionId, events)
|
|
1489
|
+
.catch(() => undefined);
|
|
1490
|
+
},
|
|
1364
1491
|
);
|
|
1365
1492
|
} catch (error) {
|
|
1366
1493
|
observability.warn("sandbox reaper: retained-process provider probe failed", {
|
|
@@ -1444,15 +1571,20 @@ async function reconcileTerminalRetainedProcesses(
|
|
|
1444
1571
|
if (settlement.process.state === "active") {
|
|
1445
1572
|
throw new Error("Retained-process reconciliation returned an active durable process");
|
|
1446
1573
|
}
|
|
1447
|
-
|
|
1448
|
-
|
|
1449
|
-
|
|
1450
|
-
|
|
1451
|
-
|
|
1452
|
-
|
|
1453
|
-
|
|
1454
|
-
|
|
1455
|
-
|
|
1574
|
+
if (settlement.backgroundCommandEvents.length > 0) {
|
|
1575
|
+
try {
|
|
1576
|
+
await bus.publish(
|
|
1577
|
+
process.workspaceId,
|
|
1578
|
+
process.sessionId,
|
|
1579
|
+
settlement.backgroundCommandEvents,
|
|
1580
|
+
);
|
|
1581
|
+
} catch (error) {
|
|
1582
|
+
observability.warn("sandbox reaper: retained-command event fanout failed", {
|
|
1583
|
+
processId: process.id,
|
|
1584
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1585
|
+
});
|
|
1586
|
+
}
|
|
1587
|
+
}
|
|
1456
1588
|
recordRetainedProcessReconciliation(observability, `settled_${proof.outcome}`);
|
|
1457
1589
|
} catch (error) {
|
|
1458
1590
|
recordRetainedProcessReconciliation(observability, "settlement_failed");
|
|
@@ -1580,11 +1712,30 @@ async function withRetainedProcessProbeTimeout<T>(promise: Promise<T>): Promise<
|
|
|
1580
1712
|
}
|
|
1581
1713
|
}
|
|
1582
1714
|
|
|
1715
|
+
/** Consumed SDK output cannot be polled again. Retain the receipt across
|
|
1716
|
+
* persistence retries in this worker; a worker crash remains unreplayable. */
|
|
1717
|
+
const pendingRetainedProbeOutput = new Map<string, { result: unknown; chunkId: string }>();
|
|
1718
|
+
|
|
1719
|
+
export async function captureRetainedProbeOutput(
|
|
1720
|
+
processId: string,
|
|
1721
|
+
result: unknown,
|
|
1722
|
+
capture: (result: unknown, chunkId: string) => Promise<void>,
|
|
1723
|
+
): Promise<void> {
|
|
1724
|
+
const pending = pendingRetainedProbeOutput.get(processId) ?? {
|
|
1725
|
+
result,
|
|
1726
|
+
chunkId: crypto.randomUUID(),
|
|
1727
|
+
};
|
|
1728
|
+
pendingRetainedProbeOutput.set(processId, pending);
|
|
1729
|
+
await capture(pending.result, pending.chunkId);
|
|
1730
|
+
pendingRetainedProbeOutput.delete(processId);
|
|
1731
|
+
}
|
|
1732
|
+
|
|
1583
1733
|
export async function probeRetainedProcessAtProvider(
|
|
1584
1734
|
settings: ActivityServices["settings"],
|
|
1585
1735
|
lease: LeaseSnapshot,
|
|
1586
1736
|
process: SandboxRetainedProcess,
|
|
1587
1737
|
mode: "observe" | "cancel" = "observe",
|
|
1738
|
+
captureOutput?: (result: unknown, chunkId: string) => Promise<void>,
|
|
1588
1739
|
): Promise<RetainedProcessProbeResult> {
|
|
1589
1740
|
if (
|
|
1590
1741
|
lease.id !== process.leaseId ||
|
|
@@ -1600,6 +1751,12 @@ export async function probeRetainedProcessAtProvider(
|
|
|
1600
1751
|
if (!lease.resumeState) {
|
|
1601
1752
|
return { status: "deferred", reason: "resume_state_missing" };
|
|
1602
1753
|
}
|
|
1754
|
+
const pending = pendingRetainedProbeOutput.get(process.id);
|
|
1755
|
+
if (pending) {
|
|
1756
|
+
if (!captureOutput) throw new Error("Pending command output requires its persistence callback");
|
|
1757
|
+
await captureRetainedProbeOutput(process.id, pending.result, captureOutput);
|
|
1758
|
+
return classifyRetainedProcessPollResult(pending.result, process.providerSessionId);
|
|
1759
|
+
}
|
|
1603
1760
|
const envelopeBackend = (lease.resumeState as { backendId?: unknown }).backendId;
|
|
1604
1761
|
if (envelopeBackend !== undefined && envelopeBackend !== process.providerBackend) {
|
|
1605
1762
|
return { status: "deferred", reason: "identity_mismatch" };
|
|
@@ -1743,6 +1900,7 @@ export async function probeRetainedProcessAtProvider(
|
|
|
1743
1900
|
}
|
|
1744
1901
|
return { status: "deferred", reason: "provider_error" };
|
|
1745
1902
|
}
|
|
1903
|
+
if (captureOutput) await captureRetainedProbeOutput(process.id, result, captureOutput);
|
|
1746
1904
|
const observation = classifyRetainedProcessPollResult(result, process.providerSessionId);
|
|
1747
1905
|
if (
|
|
1748
1906
|
observation.status === "deferred" &&
|
|
@@ -1761,6 +1919,7 @@ export async function probeRetainedProcessAtProvider(
|
|
|
1761
1919
|
maxOutputTokens: 2_000,
|
|
1762
1920
|
}),
|
|
1763
1921
|
);
|
|
1922
|
+
if (captureOutput) await captureRetainedProbeOutput(process.id, interrupted, captureOutput);
|
|
1764
1923
|
return classifyRetainedProcessPollResult(interrupted, process.providerSessionId);
|
|
1765
1924
|
} catch (error) {
|
|
1766
1925
|
if (isProviderSandboxNotFoundError(client.backendId, error)) {
|
|
@@ -7,6 +7,7 @@ import {
|
|
|
7
7
|
recoverSessionDispatch,
|
|
8
8
|
reconcileSessionAttemptQuiescence,
|
|
9
9
|
peekSessionWork as peekSessionWorkDb,
|
|
10
|
+
settleSessionInputWait as settleSessionInputWaitDb,
|
|
10
11
|
countQueuedTurns,
|
|
11
12
|
getSessionAttemptActivityRef,
|
|
12
13
|
getSessionEvent,
|
|
@@ -42,6 +43,8 @@ import type {
|
|
|
42
43
|
RecoverDispatchResult,
|
|
43
44
|
RecoverEscapedMcpTimeoutInput,
|
|
44
45
|
RecoverEscapedMcpTimeoutResult,
|
|
46
|
+
SettleSessionInputWaitInput,
|
|
47
|
+
SettleSessionInputWaitResult,
|
|
45
48
|
} from "./types";
|
|
46
49
|
|
|
47
50
|
export type SessionStateActivityOverrides = Partial<{
|
|
@@ -53,6 +56,7 @@ export type SessionStateActivityOverrides = Partial<{
|
|
|
53
56
|
recoverSessionDispatch: typeof recoverSessionDispatch;
|
|
54
57
|
reconcileSessionAttemptQuiescence: typeof reconcileSessionAttemptQuiescence;
|
|
55
58
|
peekSessionWork: typeof peekSessionWorkDb;
|
|
59
|
+
settleSessionInputWait: typeof settleSessionInputWaitDb;
|
|
56
60
|
countQueuedTurns: typeof countQueuedTurns;
|
|
57
61
|
getSessionAttemptActivityRef: typeof getSessionAttemptActivityRef;
|
|
58
62
|
getSessionEvent: typeof getSessionEvent;
|
|
@@ -93,6 +97,7 @@ export function createSessionStateActivities(
|
|
|
93
97
|
const reconcileSessionAttemptQuiescenceFn =
|
|
94
98
|
overrides.reconcileSessionAttemptQuiescence ?? reconcileSessionAttemptQuiescence;
|
|
95
99
|
const peekSessionWorkFn = overrides.peekSessionWork ?? peekSessionWorkDb;
|
|
100
|
+
const settleSessionInputWaitFn = overrides.settleSessionInputWait ?? settleSessionInputWaitDb;
|
|
96
101
|
const countQueuedTurnsFn = overrides.countQueuedTurns ?? countQueuedTurns;
|
|
97
102
|
const getSessionAttemptActivityRefFn =
|
|
98
103
|
overrides.getSessionAttemptActivityRef ?? getSessionAttemptActivityRef;
|
|
@@ -219,7 +224,9 @@ export function createSessionStateActivities(
|
|
|
219
224
|
: input.preClaimFailure?.disposition === "retryable" &&
|
|
220
225
|
input.preClaimFailure.code !== "claim_invariant"
|
|
221
226
|
? input.preClaimFailure.code
|
|
222
|
-
:
|
|
227
|
+
: input.preClaimFailureDisposition === "retryable"
|
|
228
|
+
? "legacy_retryable_preclaim_database_failure"
|
|
229
|
+
: null;
|
|
223
230
|
if (recoveredClaimCode) {
|
|
224
231
|
const recovery = await requestSessionTurnRecoveryFn(db, input.workspaceId, {
|
|
225
232
|
sessionId: input.sessionId,
|
|
@@ -486,6 +493,17 @@ export function createSessionStateActivities(
|
|
|
486
493
|
return peek;
|
|
487
494
|
}
|
|
488
495
|
|
|
496
|
+
async function settleSessionInputWait(
|
|
497
|
+
input: SettleSessionInputWaitInput,
|
|
498
|
+
): Promise<SettleSessionInputWaitResult> {
|
|
499
|
+
const { db, bus } = await services();
|
|
500
|
+
const result = await settleSessionInputWaitFn(db, input);
|
|
501
|
+
if (result.events.length > 0) {
|
|
502
|
+
await publishDurableSessionEventsFn(bus, input.workspaceId, input.sessionId, result.events);
|
|
503
|
+
}
|
|
504
|
+
return { action: result.action };
|
|
505
|
+
}
|
|
506
|
+
|
|
489
507
|
async function expireSessionHumanInput(
|
|
490
508
|
input: ExpireSessionHumanInputInput,
|
|
491
509
|
): Promise<ExpireSessionHumanInputResult> {
|
|
@@ -546,6 +564,7 @@ export function createSessionStateActivities(
|
|
|
546
564
|
recoverDispatch,
|
|
547
565
|
recoverEscapedMcpTimeout,
|
|
548
566
|
peekSessionWork,
|
|
567
|
+
settleSessionInputWait,
|
|
549
568
|
expireSessionHumanInput,
|
|
550
569
|
expireSessionInteractionIntervention,
|
|
551
570
|
markSessionIdle,
|
package/src/activities/types.ts
CHANGED
|
@@ -343,6 +343,18 @@ export type PeekSessionWorkInput = {
|
|
|
343
343
|
sessionId: string;
|
|
344
344
|
};
|
|
345
345
|
|
|
346
|
+
export type SettleSessionInputWaitInput = {
|
|
347
|
+
accountId: string;
|
|
348
|
+
workspaceId: string;
|
|
349
|
+
sessionId: string;
|
|
350
|
+
waitTurnId: string;
|
|
351
|
+
disposition: "held" | "timeout" | "superseded";
|
|
352
|
+
};
|
|
353
|
+
|
|
354
|
+
export type SettleSessionInputWaitResult = {
|
|
355
|
+
action: "held" | "timeout" | "superseded" | "stale";
|
|
356
|
+
};
|
|
357
|
+
|
|
346
358
|
export type ExpireSessionHumanInputInput = {
|
|
347
359
|
accountId: string;
|
|
348
360
|
workspaceId: string;
|
|
@@ -378,14 +390,10 @@ export type MaybeContinueGoalInput = {
|
|
|
378
390
|
};
|
|
379
391
|
|
|
380
392
|
export type MaybeContinueGoalResult = {
|
|
381
|
-
// `
|
|
382
|
-
// materialized
|
|
383
|
-
//
|
|
384
|
-
|
|
385
|
-
// `deferred`: idle backoff between consecutive no-input continuations. Same
|
|
386
|
-
// shape as `held`: nothing materialized, obligation armed, a delayed outbox
|
|
387
|
-
// wake at the pacing deadline (pulled to now by any new input) restarts it.
|
|
388
|
-
action: "none" | "queue" | "continue" | "paused" | "held" | "deferred";
|
|
393
|
+
// `deferred`: idle backoff between consecutive no-input continuations.
|
|
394
|
+
// Nothing was materialized; the obligation remains armed and a delayed
|
|
395
|
+
// outbox wake at the pacing deadline restarts it.
|
|
396
|
+
action: "none" | "queue" | "continue" | "paused" | "deferred";
|
|
389
397
|
};
|
|
390
398
|
|
|
391
399
|
export type DispatchScheduledTaskRunInput = {
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import { listXaiSubscriptionAccountsMetadata, updateXaiQuotaMetadata } from "@opengeni/db";
|
|
2
|
+
import { fetchXaiSubscriptionQuota } from "@opengeni/xai-subscription";
|
|
3
|
+
import { buildXaiTurnRequestAuthorization } from "./xai-auth";
|
|
4
|
+
|
|
5
|
+
/** Refresh billing, never inference, before treating cached exhaustion as current. */
|
|
6
|
+
export async function refreshExhaustedXaiQuota(
|
|
7
|
+
input: Omit<Parameters<typeof buildXaiTurnRequestAuthorization>[0], "credentialId">,
|
|
8
|
+
) {
|
|
9
|
+
const accounts = await listXaiSubscriptionAccountsMetadata(input.db, input);
|
|
10
|
+
for (const account of accounts) {
|
|
11
|
+
if (account.status !== "active" || !account.allocatorEnabled || !account.exhaustedUntil)
|
|
12
|
+
continue;
|
|
13
|
+
if (account.quotaCheckedAt && Date.now() - account.quotaCheckedAt.getTime() < 30_000) continue;
|
|
14
|
+
try {
|
|
15
|
+
const auth = await buildXaiTurnRequestAuthorization({ ...input, credentialId: account.id });
|
|
16
|
+
const quota = await fetchXaiSubscriptionQuota({ context: auth.context });
|
|
17
|
+
// Unknown billing is not evidence that a refusal has cleared.
|
|
18
|
+
if (quota.usedPercent === null) continue;
|
|
19
|
+
await updateXaiQuotaMetadata(input.db, {
|
|
20
|
+
workspaceId: input.workspaceId,
|
|
21
|
+
subjectId: input.subjectId,
|
|
22
|
+
credentialId: account.id,
|
|
23
|
+
quotaUsedPercent: quota.usedPercent,
|
|
24
|
+
quotaResetAt: quota.period?.end ?? null,
|
|
25
|
+
quotaCheckedAt: quota.checkedAt,
|
|
26
|
+
expectedExhaustedUntil: account.exhaustedUntil,
|
|
27
|
+
expectedQuotaCheckedAt: account.quotaCheckedAt,
|
|
28
|
+
exhaustedUntil:
|
|
29
|
+
quota.usedPercent < 100 ? null : (quota.period?.end ?? account.exhaustedUntil),
|
|
30
|
+
});
|
|
31
|
+
} catch {
|
|
32
|
+
// Preserve provider refusal on refresh failure; the durable timer retries.
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
}
|
package/src/activities.ts
CHANGED
|
@@ -101,6 +101,7 @@ export const reconcileSessionAttemptQuiescence =
|
|
|
101
101
|
export const recoverDispatch = defaultControlActivities.recoverDispatch;
|
|
102
102
|
export const recoverEscapedMcpTimeout = defaultControlActivities.recoverEscapedMcpTimeout;
|
|
103
103
|
export const peekSessionWork = defaultControlActivities.peekSessionWork;
|
|
104
|
+
export const settleSessionInputWait = defaultControlActivities.settleSessionInputWait;
|
|
104
105
|
export const expireSessionHumanInput = defaultControlActivities.expireSessionHumanInput;
|
|
105
106
|
export const expireSessionInteractionIntervention =
|
|
106
107
|
defaultControlActivities.expireSessionInteractionIntervention;
|