@opengeni/worker-bundle 0.26.0-canary.0 → 0.26.0-canary.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn/errors.d.ts +5 -4
- package/dist/activities/agent-turn/history-prefix.d.ts +8 -0
- package/dist/activities/agent-turn/history-sink.d.ts +6 -2
- package/dist/activities/agent-turn/quiescence.d.ts +4 -2
- package/dist/activities/sandbox-lease.d.ts +7 -2
- package/dist/activities/xai-quota.d.ts +3 -0
- package/dist/{activities-control-CTEMLTHG.js → activities-control-3SKLDTXQ.js} +205 -86
- package/dist/activities-control-3SKLDTXQ.js.map +1 -0
- package/dist/{activities-turn-S5IU347A.js → activities-turn-GHSFZB64.js} +176 -1077
- package/dist/activities-turn-GHSFZB64.js.map +1 -0
- package/dist/{chunk-TAUE3GYT.js → chunk-O2LQWJIK.js} +3115 -1948
- package/dist/chunk-O2LQWJIK.js.map +1 -0
- package/dist/index.js +1 -1
- package/dist/sandbox-routing.d.ts +6 -0
- package/dist/workflow-bundle.js +24 -2
- package/package.json +17 -17
- package/src/activities/agent-turn/agent-build.ts +1 -0
- package/src/activities/agent-turn/errors.ts +44 -11
- package/src/activities/agent-turn/failure-settlement.ts +21 -8
- package/src/activities/agent-turn/history-prefix.ts +51 -0
- package/src/activities/agent-turn/history-sink.ts +30 -6
- package/src/activities/agent-turn/quiescence.ts +8 -2
- package/src/activities/agent-turn/run.ts +4 -0
- package/src/activities/agent-turn/stream-attempt.ts +20 -7
- package/src/activities/agent-turn/tool-environment.ts +12 -0
- package/src/activities/agent-turn/xai-capacity.ts +11 -0
- package/src/activities/codex-capacity.ts +18 -0
- package/src/activities/run-input.ts +6 -7
- package/src/activities/sandbox-lease.ts +229 -87
- package/src/activities/session-state.ts +8 -7
- package/src/activities/xai-quota.ts +35 -0
- package/src/sandbox-routing.ts +115 -0
- package/src/workflows/session.ts +29 -1
- package/dist/activities-control-CTEMLTHG.js.map +0 -1
- package/dist/activities-turn-S5IU347A.js.map +0 -1
- package/dist/chunk-TAUE3GYT.js.map +0 -1
|
@@ -67,10 +67,11 @@ export declare function escapedMcpTimeoutRecoveryFailure(input: {
|
|
|
67
67
|
* Convert the atomic claim transaction's failure into a small, stable
|
|
68
68
|
* Temporal wire contract. The original error remains in activity diagnostics,
|
|
69
69
|
* but SQL text, parameters, and arbitrary invariant messages never enter
|
|
70
|
-
* workflow history.
|
|
71
|
-
*
|
|
72
|
-
*
|
|
73
|
-
*
|
|
70
|
+
* workflow history. Every database failure is safe to re-read after backoff
|
|
71
|
+
* because the claim transaction contains no model/tool effects. Database
|
|
72
|
+
* constraints and authorization guards can also be repaired by a deployment or
|
|
73
|
+
* authority refresh, so they must preserve pending work. Only a non-database
|
|
74
|
+
* claim invariant is terminal.
|
|
74
75
|
*/
|
|
75
76
|
export declare function preClaimAdmissionFailure(error: unknown): ApplicationFailure;
|
|
76
77
|
/**
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
/** A position is an append cursor only while the items before it stay identical. */
|
|
2
|
+
export declare class HistoryPrefixGuard {
|
|
3
|
+
private persisted;
|
|
4
|
+
seed(items: Array<Record<string, unknown>>, count: number, truncationTokens?: number): void;
|
|
5
|
+
verify(items: Array<Record<string, unknown>>, truncationTokens?: number): string[];
|
|
6
|
+
acknowledge(keys: string[], count: number): void;
|
|
7
|
+
private keys;
|
|
8
|
+
}
|
|
@@ -19,8 +19,8 @@ export type TurnHistorySinkDeps = {
|
|
|
19
19
|
* Dual-write of conversation truth (issue #35): completed items are reconciled
|
|
20
20
|
* into session_history_items after every model response and at every turn-end
|
|
21
21
|
* path (idempotent on position), and the sandbox recovery envelope is upserted
|
|
22
|
-
* alongside.
|
|
23
|
-
*
|
|
22
|
+
* alongside. Provider dispatch and settlement require durable persistence;
|
|
23
|
+
* the prefix guard rejects SDK history removal/reordering before slicing.
|
|
24
24
|
*
|
|
25
25
|
* Orphaned-tool-output guard: `stream.state.history` is NOT a plain
|
|
26
26
|
* append-only array — it is a computed getter
|
|
@@ -40,6 +40,10 @@ export type TurnHistorySinkDeps = {
|
|
|
40
40
|
*/
|
|
41
41
|
export declare class TurnHistorySink {
|
|
42
42
|
private readonly deps;
|
|
43
|
+
private readonly prefix;
|
|
44
|
+
seedHistory(input: string | readonly unknown[] | {
|
|
45
|
+
readonly history: unknown[];
|
|
46
|
+
}, count: number): void;
|
|
43
47
|
persistedHistoryCount: number;
|
|
44
48
|
nextHistoryPosition: number;
|
|
45
49
|
providerArtifactCandidates: Awaited<ReturnType<typeof turnInput>>["providerArtifactCandidates"];
|
|
@@ -142,8 +142,10 @@ export declare function assertSuccessfulAgentStreamCompletion(input: {
|
|
|
142
142
|
/** Call after any more-specific empty-stream recovery has had priority. */
|
|
143
143
|
export declare function assertAgentStreamNotCancelled(cancelled: boolean): void;
|
|
144
144
|
/** Interruption streams do not produce a normal final output. Call this only
|
|
145
|
-
* after the interruption branch has settled any required action.
|
|
146
|
-
|
|
145
|
+
* after the interruption branch has settled any required action. A successful
|
|
146
|
+
* runtime-owned input wait is a normal terminal result without assistant text;
|
|
147
|
+
* its authority comes from prepared tools, never the provider's output. */
|
|
148
|
+
export declare function requireAgentStreamFinalOutput(finalOutput: unknown | undefined, inputWaitYielded?: boolean): unknown;
|
|
147
149
|
/**
|
|
148
150
|
* Terminal settlement closes the active attempt before turn finalization. The
|
|
149
151
|
* ordinary workspace admission therefore correctly returns `attempt_fenced`
|
|
@@ -94,7 +94,7 @@ export type RetainedProcessProbeResult = {
|
|
|
94
94
|
status: "deferred";
|
|
95
95
|
reason: "identity_mismatch" | "resume_state_missing" | "backend_unsupported" | "provider_running" | "provider_unknown" | "provider_timeout" | "provider_error" | "provider_binding_missing" | "provider_binding_mismatch";
|
|
96
96
|
};
|
|
97
|
-
export type RetainedProcessProbeFn = (settings: ActivityServices["settings"], lease: LeaseSnapshot, process: SandboxRetainedProcess, mode?: "observe" | "cancel") => Promise<RetainedProcessProbeResult>;
|
|
97
|
+
export type RetainedProcessProbeFn = (settings: ActivityServices["settings"], lease: LeaseSnapshot, process: SandboxRetainedProcess, mode?: "observe" | "cancel", captureOutput?: (result: unknown, chunkId: string) => Promise<void>) => Promise<RetainedProcessProbeResult>;
|
|
98
98
|
export type HistoricalModalSandboxLifecycleProbeFn = typeof inspectModalSandboxLifecycle;
|
|
99
99
|
export type DrainableProviderProbeFn = (settings: ActivityServices["settings"], lease: LeaseSnapshot) => Promise<"ready" | "missing">;
|
|
100
100
|
export declare const RETAINED_PROCESS_RECONCILIATION_LIMIT = 20;
|
|
@@ -141,6 +141,10 @@ export declare function createSandboxLeaseActivities(services: () => Promise<Act
|
|
|
141
141
|
maintainSandboxLeaseSweep: (input: SandboxLeaseSweepMaintenanceInput) => Promise<SandboxLeaseSweepMaintenanceResult>;
|
|
142
142
|
reapSandboxLeases: () => Promise<ReapSandboxLeasesResult>;
|
|
143
143
|
};
|
|
144
|
+
/** Reconcile commands only against the immutable physical locator copied at
|
|
145
|
+
* adoption. A successor enrollment connection is intentionally irrelevant: no
|
|
146
|
+
* query or cancellation is ever rebound to it. */
|
|
147
|
+
export declare function reconcileConnectedMachineBackgroundCommands(db: ActivityServices["db"], settings: ActivityServices["settings"], observability: ActivityServices["observability"], bus: ActivityServices["bus"], controlRpcOverride?: ControlRpc): Promise<void>;
|
|
144
148
|
export declare function probeConnectedMachineBackgroundCommand(claim: ConnectedMachineBackgroundCommandClaim, controlRpc: ControlRpc, controlTimeoutMs: number, observedAt?: Date): Promise<ConnectedMachineBackgroundCommandProof | null>;
|
|
145
149
|
export declare function connectedCommandProofFromStatus(status: OpStatus, observedAt: Date): ConnectedMachineBackgroundCommandProof | null;
|
|
146
150
|
type RetainedProcessDeferralOutcome = Extract<RetainedProcessProbeResult, {
|
|
@@ -152,7 +156,8 @@ export declare function retainedProcessReconciliationDeferral(settings: Pick<Act
|
|
|
152
156
|
retryAfterMs: number;
|
|
153
157
|
quarantined: boolean;
|
|
154
158
|
};
|
|
155
|
-
export declare function
|
|
159
|
+
export declare function captureRetainedProbeOutput(processId: string, result: unknown, capture: (result: unknown, chunkId: string) => Promise<void>): Promise<void>;
|
|
160
|
+
export declare function probeRetainedProcessAtProvider(settings: ActivityServices["settings"], lease: LeaseSnapshot, process: SandboxRetainedProcess, mode?: "observe" | "cancel", captureOutput?: (result: unknown, chunkId: string) => Promise<void>): Promise<RetainedProcessProbeResult>;
|
|
156
161
|
/** Classify only exact SDK control banners. Arbitrary output, malformed text,
|
|
157
162
|
* and a running marker are observations to retry; none is physical exit proof. */
|
|
158
163
|
export declare function classifyRetainedProcessPollResult(result: unknown, providerSessionId: number): RetainedProcessProbeResult;
|
|
@@ -0,0 +1,3 @@
|
|
|
1
|
+
import { buildXaiTurnRequestAuthorization } from "./xai-auth.js";
|
|
2
|
+
/** Refresh billing, never inference, before treating cached exhaustion as current. */
|
|
3
|
+
export declare function refreshExhaustedXaiQuota(input: Omit<Parameters<typeof buildXaiTurnRequestAuthorization>[0], "credentialId">): Promise<void>;
|
|
@@ -9,6 +9,7 @@ import {
|
|
|
9
9
|
INCIDENT_TELEMETRY_AUTHORITY_FENCE_LINEAGE_KEY,
|
|
10
10
|
MAX_AUTOMATIC_PROVIDER_RECOVERIES,
|
|
11
11
|
assertReusableSessionRevivable,
|
|
12
|
+
captureConnectedCommandOutput,
|
|
12
13
|
collectWorkspaceArchiveObjectKeys,
|
|
13
14
|
createCodexCapacityActivities,
|
|
14
15
|
createGoalActivities,
|
|
@@ -32,7 +33,7 @@ import {
|
|
|
32
33
|
scheduledUserMessagePayload,
|
|
33
34
|
withFirstPartyTools,
|
|
34
35
|
workflowIdForSession
|
|
35
|
-
} from "./chunk-
|
|
36
|
+
} from "./chunk-O2LQWJIK.js";
|
|
36
37
|
import {
|
|
37
38
|
inspectOpenSandboxKubernetesInventory,
|
|
38
39
|
recordCreditBalanceGauges,
|
|
@@ -5236,6 +5237,7 @@ import {
|
|
|
5236
5237
|
recordConnectedMachineBackgroundCommandProof
|
|
5237
5238
|
} from "@opengeni/db/session-background-commands";
|
|
5238
5239
|
import { sandboxWarmRateMicrosPerSecond } from "@opengeni/config";
|
|
5240
|
+
import { appendSessionCommandOutput } from "@opengeni/db/session-command-output";
|
|
5239
5241
|
import { sandboxLeaseTelemetryKey } from "@opengeni/observability";
|
|
5240
5242
|
import {
|
|
5241
5243
|
captureVerifiedWorkspaceArchive,
|
|
@@ -5249,6 +5251,10 @@ import {
|
|
|
5249
5251
|
isExecSessionLostBanner,
|
|
5250
5252
|
isProviderSandboxNotFoundError,
|
|
5251
5253
|
NatsControlRpc,
|
|
5254
|
+
NatsOpStreamTransport,
|
|
5255
|
+
OpStreamExecClient,
|
|
5256
|
+
defaultSelfhostedRetryClock,
|
|
5257
|
+
stripExecBanner,
|
|
5252
5258
|
parseExecBannerExitCode,
|
|
5253
5259
|
prepareProviderForTeardownAfterCapture,
|
|
5254
5260
|
providerWorkspaceCapturePolicy,
|
|
@@ -5809,97 +5815,172 @@ async function adoptLegacyModalCheckpointReceipts(db, settings, observability) {
|
|
|
5809
5815
|
});
|
|
5810
5816
|
return adopted;
|
|
5811
5817
|
}
|
|
5812
|
-
async function reconcileConnectedMachineBackgroundCommands(db, settings, observability, bus) {
|
|
5813
|
-
const
|
|
5814
|
-
|
|
5815
|
-
|
|
5816
|
-
|
|
5817
|
-
|
|
5818
|
-
|
|
5819
|
-
|
|
5820
|
-
|
|
5821
|
-
|
|
5822
|
-
|
|
5823
|
-
|
|
5824
|
-
|
|
5825
|
-
|
|
5826
|
-
|
|
5827
|
-
|
|
5828
|
-
|
|
5829
|
-
|
|
5830
|
-
|
|
5831
|
-
|
|
5832
|
-
|
|
5833
|
-
|
|
5834
|
-
|
|
5835
|
-
|
|
5836
|
-
|
|
5837
|
-
|
|
5838
|
-
|
|
5839
|
-
|
|
5840
|
-
|
|
5841
|
-
|
|
5842
|
-
|
|
5843
|
-
|
|
5844
|
-
|
|
5818
|
+
async function reconcileConnectedMachineBackgroundCommands(db, settings, observability, bus, controlRpcOverride) {
|
|
5819
|
+
const dueBefore = /* @__PURE__ */ new Date();
|
|
5820
|
+
const deadline = Date.now() + settings.sandboxLeaseReaperPeriodMs;
|
|
5821
|
+
const offlineConnections = /* @__PURE__ */ new Set();
|
|
5822
|
+
const firstConnectionProbes = /* @__PURE__ */ new Map();
|
|
5823
|
+
do {
|
|
5824
|
+
const claimId = crypto.randomUUID();
|
|
5825
|
+
let claims;
|
|
5826
|
+
try {
|
|
5827
|
+
claims = await claimConnectedMachineSessionBackgroundCommands(db, {
|
|
5828
|
+
claimId,
|
|
5829
|
+
dueBefore,
|
|
5830
|
+
limit: CONNECTED_COMMAND_RECONCILIATION_LIMIT,
|
|
5831
|
+
claimTtlMs: CONNECTED_COMMAND_RECONCILIATION_CLAIM_TTL_MS
|
|
5832
|
+
});
|
|
5833
|
+
} catch (error) {
|
|
5834
|
+
recordConnectedCommandReconciliation(observability, "claim_failed");
|
|
5835
|
+
observability.warn("sandbox reaper: connected-command claim failed", {
|
|
5836
|
+
error: error instanceof Error ? error.message : String(error)
|
|
5837
|
+
});
|
|
5838
|
+
return;
|
|
5839
|
+
}
|
|
5840
|
+
const controlRpc = controlRpcOverride ?? new NatsControlRpc(
|
|
5841
|
+
async () => bus.getRequestConnection()
|
|
5842
|
+
);
|
|
5843
|
+
await forEachWithConcurrency(claims, SANDBOX_MAINTENANCE_ITEM_CONCURRENCY, async (claim) => {
|
|
5844
|
+
let proof = claim.proof;
|
|
5845
|
+
const connectionKey = JSON.stringify([
|
|
5846
|
+
claim.controlWorkspaceId,
|
|
5847
|
+
claim.enrollmentId,
|
|
5848
|
+
claim.connectionInstanceId
|
|
5849
|
+
]);
|
|
5850
|
+
if (!proof) {
|
|
5851
|
+
const firstProbe = firstConnectionProbes.get(connectionKey);
|
|
5852
|
+
if (firstProbe) await firstProbe;
|
|
5853
|
+
if (offlineConnections.has(connectionKey)) {
|
|
5854
|
+
await deferConnectedCommandClaim(db, settings, observability, claim, "provider_offline");
|
|
5845
5855
|
return;
|
|
5846
5856
|
}
|
|
5847
|
-
|
|
5848
|
-
|
|
5849
|
-
|
|
5850
|
-
|
|
5857
|
+
let finishFirstProbe;
|
|
5858
|
+
if (!firstProbe) {
|
|
5859
|
+
firstConnectionProbes.set(
|
|
5860
|
+
connectionKey,
|
|
5861
|
+
new Promise((resolve) => {
|
|
5862
|
+
finishFirstProbe = resolve;
|
|
5863
|
+
})
|
|
5864
|
+
);
|
|
5865
|
+
}
|
|
5866
|
+
try {
|
|
5867
|
+
proof = await probeConnectedMachineBackgroundCommand(
|
|
5868
|
+
claim,
|
|
5869
|
+
controlRpc,
|
|
5870
|
+
Math.min(
|
|
5871
|
+
settings.sandboxSelfhostedControlTimeoutMs,
|
|
5872
|
+
RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS
|
|
5873
|
+
)
|
|
5874
|
+
);
|
|
5875
|
+
if (proof?.outcome !== "lost") {
|
|
5876
|
+
if (!bus.getOpStreamConnection)
|
|
5877
|
+
throw new Error("Connected command output transport unavailable");
|
|
5878
|
+
const client = new OpStreamExecClient({
|
|
5879
|
+
workspaceId: claim.controlWorkspaceId,
|
|
5880
|
+
agentId: claim.enrollmentId,
|
|
5881
|
+
connectionInstanceId: claim.connectionInstanceId,
|
|
5882
|
+
epoch: 0,
|
|
5883
|
+
controlRpc,
|
|
5884
|
+
rpcSubject: subjectFor(
|
|
5885
|
+
claim.controlWorkspaceId,
|
|
5886
|
+
claim.enrollmentId,
|
|
5887
|
+
claim.connectionInstanceId
|
|
5888
|
+
),
|
|
5889
|
+
transport: new NatsOpStreamTransport(
|
|
5890
|
+
async () => bus.getOpStreamConnection?.() ?? null
|
|
5891
|
+
),
|
|
5892
|
+
controlTimeoutMs: Math.min(
|
|
5893
|
+
settings.sandboxSelfhostedControlTimeoutMs,
|
|
5894
|
+
RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS
|
|
5895
|
+
),
|
|
5896
|
+
retryClock: defaultSelfhostedRetryClock,
|
|
5897
|
+
journal: { attachGeneration: () => String(Date.now()), persistSettled: () => {
|
|
5898
|
+
} }
|
|
5899
|
+
});
|
|
5900
|
+
const replay = await client.readExisting(
|
|
5901
|
+
claim.opId,
|
|
5902
|
+
proof ? 5e3 : 250,
|
|
5903
|
+
async (frames) => {
|
|
5904
|
+
await captureConnectedCommandOutput(db, claim, bus)(claim.commandId, frames);
|
|
5905
|
+
}
|
|
5906
|
+
);
|
|
5907
|
+
if (proof && replay.status !== "completed" && !replay.terminal) {
|
|
5908
|
+
throw new Error(
|
|
5909
|
+
"Connected command terminal output replay has not reached its exit frontier"
|
|
5910
|
+
);
|
|
5911
|
+
}
|
|
5912
|
+
}
|
|
5913
|
+
if (!proof) {
|
|
5914
|
+
await deferConnectedCommandClaim(
|
|
5915
|
+
db,
|
|
5916
|
+
settings,
|
|
5917
|
+
observability,
|
|
5918
|
+
claim,
|
|
5919
|
+
"provider_running"
|
|
5920
|
+
);
|
|
5921
|
+
return;
|
|
5922
|
+
}
|
|
5923
|
+
} catch (error) {
|
|
5924
|
+
const offline = error instanceof SelfhostedControlError && (error.agentOffline || error.reason === "agent_reconnecting" || error.draining);
|
|
5925
|
+
if (offline) offlineConnections.add(connectionKey);
|
|
5926
|
+
if (!offline) {
|
|
5927
|
+
observability.warn("sandbox reaper: connected-command provider probe failed", {
|
|
5928
|
+
commandId: claim.commandId,
|
|
5929
|
+
enrollmentId: claim.enrollmentId,
|
|
5930
|
+
connectionInstanceId: claim.connectionInstanceId,
|
|
5931
|
+
error: error instanceof Error ? error.message : String(error)
|
|
5932
|
+
});
|
|
5933
|
+
}
|
|
5934
|
+
await deferConnectedCommandClaim(
|
|
5935
|
+
db,
|
|
5936
|
+
settings,
|
|
5937
|
+
observability,
|
|
5938
|
+
claim,
|
|
5939
|
+
offline ? "provider_offline" : "provider_error"
|
|
5940
|
+
);
|
|
5941
|
+
return;
|
|
5942
|
+
} finally {
|
|
5943
|
+
finishFirstProbe?.();
|
|
5944
|
+
}
|
|
5945
|
+
try {
|
|
5946
|
+
await recordConnectedMachineBackgroundCommandProof(db, { claim, proof });
|
|
5947
|
+
recordConnectedCommandReconciliation(observability, `proof_${proof.outcome}`);
|
|
5948
|
+
} catch (error) {
|
|
5949
|
+
recordConnectedCommandReconciliation(observability, "proof_checkpoint_failed");
|
|
5950
|
+
observability.warn("sandbox reaper: connected-command proof checkpoint failed", {
|
|
5851
5951
|
commandId: claim.commandId,
|
|
5852
|
-
enrollmentId: claim.enrollmentId,
|
|
5853
|
-
connectionInstanceId: claim.connectionInstanceId,
|
|
5854
5952
|
error: error instanceof Error ? error.message : String(error)
|
|
5855
5953
|
});
|
|
5954
|
+
return;
|
|
5856
5955
|
}
|
|
5857
|
-
await deferConnectedCommandClaim(
|
|
5858
|
-
db,
|
|
5859
|
-
settings,
|
|
5860
|
-
observability,
|
|
5861
|
-
claim,
|
|
5862
|
-
offline ? "provider_offline" : "provider_error"
|
|
5863
|
-
);
|
|
5864
|
-
return;
|
|
5865
5956
|
}
|
|
5866
5957
|
try {
|
|
5867
|
-
await
|
|
5868
|
-
|
|
5958
|
+
const settlement = await settleClaimedConnectedMachineBackgroundCommand(db, { claim });
|
|
5959
|
+
if (!settlement.settled) {
|
|
5960
|
+
throw new Error("Connected command settlement lost its exact claim or proof");
|
|
5961
|
+
}
|
|
5962
|
+
if (settlement.events.length > 0) {
|
|
5963
|
+
try {
|
|
5964
|
+
await bus.publish(claim.workspaceId, claim.sessionId, settlement.events);
|
|
5965
|
+
} catch (error) {
|
|
5966
|
+
observability.warn("sandbox reaper: connected-command event fanout failed", {
|
|
5967
|
+
commandId: claim.commandId,
|
|
5968
|
+
error: error instanceof Error ? error.message : String(error)
|
|
5969
|
+
});
|
|
5970
|
+
}
|
|
5971
|
+
}
|
|
5972
|
+
recordConnectedCommandReconciliation(observability, `settled_${proof.outcome}`);
|
|
5869
5973
|
} catch (error) {
|
|
5870
|
-
recordConnectedCommandReconciliation(observability, "
|
|
5871
|
-
observability.warn("sandbox reaper: connected-command
|
|
5974
|
+
recordConnectedCommandReconciliation(observability, "settlement_failed");
|
|
5975
|
+
observability.warn("sandbox reaper: connected-command settlement failed", {
|
|
5872
5976
|
commandId: claim.commandId,
|
|
5873
5977
|
error: error instanceof Error ? error.message : String(error)
|
|
5874
5978
|
});
|
|
5875
|
-
|
|
5876
|
-
}
|
|
5877
|
-
}
|
|
5878
|
-
try {
|
|
5879
|
-
const settlement = await settleClaimedConnectedMachineBackgroundCommand(db, { claim });
|
|
5880
|
-
if (!settlement.settled) {
|
|
5881
|
-
throw new Error("Connected command settlement lost its exact claim or proof");
|
|
5882
|
-
}
|
|
5883
|
-
if (settlement.events.length > 0) {
|
|
5884
|
-
try {
|
|
5885
|
-
await bus.publish(claim.workspaceId, claim.sessionId, settlement.events);
|
|
5886
|
-
} catch (error) {
|
|
5887
|
-
observability.warn("sandbox reaper: connected-command event fanout failed", {
|
|
5888
|
-
commandId: claim.commandId,
|
|
5889
|
-
error: error instanceof Error ? error.message : String(error)
|
|
5890
|
-
});
|
|
5891
|
-
}
|
|
5979
|
+
await deferConnectedCommandClaim(db, settings, observability, claim, "settlement_failed");
|
|
5892
5980
|
}
|
|
5893
|
-
|
|
5894
|
-
|
|
5895
|
-
|
|
5896
|
-
observability.warn("sandbox reaper: connected-command settlement failed", {
|
|
5897
|
-
commandId: claim.commandId,
|
|
5898
|
-
error: error instanceof Error ? error.message : String(error)
|
|
5899
|
-
});
|
|
5900
|
-
await deferConnectedCommandClaim(db, settings, observability, claim, "settlement_failed");
|
|
5901
|
-
}
|
|
5902
|
-
});
|
|
5981
|
+
});
|
|
5982
|
+
if (claims.length < CONNECTED_COMMAND_RECONCILIATION_LIMIT) break;
|
|
5983
|
+
} while (Date.now() < deadline);
|
|
5903
5984
|
}
|
|
5904
5985
|
async function probeConnectedMachineBackgroundCommand(claim, controlRpc, controlTimeoutMs, observedAt = /* @__PURE__ */ new Date()) {
|
|
5905
5986
|
const input = {
|
|
@@ -5926,11 +6007,14 @@ function connectedCommandProofFromStatus(status, observedAt) {
|
|
|
5926
6007
|
throw new Error("Connected command completed without an exit record");
|
|
5927
6008
|
}
|
|
5928
6009
|
const failureCode = exit.failureCode.replace(/[^A-Za-z0-9_-]/g, "_").slice(0, 128);
|
|
5929
|
-
const reason = exit.cancelled ? "op_cancelled" : exit.timedOut ? "op_timed_out" :
|
|
6010
|
+
const reason = failureCode ? `op_failure_${failureCode}` : exit.cancelled ? "op_cancelled" : exit.timedOut ? "op_timed_out" : "op_exit";
|
|
5930
6011
|
return {
|
|
5931
6012
|
outcome: "exited",
|
|
5932
6013
|
exitCode: exit.exitCode,
|
|
5933
6014
|
reason,
|
|
6015
|
+
...failureCode ? {
|
|
6016
|
+
failure: { code: failureCode, detail: exit.failureDetail, retryable: false }
|
|
6017
|
+
} : {},
|
|
5934
6018
|
observedAt
|
|
5935
6019
|
};
|
|
5936
6020
|
}
|
|
@@ -6075,7 +6159,23 @@ async function reconcileTerminalRetainedProcesses(db, settings, observability, p
|
|
|
6075
6159
|
settings,
|
|
6076
6160
|
lease,
|
|
6077
6161
|
process,
|
|
6078
|
-
claim.ownerState === "background_stopping" ? "cancel" : "observe"
|
|
6162
|
+
claim.ownerState === "background_stopping" ? "cancel" : "observe",
|
|
6163
|
+
async (result, chunkId) => {
|
|
6164
|
+
if (typeof result !== "string" || isExecSessionLostBanner(result, process.providerSessionId))
|
|
6165
|
+
return;
|
|
6166
|
+
const events = await appendSessionCommandOutput(db, {
|
|
6167
|
+
accountId: process.accountId,
|
|
6168
|
+
workspaceId: process.workspaceId,
|
|
6169
|
+
sessionId: process.sessionId,
|
|
6170
|
+
commandId: process.id,
|
|
6171
|
+
chunkId,
|
|
6172
|
+
stream: "stdout",
|
|
6173
|
+
streamFidelity: "merged",
|
|
6174
|
+
chunk: stripExecBanner(result)
|
|
6175
|
+
});
|
|
6176
|
+
if (events.length && bus)
|
|
6177
|
+
await bus.publish(process.workspaceId, process.sessionId, events).catch(() => void 0);
|
|
6178
|
+
}
|
|
6079
6179
|
);
|
|
6080
6180
|
} catch (error) {
|
|
6081
6181
|
observability.warn("sandbox reaper: retained-process provider probe failed", {
|
|
@@ -6253,13 +6353,29 @@ async function withRetainedProcessProbeTimeout(promise) {
|
|
|
6253
6353
|
if (timer) clearTimeout(timer);
|
|
6254
6354
|
}
|
|
6255
6355
|
}
|
|
6256
|
-
|
|
6356
|
+
var pendingRetainedProbeOutput = /* @__PURE__ */ new Map();
|
|
6357
|
+
async function captureRetainedProbeOutput(processId, result, capture) {
|
|
6358
|
+
const pending = pendingRetainedProbeOutput.get(processId) ?? {
|
|
6359
|
+
result,
|
|
6360
|
+
chunkId: crypto.randomUUID()
|
|
6361
|
+
};
|
|
6362
|
+
pendingRetainedProbeOutput.set(processId, pending);
|
|
6363
|
+
await capture(pending.result, pending.chunkId);
|
|
6364
|
+
pendingRetainedProbeOutput.delete(processId);
|
|
6365
|
+
}
|
|
6366
|
+
async function probeRetainedProcessAtProvider(settings, lease, process, mode = "observe", captureOutput) {
|
|
6257
6367
|
if (lease.id !== process.leaseId || lease.sandboxGroupId !== process.sandboxGroupId || lease.leaseEpoch !== process.leaseEpoch || lease.backend !== process.providerBackend || lease.instanceId !== process.providerInstanceId || lease.resumeBackendId !== process.providerBackend || process.routeTargetId !== null) {
|
|
6258
6368
|
return { status: "deferred", reason: "identity_mismatch" };
|
|
6259
6369
|
}
|
|
6260
6370
|
if (!lease.resumeState) {
|
|
6261
6371
|
return { status: "deferred", reason: "resume_state_missing" };
|
|
6262
6372
|
}
|
|
6373
|
+
const pending = pendingRetainedProbeOutput.get(process.id);
|
|
6374
|
+
if (pending) {
|
|
6375
|
+
if (!captureOutput) throw new Error("Pending command output requires its persistence callback");
|
|
6376
|
+
await captureRetainedProbeOutput(process.id, pending.result, captureOutput);
|
|
6377
|
+
return classifyRetainedProcessPollResult(pending.result, process.providerSessionId);
|
|
6378
|
+
}
|
|
6263
6379
|
const envelopeBackend = lease.resumeState.backendId;
|
|
6264
6380
|
if (envelopeBackend !== void 0 && envelopeBackend !== process.providerBackend) {
|
|
6265
6381
|
return { status: "deferred", reason: "identity_mismatch" };
|
|
@@ -6391,6 +6507,7 @@ async function probeRetainedProcessAtProvider(settings, lease, process, mode = "
|
|
|
6391
6507
|
}
|
|
6392
6508
|
return { status: "deferred", reason: "provider_error" };
|
|
6393
6509
|
}
|
|
6510
|
+
if (captureOutput) await captureRetainedProbeOutput(process.id, result, captureOutput);
|
|
6394
6511
|
const observation = classifyRetainedProcessPollResult(result, process.providerSessionId);
|
|
6395
6512
|
if (observation.status === "deferred" && observation.reason === "provider_running" && lease.rotationRequestedAt !== null) {
|
|
6396
6513
|
try {
|
|
@@ -6402,6 +6519,7 @@ async function probeRetainedProcessAtProvider(settings, lease, process, mode = "
|
|
|
6402
6519
|
maxOutputTokens: 2e3
|
|
6403
6520
|
})
|
|
6404
6521
|
);
|
|
6522
|
+
if (captureOutput) await captureRetainedProbeOutput(process.id, interrupted, captureOutput);
|
|
6405
6523
|
return classifyRetainedProcessPollResult(interrupted, process.providerSessionId);
|
|
6406
6524
|
} catch (error) {
|
|
6407
6525
|
if (isProviderSandboxNotFoundError(client.backendId, error)) {
|
|
@@ -9674,7 +9792,8 @@ function createSessionStateActivities(services, overrides = {}) {
|
|
|
9674
9792
|
sessionId: input.sessionId,
|
|
9675
9793
|
workflowId,
|
|
9676
9794
|
trigger: input.trigger,
|
|
9677
|
-
error: input.error ?? "Agent turn admission failed before attempt claim."
|
|
9795
|
+
error: input.error ?? "Agent turn admission failed before attempt claim.",
|
|
9796
|
+
...input.preClaimFailure ? { admissionFailure: input.preClaimFailure } : {}
|
|
9678
9797
|
});
|
|
9679
9798
|
if (failed.action === "terminal") return { action: "terminal" };
|
|
9680
9799
|
if (failed.action === "stale") return { action: "stale" };
|
|
@@ -9715,7 +9834,7 @@ function createSessionStateActivities(services, overrides = {}) {
|
|
|
9715
9834
|
if (providerRecoveryCount !== void 0 && (!Number.isSafeInteger(providerRecoveryCount) || providerRecoveryCount <= 0 || providerRecoveryCount !== providerRecoveryCountFromMetadata(turn.metadata ?? {}) + 1 || providerRecoveryCount > MAX_AUTOMATIC_PROVIDER_RECOVERIES || typeof providerFailureCode !== "string" || !/^[a-z][a-z0-9_]{0,63}$/.test(providerFailureCode))) {
|
|
9716
9835
|
return { action: "stale" };
|
|
9717
9836
|
}
|
|
9718
|
-
const recoveredClaimCode = postClaimIdentityMatches ? postClaimRecovery.code : input.preClaimFailure?.disposition === "retryable" && input.preClaimFailure.code !== "claim_invariant" ? input.preClaimFailure.code : null;
|
|
9837
|
+
const recoveredClaimCode = postClaimIdentityMatches ? postClaimRecovery.code : input.preClaimFailure?.disposition === "retryable" && input.preClaimFailure.code !== "claim_invariant" ? input.preClaimFailure.code : input.preClaimFailureDisposition === "retryable" ? "legacy_retryable_preclaim_database_failure" : null;
|
|
9719
9838
|
if (recoveredClaimCode) {
|
|
9720
9839
|
const recovery = await requestSessionTurnRecoveryFn(db, input.workspaceId, {
|
|
9721
9840
|
sessionId: input.sessionId,
|
|
@@ -9971,7 +10090,7 @@ function createSessionStateActivities(services, overrides = {}) {
|
|
|
9971
10090
|
await publishDurableSessionEventsFn(bus, input.workspaceId, input.sessionId, settled.events);
|
|
9972
10091
|
}
|
|
9973
10092
|
await refreshQueuedTurnsGauge(db, observability, countQueuedTurnsFn, recordTurnsQueuedGaugeFn);
|
|
9974
|
-
if (settled.action === "stale") {
|
|
10093
|
+
if (settled.action === "stale" || !settled.notifyParent) {
|
|
9975
10094
|
return;
|
|
9976
10095
|
}
|
|
9977
10096
|
await notifyParentOfChildIdleFn(
|
|
@@ -10081,4 +10200,4 @@ export {
|
|
|
10081
10200
|
createControlActivities,
|
|
10082
10201
|
createControlActivitiesFromServices
|
|
10083
10202
|
};
|
|
10084
|
-
//# sourceMappingURL=activities-control-
|
|
10203
|
+
//# sourceMappingURL=activities-control-3SKLDTXQ.js.map
|