@opengeni/worker-bundle 0.24.2-canary.0 → 0.26.0-canary.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn/agent-build.d.ts +3 -10
- package/dist/activities/agent-turn/claim.d.ts +7 -0
- package/dist/activities/agent-turn/credential-leases.d.ts +8 -0
- package/dist/activities/agent-turn/errors.d.ts +6 -5
- package/dist/activities/agent-turn/failure-settlement.d.ts +43 -1
- package/dist/activities/agent-turn/history-prefix.d.ts +8 -0
- package/dist/activities/agent-turn/history-sink.d.ts +6 -2
- package/dist/activities/agent-turn/quiescence.d.ts +13 -8
- package/dist/activities/agent-turn/tool-environment.d.ts +2 -13
- package/dist/activities/agent-turn/tool-policy.d.ts +4 -0
- package/dist/activities/agent-turn/turn-context.d.ts +9 -1
- package/dist/activities/codemode-dispatcher.d.ts +10 -1
- package/dist/activities/codex-capacity.d.ts +32 -3
- package/dist/activities/codex-fleet-shadow.d.ts +1 -1
- package/dist/activities/codex-image-generation.d.ts +1 -1
- package/dist/activities/codex-rotation.d.ts +8 -67
- package/dist/activities/goals.d.ts +1 -1
- package/dist/activities/google-drive-publication.d.ts +2 -13
- package/dist/activities/image-generation-operation.d.ts +13 -4
- package/dist/activities/sandbox-lease.d.ts +7 -2
- package/dist/activities/session-state.d.ts +4 -2
- package/dist/activities/types.d.ts +11 -1
- package/dist/activities/xai-quota.d.ts +3 -0
- package/dist/{activities-control-ZWH7J4KZ.js → activities-control-KS3ZPQVY.js} +233 -88
- package/dist/activities-control-KS3ZPQVY.js.map +1 -0
- package/dist/activities-control.d.ts +2 -0
- package/dist/{activities-turn-I4OPLQLS.js → activities-turn-GHSFZB64.js} +1109 -1865
- package/dist/activities-turn-GHSFZB64.js.map +1 -0
- package/dist/activities.d.ts +2 -0
- package/dist/{chunk-65JEWP2Q.js → chunk-D4ZEFOYH.js} +1 -1
- package/dist/chunk-D4ZEFOYH.js.map +1 -0
- package/dist/{chunk-RQNH6CGR.js → chunk-O2LQWJIK.js} +3250 -2061
- package/dist/chunk-O2LQWJIK.js.map +1 -0
- package/dist/github-rest-mcp.d.ts +0 -2
- package/dist/index.d.ts +1 -0
- package/dist/index.js +2 -2
- package/dist/index.js.map +1 -1
- package/dist/observability-metrics.d.ts +1 -1
- package/dist/sandbox-routing.d.ts +6 -0
- package/dist/workflow-bundle.js +94 -9
- package/dist/workflows/activities.d.ts +1 -1
- package/dist/workflows/session.d.ts +3 -0
- package/package.json +17 -17
- package/src/activities/agent-turn/agent-build.ts +21 -26
- package/src/activities/agent-turn/claim.ts +26 -1
- package/src/activities/agent-turn/codex-capacity.ts +311 -234
- package/src/activities/agent-turn/credential-leases.ts +36 -0
- package/src/activities/agent-turn/errors.ts +73 -12
- package/src/activities/agent-turn/failure-settlement.ts +472 -331
- package/src/activities/agent-turn/history-prefix.ts +51 -0
- package/src/activities/agent-turn/history-sink.ts +30 -6
- package/src/activities/agent-turn/quiescence.ts +29 -17
- package/src/activities/agent-turn/run.ts +22 -5
- package/src/activities/agent-turn/stream-attempt.ts +35 -8
- package/src/activities/agent-turn/tool-environment.ts +42 -32
- package/src/activities/agent-turn/tool-policy.ts +14 -8
- package/src/activities/agent-turn/turn-context.ts +13 -0
- package/src/activities/agent-turn/xai-capacity.ts +11 -0
- package/src/activities/codemode-dispatcher.ts +235 -59
- package/src/activities/codex-capacity.ts +107 -10
- package/src/activities/codex-fleet-shadow.ts +8 -1
- package/src/activities/codex-image-generation.ts +19 -2
- package/src/activities/codex-rotation.ts +45 -142
- package/src/activities/goals.ts +9 -9
- package/src/activities/google-drive-publication.ts +4 -81
- package/src/activities/image-generation-operation.ts +28 -3
- package/src/activities/run-input.ts +6 -7
- package/src/activities/sandbox-lease.ts +246 -87
- package/src/activities/session-state.ts +20 -1
- package/src/activities/types.ts +16 -8
- package/src/activities/xai-quota.ts +35 -0
- package/src/activities.ts +1 -0
- package/src/github-rest-mcp.ts +0 -82
- package/src/observability-metrics.ts +4 -0
- package/src/sandbox-routing.ts +186 -16
- package/src/workflows/activities.ts +1 -0
- package/src/workflows/session.ts +114 -8
- package/dist/activities/api-integrations.d.ts +0 -21
- package/dist/activities-control-ZWH7J4KZ.js.map +0 -1
- package/dist/activities-turn-I4OPLQLS.js.map +0 -1
- package/dist/chunk-65JEWP2Q.js.map +0 -1
- package/dist/chunk-RQNH6CGR.js.map +0 -1
- package/src/activities/api-integrations.ts +0 -151
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { settleSessionAttemptInterruptions, applySessionTurnSettlement, enqueueSessionWorkflowWake, failSessionWorkBeforeAttemptClaim, requestSessionTurnRecovery, recoverSessionDispatch, reconcileSessionAttemptQuiescence, peekSessionWork as peekSessionWorkDb, countQueuedTurns, getSessionAttemptActivityRef, getSessionEvent, getSessionTurnForAttempt, expireSessionInteractionIntervention as expireSessionInteractionInterventionDb, expireSessionHumanInputRequest, markSessionAttemptQuiesced, requireSession, settleSessionIdleWithParentOutbox } from "@opengeni/db";
|
|
1
|
+
import { settleSessionAttemptInterruptions, applySessionTurnSettlement, enqueueSessionWorkflowWake, failSessionWorkBeforeAttemptClaim, requestSessionTurnRecovery, recoverSessionDispatch, reconcileSessionAttemptQuiescence, peekSessionWork as peekSessionWorkDb, settleSessionInputWait as settleSessionInputWaitDb, countQueuedTurns, getSessionAttemptActivityRef, getSessionEvent, getSessionTurnForAttempt, expireSessionInteractionIntervention as expireSessionInteractionInterventionDb, expireSessionHumanInputRequest, markSessionAttemptQuiesced, requireSession, settleSessionIdleWithParentOutbox } from "@opengeni/db";
|
|
2
2
|
import { publishDurableSessionEvents } from "@opengeni/events";
|
|
3
3
|
import { deliverFailedChildTurnToParent, notifyParentOfChildIdle } from "./parent-wake.js";
|
|
4
4
|
import { recordTurnsQueuedGauge, recordWorkerDeathRecoveryMetrics } from "../observability-metrics.js";
|
|
5
|
-
import type { ControlActivityServices, ExpireSessionHumanInputInput, ExpireSessionHumanInputResult, ExpireSessionInteractionInterventionInput, ExpireSessionInteractionInterventionResult, PeekSessionWorkInput, FailSessionAttemptInput, FailSessionAttemptResult, SettleSessionInterruptionsInput, MarkSessionIdleInput, PersistSessionAttemptQuiescenceInput, ReconcileSessionAttemptQuiescenceInput, ReconcileSessionAttemptQuiescenceResult, RecoverDispatchInput, RecoverDispatchResult, RecoverEscapedMcpTimeoutInput, RecoverEscapedMcpTimeoutResult } from "./types.js";
|
|
5
|
+
import type { ControlActivityServices, ExpireSessionHumanInputInput, ExpireSessionHumanInputResult, ExpireSessionInteractionInterventionInput, ExpireSessionInteractionInterventionResult, PeekSessionWorkInput, FailSessionAttemptInput, FailSessionAttemptResult, SettleSessionInterruptionsInput, MarkSessionIdleInput, PersistSessionAttemptQuiescenceInput, ReconcileSessionAttemptQuiescenceInput, ReconcileSessionAttemptQuiescenceResult, RecoverDispatchInput, RecoverDispatchResult, RecoverEscapedMcpTimeoutInput, RecoverEscapedMcpTimeoutResult, SettleSessionInputWaitInput, SettleSessionInputWaitResult } from "./types.js";
|
|
6
6
|
export type SessionStateActivityOverrides = Partial<{
|
|
7
7
|
settleSessionAttemptInterruptions: typeof settleSessionAttemptInterruptions;
|
|
8
8
|
applySessionTurnSettlement: typeof applySessionTurnSettlement;
|
|
@@ -12,6 +12,7 @@ export type SessionStateActivityOverrides = Partial<{
|
|
|
12
12
|
recoverSessionDispatch: typeof recoverSessionDispatch;
|
|
13
13
|
reconcileSessionAttemptQuiescence: typeof reconcileSessionAttemptQuiescence;
|
|
14
14
|
peekSessionWork: typeof peekSessionWorkDb;
|
|
15
|
+
settleSessionInputWait: typeof settleSessionInputWaitDb;
|
|
15
16
|
countQueuedTurns: typeof countQueuedTurns;
|
|
16
17
|
getSessionAttemptActivityRef: typeof getSessionAttemptActivityRef;
|
|
17
18
|
getSessionEvent: typeof getSessionEvent;
|
|
@@ -38,6 +39,7 @@ export declare function createSessionStateActivities(services: () => Promise<Con
|
|
|
38
39
|
recoverDispatch: (input: RecoverDispatchInput) => Promise<RecoverDispatchResult>;
|
|
39
40
|
recoverEscapedMcpTimeout: (input: RecoverEscapedMcpTimeoutInput) => Promise<RecoverEscapedMcpTimeoutResult>;
|
|
40
41
|
peekSessionWork: (input: PeekSessionWorkInput) => Promise<import("@opengeni/db").SessionWorkPeek>;
|
|
42
|
+
settleSessionInputWait: (input: SettleSessionInputWaitInput) => Promise<SettleSessionInputWaitResult>;
|
|
41
43
|
expireSessionHumanInput: (input: ExpireSessionHumanInputInput) => Promise<ExpireSessionHumanInputResult>;
|
|
42
44
|
expireSessionInteractionIntervention: (input: ExpireSessionInteractionInterventionInput) => Promise<ExpireSessionInteractionInterventionResult>;
|
|
43
45
|
markSessionIdle: (input: MarkSessionIdleInput) => Promise<void>;
|
|
@@ -272,6 +272,16 @@ export type PeekSessionWorkInput = {
|
|
|
272
272
|
workspaceId: string;
|
|
273
273
|
sessionId: string;
|
|
274
274
|
};
|
|
275
|
+
export type SettleSessionInputWaitInput = {
|
|
276
|
+
accountId: string;
|
|
277
|
+
workspaceId: string;
|
|
278
|
+
sessionId: string;
|
|
279
|
+
waitTurnId: string;
|
|
280
|
+
disposition: "held" | "timeout" | "superseded";
|
|
281
|
+
};
|
|
282
|
+
export type SettleSessionInputWaitResult = {
|
|
283
|
+
action: "held" | "timeout" | "superseded" | "stale";
|
|
284
|
+
};
|
|
275
285
|
export type ExpireSessionHumanInputInput = {
|
|
276
286
|
accountId: string;
|
|
277
287
|
workspaceId: string;
|
|
@@ -301,7 +311,7 @@ export type MaybeContinueGoalInput = {
|
|
|
301
311
|
workflowId: string;
|
|
302
312
|
};
|
|
303
313
|
export type MaybeContinueGoalResult = {
|
|
304
|
-
action: "none" | "queue" | "continue" | "paused" | "
|
|
314
|
+
action: "none" | "queue" | "continue" | "paused" | "deferred";
|
|
305
315
|
};
|
|
306
316
|
export type DispatchScheduledTaskRunInput = {
|
|
307
317
|
workspaceId: string;
|
|
@@ -0,0 +1,3 @@
|
|
|
1
|
+
import { buildXaiTurnRequestAuthorization } from "./xai-auth.js";
|
|
2
|
+
/** Refresh billing, never inference, before treating cached exhaustion as current. */
|
|
3
|
+
export declare function refreshExhaustedXaiQuota(input: Omit<Parameters<typeof buildXaiTurnRequestAuthorization>[0], "credentialId">): Promise<void>;
|
|
@@ -9,6 +9,7 @@ import {
|
|
|
9
9
|
INCIDENT_TELEMETRY_AUTHORITY_FENCE_LINEAGE_KEY,
|
|
10
10
|
MAX_AUTOMATIC_PROVIDER_RECOVERIES,
|
|
11
11
|
assertReusableSessionRevivable,
|
|
12
|
+
captureConnectedCommandOutput,
|
|
12
13
|
collectWorkspaceArchiveObjectKeys,
|
|
13
14
|
createCodexCapacityActivities,
|
|
14
15
|
createGoalActivities,
|
|
@@ -32,7 +33,7 @@ import {
|
|
|
32
33
|
scheduledUserMessagePayload,
|
|
33
34
|
withFirstPartyTools,
|
|
34
35
|
workflowIdForSession
|
|
35
|
-
} from "./chunk-
|
|
36
|
+
} from "./chunk-O2LQWJIK.js";
|
|
36
37
|
import {
|
|
37
38
|
inspectOpenSandboxKubernetesInventory,
|
|
38
39
|
recordCreditBalanceGauges,
|
|
@@ -52,7 +53,7 @@ import {
|
|
|
52
53
|
recordTurnsQueuedGauge,
|
|
53
54
|
recordWorkerDeathRecoveryMetrics,
|
|
54
55
|
runtimeMetricsHooksForObservability
|
|
55
|
-
} from "./chunk-
|
|
56
|
+
} from "./chunk-D4ZEFOYH.js";
|
|
56
57
|
|
|
57
58
|
// src/activities/browser-state-artifact-reaper.ts
|
|
58
59
|
import {
|
|
@@ -5225,6 +5226,7 @@ import {
|
|
|
5225
5226
|
workspaceArchiveCaptureDeadlineElapsed,
|
|
5226
5227
|
retainedProcessReconciliationProof,
|
|
5227
5228
|
retainedProcessSettlementIdentity,
|
|
5229
|
+
settleClaimedConnectedMachineBackgroundCommand,
|
|
5228
5230
|
settleSandboxCheckpointArtifactGc,
|
|
5229
5231
|
rlsContextForWorkspace as rlsContextForWorkspace2,
|
|
5230
5232
|
settleRetainedProcess
|
|
@@ -5232,11 +5234,10 @@ import {
|
|
|
5232
5234
|
import {
|
|
5233
5235
|
claimConnectedMachineSessionBackgroundCommands,
|
|
5234
5236
|
deferConnectedMachineBackgroundCommandReconciliation,
|
|
5235
|
-
recordConnectedMachineBackgroundCommandProof
|
|
5236
|
-
settleClaimedConnectedMachineBackgroundCommand,
|
|
5237
|
-
settleSessionBackgroundCommandForRetainedProcess
|
|
5237
|
+
recordConnectedMachineBackgroundCommandProof
|
|
5238
5238
|
} from "@opengeni/db/session-background-commands";
|
|
5239
5239
|
import { sandboxWarmRateMicrosPerSecond } from "@opengeni/config";
|
|
5240
|
+
import { appendSessionCommandOutput } from "@opengeni/db/session-command-output";
|
|
5240
5241
|
import { sandboxLeaseTelemetryKey } from "@opengeni/observability";
|
|
5241
5242
|
import {
|
|
5242
5243
|
captureVerifiedWorkspaceArchive,
|
|
@@ -5250,6 +5251,10 @@ import {
|
|
|
5250
5251
|
isExecSessionLostBanner,
|
|
5251
5252
|
isProviderSandboxNotFoundError,
|
|
5252
5253
|
NatsControlRpc,
|
|
5254
|
+
NatsOpStreamTransport,
|
|
5255
|
+
OpStreamExecClient,
|
|
5256
|
+
defaultSelfhostedRetryClock,
|
|
5257
|
+
stripExecBanner,
|
|
5253
5258
|
parseExecBannerExitCode,
|
|
5254
5259
|
prepareProviderForTeardownAfterCapture,
|
|
5255
5260
|
providerWorkspaceCapturePolicy,
|
|
@@ -5570,7 +5575,8 @@ function createSandboxLeaseActivities(services, options = {}) {
|
|
|
5570
5575
|
settings,
|
|
5571
5576
|
observability,
|
|
5572
5577
|
probeRetainedProcess,
|
|
5573
|
-
options.inspectHistoricalModalSandbox ?? inspectModalSandboxLifecycle
|
|
5578
|
+
options.inspectHistoricalModalSandbox ?? inspectModalSandboxLifecycle,
|
|
5579
|
+
service.bus
|
|
5574
5580
|
);
|
|
5575
5581
|
try {
|
|
5576
5582
|
modalOrphansTerminated = await sweepModalOrphans(settings, db, observability);
|
|
@@ -5809,86 +5815,172 @@ async function adoptLegacyModalCheckpointReceipts(db, settings, observability) {
|
|
|
5809
5815
|
});
|
|
5810
5816
|
return adopted;
|
|
5811
5817
|
}
|
|
5812
|
-
async function reconcileConnectedMachineBackgroundCommands(db, settings, observability, bus) {
|
|
5813
|
-
const
|
|
5814
|
-
|
|
5815
|
-
|
|
5816
|
-
|
|
5817
|
-
|
|
5818
|
-
|
|
5819
|
-
|
|
5820
|
-
|
|
5821
|
-
|
|
5822
|
-
|
|
5823
|
-
|
|
5824
|
-
|
|
5825
|
-
|
|
5826
|
-
|
|
5827
|
-
|
|
5828
|
-
|
|
5829
|
-
|
|
5830
|
-
|
|
5831
|
-
|
|
5832
|
-
|
|
5833
|
-
|
|
5834
|
-
|
|
5835
|
-
|
|
5836
|
-
|
|
5837
|
-
|
|
5838
|
-
|
|
5839
|
-
|
|
5840
|
-
|
|
5841
|
-
|
|
5842
|
-
|
|
5843
|
-
|
|
5844
|
-
|
|
5818
|
+
async function reconcileConnectedMachineBackgroundCommands(db, settings, observability, bus, controlRpcOverride) {
|
|
5819
|
+
const dueBefore = /* @__PURE__ */ new Date();
|
|
5820
|
+
const deadline = Date.now() + settings.sandboxLeaseReaperPeriodMs;
|
|
5821
|
+
const offlineConnections = /* @__PURE__ */ new Set();
|
|
5822
|
+
const firstConnectionProbes = /* @__PURE__ */ new Map();
|
|
5823
|
+
do {
|
|
5824
|
+
const claimId = crypto.randomUUID();
|
|
5825
|
+
let claims;
|
|
5826
|
+
try {
|
|
5827
|
+
claims = await claimConnectedMachineSessionBackgroundCommands(db, {
|
|
5828
|
+
claimId,
|
|
5829
|
+
dueBefore,
|
|
5830
|
+
limit: CONNECTED_COMMAND_RECONCILIATION_LIMIT,
|
|
5831
|
+
claimTtlMs: CONNECTED_COMMAND_RECONCILIATION_CLAIM_TTL_MS
|
|
5832
|
+
});
|
|
5833
|
+
} catch (error) {
|
|
5834
|
+
recordConnectedCommandReconciliation(observability, "claim_failed");
|
|
5835
|
+
observability.warn("sandbox reaper: connected-command claim failed", {
|
|
5836
|
+
error: error instanceof Error ? error.message : String(error)
|
|
5837
|
+
});
|
|
5838
|
+
return;
|
|
5839
|
+
}
|
|
5840
|
+
const controlRpc = controlRpcOverride ?? new NatsControlRpc(
|
|
5841
|
+
async () => bus.getRequestConnection()
|
|
5842
|
+
);
|
|
5843
|
+
await forEachWithConcurrency(claims, SANDBOX_MAINTENANCE_ITEM_CONCURRENCY, async (claim) => {
|
|
5844
|
+
let proof = claim.proof;
|
|
5845
|
+
const connectionKey = JSON.stringify([
|
|
5846
|
+
claim.controlWorkspaceId,
|
|
5847
|
+
claim.enrollmentId,
|
|
5848
|
+
claim.connectionInstanceId
|
|
5849
|
+
]);
|
|
5850
|
+
if (!proof) {
|
|
5851
|
+
const firstProbe = firstConnectionProbes.get(connectionKey);
|
|
5852
|
+
if (firstProbe) await firstProbe;
|
|
5853
|
+
if (offlineConnections.has(connectionKey)) {
|
|
5854
|
+
await deferConnectedCommandClaim(db, settings, observability, claim, "provider_offline");
|
|
5845
5855
|
return;
|
|
5846
5856
|
}
|
|
5847
|
-
|
|
5848
|
-
|
|
5849
|
-
|
|
5850
|
-
|
|
5857
|
+
let finishFirstProbe;
|
|
5858
|
+
if (!firstProbe) {
|
|
5859
|
+
firstConnectionProbes.set(
|
|
5860
|
+
connectionKey,
|
|
5861
|
+
new Promise((resolve) => {
|
|
5862
|
+
finishFirstProbe = resolve;
|
|
5863
|
+
})
|
|
5864
|
+
);
|
|
5865
|
+
}
|
|
5866
|
+
try {
|
|
5867
|
+
proof = await probeConnectedMachineBackgroundCommand(
|
|
5868
|
+
claim,
|
|
5869
|
+
controlRpc,
|
|
5870
|
+
Math.min(
|
|
5871
|
+
settings.sandboxSelfhostedControlTimeoutMs,
|
|
5872
|
+
RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS
|
|
5873
|
+
)
|
|
5874
|
+
);
|
|
5875
|
+
if (proof?.outcome !== "lost") {
|
|
5876
|
+
if (!bus.getOpStreamConnection)
|
|
5877
|
+
throw new Error("Connected command output transport unavailable");
|
|
5878
|
+
const client = new OpStreamExecClient({
|
|
5879
|
+
workspaceId: claim.controlWorkspaceId,
|
|
5880
|
+
agentId: claim.enrollmentId,
|
|
5881
|
+
connectionInstanceId: claim.connectionInstanceId,
|
|
5882
|
+
epoch: 0,
|
|
5883
|
+
controlRpc,
|
|
5884
|
+
rpcSubject: subjectFor(
|
|
5885
|
+
claim.controlWorkspaceId,
|
|
5886
|
+
claim.enrollmentId,
|
|
5887
|
+
claim.connectionInstanceId
|
|
5888
|
+
),
|
|
5889
|
+
transport: new NatsOpStreamTransport(
|
|
5890
|
+
async () => bus.getOpStreamConnection?.() ?? null
|
|
5891
|
+
),
|
|
5892
|
+
controlTimeoutMs: Math.min(
|
|
5893
|
+
settings.sandboxSelfhostedControlTimeoutMs,
|
|
5894
|
+
RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS
|
|
5895
|
+
),
|
|
5896
|
+
retryClock: defaultSelfhostedRetryClock,
|
|
5897
|
+
journal: { attachGeneration: () => String(Date.now()), persistSettled: () => {
|
|
5898
|
+
} }
|
|
5899
|
+
});
|
|
5900
|
+
const replay = await client.readExisting(
|
|
5901
|
+
claim.opId,
|
|
5902
|
+
proof ? 5e3 : 250,
|
|
5903
|
+
async (frames) => {
|
|
5904
|
+
await captureConnectedCommandOutput(db, claim, bus)(claim.commandId, frames);
|
|
5905
|
+
}
|
|
5906
|
+
);
|
|
5907
|
+
if (proof && replay.status !== "completed" && !replay.terminal) {
|
|
5908
|
+
throw new Error(
|
|
5909
|
+
"Connected command terminal output replay has not reached its exit frontier"
|
|
5910
|
+
);
|
|
5911
|
+
}
|
|
5912
|
+
}
|
|
5913
|
+
if (!proof) {
|
|
5914
|
+
await deferConnectedCommandClaim(
|
|
5915
|
+
db,
|
|
5916
|
+
settings,
|
|
5917
|
+
observability,
|
|
5918
|
+
claim,
|
|
5919
|
+
"provider_running"
|
|
5920
|
+
);
|
|
5921
|
+
return;
|
|
5922
|
+
}
|
|
5923
|
+
} catch (error) {
|
|
5924
|
+
const offline = error instanceof SelfhostedControlError && (error.agentOffline || error.reason === "agent_reconnecting" || error.draining);
|
|
5925
|
+
if (offline) offlineConnections.add(connectionKey);
|
|
5926
|
+
if (!offline) {
|
|
5927
|
+
observability.warn("sandbox reaper: connected-command provider probe failed", {
|
|
5928
|
+
commandId: claim.commandId,
|
|
5929
|
+
enrollmentId: claim.enrollmentId,
|
|
5930
|
+
connectionInstanceId: claim.connectionInstanceId,
|
|
5931
|
+
error: error instanceof Error ? error.message : String(error)
|
|
5932
|
+
});
|
|
5933
|
+
}
|
|
5934
|
+
await deferConnectedCommandClaim(
|
|
5935
|
+
db,
|
|
5936
|
+
settings,
|
|
5937
|
+
observability,
|
|
5938
|
+
claim,
|
|
5939
|
+
offline ? "provider_offline" : "provider_error"
|
|
5940
|
+
);
|
|
5941
|
+
return;
|
|
5942
|
+
} finally {
|
|
5943
|
+
finishFirstProbe?.();
|
|
5944
|
+
}
|
|
5945
|
+
try {
|
|
5946
|
+
await recordConnectedMachineBackgroundCommandProof(db, { claim, proof });
|
|
5947
|
+
recordConnectedCommandReconciliation(observability, `proof_${proof.outcome}`);
|
|
5948
|
+
} catch (error) {
|
|
5949
|
+
recordConnectedCommandReconciliation(observability, "proof_checkpoint_failed");
|
|
5950
|
+
observability.warn("sandbox reaper: connected-command proof checkpoint failed", {
|
|
5851
5951
|
commandId: claim.commandId,
|
|
5852
|
-
enrollmentId: claim.enrollmentId,
|
|
5853
|
-
connectionInstanceId: claim.connectionInstanceId,
|
|
5854
5952
|
error: error instanceof Error ? error.message : String(error)
|
|
5855
5953
|
});
|
|
5954
|
+
return;
|
|
5856
5955
|
}
|
|
5857
|
-
await deferConnectedCommandClaim(
|
|
5858
|
-
db,
|
|
5859
|
-
settings,
|
|
5860
|
-
observability,
|
|
5861
|
-
claim,
|
|
5862
|
-
offline ? "provider_offline" : "provider_error"
|
|
5863
|
-
);
|
|
5864
|
-
return;
|
|
5865
5956
|
}
|
|
5866
5957
|
try {
|
|
5867
|
-
await
|
|
5868
|
-
|
|
5958
|
+
const settlement = await settleClaimedConnectedMachineBackgroundCommand(db, { claim });
|
|
5959
|
+
if (!settlement.settled) {
|
|
5960
|
+
throw new Error("Connected command settlement lost its exact claim or proof");
|
|
5961
|
+
}
|
|
5962
|
+
if (settlement.events.length > 0) {
|
|
5963
|
+
try {
|
|
5964
|
+
await bus.publish(claim.workspaceId, claim.sessionId, settlement.events);
|
|
5965
|
+
} catch (error) {
|
|
5966
|
+
observability.warn("sandbox reaper: connected-command event fanout failed", {
|
|
5967
|
+
commandId: claim.commandId,
|
|
5968
|
+
error: error instanceof Error ? error.message : String(error)
|
|
5969
|
+
});
|
|
5970
|
+
}
|
|
5971
|
+
}
|
|
5972
|
+
recordConnectedCommandReconciliation(observability, `settled_${proof.outcome}`);
|
|
5869
5973
|
} catch (error) {
|
|
5870
|
-
recordConnectedCommandReconciliation(observability, "
|
|
5871
|
-
observability.warn("sandbox reaper: connected-command
|
|
5974
|
+
recordConnectedCommandReconciliation(observability, "settlement_failed");
|
|
5975
|
+
observability.warn("sandbox reaper: connected-command settlement failed", {
|
|
5872
5976
|
commandId: claim.commandId,
|
|
5873
5977
|
error: error instanceof Error ? error.message : String(error)
|
|
5874
5978
|
});
|
|
5875
|
-
|
|
5979
|
+
await deferConnectedCommandClaim(db, settings, observability, claim, "settlement_failed");
|
|
5876
5980
|
}
|
|
5877
|
-
}
|
|
5878
|
-
|
|
5879
|
-
|
|
5880
|
-
throw new Error("Connected command settlement lost its exact claim or proof");
|
|
5881
|
-
}
|
|
5882
|
-
recordConnectedCommandReconciliation(observability, `settled_${proof.outcome}`);
|
|
5883
|
-
} catch (error) {
|
|
5884
|
-
recordConnectedCommandReconciliation(observability, "settlement_failed");
|
|
5885
|
-
observability.warn("sandbox reaper: connected-command settlement failed", {
|
|
5886
|
-
commandId: claim.commandId,
|
|
5887
|
-
error: error instanceof Error ? error.message : String(error)
|
|
5888
|
-
});
|
|
5889
|
-
await deferConnectedCommandClaim(db, settings, observability, claim, "settlement_failed");
|
|
5890
|
-
}
|
|
5891
|
-
});
|
|
5981
|
+
});
|
|
5982
|
+
if (claims.length < CONNECTED_COMMAND_RECONCILIATION_LIMIT) break;
|
|
5983
|
+
} while (Date.now() < deadline);
|
|
5892
5984
|
}
|
|
5893
5985
|
async function probeConnectedMachineBackgroundCommand(claim, controlRpc, controlTimeoutMs, observedAt = /* @__PURE__ */ new Date()) {
|
|
5894
5986
|
const input = {
|
|
@@ -5915,11 +6007,14 @@ function connectedCommandProofFromStatus(status, observedAt) {
|
|
|
5915
6007
|
throw new Error("Connected command completed without an exit record");
|
|
5916
6008
|
}
|
|
5917
6009
|
const failureCode = exit.failureCode.replace(/[^A-Za-z0-9_-]/g, "_").slice(0, 128);
|
|
5918
|
-
const reason = exit.cancelled ? "op_cancelled" : exit.timedOut ? "op_timed_out" :
|
|
6010
|
+
const reason = failureCode ? `op_failure_${failureCode}` : exit.cancelled ? "op_cancelled" : exit.timedOut ? "op_timed_out" : "op_exit";
|
|
5919
6011
|
return {
|
|
5920
6012
|
outcome: "exited",
|
|
5921
6013
|
exitCode: exit.exitCode,
|
|
5922
6014
|
reason,
|
|
6015
|
+
...failureCode ? {
|
|
6016
|
+
failure: { code: failureCode, detail: exit.failureDetail, retryable: false }
|
|
6017
|
+
} : {},
|
|
5923
6018
|
observedAt
|
|
5924
6019
|
};
|
|
5925
6020
|
}
|
|
@@ -5957,7 +6052,7 @@ async function deferConnectedCommandClaim(db, settings, observability, claim, ou
|
|
|
5957
6052
|
});
|
|
5958
6053
|
}
|
|
5959
6054
|
}
|
|
5960
|
-
async function reconcileTerminalRetainedProcesses(db, settings, observability, probe, inspectHistoricalModalSandbox) {
|
|
6055
|
+
async function reconcileTerminalRetainedProcesses(db, settings, observability, probe, inspectHistoricalModalSandbox, bus) {
|
|
5961
6056
|
const claimId = crypto.randomUUID();
|
|
5962
6057
|
let claims;
|
|
5963
6058
|
try {
|
|
@@ -6064,7 +6159,23 @@ async function reconcileTerminalRetainedProcesses(db, settings, observability, p
|
|
|
6064
6159
|
settings,
|
|
6065
6160
|
lease,
|
|
6066
6161
|
process,
|
|
6067
|
-
claim.ownerState === "background_stopping" ? "cancel" : "observe"
|
|
6162
|
+
claim.ownerState === "background_stopping" ? "cancel" : "observe",
|
|
6163
|
+
async (result, chunkId) => {
|
|
6164
|
+
if (typeof result !== "string" || isExecSessionLostBanner(result, process.providerSessionId))
|
|
6165
|
+
return;
|
|
6166
|
+
const events = await appendSessionCommandOutput(db, {
|
|
6167
|
+
accountId: process.accountId,
|
|
6168
|
+
workspaceId: process.workspaceId,
|
|
6169
|
+
sessionId: process.sessionId,
|
|
6170
|
+
commandId: process.id,
|
|
6171
|
+
chunkId,
|
|
6172
|
+
stream: "stdout",
|
|
6173
|
+
streamFidelity: "merged",
|
|
6174
|
+
chunk: stripExecBanner(result)
|
|
6175
|
+
});
|
|
6176
|
+
if (events.length && bus)
|
|
6177
|
+
await bus.publish(process.workspaceId, process.sessionId, events).catch(() => void 0);
|
|
6178
|
+
}
|
|
6068
6179
|
);
|
|
6069
6180
|
} catch (error) {
|
|
6070
6181
|
observability.warn("sandbox reaper: retained-process provider probe failed", {
|
|
@@ -6147,15 +6258,20 @@ async function reconcileTerminalRetainedProcesses(db, settings, observability, p
|
|
|
6147
6258
|
if (settlement.process.state === "active") {
|
|
6148
6259
|
throw new Error("Retained-process reconciliation returned an active durable process");
|
|
6149
6260
|
}
|
|
6150
|
-
|
|
6151
|
-
|
|
6152
|
-
|
|
6153
|
-
|
|
6154
|
-
|
|
6155
|
-
|
|
6156
|
-
|
|
6157
|
-
|
|
6158
|
-
|
|
6261
|
+
if (settlement.backgroundCommandEvents.length > 0) {
|
|
6262
|
+
try {
|
|
6263
|
+
await bus.publish(
|
|
6264
|
+
process.workspaceId,
|
|
6265
|
+
process.sessionId,
|
|
6266
|
+
settlement.backgroundCommandEvents
|
|
6267
|
+
);
|
|
6268
|
+
} catch (error) {
|
|
6269
|
+
observability.warn("sandbox reaper: retained-command event fanout failed", {
|
|
6270
|
+
processId: process.id,
|
|
6271
|
+
error: error instanceof Error ? error.message : String(error)
|
|
6272
|
+
});
|
|
6273
|
+
}
|
|
6274
|
+
}
|
|
6159
6275
|
recordRetainedProcessReconciliation(observability, `settled_${proof.outcome}`);
|
|
6160
6276
|
} catch (error) {
|
|
6161
6277
|
recordRetainedProcessReconciliation(observability, "settlement_failed");
|
|
@@ -6237,13 +6353,29 @@ async function withRetainedProcessProbeTimeout(promise) {
|
|
|
6237
6353
|
if (timer) clearTimeout(timer);
|
|
6238
6354
|
}
|
|
6239
6355
|
}
|
|
6240
|
-
|
|
6356
|
+
var pendingRetainedProbeOutput = /* @__PURE__ */ new Map();
|
|
6357
|
+
async function captureRetainedProbeOutput(processId, result, capture) {
|
|
6358
|
+
const pending = pendingRetainedProbeOutput.get(processId) ?? {
|
|
6359
|
+
result,
|
|
6360
|
+
chunkId: crypto.randomUUID()
|
|
6361
|
+
};
|
|
6362
|
+
pendingRetainedProbeOutput.set(processId, pending);
|
|
6363
|
+
await capture(pending.result, pending.chunkId);
|
|
6364
|
+
pendingRetainedProbeOutput.delete(processId);
|
|
6365
|
+
}
|
|
6366
|
+
async function probeRetainedProcessAtProvider(settings, lease, process, mode = "observe", captureOutput) {
|
|
6241
6367
|
if (lease.id !== process.leaseId || lease.sandboxGroupId !== process.sandboxGroupId || lease.leaseEpoch !== process.leaseEpoch || lease.backend !== process.providerBackend || lease.instanceId !== process.providerInstanceId || lease.resumeBackendId !== process.providerBackend || process.routeTargetId !== null) {
|
|
6242
6368
|
return { status: "deferred", reason: "identity_mismatch" };
|
|
6243
6369
|
}
|
|
6244
6370
|
if (!lease.resumeState) {
|
|
6245
6371
|
return { status: "deferred", reason: "resume_state_missing" };
|
|
6246
6372
|
}
|
|
6373
|
+
const pending = pendingRetainedProbeOutput.get(process.id);
|
|
6374
|
+
if (pending) {
|
|
6375
|
+
if (!captureOutput) throw new Error("Pending command output requires its persistence callback");
|
|
6376
|
+
await captureRetainedProbeOutput(process.id, pending.result, captureOutput);
|
|
6377
|
+
return classifyRetainedProcessPollResult(pending.result, process.providerSessionId);
|
|
6378
|
+
}
|
|
6247
6379
|
const envelopeBackend = lease.resumeState.backendId;
|
|
6248
6380
|
if (envelopeBackend !== void 0 && envelopeBackend !== process.providerBackend) {
|
|
6249
6381
|
return { status: "deferred", reason: "identity_mismatch" };
|
|
@@ -6375,6 +6507,7 @@ async function probeRetainedProcessAtProvider(settings, lease, process, mode = "
|
|
|
6375
6507
|
}
|
|
6376
6508
|
return { status: "deferred", reason: "provider_error" };
|
|
6377
6509
|
}
|
|
6510
|
+
if (captureOutput) await captureRetainedProbeOutput(process.id, result, captureOutput);
|
|
6378
6511
|
const observation = classifyRetainedProcessPollResult(result, process.providerSessionId);
|
|
6379
6512
|
if (observation.status === "deferred" && observation.reason === "provider_running" && lease.rotationRequestedAt !== null) {
|
|
6380
6513
|
try {
|
|
@@ -6386,6 +6519,7 @@ async function probeRetainedProcessAtProvider(settings, lease, process, mode = "
|
|
|
6386
6519
|
maxOutputTokens: 2e3
|
|
6387
6520
|
})
|
|
6388
6521
|
);
|
|
6522
|
+
if (captureOutput) await captureRetainedProbeOutput(process.id, interrupted, captureOutput);
|
|
6389
6523
|
return classifyRetainedProcessPollResult(interrupted, process.providerSessionId);
|
|
6390
6524
|
} catch (error) {
|
|
6391
6525
|
if (isProviderSandboxNotFoundError(client.backendId, error)) {
|
|
@@ -9592,6 +9726,7 @@ import {
|
|
|
9592
9726
|
recoverSessionDispatch,
|
|
9593
9727
|
reconcileSessionAttemptQuiescence,
|
|
9594
9728
|
peekSessionWork as peekSessionWorkDb,
|
|
9729
|
+
settleSessionInputWait as settleSessionInputWaitDb,
|
|
9595
9730
|
countQueuedTurns as countQueuedTurns2,
|
|
9596
9731
|
getSessionAttemptActivityRef,
|
|
9597
9732
|
getSessionEvent,
|
|
@@ -9613,6 +9748,7 @@ function createSessionStateActivities(services, overrides = {}) {
|
|
|
9613
9748
|
const recoverSessionDispatchFn = overrides.recoverSessionDispatch ?? recoverSessionDispatch;
|
|
9614
9749
|
const reconcileSessionAttemptQuiescenceFn = overrides.reconcileSessionAttemptQuiescence ?? reconcileSessionAttemptQuiescence;
|
|
9615
9750
|
const peekSessionWorkFn = overrides.peekSessionWork ?? peekSessionWorkDb;
|
|
9751
|
+
const settleSessionInputWaitFn = overrides.settleSessionInputWait ?? settleSessionInputWaitDb;
|
|
9616
9752
|
const countQueuedTurnsFn = overrides.countQueuedTurns ?? countQueuedTurns2;
|
|
9617
9753
|
const getSessionAttemptActivityRefFn = overrides.getSessionAttemptActivityRef ?? getSessionAttemptActivityRef;
|
|
9618
9754
|
const getSessionEventFn = overrides.getSessionEvent ?? getSessionEvent;
|
|
@@ -9697,7 +9833,7 @@ function createSessionStateActivities(services, overrides = {}) {
|
|
|
9697
9833
|
if (providerRecoveryCount !== void 0 && (!Number.isSafeInteger(providerRecoveryCount) || providerRecoveryCount <= 0 || providerRecoveryCount !== providerRecoveryCountFromMetadata(turn.metadata ?? {}) + 1 || providerRecoveryCount > MAX_AUTOMATIC_PROVIDER_RECOVERIES || typeof providerFailureCode !== "string" || !/^[a-z][a-z0-9_]{0,63}$/.test(providerFailureCode))) {
|
|
9698
9834
|
return { action: "stale" };
|
|
9699
9835
|
}
|
|
9700
|
-
const recoveredClaimCode = postClaimIdentityMatches ? postClaimRecovery.code : input.preClaimFailure?.disposition === "retryable" && input.preClaimFailure.code !== "claim_invariant" ? input.preClaimFailure.code : null;
|
|
9836
|
+
const recoveredClaimCode = postClaimIdentityMatches ? postClaimRecovery.code : input.preClaimFailure?.disposition === "retryable" && input.preClaimFailure.code !== "claim_invariant" ? input.preClaimFailure.code : input.preClaimFailureDisposition === "retryable" ? "legacy_retryable_preclaim_database_failure" : null;
|
|
9701
9837
|
if (recoveredClaimCode) {
|
|
9702
9838
|
const recovery = await requestSessionTurnRecoveryFn(db, input.workspaceId, {
|
|
9703
9839
|
sessionId: input.sessionId,
|
|
@@ -9915,6 +10051,14 @@ function createSessionStateActivities(services, overrides = {}) {
|
|
|
9915
10051
|
await refreshQueuedTurnsGauge(db, observability, countQueuedTurnsFn, recordTurnsQueuedGaugeFn);
|
|
9916
10052
|
return peek;
|
|
9917
10053
|
}
|
|
10054
|
+
async function settleSessionInputWait(input) {
|
|
10055
|
+
const { db, bus } = await services();
|
|
10056
|
+
const result = await settleSessionInputWaitFn(db, input);
|
|
10057
|
+
if (result.events.length > 0) {
|
|
10058
|
+
await publishDurableSessionEventsFn(bus, input.workspaceId, input.sessionId, result.events);
|
|
10059
|
+
}
|
|
10060
|
+
return { action: result.action };
|
|
10061
|
+
}
|
|
9918
10062
|
async function expireSessionHumanInput(input) {
|
|
9919
10063
|
const { db, bus } = await services();
|
|
9920
10064
|
const result = await expireSessionHumanInputRequestFn(db, input);
|
|
@@ -9963,6 +10107,7 @@ function createSessionStateActivities(services, overrides = {}) {
|
|
|
9963
10107
|
recoverDispatch,
|
|
9964
10108
|
recoverEscapedMcpTimeout,
|
|
9965
10109
|
peekSessionWork,
|
|
10110
|
+
settleSessionInputWait,
|
|
9966
10111
|
expireSessionHumanInput,
|
|
9967
10112
|
expireSessionInteractionIntervention,
|
|
9968
10113
|
markSessionIdle
|
|
@@ -10054,4 +10199,4 @@ export {
|
|
|
10054
10199
|
createControlActivities,
|
|
10055
10200
|
createControlActivitiesFromServices
|
|
10056
10201
|
};
|
|
10057
|
-
//# sourceMappingURL=activities-control-
|
|
10202
|
+
//# sourceMappingURL=activities-control-KS3ZPQVY.js.map
|