@tangle-network/agent-runtime 0.274.0 → 0.279.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{activation-DiW4KBY-.d.ts → activation-B5NExoOz.d.ts} +137 -105
- package/dist/{activation-t5vg1Zyu.js → activation-BkUdrvqh.js} +2 -2
- package/dist/{activation-t5vg1Zyu.js.map → activation-BkUdrvqh.js.map} +1 -1
- package/dist/agent.d.ts +2 -2
- package/dist/agent.js +2 -2
- package/dist/{coordination-driver-B2GTcMvx.js → coordination-driver-BBL_OgRw.js} +95 -28
- package/dist/coordination-driver-BBL_OgRw.js.map +1 -0
- package/dist/{delegate-Bmt7uynp.js → delegate-CZc7B-Tl.js} +2 -2
- package/dist/{delegate-Bmt7uynp.js.map → delegate-CZc7B-Tl.js.map} +1 -1
- package/dist/durable.d.ts +2 -2
- package/dist/durable.js +2 -2
- package/dist/{graph-9khZscYU.js → graph-Ci8Ufs_f.js} +4 -4
- package/dist/{graph-9khZscYU.js.map → graph-Ci8Ufs_f.js.map} +1 -1
- package/dist/{improvement-cycle-oQFOTgLd.js → improvement-cycle-Bi43xCVa.js} +649 -213
- package/dist/improvement-cycle-Bi43xCVa.js.map +1 -0
- package/dist/{index-CjLtO4B3.d.ts → index-Dm8SHDGW.d.ts} +51 -17
- package/dist/index.d.ts +9 -9
- package/dist/index.js +55 -14
- package/dist/index.js.map +1 -1
- package/dist/intelligence.d.ts +3 -3
- package/dist/intelligence.js +3 -3
- package/dist/kernel.d.ts +4 -4
- package/dist/kernel.js +8 -8
- package/dist/{loop-runner-bin-C-rco77U.js → loop-runner-bin-C9FJshTP.js} +3 -3
- package/dist/{loop-runner-bin-C-rco77U.js.map → loop-runner-bin-C9FJshTP.js.map} +1 -1
- package/dist/{loop-runner-bin-Ceehl2oB.d.ts → loop-runner-bin-D6khQtoC.d.ts} +3 -3
- package/dist/loop-runner-bin.d.ts +1 -1
- package/dist/loop-runner-bin.js +1 -1
- package/dist/mcp/bin.js +3 -3
- package/dist/mcp/index.d.ts +5 -5
- package/dist/mcp/index.js +4 -4
- package/dist/{openai-tools-D8ZYHReZ.d.ts → openai-tools-H8QPhTVE.d.ts} +2 -2
- package/dist/platform.d.ts +91 -1
- package/dist/platform.js +162 -6
- package/dist/platform.js.map +1 -1
- package/dist/profiles.d.ts +2 -2
- package/dist/{provision-supervisor-Ds43T8XY.js → provision-supervisor-BvnihiyB.js} +3 -3
- package/dist/{provision-supervisor-Ds43T8XY.js.map → provision-supervisor-BvnihiyB.js.map} +1 -1
- package/dist/{runtime-CtrgIqQm.js → runtime-DsAHXig2.js} +73 -26
- package/dist/runtime-DsAHXig2.js.map +1 -0
- package/dist/{server-DnSzxChA.js → server-Dyc1zvug.js} +3 -3
- package/dist/{server-DnSzxChA.js.map → server-Dyc1zvug.js.map} +1 -1
- package/dist/{stream-agent-turn-C2rtL6P2.d.ts → stream-agent-turn-Bk79CnTw.d.ts} +2 -2
- package/dist/{structural-rollout-DN8CVdpp.js → structural-rollout-D-0X6lsU.js} +2 -2
- package/dist/{structural-rollout-DN8CVdpp.js.map → structural-rollout-D-0X6lsU.js.map} +1 -1
- package/dist/{substrate-DPIUXsZ5.d.ts → substrate-CHQ5GYnU.d.ts} +2 -2
- package/dist/{supervise-CFyAWebz.js → supervise-DBQdrp7H.js} +171 -13
- package/dist/supervise-DBQdrp7H.js.map +1 -0
- package/dist/{supervisor-C50SI_Ww.js → supervisor-DtlPj9me.js} +372 -77
- package/dist/supervisor-DtlPj9me.js.map +1 -0
- package/dist/testing.d.ts +2 -2
- package/dist/testing.js +60 -60
- package/dist/tui/index.d.ts +1 -1
- package/dist/tui/index.js +1 -1
- package/dist/{types-r7Fzum3M.d.ts → types-D56jQad-.d.ts} +92 -66
- package/package.json +10 -7
- package/dist/coordination-driver-B2GTcMvx.js.map +0 -1
- package/dist/improvement-cycle-oQFOTgLd.js.map +0 -1
- package/dist/runtime-CtrgIqQm.js.map +0 -1
- package/dist/supervise-CFyAWebz.js.map +0 -1
- package/dist/supervisor-C50SI_Ww.js.map +0 -1
|
@@ -3,7 +3,7 @@ import { a as concreteProfileModel, c as profileModelExecutionSettings, d as age
|
|
|
3
3
|
import { i as verifyAgentCandidateWorkspaceArchive, x as verifyWorkspaceSnapshotArtifacts } from "./workspace-archive-C9lgf77y.js";
|
|
4
4
|
import { i as notifyRuntimeHookEvent } from "./runtime-hooks-tXpAarhW.js";
|
|
5
5
|
import { n as assertNoSymlinkDescendant, r as publishExclusiveDurableFile } from "./durable-file-DWQA4ooo.js";
|
|
6
|
-
import { AGENT_PROFILE_MATERIALIZATION_AXES, AgentEnvironmentCapabilitiesSchema, AgentExactRunControlRefSchema, AgentInteractiveSessionControlClaimAcknowledgementSchema, AgentInteractiveSessionControlClaimSchema, AgentInteractiveSessionPromptAcknowledgementSchema, AgentInteractiveSessionRefSchema, AgentInteractiveSessionStartSchema, AgentInteractiveSessionStatusSchema, AgentInteractiveSessionStopAcknowledgementSchema, AgentNativeContextContinuationResultSchema, AgentRunCancellationAcknowledgementSchema, AgentRunCancellationRequestSchema, AgentTurnInputSchema, AgentTurnResultSchema, CanonicalStreamEventSchema, ContextTransferRequestSchema, InteractionAcknowledgementSchema, InteractionResponseCommandSchema, NativeContextBoundaryProofSchema, NativeContextContinuationRequestSchema, REASONING_EFFORTS, RuntimeEventEnvelopeSchema, TerminalReplayWindowSchema, TerminalSessionRefSchema, WorkspaceRequestSchema, agentCandidateWorkspaceSnapshotEvidenceSchema, agentInteractiveSessionControlClaimAcknowledgementMatchesRequest, agentInteractiveSessionControlClaimMatchesRef, agentInteractiveSessionPromptAcknowledgementMatchesCommand, agentInteractiveSessionRefMatchesStart, agentInteractiveSessionRunRef, agentInteractiveSessionStatusMatchesRef, agentInteractiveSessionStopAcknowledgementMatchesCommand, agentNativeContextContinuationResultMatchesRequest, agentProfileSchema, agentRunCancellationAcknowledgementMatchesRequest, agentRunCancellationRequestDigest, canonicalAgentProfileDigest, canonicalCandidateDigest, canonicalWorkspaceCwd, exactAgentInteractiveSessionStart, harnessSystemPromptIntents, harnessTypeSchema, nativeContextContinuationTurnDigest, nativeReasoningControl, profileMaterializationAxes, profileMaterializationAxes as profileMaterializationAxes$1, renderInputPartsAsText, sha256DigestSchema, workspaceCwdPathForBase } from "@tangle-network/agent-interface";
|
|
6
|
+
import { AGENT_PROFILE_MATERIALIZATION_AXES, AgentEnvironmentCapabilitiesSchema, AgentExactRunControlRefSchema, AgentInteractiveSessionControlClaimAcknowledgementSchema, AgentInteractiveSessionControlClaimSchema, AgentInteractiveSessionPromptAcknowledgementSchema, AgentInteractiveSessionRefSchema, AgentInteractiveSessionStartSchema, AgentInteractiveSessionStatusSchema, AgentInteractiveSessionStopAcknowledgementSchema, AgentNativeContextContinuationResultSchema, AgentRunCancellationAcknowledgementSchema, AgentRunCancellationRequestSchema, AgentTurnInputSchema, AgentTurnResultSchema, CanonicalStreamEventSchema, ContextTransferRequestSchema, InteractionAcknowledgementSchema, InteractionResponseCommandSchema, NativeContextBoundaryProofSchema, NativeContextContinuationRequestSchema, REASONING_EFFORTS, RuntimeEventEnvelopeSchema, TerminalReplayWindowSchema, TerminalSessionRefSchema, WorkspaceRequestSchema, agentCandidateWorkspaceSnapshotEvidenceSchema, agentInteractiveSessionControlClaimAcknowledgementMatchesRequest, agentInteractiveSessionControlClaimMatchesRef, agentInteractiveSessionPromptAcknowledgementMatchesCommand, agentInteractiveSessionRefMatchesStart, agentInteractiveSessionRunRef, agentInteractiveSessionStatusMatchesRef, agentInteractiveSessionStopAcknowledgementMatchesCommand, agentNativeContextContinuationResultMatchesRequest, agentProfileSchema, agentRunCancellationAcknowledgementMatchesRequest, agentRunCancellationRequestDigest, canonicalAgentProfileDigest, canonicalCandidateDigest, canonicalWorkspaceCwd, exactAgentInteractiveSessionStart, harnessSystemPromptIntents, harnessTypeSchema, nativeContextContinuationTurnDigest, nativeReasoningControl, profileMaterializationAxes, profileMaterializationAxes as profileMaterializationAxes$1, renderInputPartsAsText, sha256DigestSchema, workspaceCheckpointRequestDigest, workspaceCheckpointResultMatchesRequest, workspaceCleanupAcknowledgementMatches, workspaceCleanupRequestDigest, workspaceCwdPathForBase } from "@tangle-network/agent-interface";
|
|
7
7
|
import { estimateCost, isModelPriced, isToolSpan, toolSpansToTraceAnalysisStore } from "@tangle-network/agent-eval";
|
|
8
8
|
import { createAgentRunOutcomeTracker } from "@tangle-network/sandbox/runtime";
|
|
9
9
|
import { createHash, randomBytes, randomUUID } from "node:crypto";
|
|
@@ -1622,7 +1622,8 @@ function buildLlmCall(data, agentRunName) {
|
|
|
1622
1622
|
const promptCache = readPromptCacheUsage(data);
|
|
1623
1623
|
const tokensKnown = explicitTokensKnown !== false && tokensIn !== void 0 && outputTokens !== void 0;
|
|
1624
1624
|
const usdKnown = explicitCostKnown !== false && costUsd !== void 0 && (costProvenance === void 0 || costProvenance === "provider-receipt" || costProvenance === "billing-receipt");
|
|
1625
|
-
|
|
1625
|
+
const tools = readOfferedTools(data.tools);
|
|
1626
|
+
if (tokensIn === void 0 && tokensOut === void 0 && costUsd === void 0 && catalogEstimate === void 0 && promptCache === void 0 && tools === void 0 && explicitTokensKnown !== false && explicitCostKnown !== false) return;
|
|
1626
1627
|
const event = {
|
|
1627
1628
|
type: "llm_call",
|
|
1628
1629
|
model: typeof data.model === "string" && data.model.length > 0 ? data.model : agentRunName
|
|
@@ -1634,8 +1635,17 @@ function buildLlmCall(data, agentRunName) {
|
|
|
1634
1635
|
if (!usdKnown) event.usdKnown = false;
|
|
1635
1636
|
if (catalogEstimate !== void 0) event.estimatedCostUsd = catalogEstimate;
|
|
1636
1637
|
if (promptCache !== void 0) event.promptCache = promptCache;
|
|
1638
|
+
if (tools !== void 0) event.tools = tools;
|
|
1637
1639
|
return event;
|
|
1638
1640
|
}
|
|
1641
|
+
/** `data.tools`, when it is a non-empty array of non-empty tool-name strings. Anything else
|
|
1642
|
+
* (absent, empty, wrong element type) is treated as "the producer did not report its offered
|
|
1643
|
+
* set" — never a placeholder empty list, which would read as "offered zero tools" downstream. */
|
|
1644
|
+
function readOfferedTools(value) {
|
|
1645
|
+
if (!Array.isArray(value) || value.length === 0) return void 0;
|
|
1646
|
+
const names = value.filter((v) => typeof v === "string" && v.length > 0);
|
|
1647
|
+
return names.length === value.length ? names : void 0;
|
|
1648
|
+
}
|
|
1639
1649
|
/**
|
|
1640
1650
|
* Every wire spelling of a prompt-cache READ counter, in precedence order: the canonical name
|
|
1641
1651
|
* first, then the spellings observed on the paths that reach this extractor — the sandbox terminal
|
|
@@ -5142,6 +5152,10 @@ function createRuntimeStreamEventCollector(options = {}) {
|
|
|
5142
5152
|
* The exporter accepts both raw OtelSpan objects and LoopTraceEvents
|
|
5143
5153
|
* (which get converted to OTLP spans automatically).
|
|
5144
5154
|
*/
|
|
5155
|
+
/** OTel GenAI: the tool definitions offered to the model. Matches the constant agent-eval and
|
|
5156
|
+
* traces each define locally for the same external convention (there is no shared semconv
|
|
5157
|
+
* package to import it from). */
|
|
5158
|
+
const TOOL_DEFINITIONS_ATTR = "gen_ai.tool.definitions";
|
|
5145
5159
|
const SCOPE = {
|
|
5146
5160
|
name: "@tangle-network/agent-runtime",
|
|
5147
5161
|
version: "0.83.0"
|
|
@@ -5519,6 +5533,10 @@ function buildRuntimeEventOtelSpans(events, traceId, parentSpanId, options = {})
|
|
|
5519
5533
|
if (event.costUsd !== void 0) attrs["tangle.cost.usd"] = event.costUsd;
|
|
5520
5534
|
if (event.latencyMs !== void 0) attrs["tangle.latency_ms"] = event.latencyMs;
|
|
5521
5535
|
if (event.finishReason !== void 0) attrs["gen_ai.response.finish_reasons"] = event.finishReason;
|
|
5536
|
+
if (event.tools !== void 0 && event.tools.length > 0) attrs[TOOL_DEFINITIONS_ATTR] = JSON.stringify(event.tools.map((toolName) => ({
|
|
5537
|
+
type: "function",
|
|
5538
|
+
name: toolName
|
|
5539
|
+
})));
|
|
5522
5540
|
} else if (event.type === "backend_error") {
|
|
5523
5541
|
attrs["error.type"] = event.error?.kind ?? "backend";
|
|
5524
5542
|
attrs["error.message"] = event.message;
|
|
@@ -5821,71 +5839,6 @@ function generateSpanId() {
|
|
|
5821
5839
|
else for (let i = 0; i < 8; i++) bytes[i] = Math.floor(Math.random() * 256);
|
|
5822
5840
|
return Array.from(bytes).map((b) => b.toString(16).padStart(2, "0")).join("");
|
|
5823
5841
|
}
|
|
5824
|
-
/** Wire version the eval-runs ingest enforces (X-Tangle-Wire-Version + body). */
|
|
5825
|
-
const INTELLIGENCE_WIRE_VERSION = "2026-05-26.v1";
|
|
5826
|
-
const DEFAULT_INTELLIGENCE_BASE = "https://intelligence.tangle.tools";
|
|
5827
|
-
/**
|
|
5828
|
-
* Ship self-improvement eval-run events to Tangle Intelligence. Unlike the
|
|
5829
|
-
* best-effort span exporter, this RESOLVES with the ingest verdict (accepted /
|
|
5830
|
-
* rejected per event) so a consumer's loop can assert its provenance landed.
|
|
5831
|
-
* Success requires a complete, valid acknowledgement accepting every event.
|
|
5832
|
-
* Throws on a missing key, network failure, or unreadable/inconsistent acknowledgement.
|
|
5833
|
-
*/
|
|
5834
|
-
async function exportEvalRuns(events, config) {
|
|
5835
|
-
if (events.length === 0) return {
|
|
5836
|
-
ok: true,
|
|
5837
|
-
status: 0,
|
|
5838
|
-
accepted: 0,
|
|
5839
|
-
rejected: []
|
|
5840
|
-
};
|
|
5841
|
-
const apiKey = config?.apiKey ?? (typeof process !== "undefined" ? process.env.TANGLE_API_KEY : void 0);
|
|
5842
|
-
if (!apiKey?.trim()) throw new Error("exportEvalRuns: apiKey required (pass config.apiKey or set TANGLE_API_KEY)");
|
|
5843
|
-
const url = `${(config?.base ?? (typeof process !== "undefined" ? process.env.TANGLE_INTELLIGENCE_URL : void 0) ?? DEFAULT_INTELLIGENCE_BASE).replace(/\/+$/, "")}/v1/ingest/eval-runs`;
|
|
5844
|
-
const count = events.length;
|
|
5845
|
-
const res = await fetch(url, {
|
|
5846
|
-
method: "POST",
|
|
5847
|
-
headers: {
|
|
5848
|
-
"content-type": "application/json",
|
|
5849
|
-
authorization: `Bearer ${apiKey}`,
|
|
5850
|
-
"X-Tangle-Wire-Version": INTELLIGENCE_WIRE_VERSION,
|
|
5851
|
-
...config?.idempotencyKey ? { "Idempotency-Key": config.idempotencyKey } : {}
|
|
5852
|
-
},
|
|
5853
|
-
body: JSON.stringify({
|
|
5854
|
-
wireVersion: INTELLIGENCE_WIRE_VERSION,
|
|
5855
|
-
events
|
|
5856
|
-
})
|
|
5857
|
-
});
|
|
5858
|
-
const invalid = (reason) => {
|
|
5859
|
-
throw new Error(`exportEvalRuns: ${reason} (HTTP ${res.status})`);
|
|
5860
|
-
};
|
|
5861
|
-
let parsed;
|
|
5862
|
-
try {
|
|
5863
|
-
parsed = await res.json();
|
|
5864
|
-
} catch {
|
|
5865
|
-
return invalid("Unreadable eval-runs acknowledgement");
|
|
5866
|
-
}
|
|
5867
|
-
if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) return invalid("Invalid eval-runs acknowledgement");
|
|
5868
|
-
const { accepted, rejected } = parsed;
|
|
5869
|
-
if (!Number.isSafeInteger(accepted) || accepted < 0 || accepted > count || !Array.isArray(rejected)) return invalid("Invalid eval-runs acknowledgement counts");
|
|
5870
|
-
const indices = /* @__PURE__ */ new Set();
|
|
5871
|
-
const validated = [];
|
|
5872
|
-
for (const item of rejected) {
|
|
5873
|
-
if (item === null || typeof item !== "object" || Array.isArray(item) || !Number.isSafeInteger(item.index) || item.index < 0 || item.index >= count || indices.has(item.index) || typeof item.reason !== "string" || !item.reason.trim()) return invalid("Invalid eval-runs acknowledgement rejection");
|
|
5874
|
-
indices.add(item.index);
|
|
5875
|
-
validated.push({
|
|
5876
|
-
index: item.index,
|
|
5877
|
-
reason: item.reason
|
|
5878
|
-
});
|
|
5879
|
-
}
|
|
5880
|
-
if (accepted + validated.length !== count) return invalid("Incomplete eval-runs acknowledgement");
|
|
5881
|
-
if (!res.ok && accepted !== 0) return invalid("Failed HTTP response claimed accepted eval-runs");
|
|
5882
|
-
return {
|
|
5883
|
-
ok: res.ok && accepted === count && validated.length === 0,
|
|
5884
|
-
status: res.status,
|
|
5885
|
-
accepted,
|
|
5886
|
-
rejected: validated
|
|
5887
|
-
};
|
|
5888
|
-
}
|
|
5889
5842
|
//#endregion
|
|
5890
5843
|
//#region src/mcp/trace-propagation.ts
|
|
5891
5844
|
/**
|
|
@@ -8313,6 +8266,44 @@ function unavailablePauseMs(consecutive, policy = {}) {
|
|
|
8313
8266
|
return Math.min(ceiling, first * 2 ** Math.max(0, consecutive - 1));
|
|
8314
8267
|
}
|
|
8315
8268
|
//#endregion
|
|
8269
|
+
//#region src/runtime/supervise/workspace-checkpoint.ts
|
|
8270
|
+
/** Workspace-relative path of the marker Runtime writes before every checkpoint. */
|
|
8271
|
+
const WORKSPACE_CHECKPOINT_MARKER_PATH = ".agent-runtime-checkpoint";
|
|
8272
|
+
/** Bound on one checkpoint: environment lookup, marker write, and the provider operation. */
|
|
8273
|
+
const WORKSPACE_CHECKPOINT_TIMEOUT_MS = 12e4;
|
|
8274
|
+
/** Write the marker. False when the environment cannot take it, so no restore can be verified. */
|
|
8275
|
+
async function writeWorkspaceMarker(environment, marker, signal) {
|
|
8276
|
+
if (typeof environment.write !== "function") return false;
|
|
8277
|
+
try {
|
|
8278
|
+
await environment.write(marker.path, marker.content, { signal });
|
|
8279
|
+
return true;
|
|
8280
|
+
} catch {
|
|
8281
|
+
return false;
|
|
8282
|
+
}
|
|
8283
|
+
}
|
|
8284
|
+
/** Read the marker back from a restored environment and compare it byte for byte. */
|
|
8285
|
+
async function verifyWorkspaceMarker(environment, marker, signal) {
|
|
8286
|
+
if (marker === void 0) return {
|
|
8287
|
+
verified: false,
|
|
8288
|
+
detail: "the checkpoint has no marker: the source could not take one"
|
|
8289
|
+
};
|
|
8290
|
+
if (typeof environment.read !== "function") return {
|
|
8291
|
+
verified: false,
|
|
8292
|
+
detail: "the restored environment does not expose read"
|
|
8293
|
+
};
|
|
8294
|
+
try {
|
|
8295
|
+
return await environment.read(marker.path, { signal }) === marker.content ? { verified: true } : {
|
|
8296
|
+
verified: false,
|
|
8297
|
+
detail: "the marker in the restored environment differs from the checkpoint"
|
|
8298
|
+
};
|
|
8299
|
+
} catch (error) {
|
|
8300
|
+
return {
|
|
8301
|
+
verified: false,
|
|
8302
|
+
detail: `the marker could not be read: ${error instanceof Error ? error.message : String(error)}`.slice(0, 500)
|
|
8303
|
+
};
|
|
8304
|
+
}
|
|
8305
|
+
}
|
|
8306
|
+
//#endregion
|
|
8316
8307
|
//#region src/runtime/environment-provider.ts
|
|
8317
8308
|
var ValidationError = class extends Error {
|
|
8318
8309
|
constructor(message, options) {
|
|
@@ -9245,9 +9236,14 @@ async function providerExecutionSource(args, turn, signal) {
|
|
|
9245
9236
|
const priorSession = retention.priorSession;
|
|
9246
9237
|
const environmentKey = priorSession?.idempotencyKey ?? `runtime:${args.executionId}`;
|
|
9247
9238
|
const turnId = `${args.executionId}:turn:0`;
|
|
9239
|
+
const restore = priorSession === void 0 ? retention.restoreWorkspace : void 0;
|
|
9248
9240
|
const material = {
|
|
9249
9241
|
environment: {
|
|
9250
9242
|
...args.options.defaults,
|
|
9243
|
+
...restore === void 0 ? {} : { workspace: {
|
|
9244
|
+
...args.options.defaults?.workspace,
|
|
9245
|
+
checkpoint: restore.checkpoint
|
|
9246
|
+
} },
|
|
9251
9247
|
profile: args.createProfile,
|
|
9252
9248
|
idempotencyKey: environmentKey,
|
|
9253
9249
|
signal
|
|
@@ -9328,6 +9324,14 @@ async function providerExecutionSource(args, turn, signal) {
|
|
|
9328
9324
|
args.onPending(true);
|
|
9329
9325
|
const environment = await args.provider.get?.(handle.controlRef.environmentId);
|
|
9330
9326
|
if (!environment) throw new Error("retained provider environment is unavailable");
|
|
9327
|
+
if (restore !== void 0 && !args.recovering) {
|
|
9328
|
+
const found = await verifyWorkspaceMarker(environment, restore.marker, signal);
|
|
9329
|
+
await retention.onWorkspaceRestored?.({
|
|
9330
|
+
environmentId: environment.id,
|
|
9331
|
+
checkpoint: restore.checkpoint,
|
|
9332
|
+
...found
|
|
9333
|
+
});
|
|
9334
|
+
}
|
|
9331
9335
|
const session = exactSession(environment, handle.controlRef).session;
|
|
9332
9336
|
async function* events() {
|
|
9333
9337
|
let observationFailure;
|
|
@@ -17531,6 +17535,10 @@ const outsideKinds = /* @__PURE__ */ new Set([
|
|
|
17531
17535
|
"teardown-pending",
|
|
17532
17536
|
"teardown-confirmed",
|
|
17533
17537
|
"environment-teardown",
|
|
17538
|
+
"workspace-checkpoint-requested",
|
|
17539
|
+
"workspace-checkpoint",
|
|
17540
|
+
"workspace-checkpoint-cleanup",
|
|
17541
|
+
"workspace-restored",
|
|
17534
17542
|
"trace-unpropagated",
|
|
17535
17543
|
"paused",
|
|
17536
17544
|
"node-inputs-resolved",
|
|
@@ -17602,6 +17610,10 @@ async function replaySpawnTree(journal, blobs, root) {
|
|
|
17602
17610
|
if (ev.kind === "teardown-unconfirmed") continue;
|
|
17603
17611
|
if (ev.kind === "teardown-pending" || ev.kind === "teardown-confirmed") continue;
|
|
17604
17612
|
if (ev.kind === "environment-teardown") continue;
|
|
17613
|
+
if (ev.kind === "workspace-checkpoint-requested") continue;
|
|
17614
|
+
if (ev.kind === "workspace-checkpoint") continue;
|
|
17615
|
+
if (ev.kind === "workspace-checkpoint-cleanup") continue;
|
|
17616
|
+
if (ev.kind === "workspace-restored") continue;
|
|
17605
17617
|
if (ev.kind === "trace-unpropagated") continue;
|
|
17606
17618
|
if (ev.kind === "paused") continue;
|
|
17607
17619
|
if (ev.kind === "node-inputs-resolved") continue;
|
|
@@ -22183,6 +22195,23 @@ function registerScopeRetainedOwner(scope, args) {
|
|
|
22183
22195
|
},
|
|
22184
22196
|
preserveEnvironment: true,
|
|
22185
22197
|
admissions,
|
|
22198
|
+
get restoreWorkspace() {
|
|
22199
|
+
return state.restore;
|
|
22200
|
+
},
|
|
22201
|
+
onWorkspaceRestored: async (receipt) => {
|
|
22202
|
+
await args.journal.appendEvent(args.rootId, {
|
|
22203
|
+
kind: "workspace-restored",
|
|
22204
|
+
id: args.nodeId,
|
|
22205
|
+
provider: receipt.checkpoint.provider,
|
|
22206
|
+
environmentId: receipt.environmentId,
|
|
22207
|
+
checkpointId: receipt.checkpoint.checkpointId,
|
|
22208
|
+
sourceEnvironmentId: receipt.checkpoint.source.environmentId,
|
|
22209
|
+
verified: receipt.verified,
|
|
22210
|
+
...receipt.detail === void 0 ? {} : { detail: receipt.detail },
|
|
22211
|
+
seq: state.nextSequence(),
|
|
22212
|
+
at: new Date(args.now()).toISOString()
|
|
22213
|
+
});
|
|
22214
|
+
},
|
|
22186
22215
|
onAdmission: async (admission) => {
|
|
22187
22216
|
scope.signal.throwIfAborted();
|
|
22188
22217
|
const prior = admissions.find((record) => record.phase === admission.phase);
|
|
@@ -22317,6 +22346,222 @@ function scopeRetainedOwnerResourceReader(scope) {
|
|
|
22317
22346
|
}
|
|
22318
22347
|
};
|
|
22319
22348
|
}
|
|
22349
|
+
/**
|
|
22350
|
+
* Note that the owner's manager made a coordination call, and checkpoint its workspace when the
|
|
22351
|
+
* last checkpoint is older than {@link WORKSPACE_CHECKPOINT_MIN_INTERVAL_MS}.
|
|
22352
|
+
*
|
|
22353
|
+
* A coordination call is where a manager commits work: it spawns, waits for, reads, or submits.
|
|
22354
|
+
* The checkpoint runs in the background and never delays the call. A failed checkpoint leaves the
|
|
22355
|
+
* previous one as the restore point; a provider or environment that cannot checkpoint stops the
|
|
22356
|
+
* attempts for this owner.
|
|
22357
|
+
*/
|
|
22358
|
+
function noteScopeRetainedOwnerCoordination(scope) {
|
|
22359
|
+
const state = owners.get(scope);
|
|
22360
|
+
if (state?.provider === void 0 || state.checkpointsUnavailable !== void 0) return;
|
|
22361
|
+
if (state.checkpointing !== void 0 || scope.signal.aborted) return;
|
|
22362
|
+
const now = state.args.now();
|
|
22363
|
+
if (state.lastCheckpointAt !== void 0 && now - state.lastCheckpointAt < 6e4) return;
|
|
22364
|
+
state.lastCheckpointAt = now;
|
|
22365
|
+
state.checkpointing = checkpointOwnerWorkspace(scope, state).catch(() => void 0).finally(() => {
|
|
22366
|
+
delete state.checkpointing;
|
|
22367
|
+
});
|
|
22368
|
+
}
|
|
22369
|
+
async function checkpointOwnerWorkspace(scope, state) {
|
|
22370
|
+
const { provider, args } = state;
|
|
22371
|
+
if (provider === void 0) return;
|
|
22372
|
+
if (!provider.get) {
|
|
22373
|
+
state.checkpointsUnavailable = `provider ${provider.name} cannot reconstruct an environment`;
|
|
22374
|
+
return;
|
|
22375
|
+
}
|
|
22376
|
+
if ((await provider.capabilities()).create?.workspaceCheckpoint !== true) {
|
|
22377
|
+
state.checkpointsUnavailable = `provider ${provider.name} cannot create an environment from a checkpoint`;
|
|
22378
|
+
return;
|
|
22379
|
+
}
|
|
22380
|
+
const dispatched = [...state.admissions].reverse().find((admission) => admission.phase === "dispatched");
|
|
22381
|
+
if (dispatched?.phase !== "dispatched") return;
|
|
22382
|
+
const source = dispatched.controlRef;
|
|
22383
|
+
const signal = AbortSignal.any([scope.signal, AbortSignal.timeout(WORKSPACE_CHECKPOINT_TIMEOUT_MS)]);
|
|
22384
|
+
const get = provider.get.bind(provider);
|
|
22385
|
+
const environment = await runAbortable(() => get(source.environmentId), signal, "retained owner checkpoint timed out");
|
|
22386
|
+
if (environment === null || environment.id !== source.environmentId || environment.provider !== provider.name) return;
|
|
22387
|
+
const branching = environment.workspaceBranching;
|
|
22388
|
+
if (branching === void 0) {
|
|
22389
|
+
state.checkpointsUnavailable = `environment ${environment.id} offers no durable checkpoint`;
|
|
22390
|
+
return;
|
|
22391
|
+
}
|
|
22392
|
+
const seq = state.nextSequence();
|
|
22393
|
+
const at = new Date(args.now()).toISOString();
|
|
22394
|
+
const name = `owner-checkpoint-${seq}`;
|
|
22395
|
+
const marker = {
|
|
22396
|
+
path: WORKSPACE_CHECKPOINT_MARKER_PATH,
|
|
22397
|
+
content: `${JSON.stringify({
|
|
22398
|
+
node: args.nodeId,
|
|
22399
|
+
checkpoint: name,
|
|
22400
|
+
at
|
|
22401
|
+
})}\n`
|
|
22402
|
+
};
|
|
22403
|
+
const marked = await writeWorkspaceMarker(environment, marker, signal);
|
|
22404
|
+
const material = {
|
|
22405
|
+
source,
|
|
22406
|
+
name
|
|
22407
|
+
};
|
|
22408
|
+
const request = {
|
|
22409
|
+
...material,
|
|
22410
|
+
idempotencyKey: `${args.nodeId}:workspace-checkpoint:${seq}`,
|
|
22411
|
+
requestDigest: workspaceCheckpointRequestDigest(material)
|
|
22412
|
+
};
|
|
22413
|
+
state.checkpointHandles ??= /* @__PURE__ */ new Map();
|
|
22414
|
+
state.checkpointHandles.set(environment.id, branching);
|
|
22415
|
+
await args.journal.appendEvent(args.rootId, {
|
|
22416
|
+
kind: "workspace-checkpoint-requested",
|
|
22417
|
+
id: args.nodeId,
|
|
22418
|
+
provider: provider.name,
|
|
22419
|
+
environmentId: environment.id,
|
|
22420
|
+
request,
|
|
22421
|
+
...marked ? { marker } : {},
|
|
22422
|
+
seq,
|
|
22423
|
+
at
|
|
22424
|
+
});
|
|
22425
|
+
const result = await runAbortable(() => branching.checkpoint(request, { signal }), signal, "retained owner checkpoint timed out");
|
|
22426
|
+
if (result.status !== "created" && result.status !== "replayed") return;
|
|
22427
|
+
if (!workspaceCheckpointResultMatchesRequest(request, result)) return;
|
|
22428
|
+
scope.signal.throwIfAborted();
|
|
22429
|
+
await args.journal.appendEvent(args.rootId, {
|
|
22430
|
+
kind: "workspace-checkpoint",
|
|
22431
|
+
id: args.nodeId,
|
|
22432
|
+
provider: provider.name,
|
|
22433
|
+
environmentId: environment.id,
|
|
22434
|
+
checkpoint: result.checkpoint,
|
|
22435
|
+
...marked ? { marker } : {},
|
|
22436
|
+
seq: state.nextSequence(),
|
|
22437
|
+
at
|
|
22438
|
+
});
|
|
22439
|
+
await deleteCheckpoints(state, branching, pendingCheckpoints(await args.journal.loadTree(args.rootId) ?? [], state).filter((event) => event.environmentId === environment.id).slice(0, -2), signal);
|
|
22440
|
+
}
|
|
22441
|
+
/** Reconstruct authority for this source only; a replacement cannot own its snapshots. */
|
|
22442
|
+
async function checkpointHandle(state, sourceEnvironmentId, signal) {
|
|
22443
|
+
const cached = state.checkpointHandles?.get(sourceEnvironmentId);
|
|
22444
|
+
if (cached !== void 0) return cached;
|
|
22445
|
+
const provider = state.provider;
|
|
22446
|
+
if (provider === void 0) return void 0;
|
|
22447
|
+
try {
|
|
22448
|
+
const branching = await runAbortable(async () => {
|
|
22449
|
+
if (provider.workspaceBranching) return await provider.workspaceBranching.forEnvironment(sourceEnvironmentId, { signal }) ?? void 0;
|
|
22450
|
+
const source = await provider.get?.(sourceEnvironmentId);
|
|
22451
|
+
return source?.id === sourceEnvironmentId && source.provider === provider.name ? source.workspaceBranching : void 0;
|
|
22452
|
+
}, signal, "retained owner checkpoint handle lookup timed out");
|
|
22453
|
+
if (branching !== void 0) {
|
|
22454
|
+
state.checkpointHandles ??= /* @__PURE__ */ new Map();
|
|
22455
|
+
state.checkpointHandles.set(sourceEnvironmentId, branching);
|
|
22456
|
+
}
|
|
22457
|
+
return branching;
|
|
22458
|
+
} catch {
|
|
22459
|
+
return;
|
|
22460
|
+
}
|
|
22461
|
+
}
|
|
22462
|
+
/** An aborted observation cannot prove that the provider never created a snapshot. */
|
|
22463
|
+
async function reconcileCheckpointRequests(state, events) {
|
|
22464
|
+
const operationKey = (operation) => JSON.stringify([operation.idempotencyKey, operation.requestDigest]);
|
|
22465
|
+
const resolved = new Set(events.flatMap((event) => event.kind === "workspace-checkpoint" && event.id === state.args.nodeId && event.provider === state.provider?.name ? [operationKey(event.checkpoint)] : []));
|
|
22466
|
+
const pending = events.filter((event) => event.kind === "workspace-checkpoint-requested" && event.id === state.args.nodeId && event.provider === state.provider?.name && !resolved.has(operationKey(event.request)));
|
|
22467
|
+
const unconfirmed = [];
|
|
22468
|
+
for (const event of pending) {
|
|
22469
|
+
const signal = AbortSignal.timeout(3e4);
|
|
22470
|
+
try {
|
|
22471
|
+
const branching = await checkpointHandle(state, event.environmentId, signal);
|
|
22472
|
+
if (branching === void 0) throw new Error("source-scoped checkpoint handle unavailable");
|
|
22473
|
+
const result = await runAbortable(() => branching.lookupCheckpoint({
|
|
22474
|
+
idempotencyKey: event.request.idempotencyKey,
|
|
22475
|
+
requestDigest: event.request.requestDigest
|
|
22476
|
+
}, { signal }), signal, "retained owner checkpoint lookup timed out");
|
|
22477
|
+
if (result.status !== "found" || !workspaceCheckpointResultMatchesRequest(event.request, result)) throw new Error("checkpoint operation outcome is unresolved");
|
|
22478
|
+
const recovered = {
|
|
22479
|
+
kind: "workspace-checkpoint",
|
|
22480
|
+
id: event.id,
|
|
22481
|
+
provider: event.provider,
|
|
22482
|
+
environmentId: event.environmentId,
|
|
22483
|
+
checkpoint: result.checkpoint,
|
|
22484
|
+
...event.marker === void 0 ? {} : { marker: event.marker },
|
|
22485
|
+
seq: state.nextSequence(),
|
|
22486
|
+
at: event.at
|
|
22487
|
+
};
|
|
22488
|
+
await state.args.journal.appendEvent(state.args.rootId, recovered);
|
|
22489
|
+
events.push(recovered);
|
|
22490
|
+
} catch {
|
|
22491
|
+
unconfirmed.push(event);
|
|
22492
|
+
}
|
|
22493
|
+
}
|
|
22494
|
+
return unconfirmed;
|
|
22495
|
+
}
|
|
22496
|
+
function pendingCheckpoints(events, state) {
|
|
22497
|
+
const confirmed = new Set(events.flatMap((event) => event.kind === "workspace-checkpoint-cleanup" && event.id === state.args.nodeId && event.provider === state.provider?.name && event.confirmed ? [JSON.stringify([event.environmentId, event.checkpointId])] : []));
|
|
22498
|
+
return events.filter((event) => event.kind === "workspace-checkpoint" && event.id === state.args.nodeId && event.provider === state.provider?.name && !confirmed.has(JSON.stringify([event.environmentId, event.checkpoint.checkpointId])));
|
|
22499
|
+
}
|
|
22500
|
+
/** Only the source-scoped handle can attest deletion, including after its source is lost. */
|
|
22501
|
+
async function deleteCheckpoints(state, branching, checkpoints, signal) {
|
|
22502
|
+
const unconfirmed = [];
|
|
22503
|
+
for (const event of checkpoints) {
|
|
22504
|
+
const material = {
|
|
22505
|
+
kind: "checkpoint",
|
|
22506
|
+
targetId: event.checkpoint.checkpointId,
|
|
22507
|
+
provider: event.checkpoint.provider
|
|
22508
|
+
};
|
|
22509
|
+
const request = {
|
|
22510
|
+
...material,
|
|
22511
|
+
operationId: `${event.checkpoint.idempotencyKey}:delete`,
|
|
22512
|
+
requestDigest: workspaceCleanupRequestDigest(material)
|
|
22513
|
+
};
|
|
22514
|
+
let confirmed = false;
|
|
22515
|
+
try {
|
|
22516
|
+
if (branching === void 0) throw new Error("source-scoped checkpoint handle unavailable");
|
|
22517
|
+
const acknowledgement = await runAbortable(() => branching.deleteCheckpoint(request, { signal }), signal, "retained owner checkpoint cleanup timed out");
|
|
22518
|
+
confirmed = workspaceCleanupAcknowledgementMatches(request, acknowledgement) && (acknowledgement.status === "deleted" || acknowledgement.status === "already_absent");
|
|
22519
|
+
} catch {}
|
|
22520
|
+
if (!confirmed) unconfirmed.push(event);
|
|
22521
|
+
await state.args.journal.appendEvent(state.args.rootId, {
|
|
22522
|
+
kind: "workspace-checkpoint-cleanup",
|
|
22523
|
+
id: state.args.nodeId,
|
|
22524
|
+
provider: event.provider,
|
|
22525
|
+
environmentId: event.environmentId,
|
|
22526
|
+
checkpointId: event.checkpoint.checkpointId,
|
|
22527
|
+
confirmed,
|
|
22528
|
+
seq: state.nextSequence(),
|
|
22529
|
+
at: new Date(state.args.now()).toISOString()
|
|
22530
|
+
});
|
|
22531
|
+
}
|
|
22532
|
+
return unconfirmed;
|
|
22533
|
+
}
|
|
22534
|
+
/**
|
|
22535
|
+
* The checkpoint a new environment of this owner starts from: the latest one journaled before
|
|
22536
|
+
* the invocation's input, taken from an environment the provider has since lost. Only such an
|
|
22537
|
+
* environment needs one; a live environment is continued, never copied.
|
|
22538
|
+
*/
|
|
22539
|
+
function restoreBefore(owned, inputIndex) {
|
|
22540
|
+
const destroyed = destroyedEnvironmentIds(owned);
|
|
22541
|
+
return owned.slice(0, inputIndex < 0 ? owned.length : inputIndex).filter((event) => event.kind === "workspace-checkpoint" && destroyed.has(event.environmentId)).at(-1);
|
|
22542
|
+
}
|
|
22543
|
+
async function restoreCapable(provider) {
|
|
22544
|
+
if (provider === void 0) return false;
|
|
22545
|
+
try {
|
|
22546
|
+
return (await provider.capabilities()).create?.workspaceCheckpoint === true;
|
|
22547
|
+
} catch {
|
|
22548
|
+
return false;
|
|
22549
|
+
}
|
|
22550
|
+
}
|
|
22551
|
+
/**
|
|
22552
|
+
* What the owner's next new environment will hold of the lost one, read before the drive so the
|
|
22553
|
+
* re-entered director is told: the checkpoint's time, or undefined when there is nothing to
|
|
22554
|
+
* restore.
|
|
22555
|
+
*/
|
|
22556
|
+
async function scopeRetainedOwnerRestorePoint(scope) {
|
|
22557
|
+
const state = owners.get(scope);
|
|
22558
|
+
if (state === void 0 || !await restoreCapable(state.provider)) return void 0;
|
|
22559
|
+
const latest = restoreBefore((await state.args.journal.loadTree(state.args.rootId) ?? []).filter((event) => event.id === state.args.nodeId), -1);
|
|
22560
|
+
return latest === void 0 ? void 0 : {
|
|
22561
|
+
checkpointId: latest.checkpoint.checkpointId,
|
|
22562
|
+
takenAt: latest.at
|
|
22563
|
+
};
|
|
22564
|
+
}
|
|
22320
22565
|
/** @internal Whether another owner release could still destroy what the last one left. */
|
|
22321
22566
|
function retainedOwnerReleaseRetriable(scope) {
|
|
22322
22567
|
return owners.get(scope)?.releaseRetriable ?? false;
|
|
@@ -22326,7 +22571,11 @@ async function releaseScopeRetainedOwnerEnvironment(scope) {
|
|
|
22326
22571
|
const state = owners.get(scope);
|
|
22327
22572
|
if (!state?.provider) return [];
|
|
22328
22573
|
const { provider, args } = state;
|
|
22329
|
-
|
|
22574
|
+
state.checkpointsUnavailable ??= "released";
|
|
22575
|
+
await state.checkpointing?.catch(() => void 0);
|
|
22576
|
+
const events = [...await args.journal.loadTree(args.rootId) ?? []];
|
|
22577
|
+
const requestFailures = await reconcileCheckpointRequests(state, events);
|
|
22578
|
+
const pendingSources = new Set(requestFailures.map((event) => event.environmentId));
|
|
22330
22579
|
const released = new Set(events.flatMap((event) => event.kind === "environment-teardown" && event.id === args.nodeId && event.provider === provider.name && event.destroyed ? [event.environmentId] : []));
|
|
22331
22580
|
const environments = new Set(events.flatMap((event) => event.id === args.nodeId && event.kind === "execution-admitted" && event.admission.phase === "environment" ? [event.admission.environmentId] : []));
|
|
22332
22581
|
const retentionFailures = state.workspaceRetention || events.some((event) => event.kind === "execution-input" && event.id === args.nodeId && event.workspaceRetention === true) ? await ownerWorkspaceRetentionFailures(events, args.nodeId, args.blobs) : /* @__PURE__ */ new Set();
|
|
@@ -22334,12 +22583,20 @@ async function releaseScopeRetainedOwnerEnvironment(scope) {
|
|
|
22334
22583
|
const preserved = [];
|
|
22335
22584
|
let lastDetail;
|
|
22336
22585
|
let retriable = false;
|
|
22586
|
+
const checkpointFailures = [];
|
|
22587
|
+
const pending = pendingCheckpoints(events, state).filter((event) => !retentionFailures.has(event.environmentId));
|
|
22588
|
+
for (const sourceEnvironmentId of new Set(pending.map((event) => event.environmentId))) {
|
|
22589
|
+
const signal = AbortSignal.timeout(3e4);
|
|
22590
|
+
const branching = await checkpointHandle(state, sourceEnvironmentId, signal);
|
|
22591
|
+
checkpointFailures.push(...await deleteCheckpoints(state, branching, pending.filter((event) => event.environmentId === sourceEnvironmentId), signal));
|
|
22592
|
+
}
|
|
22593
|
+
for (const event of checkpointFailures) pendingSources.add(event.environmentId);
|
|
22337
22594
|
for (const environmentId of environments) {
|
|
22338
22595
|
if (released.has(environmentId)) continue;
|
|
22339
22596
|
let destroyed = false;
|
|
22340
22597
|
let detail;
|
|
22341
|
-
if (retentionFailures.has(environmentId)) {
|
|
22342
|
-
detail = "provider workspace retention: source preserved because the owner execution has no verified workspace receipt";
|
|
22598
|
+
if (retentionFailures.has(environmentId) || pendingSources.has(environmentId)) {
|
|
22599
|
+
detail = pendingSources.has(environmentId) ? "checkpoint cleanup unresolved: source preserved for exact lookup and cleanup" : "provider workspace retention: source preserved because the owner execution has no verified workspace receipt";
|
|
22343
22600
|
preserved.push(environmentId);
|
|
22344
22601
|
lastDetail = detail;
|
|
22345
22602
|
} else {
|
|
@@ -22382,6 +22639,19 @@ async function releaseScopeRetainedOwnerEnvironment(scope) {
|
|
|
22382
22639
|
});
|
|
22383
22640
|
}
|
|
22384
22641
|
state.releaseRetriable = retriable;
|
|
22642
|
+
const checkpointTeardowns = [...checkpointFailures.map((event) => ({
|
|
22643
|
+
id: args.nodeId,
|
|
22644
|
+
label: "scope owner workspace checkpoint",
|
|
22645
|
+
runtime: provider.name,
|
|
22646
|
+
status: "done",
|
|
22647
|
+
detail: `checkpoint cleanup unconfirmed: ${event.checkpoint.checkpointId} from ${event.environmentId}`
|
|
22648
|
+
})), ...requestFailures.map((event) => ({
|
|
22649
|
+
id: args.nodeId,
|
|
22650
|
+
label: "scope owner checkpoint request",
|
|
22651
|
+
runtime: provider.name,
|
|
22652
|
+
status: "done",
|
|
22653
|
+
detail: `checkpoint request unresolved: ${event.request.idempotencyKey} from ${event.environmentId}`
|
|
22654
|
+
}))];
|
|
22385
22655
|
return unconfirmed.length > 0 || preserved.length > 0 ? [{
|
|
22386
22656
|
id: args.nodeId,
|
|
22387
22657
|
label: "scope owner",
|
|
@@ -22397,7 +22667,7 @@ async function releaseScopeRetainedOwnerEnvironment(scope) {
|
|
|
22397
22667
|
keptFor: "evidence"
|
|
22398
22668
|
})) },
|
|
22399
22669
|
...lastDetail === void 0 ? {} : { detail: lastDetail }
|
|
22400
|
-
}] :
|
|
22670
|
+
}, ...checkpointTeardowns] : checkpointTeardowns;
|
|
22401
22671
|
}
|
|
22402
22672
|
/** Group owner admissions by input and require a verified workspace receipt before deletion. */
|
|
22403
22673
|
async function ownerWorkspaceRetentionFailures(events, nodeId, blobs) {
|
|
@@ -22566,11 +22836,26 @@ async function prepareScopeRetainedOwnerTask(scope, task) {
|
|
|
22566
22836
|
}
|
|
22567
22837
|
if (state.priorSession !== void 0 && destroyed.has(state.priorSession.environmentId)) delete state.priorSession;
|
|
22568
22838
|
state.prepared = true;
|
|
22839
|
+
const capable = await restoreCapable(state.provider);
|
|
22840
|
+
const restoreFor = (inputIndex) => {
|
|
22841
|
+
if (!capable || state.priorSession !== void 0) return void 0;
|
|
22842
|
+
const point = restoreBefore(owned, inputIndex);
|
|
22843
|
+
return point === void 0 ? void 0 : {
|
|
22844
|
+
checkpoint: point.checkpoint,
|
|
22845
|
+
...point.marker === void 0 ? {} : { marker: point.marker }
|
|
22846
|
+
};
|
|
22847
|
+
};
|
|
22569
22848
|
if (state.taskRef !== void 0) {
|
|
22570
22849
|
const original = await args.blobs.get(state.taskRef);
|
|
22571
22850
|
if (original === void 0 || contentAddress(original) !== state.taskRef) throw new ValidationError$1("retained owner task is missing or corrupt");
|
|
22851
|
+
const restore = restoreFor(owned.findIndex((event) => event.kind === "execution-input" && event.seq === state.inputSequence));
|
|
22852
|
+
if (restore === void 0) delete state.restore;
|
|
22853
|
+
else state.restore = restore;
|
|
22572
22854
|
return original;
|
|
22573
22855
|
}
|
|
22856
|
+
const restore = restoreFor(-1);
|
|
22857
|
+
if (restore === void 0) delete state.restore;
|
|
22858
|
+
else state.restore = restore;
|
|
22574
22859
|
const snapshot = detachedSnapshot(task, "retained owner task");
|
|
22575
22860
|
const taskRef = contentAddress(snapshot);
|
|
22576
22861
|
await args.blobs.put(taskRef, snapshot);
|
|
@@ -26315,14 +26600,23 @@ function createContinuationKeeper(input) {
|
|
|
26315
26600
|
const questions = expandQuestions(policy.profile.questions ?? [], failing, workers);
|
|
26316
26601
|
const left = policy.panelUsd.perRun - panelUsd;
|
|
26317
26602
|
const usdCap = Math.min(policy.panelUsd.perContinuation, left);
|
|
26318
|
-
|
|
26603
|
+
const asking = questions.length > 0 && usdCap > 0 && panelUsdKnown;
|
|
26604
|
+
let traces;
|
|
26605
|
+
let unavailable;
|
|
26606
|
+
if (asking) try {
|
|
26607
|
+
traces = await input.traces?.();
|
|
26608
|
+
if (traces === void 0) unavailable = "the run has recorded no trace yet";
|
|
26609
|
+
} catch (error) {
|
|
26610
|
+
unavailable = `the run's traces could not be read: ${error instanceof Error ? error.message : String(error)}`;
|
|
26611
|
+
}
|
|
26612
|
+
if (asking && traces !== void 0) {
|
|
26319
26613
|
const result = await policy.runPanel({
|
|
26320
26614
|
continuation,
|
|
26321
26615
|
task: input.task,
|
|
26322
26616
|
...verdict === void 0 ? {} : { verdict },
|
|
26323
26617
|
reads: input.reads(),
|
|
26324
26618
|
questions,
|
|
26325
|
-
|
|
26619
|
+
traces,
|
|
26326
26620
|
workers,
|
|
26327
26621
|
...policy.best === void 0 ? {} : { bar: policy.best },
|
|
26328
26622
|
...policy.reference === void 0 ? {} : { reference: policy.reference },
|
|
@@ -26350,7 +26644,8 @@ function createContinuationKeeper(input) {
|
|
|
26350
26644
|
asked: 0,
|
|
26351
26645
|
proposed: 0,
|
|
26352
26646
|
admitted: 0,
|
|
26353
|
-
usd: 0
|
|
26647
|
+
usd: 0,
|
|
26648
|
+
...unavailable === void 0 ? {} : { unavailable }
|
|
26354
26649
|
};
|
|
26355
26650
|
}
|
|
26356
26651
|
const appended = policy.append ? await policy.append({
|
|
@@ -26501,7 +26796,7 @@ function createContinuationKeeper(input) {
|
|
|
26501
26796
|
* them) and each run's budget and 8-hour deadline were almost untouched.
|
|
26502
26797
|
*/
|
|
26503
26798
|
const DEFAULT_MAX_CONSECUTIVE_FAILURES = 3;
|
|
26504
|
-
const DEFAULT_MAX_ATTEMPTS =
|
|
26799
|
+
const DEFAULT_MAX_ATTEMPTS = Number.POSITIVE_INFINITY;
|
|
26505
26800
|
const DEFAULT_INITIAL_BACKOFF_MS = 2e3;
|
|
26506
26801
|
const DEFAULT_MAX_BACKOFF_MS = 3e4;
|
|
26507
26802
|
/**
|
|
@@ -27937,6 +28232,6 @@ function isNonEmptySpend(s) {
|
|
|
27937
28232
|
return s.iterations > 0 || s.tokens.input > 0 || s.tokens.output > 0 || s.usd > 0 || s.ms > 0 || s.tokensKnown === false || s.usdKnown === false || Object.values(s.resources ?? {}).some((resource) => resource.amount > 0 || !resource.known);
|
|
27938
28233
|
}
|
|
27939
28234
|
//#endregion
|
|
27940
|
-
export { bindScopeRetainedOwnerProvider as $, detachedSnapshot as $i,
|
|
28235
|
+
export { bindScopeRetainedOwnerProvider as $, detachedSnapshot as $i, LEAF_CONTINUATION_TASK as $n, runWorktreeHarness as $r, parseWorkerToolTraceArtifact as $t, driverExecutorFactory as A, resourceTelemetry as Aa, readCommittedJsonLines as Ai, bridgeAdmissionRefusal as An, readHarnessTranscript as Ar, createExecutor as At, recordScopeOwnerMaterialization as B, promptOptionsFromAgentTurnInput as Ba, providerAttemptEvidence as Bi, defaultSelectWinner as Bn, createOtelExporter as Br, InMemoryResultBlobStore as Bt, createContinuationKeeper as C, stringifySafe as Ca, sandboxActProfileMaterialization as Ci, createPeerMailbox as Cn, runAbortable as Cr, freeSlots as Ct, passedItems as D, withTimeout as Da, isNoEntError as Di, isLiveNodeStatus as Dn, captureHarnessTranscript as Dr, captureReusableExecutorConfig as Dt, failedItems as E, unmeteredSpend as Ea, worktreeCliProfileMaterialization as Ei, peerMailVerbNames as En, resolveRedactor as Er, bindReusableExecutorExecutionId as Et, beginScopeOwnerAttempt as F, DEFAULT_SUCCESSFUL_SHUTDOWN_MS as Fa, finalizeRuntimeOwnedPendingExecutor as Fi, readCodexRolloutSession as Fn, traceContextToEnv as Fr, createSteerableSandboxSession as Ft, createWorkerSlots as G, runtimeOwnedExecutorProviderEvidence as Gi, acquireSandbox as Gn, padTraceId as Gr, insideCursorNamespace as Gt, restoreScopeOwnerAcceptedExecution as H, runtimeOwnedDriveHarnessProviderEvidence as Hi, runAgentRounds as Hn, generateSpanId as Hr, SpawnEventIndex as Ht, createScope as I, armDeadlineTimer as Ia, inheritRuntimeOwnedExecutorAttestation as Ii, bridgeRuntimeAttachmentsKey as In, buildLoopOtelSpans as Ir, createRouterTranscript as It, pollFor as J, unknownExecutionBindingReceipt as Ji, routerBrain as Jn, createRuntimeStreamEventCollector as Jr, pendingWaits as Jt, createWaitProbes as K, runtimeOwnedPendingExecutorMaterialization as Ki, assertBoxlessPromptOptions as Kn, toOtelAttributes as Kr, loadSpawnForest as Kt, deriveNodeExecutionIdentity as L, teardownExecutor as La, knownExecutionBindingReceipt as Li, bridgeStopSignalKey as Ln, buildLoopSpanNodes as Lr, createInPlaceCliExecutor as Lt, withDriverExecutor as M, decodeHarnessUsage as Ma, attestRuntimeOwnedPendingExecutor as Mi, addHarnessUsage as Mn, createPropagatingTraceEmitter as Mr, snapshotExecutorConfig as Mt, SUBTREE_RESULT_LIMIT as N, abortError$2 as Na, attestRuntimeOwnedScopeOwner as Ni, createCodexRolloutStoreReader as Nn, mergeTraceEnv as Nr, createWorktreeCliExecutor as Nt, verdictFromJudgeScore as O, zeroSpend as Oa, parseCommittedJsonLines as Oi, isTerminalNodeStatus as On, harnessTranscriptArtifact as Or, cliInPlaceExecutor as Ot, assertRecursiveReservationPolicy as P, awaitAbortable$1 as Pa, authoredProfileDigest as Pi, harnessUsageIsEmpty as Pn, readTraceContextFromEnv as Pr, DEFAULT_SANDBOX_STEERING_MAX_TURNS as Pt, bindScopeRetainedOwnerEnvironmentId as Q, detachedFrozen as Qi, DEFAULT_SANDBOX_IDLE_TIMEOUT_SECONDS as Qn, runSettledCommand as Qr, isTraceAnalysisStore as Qt, meterRuntimeOwnedAccounting as R, teardownSurfaces as Ra, knownMaterializationReceipt as Ri, TERMINAL_DECISIONS as Rn, buildRuntimeEventOtelSpans as Rr, FileResultBlobStore as Rt, composeContinuationNote as S, sleep as Sa, renderUnsupported as Si, claimsAuthority as Sn, linkAbort as Sr, startRetainedInteractiveRun as St, expandQuestions as T, throwIfAborted as Ta, validateProfileMaterialization as Ti, peerMailTools as Tn, defaultRedactorIdentityMaterial as Tr, rollingDispatch as Tt, scopeOwnerExecutorNodeContext as U, runtimeOwnedExecutorExecutionBinding as Ui, createSandboxLineage as Un, loopEventToOtelSpan as Ur, assertContentAddress as Ut, recordScopeOwnerPause as V, providerMessageText as Va, recordRuntimeOwnedDriveHarnessProviderEvidence as Vi, isTerminalDecision as Vn, flatOtelSpan as Vr, InMemorySpawnJournal as Vt, settledToIteration as W, runtimeOwnedExecutorMaterialization as Wi, probeSandboxCapabilities as Wn, padSpanId as Wr, encodeResultBlob as Wt, validateWaitSpec as X, executableAgentProfileSnapshot as Xi, canonicalObservedModelParts as Xn, sanitizeKnowledgeReadinessReport as Xr, WORKER_TOOL_TRACE_SCHEMA_VERSION as Xt, timerAt as Y, unknownMaterializationReceipt as Yi, runBrainLoop as Yn, sanitizeAgentRuntimeEvent as Yr, replaySpawnTree as Yt, waitUntil as Z, executableAgentSpecSnapshot as Zi, observedModelMatchesDeclared as Zn, sanitizeRuntimeStreamEvent as Zr, captureWorkerTraceEvidence as Zt, CheckUnavailableError as _, deleteBoxSafe as _a, promptControlProfileMaterialization as _i, readWorkerProgress as _n, retainedCreateMaterial as _r, readWorkerInteractiveBinding as _t, collectDelivered as a, createSandboxUsageLedger as aa, harnessInvocation as ai, meterUsageEvent as an, unavailablePauseMs as ar, scopeRetainedOwnerPriorSpend as at, bestComposite as b, promptCacheTokenClasses as ba, promptResourceProfileMaterialization as bi, DEFAULT_PEER_MAIL_LIMITS as bn, RunCancellationReason as br, reconnectRetainedInteractiveRun as bt, runTree as c, mapSandboxEvent as ca, parseCodexTokenUsage as ci, spendFromUsageTotals as cn, decodeToolPart as cr, scopeRetainedOwnerResult as ct, classifyDriverFailure as d, sandboxEventServedBackend as da, AGENT_PROFILE_MATERIALIZATION_AXES as di, workerTraceEnv as dn, retainedExecutorSeamKey as dr, hostDirectoryReader as dt, projectSandboxOutcome as ea, captureWorktreeDiff as ei, workerTraceAnalysisStore as en, createAgentEnvironmentProviderRegistry as er, consumeScopeRetainedOwnerResult as et, runDriverWithRetry as f, sandboxProgressEvents as fa, assertProfileMaterialization as fi, workerTraceHeaders as fn, errMessage as fr, resolveSpawnResourcePaths as ft, CONTINUATION_FACTS as g, cloneSpend as ga, profileMaterializationAxes$1 as gi, createActivityLog as gn, startRetainedRunInEnvironment as gr, attachWorker as gt, CONTINUATIONS_DIR as h, chargedTokens as ha, fullProfileMaterialization as hi, DEFAULT_STALL_AFTER_MS as hn, startRetainedRun as hr, workerInteractiveAdmissionFile as ht, bestDelivered as i, createSandboxToolPartState as ia, LOCAL_HARNESSES as ii, createBudgetPool as in, sandboxClientAsProvider as ir, scopeRetainedOwnerContext as it, isDriverSpec as j, withBudgetResources as ja, writeAllBytes as ji, bridgeModelRouteRefusal as jn, contentAddress as jr, createExecutorRegistry as jt, driverChild as k, addResourceSpend as ka, prepareJsonlAppend as ki, executorFailure as kn, persistHarnessTranscript as kr, cliWorktreeExecutor as kt, DriverAttemptsExhaustedError as l, mapSandboxToolEvent as la, CodexExecutionDiagnosticError as li, WORKER_TRACE_PROPAGATION as ln, sandboxSessionTraceSource as lr, SPAWN_RESOURCE_PATH_MAX_BYTES as lt, upstreamUnavailableSignal as m, addSpend as ma, defineProfileMaterializationContract as mi, taskToPrompt as mn, recoverRetainedRun as mr, readWorkerInteractiveAdmissions as mt, createRootHandle as n, assertSandboxServedModel as na, removeWorktree as ni, ReservationWaitRefused as nn, providerAsSandboxClient as nr, prepareScopeRetainedOwnerTask as nt, pickBestDelivered as o, extractLlmCallEvent as oa, harnessSupportsReasoningEffort as oi, newUsageTotals as on, unavailableSignalOfFailure as or, scopeRetainedOwnerResourceReader as ot, summarizeDriverAttempts as p, sumSandboxUsage as pa, controlProfileMaterialization as pi, workerTraceSeamKey as pn, reconnectRetainedRun as pr, interactiveAdmissionSeamKey as pt, isWaitOutcome as q, runtimeOwnedScopeOwnerRuntime as qi, readPromptOptions as qn, createRuntimeEventCollector as qr, materializeTreeView as qt, createSupervisor as r, canonicalStreamEventFromSandboxEvent as ra, DEFAULT_LOCAL_HARNESS as ri, assertValidBudget as rn, resolveAgentEnvironmentProvider as rr, reconcileScopeRetainedOwnerEnvironment as rt, runFinalizer as s, isSandboxTerminalEvent as sa, localHarnessExecutable as si, spendFromUsageEvents as sn, createPushTraceSource as sr, scopeRetainedOwnerRestorePoint as st, DEFAULT_MAX_DEPTH as t, readSandboxOutcome as ta, createWorktree as ti, BudgetReconcileFault as tn, providerAsExecutor as tr, noteScopeRetainedOwnerCoordination as tt, HarnessTurnFailedError as u, notifySandboxEventObserver as ua, terminateProcessTreeAndConfirm as ui, readWorkerTraceContext as un, registerRetainedExecutorPreparation as ur, environmentReader as ut, admitContinuationPolicy as v, hasCompleteCacheBreakdown as va, promptModelProfileMaterialization as vi, createInbox as vn, destroyInteractiveEnvironment as vr, workerInteractiveBindingFile as vt, distillFindings as w, throwAbort as wa, unsupportedProfileDimensions as wi, isPeerMailEnvelope as wn, defaultRedactor as wr, queueOf as wt, checkVerdictOf as x, randomSuffix as xa, renderProfileMaterializationIssues as xi, PEER_MAIL_WIRE_KEY as xn, abortError as xr, recoverRetainedInteractiveRun as xt, admitFinding as y, isAbortError$2 as ya, promptOnlyProfileMaterialization as yi, AUTHORITY_MARKERS as yn, environmentGone as yr, workerInteractiveBindingsDir as yt, meterRuntimeOwnedProviderAttempt as z, promptFromAgentTurnInput as za, newExecutionAttemptId as zi, createSandboxForSpec as zn, createOpenInferenceFileExporter as zr, FileSpawnJournal as zt };
|
|
27941
28236
|
|
|
27942
|
-
//# sourceMappingURL=supervisor-
|
|
28237
|
+
//# sourceMappingURL=supervisor-DtlPj9me.js.map
|