@tangle-network/agent-runtime 0.264.0 → 0.271.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{activation-Bx4kfVU3.d.ts → activation-BrJmwTLB.d.ts} +4 -4
- package/dist/{activation-DHFdvps3.js → activation-DGbFPGR3.js} +2 -2
- package/dist/{activation-DHFdvps3.js.map → activation-DGbFPGR3.js.map} +1 -1
- package/dist/agent.d.ts +2 -2
- package/dist/agent.js +2 -2
- package/dist/{coordination-driver-Bz-H17bC.js → coordination-driver-iBGMKDvB.js} +90 -50
- package/dist/coordination-driver-iBGMKDvB.js.map +1 -0
- package/dist/{delegate-CwbWhX5W.js → delegate-DOqUvaIz.js} +3 -3
- package/dist/delegate-DOqUvaIz.js.map +1 -0
- package/dist/durable.d.ts +5 -4
- package/dist/durable.js +3 -3
- package/dist/durable.js.map +1 -1
- package/dist/{graph-5NrpTg0h.js → graph-DnMcepkN.js} +12 -7
- package/dist/graph-DnMcepkN.js.map +1 -0
- package/dist/{improvement-cycle-WRWcFyuO.js → improvement-cycle-1IyRJo5h.js} +9 -9
- package/dist/improvement-cycle-1IyRJo5h.js.map +1 -0
- package/dist/{index-9wXpbjbX.d.ts → index-CL1dduJd.d.ts} +123 -73
- package/dist/index.d.ts +7 -7
- package/dist/index.js +7 -7
- package/dist/intelligence.d.ts +9 -3
- package/dist/intelligence.js +11 -5
- package/dist/intelligence.js.map +1 -1
- package/dist/kernel.d.ts +4 -4
- package/dist/kernel.js +9 -9
- package/dist/{loop-runner-bin-DC1Lm0nM.js → loop-runner-bin-BPZ9tt1W.js} +3 -3
- package/dist/{loop-runner-bin-DC1Lm0nM.js.map → loop-runner-bin-BPZ9tt1W.js.map} +1 -1
- package/dist/{loop-runner-bin-DPewOZMn.d.ts → loop-runner-bin-O_vM1w8Y.d.ts} +3 -3
- package/dist/loop-runner-bin.d.ts +1 -1
- package/dist/loop-runner-bin.js +1 -1
- package/dist/mcp/bin.js +3 -3
- package/dist/mcp/index.d.ts +12 -6
- package/dist/mcp/index.js +7 -7
- package/dist/mcp/memory-bin.js +1 -1
- package/dist/{memory-server-Cn8LbAxO.js → memory-server-DXQ8IxGR.js} +2 -2
- package/dist/{memory-server-Cn8LbAxO.js.map → memory-server-DXQ8IxGR.js.map} +1 -1
- package/dist/{openai-tools-ByTMUJcD.d.ts → openai-tools-DWHxpwx2.d.ts} +2 -2
- package/dist/profiles.d.ts +2 -2
- package/dist/{provision-supervisor-BZ6_uBGR.js → provision-supervisor-Cy8pqE95.js} +4 -4
- package/dist/{provision-supervisor-BZ6_uBGR.js.map → provision-supervisor-Cy8pqE95.js.map} +1 -1
- package/dist/{runtime-CUZC75EX.js → runtime-DVjIH1eo.js} +1160 -18
- package/dist/runtime-DVjIH1eo.js.map +1 -0
- package/dist/{server-KpdpN627.js → server-CN9vzbSe.js} +4 -4
- package/dist/{server-KpdpN627.js.map → server-CN9vzbSe.js.map} +1 -1
- package/dist/{stream-agent-turn-D1zelVx6.d.ts → stream-agent-turn-DbEsKgb_.d.ts} +200 -3
- package/dist/{structural-rollout-D17I5CwF.js → structural-rollout-I92sIfNl.js} +2 -2
- package/dist/{structural-rollout-D17I5CwF.js.map → structural-rollout-I92sIfNl.js.map} +1 -1
- package/dist/{substrate-qQphD9f0.d.ts → substrate-CbC03Vzo.d.ts} +2 -2
- package/dist/{supervise-_DTVHXPC.js → supervise-DJ_1i9vj.js} +85 -30
- package/dist/supervise-DJ_1i9vj.js.map +1 -0
- package/dist/{supervisor-C26k7X5-.js → supervisor-Cvd5mf7x.js} +1276 -488
- package/dist/supervisor-Cvd5mf7x.js.map +1 -0
- package/dist/testing.d.ts +2 -2
- package/dist/testing.js +14 -14
- package/dist/testing.js.map +1 -1
- package/dist/{tool-server-x9NEhvHZ.js → tool-server-Dqer2M4_.js} +30 -12
- package/dist/tool-server-Dqer2M4_.js.map +1 -0
- package/dist/tui/index.d.ts +1 -1
- package/dist/tui/index.js +1 -1
- package/dist/{types-BV14WTyW.d.ts → types-Bo7z5aG5.d.ts} +311 -75
- package/package.json +3 -2
- package/dist/coordination-driver-Bz-H17bC.js.map +0 -1
- package/dist/delegate-CwbWhX5W.js.map +0 -1
- package/dist/graph-5NrpTg0h.js.map +0 -1
- package/dist/improvement-cycle-WRWcFyuO.js.map +0 -1
- package/dist/runtime-CUZC75EX.js.map +0 -1
- package/dist/supervise-_DTVHXPC.js.map +0 -1
- package/dist/supervisor-C26k7X5-.js.map +0 -1
- package/dist/tool-server-x9NEhvHZ.js.map +0 -1
|
@@ -14,7 +14,7 @@ import { access, chmod, copyFile, mkdir, mkdtemp, open, readFile, readdir, realp
|
|
|
14
14
|
import { applyWorkspacePlan, hashWorkspacePlan, materializeProfile, resolveAgentProfileResources } from "@tangle-network/agent-profile-materialize";
|
|
15
15
|
import { homedir, hostname, tmpdir } from "node:os";
|
|
16
16
|
import { stripVTControlCharacters } from "node:util";
|
|
17
|
-
import { deriveHexId, isW3CSpanId, isW3CTraceId } from "@tangle-network/agent-trace-contract";
|
|
17
|
+
import { ATTR, deriveHexId, isW3CSpanId, isW3CTraceId } from "@tangle-network/agent-trace-contract";
|
|
18
18
|
import { SDKError, generateIdempotencyKey, withRetry } from "@tangle-network/agent-core";
|
|
19
19
|
import { collectAgentFinalMessageText, collectAgentResponseText } from "@tangle-network/sandbox";
|
|
20
20
|
import { request } from "node:http";
|
|
@@ -5171,25 +5171,30 @@ const GEN_AI = {
|
|
|
5171
5171
|
* useless for the runs you most want to look at.
|
|
5172
5172
|
*/
|
|
5173
5173
|
function createOpenInferenceFileExporter(filePath) {
|
|
5174
|
-
|
|
5174
|
+
let written = 0;
|
|
5175
|
+
let dropped = 0;
|
|
5175
5176
|
let failed;
|
|
5176
|
-
const append = (line) => {
|
|
5177
|
-
try {
|
|
5178
|
-
appendFileSync(filePath, `${line}\n`, "utf8");
|
|
5179
|
-
} catch (error) {
|
|
5180
|
-
failed ??= error instanceof Error ? error : new Error(String(error));
|
|
5181
|
-
}
|
|
5182
|
-
};
|
|
5183
5177
|
return {
|
|
5184
5178
|
exportSpan(span) {
|
|
5185
|
-
|
|
5186
|
-
|
|
5179
|
+
try {
|
|
5180
|
+
appendFileSync(filePath, `${JSON.stringify(toOpenInferenceLine(span))}\n`, "utf8");
|
|
5181
|
+
written += 1;
|
|
5182
|
+
} catch (error) {
|
|
5183
|
+
dropped += 1;
|
|
5184
|
+
failed ??= error instanceof Error ? error : new Error(String(error));
|
|
5185
|
+
}
|
|
5187
5186
|
},
|
|
5188
5187
|
async flush() {
|
|
5189
|
-
if (failed) throw new Error(`OpenInference file exporter failed writing ${filePath}: ${failed.message}`);
|
|
5188
|
+
if (failed) throw new Error(`OpenInference file exporter failed writing ${filePath} (${dropped} spans dropped): ${failed.message}`);
|
|
5190
5189
|
},
|
|
5191
|
-
async shutdown() {
|
|
5192
|
-
|
|
5190
|
+
async shutdown() {},
|
|
5191
|
+
stats() {
|
|
5192
|
+
return {
|
|
5193
|
+
written,
|
|
5194
|
+
dropped,
|
|
5195
|
+
pending: 0,
|
|
5196
|
+
...failed ? { lastError: failed.message } : {}
|
|
5197
|
+
};
|
|
5193
5198
|
}
|
|
5194
5199
|
};
|
|
5195
5200
|
}
|
|
@@ -5240,68 +5245,172 @@ function toOpenInferenceLine(span) {
|
|
|
5240
5245
|
};
|
|
5241
5246
|
}
|
|
5242
5247
|
/**
|
|
5243
|
-
* Create an
|
|
5248
|
+
* Create an OTLP/HTTP exporter. Returns undefined when no endpoint is configured.
|
|
5249
|
+
*
|
|
5250
|
+
* One batch is in flight at a time; spans that arrive meanwhile wait in a queue bounded by
|
|
5251
|
+
* `maxQueueSize`. Every span ends in exactly one of `written` or `dropped`: a non-2xx response, a
|
|
5252
|
+
* network error, a timeout, an OTLP `partialSuccess.rejectedSpans` count and a full queue each count
|
|
5253
|
+
* as drops and set `lastError`. Nothing here throws into the caller's run; `flush()` reports the
|
|
5254
|
+
* loss, the same rule {@link createOpenInferenceFileExporter} follows.
|
|
5244
5255
|
*/
|
|
5245
5256
|
function createOtelExporter(config) {
|
|
5246
5257
|
const resolvedEndpoint = config?.endpoint ?? (typeof process !== "undefined" ? process.env.OTEL_EXPORTER_OTLP_ENDPOINT : void 0);
|
|
5247
5258
|
if (!resolvedEndpoint) return void 0;
|
|
5248
|
-
const
|
|
5259
|
+
const url = `${resolvedEndpoint.replace(/\/+$/, "")}/v1/traces`;
|
|
5249
5260
|
const headers = config?.headers ?? parseHeadersFromEnv();
|
|
5250
|
-
const batchSize = config?.batchSize
|
|
5261
|
+
const batchSize = positiveInteger(config?.batchSize, 64);
|
|
5251
5262
|
const flushIntervalMs = config?.flushIntervalMs ?? 5e3;
|
|
5252
|
-
const
|
|
5253
|
-
const
|
|
5254
|
-
const
|
|
5255
|
-
|
|
5263
|
+
const maxQueueSize = Math.max(positiveInteger(config?.maxQueueSize, 2048), batchSize);
|
|
5264
|
+
const timeoutMs = positiveInteger(config?.timeoutMs, 1e4);
|
|
5265
|
+
const resource = { attributes: toOtelAttributes({
|
|
5266
|
+
"service.name": config?.serviceName ?? "agent-runtime",
|
|
5267
|
+
...config?.resourceAttributes ?? {}
|
|
5268
|
+
}) };
|
|
5269
|
+
const queue = [];
|
|
5270
|
+
let inFlight = 0;
|
|
5271
|
+
let written = 0;
|
|
5272
|
+
let dropped = 0;
|
|
5273
|
+
let droppedAtLastFlush = 0;
|
|
5274
|
+
let lastError;
|
|
5275
|
+
let sending;
|
|
5256
5276
|
let stopped = false;
|
|
5257
|
-
const
|
|
5258
|
-
|
|
5259
|
-
|
|
5260
|
-
pending.push(span);
|
|
5261
|
-
if (pending.length >= batchSize) doFlush();
|
|
5262
|
-
},
|
|
5263
|
-
async flush() {
|
|
5264
|
-
await doFlush();
|
|
5265
|
-
},
|
|
5266
|
-
async shutdown() {
|
|
5267
|
-
stopped = true;
|
|
5268
|
-
if (timer !== void 0) {
|
|
5269
|
-
clearInterval(timer);
|
|
5270
|
-
timer = void 0;
|
|
5271
|
-
}
|
|
5272
|
-
await doFlush();
|
|
5273
|
-
}
|
|
5277
|
+
const drop = (count, reason) => {
|
|
5278
|
+
dropped += count;
|
|
5279
|
+
lastError = reason;
|
|
5274
5280
|
};
|
|
5275
|
-
|
|
5276
|
-
if (pending.length > 0) doFlush();
|
|
5277
|
-
}, flushIntervalMs);
|
|
5278
|
-
if (typeof timer === "object" && "unref" in timer) timer.unref();
|
|
5279
|
-
async function doFlush() {
|
|
5280
|
-
if (pending.length === 0) return;
|
|
5281
|
-
const batch = pending.splice(0);
|
|
5281
|
+
async function send(batch) {
|
|
5282
5282
|
const body = { resourceSpans: [{
|
|
5283
|
-
resource
|
|
5284
|
-
"service.name": serviceName,
|
|
5285
|
-
...resourceAttrs
|
|
5286
|
-
}) },
|
|
5283
|
+
resource,
|
|
5287
5284
|
scopeSpans: [{
|
|
5288
5285
|
scope: SCOPE,
|
|
5289
5286
|
spans: batch
|
|
5290
5287
|
}]
|
|
5291
5288
|
}] };
|
|
5292
|
-
const url = `${endpoint.replace(/\/+$/, "")}/v1/traces`;
|
|
5293
5289
|
try {
|
|
5294
|
-
await fetch(url, {
|
|
5290
|
+
const response = await fetch(url, {
|
|
5295
5291
|
method: "POST",
|
|
5296
5292
|
headers: {
|
|
5297
5293
|
"content-type": "application/json",
|
|
5298
5294
|
...headers
|
|
5299
5295
|
},
|
|
5300
|
-
body: JSON.stringify(body)
|
|
5296
|
+
body: JSON.stringify(body),
|
|
5297
|
+
signal: AbortSignal.timeout(timeoutMs)
|
|
5301
5298
|
});
|
|
5302
|
-
|
|
5299
|
+
const text = await response.text().catch(() => "");
|
|
5300
|
+
if (!response.ok) {
|
|
5301
|
+
const detail = text ? `: ${text.slice(0, 200)}` : "";
|
|
5302
|
+
drop(batch.length, `HTTP ${response.status} from ${url}${detail}`);
|
|
5303
|
+
return;
|
|
5304
|
+
}
|
|
5305
|
+
const rejected = rejectedSpans(text, batch.length);
|
|
5306
|
+
if (rejected.count > 0) drop(rejected.count, `collector rejected ${rejected.count} spans: ${rejected.message}`);
|
|
5307
|
+
written += batch.length - rejected.count;
|
|
5308
|
+
} catch (error) {
|
|
5309
|
+
drop(batch.length, `POST ${url} failed: ${describeFetchError(error)}`);
|
|
5310
|
+
}
|
|
5311
|
+
}
|
|
5312
|
+
/** The single sender: it drains the queue batch by batch until the queue is empty. */
|
|
5313
|
+
function drain() {
|
|
5314
|
+
sending ??= (async () => {
|
|
5315
|
+
try {
|
|
5316
|
+
while (queue.length > 0) {
|
|
5317
|
+
const batch = queue.splice(0, batchSize);
|
|
5318
|
+
inFlight = batch.length;
|
|
5319
|
+
await send(batch);
|
|
5320
|
+
inFlight = 0;
|
|
5321
|
+
}
|
|
5322
|
+
} finally {
|
|
5323
|
+
sending = void 0;
|
|
5324
|
+
}
|
|
5325
|
+
})();
|
|
5326
|
+
return sending;
|
|
5327
|
+
}
|
|
5328
|
+
const timer = setInterval(() => {
|
|
5329
|
+
if (queue.length > 0) drain();
|
|
5330
|
+
}, flushIntervalMs);
|
|
5331
|
+
if (typeof timer === "object" && "unref" in timer) timer.unref();
|
|
5332
|
+
return {
|
|
5333
|
+
exportSpan(span) {
|
|
5334
|
+
if (stopped) {
|
|
5335
|
+
drop(1, "span exported after the exporter stopped");
|
|
5336
|
+
return;
|
|
5337
|
+
}
|
|
5338
|
+
if (queue.length + inFlight >= maxQueueSize) {
|
|
5339
|
+
drop(1, `queue full: ${maxQueueSize} spans already waiting on ${url}`);
|
|
5340
|
+
return;
|
|
5341
|
+
}
|
|
5342
|
+
queue.push(span);
|
|
5343
|
+
if (queue.length >= batchSize) drain();
|
|
5344
|
+
},
|
|
5345
|
+
async flush() {
|
|
5346
|
+
await drain();
|
|
5347
|
+
const lost = dropped - droppedAtLastFlush;
|
|
5348
|
+
droppedAtLastFlush = dropped;
|
|
5349
|
+
if (lost > 0) throw new Error(`OTLP exporter dropped ${lost} spans since the last flush: ${lastError}`);
|
|
5350
|
+
},
|
|
5351
|
+
async shutdown() {
|
|
5352
|
+
stopped = true;
|
|
5353
|
+
clearInterval(timer);
|
|
5354
|
+
await drain();
|
|
5355
|
+
},
|
|
5356
|
+
stats() {
|
|
5357
|
+
return {
|
|
5358
|
+
written,
|
|
5359
|
+
dropped,
|
|
5360
|
+
pending: queue.length + inFlight,
|
|
5361
|
+
...lastError !== void 0 ? { lastError } : {}
|
|
5362
|
+
};
|
|
5363
|
+
}
|
|
5364
|
+
};
|
|
5365
|
+
}
|
|
5366
|
+
/**
|
|
5367
|
+
* Spans a 2xx OTLP/HTTP response still refused. The spec lets a collector accept a request and
|
|
5368
|
+
* reject part of it through `partialSuccess.rejectedSpans`; reading only the status would count
|
|
5369
|
+
* those spans as delivered. An empty or non-JSON body is a full success.
|
|
5370
|
+
*/
|
|
5371
|
+
function rejectedSpans(text, batchLength) {
|
|
5372
|
+
if (!text.trim()) return {
|
|
5373
|
+
count: 0,
|
|
5374
|
+
message: ""
|
|
5375
|
+
};
|
|
5376
|
+
let parsed;
|
|
5377
|
+
try {
|
|
5378
|
+
parsed = JSON.parse(text);
|
|
5379
|
+
} catch {
|
|
5380
|
+
return {
|
|
5381
|
+
count: 0,
|
|
5382
|
+
message: ""
|
|
5383
|
+
};
|
|
5303
5384
|
}
|
|
5304
|
-
return
|
|
5385
|
+
if (parsed === null || typeof parsed !== "object") return {
|
|
5386
|
+
count: 0,
|
|
5387
|
+
message: ""
|
|
5388
|
+
};
|
|
5389
|
+
const partial = parsed.partialSuccess;
|
|
5390
|
+
if (partial === null || typeof partial !== "object") return {
|
|
5391
|
+
count: 0,
|
|
5392
|
+
message: ""
|
|
5393
|
+
};
|
|
5394
|
+
const { rejectedSpans: raw, errorMessage } = partial;
|
|
5395
|
+
const count = typeof raw === "string" ? Number(raw) : typeof raw === "number" ? raw : 0;
|
|
5396
|
+
if (!Number.isSafeInteger(count) || count <= 0) return {
|
|
5397
|
+
count: 0,
|
|
5398
|
+
message: ""
|
|
5399
|
+
};
|
|
5400
|
+
return {
|
|
5401
|
+
count: Math.min(count, batchLength),
|
|
5402
|
+
message: typeof errorMessage === "string" && errorMessage ? errorMessage : "no message"
|
|
5403
|
+
};
|
|
5404
|
+
}
|
|
5405
|
+
/** Node's fetch reports every socket failure as "fetch failed"; the cause names which one. */
|
|
5406
|
+
function describeFetchError(error) {
|
|
5407
|
+
if (!(error instanceof Error)) return String(error);
|
|
5408
|
+
const cause = error.cause;
|
|
5409
|
+
const code = cause && typeof cause === "object" && "code" in cause && typeof cause.code === "string" ? cause.code : void 0;
|
|
5410
|
+
return code ? `${error.message} (${code})` : error.message;
|
|
5411
|
+
}
|
|
5412
|
+
function positiveInteger(value, fallback) {
|
|
5413
|
+
return value !== void 0 && Number.isSafeInteger(value) && value > 0 ? value : fallback;
|
|
5305
5414
|
}
|
|
5306
5415
|
/**
|
|
5307
5416
|
* Convert a LoopTraceEvent into an OtelSpan for export.
|
|
@@ -5384,6 +5493,7 @@ function buildRuntimeEventOtelSpans(events, traceId, parentSpanId, options = {})
|
|
|
5384
5493
|
let name = `tangle.runtime.${event.type}`;
|
|
5385
5494
|
if (event.type === "tool_call" || event.type === "tool_result") {
|
|
5386
5495
|
name = `agent.${event.type}`;
|
|
5496
|
+
attrs[ATTR.spanKind] = "TOOL";
|
|
5387
5497
|
attrs["tool.name"] = event.toolName;
|
|
5388
5498
|
if (event.toolCallId) attrs["tool.call_id"] = event.toolCallId;
|
|
5389
5499
|
const mcp = mcpIdentity(event.toolName);
|
|
@@ -5393,6 +5503,7 @@ function buildRuntimeEventOtelSpans(events, traceId, parentSpanId, options = {})
|
|
|
5393
5503
|
if (payload !== void 0) attrs[event.type === "tool_call" ? "tool.input" : "tool.output"] = serialized(payload);
|
|
5394
5504
|
} else if (event.type === "llm_call") {
|
|
5395
5505
|
name = "gen_ai.client.inference";
|
|
5506
|
+
attrs[ATTR.spanKind] = "LLM";
|
|
5396
5507
|
attrs["gen_ai.request.model"] = event.model;
|
|
5397
5508
|
if (event.tokensIn !== void 0) attrs["gen_ai.usage.input_tokens"] = event.tokensIn;
|
|
5398
5509
|
if (event.tokensOut !== void 0) attrs["gen_ai.usage.output_tokens"] = event.tokensOut;
|
|
@@ -5446,11 +5557,24 @@ function buildLoopOtelSpans(events, traceId, rootParentSpanId) {
|
|
|
5446
5557
|
kind: 1,
|
|
5447
5558
|
startTimeUnixNano: msToNs(node.startMs),
|
|
5448
5559
|
endTimeUnixNano: msToNs(node.endMs),
|
|
5449
|
-
attributes: toOtelAttributes(
|
|
5560
|
+
attributes: toOtelAttributes({
|
|
5561
|
+
...node.attrs,
|
|
5562
|
+
[ATTR.spanKind]: LOOP_SPAN_KIND[node.kind]
|
|
5563
|
+
}),
|
|
5450
5564
|
status: { code: node.error ? 2 : 1 }
|
|
5451
5565
|
}));
|
|
5452
5566
|
}
|
|
5453
5567
|
/**
|
|
5568
|
+
* The declared kind of each loop level. An iteration span carries its own token total, and an
|
|
5569
|
+
* undeclared span with tokens reads as an LLM call, so declaring AGENT keeps those tokens from
|
|
5570
|
+
* being counted a second time beside the model-call spans beneath it.
|
|
5571
|
+
*/
|
|
5572
|
+
const LOOP_SPAN_KIND = {
|
|
5573
|
+
loop: "CHAIN",
|
|
5574
|
+
round: "CHAIN",
|
|
5575
|
+
branch: "AGENT"
|
|
5576
|
+
};
|
|
5577
|
+
/**
|
|
5454
5578
|
* Sink-neutral core behind {@link buildLoopOtelSpans}: reconstruct the
|
|
5455
5579
|
* loop → round → branch span tree from one run's ordered `LoopTraceEvent`
|
|
5456
5580
|
* stream. Consumed by the OTEL mapper above and by the MCP delegation
|
|
@@ -5930,7 +6054,13 @@ const HARNESS_ROOTS = Object.freeze({
|
|
|
5930
6054
|
".claude/todos"
|
|
5931
6055
|
]),
|
|
5932
6056
|
codex: Object.freeze([".codex/sessions", ".codex/history.jsonl"]),
|
|
5933
|
-
opencode: Object.freeze([
|
|
6057
|
+
opencode: Object.freeze([
|
|
6058
|
+
".local/share/opencode/storage",
|
|
6059
|
+
".local/share/opencode/export",
|
|
6060
|
+
".local/share/opencode/log",
|
|
6061
|
+
".opencode/sessions",
|
|
6062
|
+
".opencode/messages"
|
|
6063
|
+
]),
|
|
5934
6064
|
pi: Object.freeze([".pi/agent/sessions"])
|
|
5935
6065
|
});
|
|
5936
6066
|
/**
|
|
@@ -7398,7 +7528,8 @@ function assertRetainedRunReplayMaterial(provider, replay, admission) {
|
|
|
7398
7528
|
}, identity, retainedContextTransfer(replay.turn.contextTransfer)));
|
|
7399
7529
|
}
|
|
7400
7530
|
function retainedRunIntent(options, identity, contextTransfer) {
|
|
7401
|
-
const
|
|
7531
|
+
const { profile } = options.environment;
|
|
7532
|
+
const requestedProfileDigest = typeof profile === "string" ? canonicalCandidateDigest(profile) : canonicalAgentProfileDigest(profile);
|
|
7402
7533
|
const requestDigest = canonicalCandidateDigest({
|
|
7403
7534
|
kind: "retained-run-intent.v1",
|
|
7404
7535
|
provider: options.provider.name,
|
|
@@ -7533,6 +7664,67 @@ function priceUnreceiptedWork(work) {
|
|
|
7533
7664
|
};
|
|
7534
7665
|
}
|
|
7535
7666
|
//#endregion
|
|
7667
|
+
//#region src/runtime/supervise/error-message.ts
|
|
7668
|
+
const maxCauseDepth = 4;
|
|
7669
|
+
const maxMessageLength = 32768;
|
|
7670
|
+
const maxPartLength = 2048;
|
|
7671
|
+
/** A persisted failure must explain its wrappers without exporting credentials or unbounded data. */
|
|
7672
|
+
function errorText(value) {
|
|
7673
|
+
let text;
|
|
7674
|
+
try {
|
|
7675
|
+
text = typeof value === "string" ? value : JSON.stringify(value) ?? String(value);
|
|
7676
|
+
} catch {
|
|
7677
|
+
text = "[unprintable rejection]";
|
|
7678
|
+
}
|
|
7679
|
+
return String(defaultRedactor(text)).slice(0, maxMessageLength);
|
|
7680
|
+
}
|
|
7681
|
+
function errorProperty(error, field) {
|
|
7682
|
+
try {
|
|
7683
|
+
const value = error[field];
|
|
7684
|
+
return value === void 0 ? void 0 : errorText(value).slice(0, field === "name" ? 256 : maxMessageLength);
|
|
7685
|
+
} catch {
|
|
7686
|
+
return `[unreadable error ${field}]`;
|
|
7687
|
+
}
|
|
7688
|
+
}
|
|
7689
|
+
/** An HTTP status carried on a thrown error, read without trusting its getter. Provider SDKs put
|
|
7690
|
+
* one on their own error classes, so this is how a refusal is recognised outside the transport
|
|
7691
|
+
* taxonomy — for the persisted message here, and for the driver's retry verdict. */
|
|
7692
|
+
function errorHttpStatus(error) {
|
|
7693
|
+
try {
|
|
7694
|
+
const status = Reflect.get(error, "status");
|
|
7695
|
+
return typeof status === "number" && Number.isInteger(status) && status >= 100 && status <= 599 ? status : void 0;
|
|
7696
|
+
} catch {
|
|
7697
|
+
return;
|
|
7698
|
+
}
|
|
7699
|
+
}
|
|
7700
|
+
/** Shared by child settlements, driver attempts, and the final no-winner result. */
|
|
7701
|
+
function errMessage(error) {
|
|
7702
|
+
const parts = [];
|
|
7703
|
+
const seen = /* @__PURE__ */ new Set();
|
|
7704
|
+
let current = error;
|
|
7705
|
+
for (let depth = 0; depth <= maxCauseDepth; depth += 1) try {
|
|
7706
|
+
if (!(current instanceof Error)) {
|
|
7707
|
+
parts.push(errorText(current).slice(0, maxPartLength));
|
|
7708
|
+
break;
|
|
7709
|
+
}
|
|
7710
|
+
if (seen.has(current)) {
|
|
7711
|
+
parts.push("[circular]");
|
|
7712
|
+
break;
|
|
7713
|
+
}
|
|
7714
|
+
seen.add(current);
|
|
7715
|
+
const status = errorHttpStatus(current);
|
|
7716
|
+
const message = (status === void 0 ? "" : `HTTP ${status}: `) + (errorProperty(current, "message") ?? "");
|
|
7717
|
+
parts.push((depth === 0 ? message : `${errorProperty(current, "name")}: ${message}`).slice(0, maxPartLength));
|
|
7718
|
+
current = current.cause;
|
|
7719
|
+
if (current === void 0 || current === null) break;
|
|
7720
|
+
if (depth === maxCauseDepth) parts.push("[cause chain truncated]");
|
|
7721
|
+
} catch {
|
|
7722
|
+
parts.push("[unreadable error cause]");
|
|
7723
|
+
break;
|
|
7724
|
+
}
|
|
7725
|
+
return parts.join(": caused by ").slice(0, maxMessageLength);
|
|
7726
|
+
}
|
|
7727
|
+
//#endregion
|
|
7536
7728
|
//#region src/runtime/supervise/retained-executor.ts
|
|
7537
7729
|
/** Scope owns the durable writer; provider executors only publish sanitized admissions. */
|
|
7538
7730
|
const retainedExecutorSeamKey = "runtime.retainedExecutor";
|
|
@@ -7637,6 +7829,11 @@ var RetainedExecutionPendingError = class extends Error {
|
|
|
7637
7829
|
this.pendingCause = resolved;
|
|
7638
7830
|
}
|
|
7639
7831
|
};
|
|
7832
|
+
/** The execution id of a leaf's later invocation, derived from its `execution-input` sequence so
|
|
7833
|
+
* a recovery derives the same one. A first invocation keeps the node id. */
|
|
7834
|
+
function leafContinuationExecutionId(nodeId, inputSeq) {
|
|
7835
|
+
return `${nodeId}:input:${inputSeq}`;
|
|
7836
|
+
}
|
|
7640
7837
|
const retainedPreparations = /* @__PURE__ */ new WeakMap();
|
|
7641
7838
|
/** Reconstruct an admitted manager through its original composition boundary. @internal */
|
|
7642
7839
|
function registerRetainedExecutorPreparation(factory, prepare) {
|
|
@@ -7875,6 +8072,75 @@ function sandboxSessionTraceSource(box, sessionId, opts = {}) {
|
|
|
7875
8072
|
});
|
|
7876
8073
|
}
|
|
7877
8074
|
//#endregion
|
|
8075
|
+
//#region src/runtime/supervise/upstream-unavailable.ts
|
|
8076
|
+
/**
|
|
8077
|
+
* An upstream that cannot serve NOW: how Runtime recognizes one, and how long an agent waits for it.
|
|
8078
|
+
*
|
|
8079
|
+
* A model provider's exhausted quota, a rate limit, an overload, or the router's own refused
|
|
8080
|
+
* credential is not a fault in the agent: the same request succeeds when capacity returns. A
|
|
8081
|
+
* driver that meets one re-enters after a pause (`runDriverWithRetry`), and a leaf continues in
|
|
8082
|
+
* its environment after the same pause (`providerAsExecutor`). Both read the refusal and the pause
|
|
8083
|
+
* from here, so an outage costs a director and its workers the same number of turns.
|
|
8084
|
+
*
|
|
8085
|
+
* This module imports nothing, so the provider executor can read it without the error classes of
|
|
8086
|
+
* `../../errors`.
|
|
8087
|
+
*/
|
|
8088
|
+
/**
|
|
8089
|
+
* Machine codes that say the upstream cannot serve now rather than that the request is wrong.
|
|
8090
|
+
* Each is a code an upstream publishes, not prose: the router's
|
|
8091
|
+
* (`lib/model-substitution.ts`, `lib/upstream-error-triage.ts`) and the OpenAI- and Anthropic-shaped
|
|
8092
|
+
* error types a relayed body carries.
|
|
8093
|
+
*/
|
|
8094
|
+
const UNAVAILABLE_CODES = /* @__PURE__ */ new Set([
|
|
8095
|
+
"provider_quota_exhausted",
|
|
8096
|
+
"provider_quota_exceeded",
|
|
8097
|
+
"provider_rate_limit",
|
|
8098
|
+
"upstream_unavailable",
|
|
8099
|
+
"provider_key_invalid",
|
|
8100
|
+
"rate_limit_exceeded",
|
|
8101
|
+
"rate_limit_error",
|
|
8102
|
+
"overloaded_error"
|
|
8103
|
+
]);
|
|
8104
|
+
/** 429 Too Many Requests, 503 Service Unavailable, and 529, which Anthropic uses for overload. */
|
|
8105
|
+
const UNAVAILABLE_STATUSES = /* @__PURE__ */ new Set([
|
|
8106
|
+
429,
|
|
8107
|
+
503,
|
|
8108
|
+
529
|
|
8109
|
+
]);
|
|
8110
|
+
/**
|
|
8111
|
+
* A harness that flattens the router's JSON body into its own failure text leaves the router's
|
|
8112
|
+
* code in that text: `opencode execution failed: No provider served model "..."
|
|
8113
|
+
* (provider_quota_exhausted)`. The code is matched as a whole token, and a status only in the
|
|
8114
|
+
* `status code <n>` framing a harness prints, so prose that merely mentions a quota does not match.
|
|
8115
|
+
*/
|
|
8116
|
+
function unavailableSignalInText(text) {
|
|
8117
|
+
for (const token of text.toLowerCase().matchAll(/[a-z_]+/gu)) if (UNAVAILABLE_CODES.has(token[0])) return token[0];
|
|
8118
|
+
const status = text.match(/\bstatus code (\d{3})\b/u)?.[1];
|
|
8119
|
+
return status !== void 0 && UNAVAILABLE_STATUSES.has(Number(status)) ? `http-${status}` : void 0;
|
|
8120
|
+
}
|
|
8121
|
+
/**
|
|
8122
|
+
* The code or status that marks a failed turn outcome as an upstream capacity refusal, or
|
|
8123
|
+
* `undefined`. The outcome's own `errorCode` decides first; the failure text is read only when no
|
|
8124
|
+
* code decided, because a harness CLI reports the router's refusal as text.
|
|
8125
|
+
*/
|
|
8126
|
+
function unavailableSignalOfFailure(failure) {
|
|
8127
|
+
const code = failure.errorCode?.toLowerCase();
|
|
8128
|
+
if (code !== void 0 && UNAVAILABLE_CODES.has(code)) return code;
|
|
8129
|
+
return unavailableSignalInText(failure.error);
|
|
8130
|
+
}
|
|
8131
|
+
const DEFAULT_UNAVAILABLE_PAUSE_MS = 15e3;
|
|
8132
|
+
const DEFAULT_MAX_UNAVAILABLE_PAUSE_MS = 3e5;
|
|
8133
|
+
/**
|
|
8134
|
+
* The pause after the `consecutive`-th refusal in a row (1-based): the first pause, doubled per
|
|
8135
|
+
* earlier refusal, up to the ceiling. The one rule for a driver that re-enters and a leaf that
|
|
8136
|
+
* continues in its environment, so an outage costs both the same number of turns.
|
|
8137
|
+
*/
|
|
8138
|
+
function unavailablePauseMs(consecutive, policy = {}) {
|
|
8139
|
+
const first = Math.max(0, policy.unavailablePauseMs ?? DEFAULT_UNAVAILABLE_PAUSE_MS);
|
|
8140
|
+
const ceiling = Math.max(first, policy.maxUnavailablePauseMs ?? DEFAULT_MAX_UNAVAILABLE_PAUSE_MS);
|
|
8141
|
+
return Math.min(ceiling, first * 2 ** Math.max(0, consecutive - 1));
|
|
8142
|
+
}
|
|
8143
|
+
//#endregion
|
|
7878
8144
|
//#region src/runtime/environment-provider.ts
|
|
7879
8145
|
var ValidationError = class extends Error {
|
|
7880
8146
|
constructor(message, options) {
|
|
@@ -8043,6 +8309,11 @@ function providerTurnDefaults(options, context) {
|
|
|
8043
8309
|
const { prompt: _prompt, parts: _parts, model: _model, sessionId: _sessionId, signal: _signal, ...turn } = turnInputFromPrompt("", options);
|
|
8044
8310
|
return turn;
|
|
8045
8311
|
}
|
|
8312
|
+
/** What a leaf is told when it continues after a refused turn. The session carries its
|
|
8313
|
+
* conversation and the environment its files, so the instruction names neither the provider nor
|
|
8314
|
+
* the refusal: a director told the details of a provider failure spent its children probing the
|
|
8315
|
+
* infrastructure (discovery-lab AGENTS.md, measured 2026-09-11). */
|
|
8316
|
+
const LEAF_CONTINUATION_TASK = "Your previous turn was interrupted before it finished. This is the same task in the same session: your conversation so far and the files in your workspace are where you left them. Continue from where you stopped. Do not start over.";
|
|
8046
8317
|
/**
|
|
8047
8318
|
* Merge the declared turn defaults under one mapped turn.
|
|
8048
8319
|
*
|
|
@@ -8089,6 +8360,16 @@ function providerAsExecutor(provider, options = {}) {
|
|
|
8089
8360
|
return createProviderExecutor(provider, profile, ctx, selected.options, selected.identity);
|
|
8090
8361
|
};
|
|
8091
8362
|
}
|
|
8363
|
+
/**
|
|
8364
|
+
* Run a provider under a placement Runtime chose for the whole execution, such as a shared box.
|
|
8365
|
+
* The identity lands in the execution's materialization plan and binding, exactly where a
|
|
8366
|
+
* declared {@link ProviderPlacement} lands, so a record says which placement served it.
|
|
8367
|
+
* @internal
|
|
8368
|
+
*/
|
|
8369
|
+
function placedProviderExecutor(provider, options, identity) {
|
|
8370
|
+
if (options.placements !== void 0) throw new ValidationError("placedProviderExecutor: placements select within one provider");
|
|
8371
|
+
return (spec, ctx) => createProviderExecutor(provider, spec.profile, ctx, options, identity);
|
|
8372
|
+
}
|
|
8092
8373
|
function createProviderExecutor(provider, profile, ctx, options, placement) {
|
|
8093
8374
|
if (options.workspaceRetention !== void 0) assertProviderWorkspaceRetentionPort(options.workspaceRetention, `providerAsExecutor(${provider.name})`);
|
|
8094
8375
|
const controller = new AbortController();
|
|
@@ -8107,6 +8388,7 @@ function createProviderExecutor(provider, profile, ctx, options, placement) {
|
|
|
8107
8388
|
let workspaceCapturePromise;
|
|
8108
8389
|
let workspaceCleanupPromise;
|
|
8109
8390
|
let retainedReleasePromise;
|
|
8391
|
+
let releaseReadTried = false;
|
|
8110
8392
|
let workspaceOutcome;
|
|
8111
8393
|
let workspacePreservationRequired = false;
|
|
8112
8394
|
let workspaceRunActive = false;
|
|
@@ -8283,51 +8565,95 @@ function createProviderExecutor(provider, profile, ctx, options, placement) {
|
|
|
8283
8565
|
beginWorkspaceExecution();
|
|
8284
8566
|
const linked = linkAbort(ctx.signal, signal, controller.signal);
|
|
8285
8567
|
try {
|
|
8286
|
-
|
|
8287
|
-
|
|
8288
|
-
profile,
|
|
8289
|
-
createProfile,
|
|
8568
|
+
const executionStartedAt = Date.now();
|
|
8569
|
+
let invocation = {
|
|
8290
8570
|
task,
|
|
8291
|
-
signal: linked.signal,
|
|
8292
|
-
...node === void 0 ? {} : { node },
|
|
8293
|
-
options,
|
|
8294
8571
|
retention,
|
|
8295
8572
|
executionId,
|
|
8296
|
-
|
|
8297
|
-
|
|
8298
|
-
|
|
8299
|
-
|
|
8300
|
-
|
|
8301
|
-
|
|
8302
|
-
|
|
8303
|
-
|
|
8304
|
-
|
|
8305
|
-
|
|
8306
|
-
|
|
8307
|
-
|
|
8308
|
-
|
|
8309
|
-
|
|
8310
|
-
|
|
8311
|
-
|
|
8312
|
-
|
|
8573
|
+
recovering
|
|
8574
|
+
};
|
|
8575
|
+
let carried;
|
|
8576
|
+
let refusalsInARow = 0;
|
|
8577
|
+
for (let attempt = 1;; attempt += 1) {
|
|
8578
|
+
const continued = carried !== void 0;
|
|
8579
|
+
const ended = yield* streamProviderExecutor({
|
|
8580
|
+
provider,
|
|
8581
|
+
profile,
|
|
8582
|
+
createProfile,
|
|
8583
|
+
task: invocation.task,
|
|
8584
|
+
signal: linked.signal,
|
|
8585
|
+
...node === void 0 ? {} : { node },
|
|
8586
|
+
options,
|
|
8587
|
+
retention: invocation.retention,
|
|
8588
|
+
executionId: invocation.executionId,
|
|
8589
|
+
trace,
|
|
8590
|
+
recovering: invocation.recovering,
|
|
8591
|
+
...carried === void 0 ? {} : { carried },
|
|
8592
|
+
executionStartedAt,
|
|
8593
|
+
mayContinue: options.unavailablePause !== false && options.workspaceRetention === void 0 && invocation.retention?.continueInvocation !== void 0,
|
|
8594
|
+
onRetained: (handle) => {
|
|
8595
|
+
retained = handle;
|
|
8596
|
+
},
|
|
8597
|
+
onPending: (value) => {
|
|
8598
|
+
pending = value;
|
|
8599
|
+
},
|
|
8600
|
+
onEnvironment: (env) => {
|
|
8601
|
+
if (continued) {
|
|
8602
|
+
if (env.id !== environment?.id) throw new ValidationError(`providerAsExecutor(${provider.name}): a continuation ran in environment ${env.id}, not ${environment?.id ?? "<none>"}`);
|
|
8603
|
+
environment = env;
|
|
8604
|
+
return;
|
|
8313
8605
|
}
|
|
8314
|
-
|
|
8315
|
-
|
|
8316
|
-
|
|
8317
|
-
|
|
8318
|
-
|
|
8319
|
-
|
|
8320
|
-
|
|
8321
|
-
|
|
8322
|
-
|
|
8323
|
-
|
|
8324
|
-
|
|
8325
|
-
|
|
8326
|
-
|
|
8327
|
-
|
|
8328
|
-
|
|
8329
|
-
|
|
8330
|
-
|
|
8606
|
+
environment = env;
|
|
8607
|
+
resetWorkspaceState(env);
|
|
8608
|
+
if (harnessTranscript.status === "unavailable" && harnessTranscript.reason === "execution-never-started") harnessTranscript = harnessTranscriptUnavailable("capture-did-not-run");
|
|
8609
|
+
finalizeRuntimeOwnedPendingExecutor(executor, {
|
|
8610
|
+
...plannedDeclaration,
|
|
8611
|
+
execution: {
|
|
8612
|
+
kind: "environment",
|
|
8613
|
+
id: env.id
|
|
8614
|
+
}
|
|
8615
|
+
}, plannedBinding);
|
|
8616
|
+
},
|
|
8617
|
+
onArtifact: (next) => {
|
|
8618
|
+
artifact = next;
|
|
8619
|
+
},
|
|
8620
|
+
onHarnessTranscript: (next) => {
|
|
8621
|
+
harnessTranscript = next;
|
|
8622
|
+
},
|
|
8623
|
+
onUnsettledFailure: () => {
|
|
8624
|
+
workspacePreservationRequired = true;
|
|
8625
|
+
},
|
|
8626
|
+
onPublishedSnapshot: (snapshot) => {
|
|
8627
|
+
workspacePublishedSnapshot = snapshot;
|
|
8628
|
+
},
|
|
8629
|
+
captureWorkspace,
|
|
8630
|
+
destroyEnvironment
|
|
8631
|
+
});
|
|
8632
|
+
const refusal = ended.unavailable;
|
|
8633
|
+
const continueInvocation = invocation.retention?.continueInvocation;
|
|
8634
|
+
if (refusal === void 0 || continueInvocation === void 0) return;
|
|
8635
|
+
refusalsInARow = ended.madeProgress ? 1 : refusalsInARow + 1;
|
|
8636
|
+
const pauseMs = unavailablePauseMs(refusalsInARow, options.unavailablePause || {});
|
|
8637
|
+
await invocation.retention?.onPause?.({
|
|
8638
|
+
attempt,
|
|
8639
|
+
signal: refusal.signal,
|
|
8640
|
+
cause: refusal.cause,
|
|
8641
|
+
attemptMs: ended.durationMs,
|
|
8642
|
+
pauseMs,
|
|
8643
|
+
madeProgress: ended.madeProgress
|
|
8644
|
+
});
|
|
8645
|
+
await sleep(pauseMs, linked.signal);
|
|
8646
|
+
linked.signal.throwIfAborted();
|
|
8647
|
+
const next = await continueInvocation(LEAF_CONTINUATION_TASK);
|
|
8648
|
+
if (next.executionId === void 0) throw new ValidationError(`providerAsExecutor(${provider.name}): a continuation has no execution id of its own`);
|
|
8649
|
+
carried = ended.spent;
|
|
8650
|
+
invocation = {
|
|
8651
|
+
task: LEAF_CONTINUATION_TASK,
|
|
8652
|
+
retention: next,
|
|
8653
|
+
executionId: next.executionId,
|
|
8654
|
+
recovering: false
|
|
8655
|
+
};
|
|
8656
|
+
}
|
|
8331
8657
|
} finally {
|
|
8332
8658
|
linked.release();
|
|
8333
8659
|
workspaceRunActive = false;
|
|
@@ -8415,6 +8741,11 @@ function createProviderExecutor(provider, profile, ctx, options, placement) {
|
|
|
8415
8741
|
environment = target;
|
|
8416
8742
|
resetWorkspaceState(target);
|
|
8417
8743
|
}
|
|
8744
|
+
if (harnessTranscript.status !== "captured" && !releaseReadTried) {
|
|
8745
|
+
releaseReadTried = true;
|
|
8746
|
+
const capture = await captureHarnessTranscript(target, profile.harness, signal);
|
|
8747
|
+
if (capture.status === "captured") harnessTranscript = capture;
|
|
8748
|
+
}
|
|
8418
8749
|
const result = await destroyEnvironment(signal);
|
|
8419
8750
|
if (!result.destroyed) return [receipt(false, result.detail, result.permanent)];
|
|
8420
8751
|
pending = false;
|
|
@@ -8487,6 +8818,7 @@ async function* streamProviderExecutor(args) {
|
|
|
8487
8818
|
let settled;
|
|
8488
8819
|
let failure;
|
|
8489
8820
|
let failed = false;
|
|
8821
|
+
let unavailable;
|
|
8490
8822
|
try {
|
|
8491
8823
|
const toolParts = createSandboxToolPartState();
|
|
8492
8824
|
for await (const event of source.events) {
|
|
@@ -8531,7 +8863,7 @@ async function* streamProviderExecutor(args) {
|
|
|
8531
8863
|
};
|
|
8532
8864
|
}
|
|
8533
8865
|
if ((args.options.requireTerminalEvent ?? true) && !terminal) throw new ValidationError(`providerAsExecutor(${args.provider.name}): stream ended without a terminal result/done/status event`);
|
|
8534
|
-
yield { kind: "iteration" };
|
|
8866
|
+
if (args.carried === void 0) yield { kind: "iteration" };
|
|
8535
8867
|
const harnessTranscript = await captureHarnessTranscript(environment, args.profile.harness, linked);
|
|
8536
8868
|
args.onHarnessTranscript(harnessTranscript);
|
|
8537
8869
|
const outcome = failures.finish();
|
|
@@ -8549,6 +8881,13 @@ async function* streamProviderExecutor(args) {
|
|
|
8549
8881
|
...sawCostEstimate ? { usdEstimated } : {},
|
|
8550
8882
|
ms: Date.now() - started
|
|
8551
8883
|
};
|
|
8884
|
+
const executionSpent = args.carried === void 0 ? spent : {
|
|
8885
|
+
...addSpend(args.carried, {
|
|
8886
|
+
...spent,
|
|
8887
|
+
iterations: 0
|
|
8888
|
+
}),
|
|
8889
|
+
ms: Date.now() - args.executionStartedAt
|
|
8890
|
+
};
|
|
8552
8891
|
const verdict = await args.options.validator?.validate(result, {
|
|
8553
8892
|
iteration: 0,
|
|
8554
8893
|
...args.node === void 0 ? {} : { node: args.node },
|
|
@@ -8570,7 +8909,15 @@ async function* streamProviderExecutor(args) {
|
|
|
8570
8909
|
outRef: contentRef$1(`provider:${args.provider.name}`, settledResult),
|
|
8571
8910
|
out: settledResult,
|
|
8572
8911
|
...verdict ? { verdict } : {},
|
|
8573
|
-
spent
|
|
8912
|
+
spent: executionSpent
|
|
8913
|
+
};
|
|
8914
|
+
const refusal = args.mayContinue && source.retained && result.outcome?.status === "failed" ? unavailableSignalOfFailure({
|
|
8915
|
+
error: result.outcome.error ?? "",
|
|
8916
|
+
...result.outcome.errorCode ? { errorCode: result.outcome.errorCode } : {}
|
|
8917
|
+
}) : void 0;
|
|
8918
|
+
if (refusal !== void 0) unavailable = {
|
|
8919
|
+
signal: refusal,
|
|
8920
|
+
cause: errorText(result.outcome?.error ?? "")
|
|
8574
8921
|
};
|
|
8575
8922
|
if (source.retained) await args.retention?.onResult(settled);
|
|
8576
8923
|
args.onPending(false);
|
|
@@ -8592,7 +8939,7 @@ async function* streamProviderExecutor(args) {
|
|
|
8592
8939
|
}
|
|
8593
8940
|
args.onHarnessTranscript(await captureHarnessTranscript(environment, args.profile.harness, linked));
|
|
8594
8941
|
} finally {
|
|
8595
|
-
if ((!source.retained || settled !== void 0 && !failed) && !(source.retained && args.retention?.preserveEnvironment) && (args.options.destroyOnSettle ?? true)) {
|
|
8942
|
+
if ((!source.retained || settled !== void 0 && !failed) && !(source.retained && args.retention?.preserveEnvironment) && !(unavailable !== void 0 && !failed) && (args.options.destroyOnSettle ?? true)) {
|
|
8596
8943
|
const cleanup = await args.destroyEnvironment();
|
|
8597
8944
|
if (!cleanup.destroyed) {
|
|
8598
8945
|
const error = new Error(cleanup.detail ?? "provider environment teardown was not confirmed");
|
|
@@ -8612,6 +8959,12 @@ async function* streamProviderExecutor(args) {
|
|
|
8612
8959
|
}
|
|
8613
8960
|
}
|
|
8614
8961
|
if (failed) throw failure;
|
|
8962
|
+
return {
|
|
8963
|
+
...unavailable === void 0 ? {} : { unavailable },
|
|
8964
|
+
...settled === void 0 ? {} : { spent: settled.spent },
|
|
8965
|
+
madeProgress: tokens.input > 0 || tokens.output > 0 || usd > 0,
|
|
8966
|
+
durationMs: Date.now() - started
|
|
8967
|
+
};
|
|
8615
8968
|
/**
|
|
8616
8969
|
* The catalog price of this execution's whole token total, under the first model id the catalog
|
|
8617
8970
|
* knows. Undefined when none prices, so an unpriced model settles with no estimate at all rather
|
|
@@ -8636,6 +8989,14 @@ async function* streamProviderExecutor(args) {
|
|
|
8636
8989
|
if (priced.usdKnown === false && priced.usdEstimated !== void 0) return priced.usdEstimated;
|
|
8637
8990
|
}
|
|
8638
8991
|
}
|
|
8992
|
+
/** This invocation's cumulative tokens as the execution's: a continuation adds what its earlier
|
|
8993
|
+
* invocations already reported. */
|
|
8994
|
+
function executionTokens(own) {
|
|
8995
|
+
if (args.carried === void 0) return own;
|
|
8996
|
+
const total = cloneTokenUsage(args.carried.tokens);
|
|
8997
|
+
addTokenUsage(total, own);
|
|
8998
|
+
return total;
|
|
8999
|
+
}
|
|
8639
9000
|
function* creditUsage(receipt, event) {
|
|
8640
9001
|
if (receipt === void 0) return;
|
|
8641
9002
|
if (receipt.model !== (args.profile.name ?? "agent")) observedModel = receipt.model;
|
|
@@ -8649,7 +9010,7 @@ async function* streamProviderExecutor(args) {
|
|
|
8649
9010
|
yield {
|
|
8650
9011
|
kind: "tokens",
|
|
8651
9012
|
mode: "cumulative",
|
|
8652
|
-
...tokens,
|
|
9013
|
+
...executionTokens(tokens),
|
|
8653
9014
|
...receipt.tokensKnown === false ? { tokensKnown: false } : {}
|
|
8654
9015
|
};
|
|
8655
9016
|
}
|
|
@@ -14468,7 +14829,7 @@ function readWorkerProgress(scope, executor, now, stallAfterMs = DEFAULT_STALL_A
|
|
|
14468
14829
|
startedAt: scope.startedAt,
|
|
14469
14830
|
lastActivityAt,
|
|
14470
14831
|
idleMs,
|
|
14471
|
-
stalled: live && idleMs > stallAfterMs,
|
|
14832
|
+
stalled: live && scope.status !== "queued" && idleMs > stallAfterMs,
|
|
14472
14833
|
stallAfterMs,
|
|
14473
14834
|
turns: executor?.turns ?? scope.turns,
|
|
14474
14835
|
tokens: scope.tokens,
|
|
@@ -14478,7 +14839,8 @@ function readWorkerProgress(scope, executor, now, stallAfterMs = DEFAULT_STALL_A
|
|
|
14478
14839
|
pendingMessages: executor?.pendingMessages ?? 0,
|
|
14479
14840
|
recentActivity: executor?.recentActivity ?? [],
|
|
14480
14841
|
...executor?.derived?.length ? { derived: executor.derived } : {},
|
|
14481
|
-
...note ? { note } : {}
|
|
14842
|
+
...note ? { note } : {},
|
|
14843
|
+
...scope.team ? { team: scope.team } : {}
|
|
14482
14844
|
};
|
|
14483
14845
|
}
|
|
14484
14846
|
//#endregion
|
|
@@ -15646,6 +16008,12 @@ function mergeBridgeObservedModel(current, next) {
|
|
|
15646
16008
|
* then a ceiling, not a measurement. It does not close admission the way the dollar channel does:
|
|
15647
16009
|
* tokens are always capped, so one unreported turn must not end the run.
|
|
15648
16010
|
*
|
|
16011
|
+
* A reservation the free balance cannot cover now can WAIT instead of failing (`reserve(b, holder,
|
|
16012
|
+
* { wait: true })`) when it would fit once the reservations still open return. It holds nothing
|
|
16013
|
+
* while it waits, so conservation is untouched. Waiting reservations are granted in the order they
|
|
16014
|
+
* asked, as settlements return budget, and are refused once no open reservation is left to return
|
|
16015
|
+
* any: then nothing could ever cover them.
|
|
16016
|
+
*
|
|
15649
16017
|
* A pool can be RESTORED from a prior process's durable record (same-host restart): measured
|
|
15650
16018
|
* committed spend is debited exactly once, and each child journaled as started but never settled
|
|
15651
16019
|
* is charged at its full declared ceiling — restart never mints capacity, and the unknown flags
|
|
@@ -15667,6 +16035,19 @@ var BudgetReconcileFault = class extends Error {
|
|
|
15667
16035
|
if (budgetViolation !== void 0) this.budgetViolation = budgetViolation;
|
|
15668
16036
|
}
|
|
15669
16037
|
};
|
|
16038
|
+
/**
|
|
16039
|
+
* A waiting reservation that can never be granted: every open reservation settled and the free
|
|
16040
|
+
* balance still cannot cover it. `shortfalls` are the channels that did not fit at that moment.
|
|
16041
|
+
*/
|
|
16042
|
+
var ReservationWaitRefused = class extends Error {
|
|
16043
|
+
reason = "budget-exhausted";
|
|
16044
|
+
shortfalls;
|
|
16045
|
+
constructor(shortfalls) {
|
|
16046
|
+
super(`budget-exhausted: nothing left running could return enough budget for this worker (${shortfalls.map((shortfall) => `${shortfall.channel} requested ${shortfall.requested}, free ${shortfall.free}`).join("; ")})`);
|
|
16047
|
+
this.name = "ReservationWaitRefused";
|
|
16048
|
+
this.shortfalls = shortfalls;
|
|
16049
|
+
}
|
|
16050
|
+
};
|
|
15670
16051
|
/** Reject malformed ceilings before they can mint capacity through negative reservations. */
|
|
15671
16052
|
function assertValidBudget(budget, label = "budget") {
|
|
15672
16053
|
const safeInteger = (value, field) => {
|
|
@@ -15877,74 +16258,146 @@ function createBudgetPool(root, runStartedAtMs, restore = {}) {
|
|
|
15877
16258
|
}
|
|
15878
16259
|
let nextTicketId = 0;
|
|
15879
16260
|
const open = /* @__PURE__ */ new Map();
|
|
15880
|
-
|
|
15881
|
-
|
|
15882
|
-
|
|
15883
|
-
|
|
15884
|
-
|
|
15885
|
-
|
|
15886
|
-
}
|
|
15887
|
-
for (const name of Object.keys(b.resources ?? {})) if (!resources.has(name)) throw new ValidationError$1(`resource ${name}: root must declare its limit`);
|
|
16261
|
+
/** Tickets admitted to wait for budget, in the order they asked. Each is also in `open`. */
|
|
16262
|
+
const waiting = [];
|
|
16263
|
+
/** Open tickets that hold nothing: waiting, or refused and not yet reconciled. */
|
|
16264
|
+
const holdingNothing = /* @__PURE__ */ new Set();
|
|
16265
|
+
/** Every channel `b` does not fit, measured against the free balance less `keep`. */
|
|
16266
|
+
function shortfallsFor(b, keep) {
|
|
15888
16267
|
const wantTokens = b.maxTokens;
|
|
15889
16268
|
const wantUsd = b.maxUsd ?? 0;
|
|
15890
16269
|
const wantIterations = b.maxIterations;
|
|
15891
16270
|
const shortfalls = [];
|
|
15892
|
-
const short = (channel, requested, free, closedByUnknownSpend = false) => {
|
|
16271
|
+
const short = (channel, requested, free, held, closedByUnknownSpend = false) => {
|
|
15893
16272
|
shortfalls.push({
|
|
15894
16273
|
channel,
|
|
15895
16274
|
requested,
|
|
15896
16275
|
free: closedByUnknownSpend ? 0 : Math.max(0, free),
|
|
16276
|
+
...closedByUnknownSpend || held <= 0 ? {} : { held },
|
|
15897
16277
|
...closedByUnknownSpend ? { closedByUnknownSpend: true } : {}
|
|
15898
16278
|
});
|
|
15899
16279
|
};
|
|
15900
16280
|
for (const [name, state] of resources) {
|
|
15901
16281
|
const limit = b.resources[name].limit;
|
|
15902
|
-
if (!state.known) short(`resource:${name}`, limit, 0, true);
|
|
15903
|
-
else if (limit > state.remaining) short(`resource:${name}`, limit, state.remaining);
|
|
15904
|
-
}
|
|
15905
|
-
if (usdCapped && usdTainted) short("usd", wantUsd, 0, true);
|
|
15906
|
-
|
|
15907
|
-
|
|
15908
|
-
if (
|
|
15909
|
-
|
|
15910
|
-
|
|
15911
|
-
|
|
15912
|
-
|
|
15913
|
-
|
|
15914
|
-
|
|
15915
|
-
|
|
15916
|
-
|
|
15917
|
-
|
|
16282
|
+
if (!state.known) short(`resource:${name}`, limit, 0, 0, true);
|
|
16283
|
+
else if (limit > state.remaining) short(`resource:${name}`, limit, state.remaining, state.reserved);
|
|
16284
|
+
}
|
|
16285
|
+
if (usdCapped && usdTainted) short("usd", wantUsd, 0, 0, true);
|
|
16286
|
+
const tokensFree = freeTokens - (keep?.tokens ?? 0);
|
|
16287
|
+
const iterationsFree = freeIterations - (keep?.iterations ?? 0);
|
|
16288
|
+
if (wantTokens > tokensFree) short("tokens", wantTokens, tokensFree, reservedTokens);
|
|
16289
|
+
if (wantIterations > iterationsFree) short("iterations", wantIterations, iterationsFree, reservedIterations);
|
|
16290
|
+
const usdFree = freeUsd - (keep?.usd ?? 0);
|
|
16291
|
+
if (usdCapped && !usdTainted && wantUsd > usdFree) short("usd", wantUsd, usdFree, reservedUsd);
|
|
16292
|
+
return shortfalls;
|
|
16293
|
+
}
|
|
16294
|
+
/** Whether every short channel would fit once the open reservations return what they hold. */
|
|
16295
|
+
function fitsAfterRefunds(shortfalls) {
|
|
16296
|
+
return shortfalls.every((shortfall) => shortfall.closedByUnknownSpend !== true && shortfall.requested <= shortfall.free + (shortfall.held ?? 0));
|
|
16297
|
+
}
|
|
16298
|
+
/** Move `b` from free to reserved. The caller has checked that it fits. */
|
|
16299
|
+
function take(b) {
|
|
15918
16300
|
for (const [name, state] of resources) {
|
|
15919
16301
|
const amount = b.resources[name].limit;
|
|
15920
16302
|
state.remaining -= amount;
|
|
15921
16303
|
state.reserved += amount;
|
|
15922
16304
|
}
|
|
15923
|
-
freeTokens -=
|
|
15924
|
-
reservedTokens +=
|
|
15925
|
-
freeIterations -=
|
|
15926
|
-
reservedIterations +=
|
|
16305
|
+
freeTokens -= b.maxTokens;
|
|
16306
|
+
reservedTokens += b.maxTokens;
|
|
16307
|
+
freeIterations -= b.maxIterations;
|
|
16308
|
+
reservedIterations += b.maxIterations;
|
|
16309
|
+
const wantUsd = b.maxUsd ?? 0;
|
|
15927
16310
|
if (wantUsd > 0) {
|
|
15928
16311
|
freeUsd -= wantUsd;
|
|
15929
16312
|
reservedUsd += wantUsd;
|
|
15930
16313
|
}
|
|
15931
|
-
|
|
15932
|
-
|
|
16314
|
+
}
|
|
16315
|
+
function ticketFor(id, b) {
|
|
15933
16316
|
return {
|
|
15934
|
-
|
|
15935
|
-
|
|
15936
|
-
|
|
15937
|
-
|
|
15938
|
-
|
|
15939
|
-
|
|
15940
|
-
|
|
15941
|
-
iterations: wantIterations,
|
|
15942
|
-
usdBudgeted: b.maxUsd !== void 0
|
|
15943
|
-
}
|
|
16317
|
+
id,
|
|
16318
|
+
reserved: {
|
|
16319
|
+
...b.resources === void 0 ? {} : { resources: Object.fromEntries(Object.entries(b.resources).map(([name, value]) => [name, { ...value }])) },
|
|
16320
|
+
tokens: b.maxTokens,
|
|
16321
|
+
usd: b.maxUsd ?? 0,
|
|
16322
|
+
iterations: b.maxIterations,
|
|
16323
|
+
usdBudgeted: b.maxUsd !== void 0
|
|
15944
16324
|
}
|
|
15945
16325
|
};
|
|
15946
16326
|
}
|
|
15947
|
-
|
|
16327
|
+
/** Grant waiting tickets in the order they asked while the head fits. When the head does not
|
|
16328
|
+
* fit and no reservation is held that could return budget, nothing can ever cover it: refuse
|
|
16329
|
+
* it, and go on with the next. */
|
|
16330
|
+
function grantWaiting() {
|
|
16331
|
+
while (waiting.length > 0) {
|
|
16332
|
+
const head = waiting[0];
|
|
16333
|
+
const shortfalls = shortfallsFor(head.budget, head.keep);
|
|
16334
|
+
if (shortfalls.length === 0) {
|
|
16335
|
+
waiting.shift();
|
|
16336
|
+
take(head.budget);
|
|
16337
|
+
holdingNothing.delete(head.ticket.id);
|
|
16338
|
+
head.grant();
|
|
16339
|
+
continue;
|
|
16340
|
+
}
|
|
16341
|
+
if (open.size > holdingNothing.size) return;
|
|
16342
|
+
waiting.shift();
|
|
16343
|
+
head.refuse(new ReservationWaitRefused(shortfalls));
|
|
16344
|
+
}
|
|
16345
|
+
}
|
|
16346
|
+
function reserve(b, holder = { stage: "admitted" }, options = {}) {
|
|
16347
|
+
assertValidBudget(b, "reservation budget");
|
|
16348
|
+
for (const [name, state] of resources) {
|
|
16349
|
+
const wanted = b.resources?.[name];
|
|
16350
|
+
if (!wanted) throw new ValidationError$1(`resource ${name}: child must declare its limit`);
|
|
16351
|
+
if (wanted.unit !== state.unit) throw new ValidationError$1(`resource ${name}: unit mismatch`);
|
|
16352
|
+
}
|
|
16353
|
+
for (const name of Object.keys(b.resources ?? {})) if (!resources.has(name)) throw new ValidationError$1(`resource ${name}: root must declare its limit`);
|
|
16354
|
+
const wantUsd = b.maxUsd ?? 0;
|
|
16355
|
+
const keep = options.keep;
|
|
16356
|
+
const shortfalls = shortfallsFor(b, keep);
|
|
16357
|
+
if (shortfalls.length === 0 && wantUsd > 0 && !usdCapped) return {
|
|
16358
|
+
ok: false,
|
|
16359
|
+
reason: "usd-unbudgeted"
|
|
16360
|
+
};
|
|
16361
|
+
const queueAhead = options.wait === true && waiting.length > 0;
|
|
16362
|
+
if (shortfalls.length === 0 && !queueAhead) {
|
|
16363
|
+
take(b);
|
|
16364
|
+
const id = nextTicketId++;
|
|
16365
|
+
open.set(id, holder);
|
|
16366
|
+
return {
|
|
16367
|
+
ok: true,
|
|
16368
|
+
ticket: ticketFor(id, b)
|
|
16369
|
+
};
|
|
16370
|
+
}
|
|
16371
|
+
if (options.wait !== true || !fitsAfterRefunds(shortfalls)) return {
|
|
16372
|
+
ok: false,
|
|
16373
|
+
reason: "budget-exhausted",
|
|
16374
|
+
shortfalls
|
|
16375
|
+
};
|
|
16376
|
+
const id = nextTicketId++;
|
|
16377
|
+
open.set(id, holder);
|
|
16378
|
+
const ticket = ticketFor(id, b);
|
|
16379
|
+
let grant;
|
|
16380
|
+
let refuse;
|
|
16381
|
+
const granted = new Promise((resolve, reject) => {
|
|
16382
|
+
grant = resolve;
|
|
16383
|
+
refuse = reject;
|
|
16384
|
+
});
|
|
16385
|
+
granted.catch(() => void 0);
|
|
16386
|
+
holdingNothing.add(id);
|
|
16387
|
+
waiting.push({
|
|
16388
|
+
ticket,
|
|
16389
|
+
budget: b,
|
|
16390
|
+
keep,
|
|
16391
|
+
grant,
|
|
16392
|
+
refuse
|
|
16393
|
+
});
|
|
16394
|
+
return {
|
|
16395
|
+
ok: true,
|
|
16396
|
+
ticket,
|
|
16397
|
+
granted
|
|
16398
|
+
};
|
|
16399
|
+
}
|
|
16400
|
+
function attribute(ticket, holder) {
|
|
15948
16401
|
const current = open.get(ticket.id);
|
|
15949
16402
|
if (current === void 0) return;
|
|
15950
16403
|
open.set(ticket.id, {
|
|
@@ -15956,7 +16409,18 @@ function createBudgetPool(root, runStartedAtMs, restore = {}) {
|
|
|
15956
16409
|
if (!open.has(ticket.id)) throw new Error(`budget pool: reconcile of unknown or already-settled ticket ${ticket.id}`);
|
|
15957
16410
|
assertValidSpend(spent, `budget pool ticket ${ticket.id} spend`);
|
|
15958
16411
|
validateResourceUnits(spent);
|
|
15959
|
-
const
|
|
16412
|
+
const heldNothing = holdingNothing.delete(ticket.id);
|
|
16413
|
+
if (heldNothing) {
|
|
16414
|
+
const index = waiting.findIndex((entry) => entry.ticket.id === ticket.id);
|
|
16415
|
+
if (index >= 0) waiting.splice(index, 1);
|
|
16416
|
+
}
|
|
16417
|
+
const held = heldNothing ? {
|
|
16418
|
+
tokens: 0,
|
|
16419
|
+
usd: 0,
|
|
16420
|
+
iterations: 0,
|
|
16421
|
+
resources: void 0
|
|
16422
|
+
} : ticket.reserved;
|
|
16423
|
+
const { tokens: rTokens, usd: rUsd, iterations: rIterations } = held;
|
|
15960
16424
|
const unknownUnderCap = usdCapped && spent.usdKnown === false;
|
|
15961
16425
|
const spentTokens = chargedTokens(spent.tokens);
|
|
15962
16426
|
const usdBudgeted = ticket.reserved.usdBudgeted !== false;
|
|
@@ -15995,9 +16459,10 @@ function createBudgetPool(root, runStartedAtMs, restore = {}) {
|
|
|
15995
16459
|
freeUsd = 0;
|
|
15996
16460
|
} else freeUsd += rUsd - spent.usd;
|
|
15997
16461
|
} else committedUsd += spent.usd;
|
|
15998
|
-
const resourceSettlement = commitResources(spent,
|
|
16462
|
+
const resourceSettlement = commitResources(spent, held.resources ?? {});
|
|
15999
16463
|
fault ??= resourceSettlement.fault;
|
|
16000
16464
|
overspent.push(...resourceSettlement.overspent);
|
|
16465
|
+
grantWaiting();
|
|
16001
16466
|
const violation = overspent.length === 0 ? void 0 : Object.freeze({ overspent: Object.freeze(overspent.map((entry) => Object.freeze(entry))) });
|
|
16002
16467
|
if (fault !== void 0) throw new BudgetReconcileFault(`budget pool: ${fault}`, violation);
|
|
16003
16468
|
return violation;
|
|
@@ -16302,6 +16767,7 @@ var FileResultBlobStore = class {
|
|
|
16302
16767
|
return `${this.dir}/${outRef.replace(/:/g, "-")}.json`;
|
|
16303
16768
|
}
|
|
16304
16769
|
};
|
|
16770
|
+
/** @internal shared with the SQL-backed store. */
|
|
16305
16771
|
function encodeResultBlob(outRef, artifact) {
|
|
16306
16772
|
const detached = detachedSnapshot(artifact, "result blob");
|
|
16307
16773
|
assertContentAddress(outRef, detached);
|
|
@@ -16314,6 +16780,7 @@ function encodeResultBlob(outRef, artifact) {
|
|
|
16314
16780
|
text
|
|
16315
16781
|
};
|
|
16316
16782
|
}
|
|
16783
|
+
/** @internal shared with the SQL-backed store. */
|
|
16317
16784
|
function assertContentAddress(outRef, artifact) {
|
|
16318
16785
|
const expected = contentAddress(artifact);
|
|
16319
16786
|
if (outRef !== expected) throw new Error(`blob outRef '${outRef}' does not match the artifact content hash '${expected}'; a content-addressed store refuses a mismatched ref (breaks the replay invariant)`);
|
|
@@ -16751,6 +17218,7 @@ function servedIdentityParts(model) {
|
|
|
16751
17218
|
};
|
|
16752
17219
|
}
|
|
16753
17220
|
/** One validation implementation for memory, durable append, and cold replay. */
|
|
17221
|
+
/** @internal shared with the SQL-backed store. */
|
|
16754
17222
|
var SpawnEventIndex = class {
|
|
16755
17223
|
root;
|
|
16756
17224
|
nodes = /* @__PURE__ */ new Map();
|
|
@@ -17007,6 +17475,7 @@ async function replaySpawnTree(journal, blobs, root) {
|
|
|
17007
17475
|
...ev.harnessTranscript === void 0 ? {} : { harnessTranscript: ev.harnessTranscript },
|
|
17008
17476
|
...ev.retainedExecution === void 0 ? {} : { retainedExecution: ev.retainedExecution },
|
|
17009
17477
|
...ev.retainedPendingCause === void 0 ? {} : { retainedPendingCause: ev.retainedPendingCause },
|
|
17478
|
+
...ev.subtree === void 0 ? {} : { subtree: ev.subtree },
|
|
17010
17479
|
...settlementTime(ev.at),
|
|
17011
17480
|
seq: ev.seq
|
|
17012
17481
|
});
|
|
@@ -17027,6 +17496,7 @@ async function replaySpawnTree(journal, blobs, root) {
|
|
|
17027
17496
|
...ev.harnessTranscript === void 0 ? {} : { harnessTranscript: ev.harnessTranscript },
|
|
17028
17497
|
...ev.retainedExecution === void 0 ? {} : { retainedExecution: ev.retainedExecution },
|
|
17029
17498
|
...ev.retainedPendingCause === void 0 ? {} : { retainedPendingCause: ev.retainedPendingCause },
|
|
17499
|
+
...ev.subtree === void 0 ? {} : { subtree: ev.subtree },
|
|
17030
17500
|
...settlementTime(ev.at),
|
|
17031
17501
|
seq: ev.seq
|
|
17032
17502
|
});
|
|
@@ -17048,6 +17518,7 @@ async function replaySpawnTree(journal, blobs, root) {
|
|
|
17048
17518
|
...budgetViolationOf(ev),
|
|
17049
17519
|
trace,
|
|
17050
17520
|
...ev.harnessTranscript === void 0 ? {} : { harnessTranscript: ev.harnessTranscript },
|
|
17521
|
+
...ev.subtree === void 0 ? {} : { subtree: ev.subtree },
|
|
17051
17522
|
...settlementTime(ev.at),
|
|
17052
17523
|
seq: ev.seq
|
|
17053
17524
|
});
|
|
@@ -19289,7 +19760,8 @@ function snapshotExecutorConfig(config) {
|
|
|
19289
19760
|
});
|
|
19290
19761
|
}
|
|
19291
19762
|
case "provider": {
|
|
19292
|
-
const { provider, registry, profileForCreate, taskToTurn, validator, workspaceRetention, ...decisionData } = config;
|
|
19763
|
+
const { provider, registry, profileForCreate, taskToTurn, validator, workspaceRetention, shared, ...decisionData } = config;
|
|
19764
|
+
if (shared !== void 0 && (decisionData.steering !== void 0 || decisionData.placements !== void 0 || workspaceRetention !== void 0)) throw new ValidationError$1("createExecutor(provider): shared placement cannot combine with steering, placements, or workspaceRetention");
|
|
19293
19765
|
const snapshot = detachedSnapshot(decisionData, "createExecutor provider config");
|
|
19294
19766
|
const resolvedProvider = resolveAgentEnvironmentProvider(provider, registry);
|
|
19295
19767
|
return Object.freeze({
|
|
@@ -19298,7 +19770,8 @@ function snapshotExecutorConfig(config) {
|
|
|
19298
19770
|
...profileForCreate === void 0 ? {} : { profileForCreate },
|
|
19299
19771
|
...taskToTurn === void 0 ? {} : { taskToTurn },
|
|
19300
19772
|
...validator === void 0 ? {} : { validator },
|
|
19301
|
-
...workspaceRetention === void 0 ? {} : { workspaceRetention }
|
|
19773
|
+
...workspaceRetention === void 0 ? {} : { workspaceRetention },
|
|
19774
|
+
...shared === void 0 ? {} : { shared }
|
|
19302
19775
|
});
|
|
19303
19776
|
}
|
|
19304
19777
|
case "sandbox": {
|
|
@@ -19490,13 +19963,22 @@ function createExecutor(config) {
|
|
|
19490
19963
|
return wrapper;
|
|
19491
19964
|
}
|
|
19492
19965
|
const profileForCreate = providerSeam.profileForCreate;
|
|
19966
|
+
const exactProfileForCreate = (profile) => {
|
|
19967
|
+
const prepared = executableAgentProfileSnapshot(profileForCreate?.(profile) ?? profile, "createExecutor(provider)");
|
|
19968
|
+
if (canonicalAgentProfileDigest(prepared) !== canonicalAgentProfileDigest(profile)) throw new ValidationError$1("createExecutor(provider): profileForCreate changed the exact AgentProfile; execution overlays are not allowed");
|
|
19969
|
+
return prepared;
|
|
19970
|
+
};
|
|
19971
|
+
const shared = providerSeam.shared;
|
|
19972
|
+
if (shared !== void 0 && shared.refusal(spec.profile) === void 0) {
|
|
19973
|
+
const { provider: _provider, registry: _registry, shared: _shared, defaults: _defaults, ...turnOptions } = providerSeam;
|
|
19974
|
+
return placedProviderExecutor(shared.providerFor(seamed.node === void 0 ? {} : { nodeId: seamed.node.nodeId }), {
|
|
19975
|
+
...turnOptions,
|
|
19976
|
+
profileForCreate: exactProfileForCreate
|
|
19977
|
+
}, shared.identity)(spec, seamed);
|
|
19978
|
+
}
|
|
19493
19979
|
return providerAsExecutor(provider, {
|
|
19494
19980
|
...providerSeam,
|
|
19495
|
-
profileForCreate:
|
|
19496
|
-
const prepared = executableAgentProfileSnapshot(profileForCreate?.(profile) ?? profile, "createExecutor(provider)");
|
|
19497
|
-
if (canonicalAgentProfileDigest(prepared) !== canonicalAgentProfileDigest(profile)) throw new ValidationError$1("createExecutor(provider): profileForCreate changed the exact AgentProfile; execution overlays are not allowed");
|
|
19498
|
-
return prepared;
|
|
19499
|
-
}
|
|
19981
|
+
profileForCreate: exactProfileForCreate
|
|
19500
19982
|
})(spec, seamed);
|
|
19501
19983
|
}
|
|
19502
19984
|
case "sandbox": {
|
|
@@ -19710,28 +20192,12 @@ function singleShotDriver(maxIterations) {
|
|
|
19710
20192
|
* recorded and STOPS further admission — the loop then drains what is already live and returns.
|
|
19711
20193
|
* It never retries a rejected spawn against the same pool, and it never spawns past `width`.
|
|
19712
20194
|
*
|
|
19713
|
-
* ──
|
|
19714
|
-
*
|
|
19715
|
-
*
|
|
19716
|
-
*
|
|
19717
|
-
*
|
|
19718
|
-
*
|
|
19719
|
-
* level. One shared Scope counter bounds every spawned manager and leaf in the recursive tree;
|
|
19720
|
-
* `spawn_worker` fails closed with `error: 'max-live-workers'` past it. Unset ⇒ NO tree cap.
|
|
19721
|
-
* `CoordinationToolsOptions.maxLiveWorkers` remains the local form for a toolbox mounted on a
|
|
19722
|
-
* caller-owned Scope that has no tree limit.
|
|
19723
|
-
* 2. `SandboxLineage`'s `maxConcurrency` / `DEFAULT_FORK_CONCURRENCY = 4`
|
|
19724
|
-
* (`src/runtime/sandbox-lineage.ts`) — kernel level. How many BOXES one `runAgentRounds` fork wave
|
|
19725
|
-
* provisions at once. It bounds a single leaf's fanout, not the supervisor's worker count.
|
|
19726
|
-
* 3. A host's own live-box governor (e.g. the kernel's `ComputeGovernor`, `maxSandboxes = 4`) — fleet
|
|
19727
|
-
* level. How many sandboxes may exist across the whole host process.
|
|
19728
|
-
*
|
|
19729
|
-
* The honest effective limit on simultaneous WORKERS is the minimum of the caps that apply to the
|
|
19730
|
-
* worker layer — (1) and (3). (2) is a different unit (boxes per fork wave inside one leaf) and
|
|
19731
|
-
* must not be min'd into it, or a 4-way fork inside one worker reads as a 4-worker ceiling.
|
|
19732
|
-
* `effectiveConcurrency` computes that minimum in one place so a host derives ONE number and
|
|
19733
|
-
* passes it to BOTH `maxLiveWorkers` and this dispatcher's `width`, instead of leaving a fleet
|
|
19734
|
-
* governor of 4 and an unset worker fence as unrelated numbers.
|
|
20195
|
+
* ── Concurrency ────────────────────────────────────────────────────────────────────────────────
|
|
20196
|
+
*
|
|
20197
|
+
* `width` bounds the children THIS dispatcher holds in flight. The tree-wide bound on working
|
|
20198
|
+
* agents is `SuperviseOptions.workerSlots` (`./worker-slots`): a spawn past it is queued by the
|
|
20199
|
+
* scope, not refused, so a `width` above the free slots only lengthens the allocator's queue.
|
|
20200
|
+
* `SandboxLineage`'s fork concurrency bounds boxes inside ONE leaf's fork wave, a different unit.
|
|
19735
20201
|
*
|
|
19736
20202
|
* ── Why this is not a copy of the kernel's batch loop ──────────────────────────────────────────
|
|
19737
20203
|
*
|
|
@@ -19820,23 +20286,6 @@ function freeSlots(liveCount, cap) {
|
|
|
19820
20286
|
if (cap === void 0 || cap <= 0) return null;
|
|
19821
20287
|
return Math.max(0, cap - liveCount);
|
|
19822
20288
|
}
|
|
19823
|
-
/**
|
|
19824
|
-
* The ONE honest effective limit on simultaneous workers: the minimum of the caps that actually
|
|
19825
|
-
* bound the worker layer. Ignores unset/non-positive caps; returns `undefined` when no cap applies
|
|
19826
|
-
* (uncapped — the conserved pool remains the only fence).
|
|
19827
|
-
*
|
|
19828
|
-
* Deliberately does NOT fold in `SandboxLineage`'s fork concurrency: that bounds boxes inside ONE
|
|
19829
|
-
* leaf's fork wave, a different unit. Folding it in would report a 4-worker ceiling for what is
|
|
19830
|
-
* really a 4-box fanout inside a single worker.
|
|
19831
|
-
*
|
|
19832
|
-
* Use it once, at the top of a run, and pass the result to BOTH `maxLiveWorkers` and a
|
|
19833
|
-
* dispatcher's `width` — that is what turns three unrelated numbers into one.
|
|
19834
|
-
*/
|
|
19835
|
-
function effectiveConcurrency(caps) {
|
|
19836
|
-
const applicable = [caps.maxLiveWorkers, caps.maxSandboxes].filter((c) => typeof c === "number" && c > 0);
|
|
19837
|
-
if (applicable.length === 0) return void 0;
|
|
19838
|
-
return Math.min(...applicable);
|
|
19839
|
-
}
|
|
19840
20289
|
/** Convenience: a `DispatchUnit` factory over a fixed array of tasks, for the common case where
|
|
19841
20290
|
* the queue is known up front and only the refill behavior is wanted. */
|
|
19842
20291
|
function queueOf(units, budget) {
|
|
@@ -19856,67 +20305,6 @@ function queueOf(units, budget) {
|
|
|
19856
20305
|
};
|
|
19857
20306
|
}
|
|
19858
20307
|
//#endregion
|
|
19859
|
-
//#region src/runtime/supervise/error-message.ts
|
|
19860
|
-
const maxCauseDepth = 4;
|
|
19861
|
-
const maxMessageLength = 32768;
|
|
19862
|
-
const maxPartLength = 2048;
|
|
19863
|
-
/** A persisted failure must explain its wrappers without exporting credentials or unbounded data. */
|
|
19864
|
-
function errorText(value) {
|
|
19865
|
-
let text;
|
|
19866
|
-
try {
|
|
19867
|
-
text = typeof value === "string" ? value : JSON.stringify(value) ?? String(value);
|
|
19868
|
-
} catch {
|
|
19869
|
-
text = "[unprintable rejection]";
|
|
19870
|
-
}
|
|
19871
|
-
return String(defaultRedactor(text)).slice(0, maxMessageLength);
|
|
19872
|
-
}
|
|
19873
|
-
function errorProperty(error, field) {
|
|
19874
|
-
try {
|
|
19875
|
-
const value = error[field];
|
|
19876
|
-
return value === void 0 ? void 0 : errorText(value).slice(0, field === "name" ? 256 : maxMessageLength);
|
|
19877
|
-
} catch {
|
|
19878
|
-
return `[unreadable error ${field}]`;
|
|
19879
|
-
}
|
|
19880
|
-
}
|
|
19881
|
-
/** An HTTP status carried on a thrown error, read without trusting its getter. Provider SDKs put
|
|
19882
|
-
* one on their own error classes, so this is how a refusal is recognised outside the transport
|
|
19883
|
-
* taxonomy — for the persisted message here, and for the driver's retry verdict. */
|
|
19884
|
-
function errorHttpStatus(error) {
|
|
19885
|
-
try {
|
|
19886
|
-
const status = Reflect.get(error, "status");
|
|
19887
|
-
return typeof status === "number" && Number.isInteger(status) && status >= 100 && status <= 599 ? status : void 0;
|
|
19888
|
-
} catch {
|
|
19889
|
-
return;
|
|
19890
|
-
}
|
|
19891
|
-
}
|
|
19892
|
-
/** Shared by child settlements, driver attempts, and the final no-winner result. */
|
|
19893
|
-
function errMessage(error) {
|
|
19894
|
-
const parts = [];
|
|
19895
|
-
const seen = /* @__PURE__ */ new Set();
|
|
19896
|
-
let current = error;
|
|
19897
|
-
for (let depth = 0; depth <= maxCauseDepth; depth += 1) try {
|
|
19898
|
-
if (!(current instanceof Error)) {
|
|
19899
|
-
parts.push(errorText(current).slice(0, maxPartLength));
|
|
19900
|
-
break;
|
|
19901
|
-
}
|
|
19902
|
-
if (seen.has(current)) {
|
|
19903
|
-
parts.push("[circular]");
|
|
19904
|
-
break;
|
|
19905
|
-
}
|
|
19906
|
-
seen.add(current);
|
|
19907
|
-
const status = errorHttpStatus(current);
|
|
19908
|
-
const message = (status === void 0 ? "" : `HTTP ${status}: `) + (errorProperty(current, "message") ?? "");
|
|
19909
|
-
parts.push((depth === 0 ? message : `${errorProperty(current, "name")}: ${message}`).slice(0, maxPartLength));
|
|
19910
|
-
current = current.cause;
|
|
19911
|
-
if (current === void 0 || current === null) break;
|
|
19912
|
-
if (depth === maxCauseDepth) parts.push("[cause chain truncated]");
|
|
19913
|
-
} catch {
|
|
19914
|
-
parts.push("[unreadable error cause]");
|
|
19915
|
-
break;
|
|
19916
|
-
}
|
|
19917
|
-
return parts.join(": caused by ").slice(0, maxMessageLength);
|
|
19918
|
-
}
|
|
19919
|
-
//#endregion
|
|
19920
20308
|
//#region src/runtime/retained-interactive-handle.ts
|
|
19921
20309
|
function createRetainedInteractiveRunHandle(environment, inputRef, capabilities, requestedStart) {
|
|
19922
20310
|
const ref = freezeInteractiveRef(inputRef);
|
|
@@ -20727,7 +21115,8 @@ function settlementFields(subject, settlement) {
|
|
|
20727
21115
|
...settlement.providerModel ? { providerModel: settlement.providerModel } : {},
|
|
20728
21116
|
...subject.budgetViolation ? { budgetViolation: subject.budgetViolation } : {},
|
|
20729
21117
|
trace: settlement.trace,
|
|
20730
|
-
...settlement.harnessTranscript ? { harnessTranscript: settlement.harnessTranscript } : {}
|
|
21118
|
+
...settlement.harnessTranscript ? { harnessTranscript: settlement.harnessTranscript } : {},
|
|
21119
|
+
...subject.subtree === void 0 ? {} : { subtree: subject.subtree }
|
|
20731
21120
|
};
|
|
20732
21121
|
}
|
|
20733
21122
|
/** `cancelled` keeps its `source` from the subject's own cancellation reason, so a released
|
|
@@ -20770,14 +21159,15 @@ function settledNodeEvidence(subject, settlement, settledAt) {
|
|
|
20770
21159
|
trace: detachedSnapshot(settlement.trace, "worker trace evidence")
|
|
20771
21160
|
};
|
|
20772
21161
|
}
|
|
20773
|
-
/** The `agent.child` payload of
|
|
20774
|
-
* kept at the settlement instant and `metered` omitted so the
|
|
20775
|
-
* twice. The sweep and the resume heal emit
|
|
20776
|
-
|
|
21162
|
+
/** The `agent.child` payload of a closed retained slot: the settlement restated with the state
|
|
21163
|
+
* the slot closed in, `settledAt` kept at the settlement instant and `metered` omitted so the
|
|
21164
|
+
* driver's inference is not summed twice. The sweep, the final close and the resume heal emit
|
|
21165
|
+
* it from the same builder. */
|
|
21166
|
+
function releasedChildPayload(subject, settlement, settledAt, releasedAt, retainedExecution = "released") {
|
|
20777
21167
|
return {
|
|
20778
21168
|
childId: subject.id,
|
|
20779
21169
|
status: "down",
|
|
20780
|
-
retainedExecution
|
|
21170
|
+
retainedExecution,
|
|
20781
21171
|
...settlement.retainedPendingCause === void 0 ? {} : { retainedPendingCause: settlement.retainedPendingCause },
|
|
20782
21172
|
releasedAt,
|
|
20783
21173
|
...settlement.outRef === void 0 ? {} : { outRef: settlement.outRef },
|
|
@@ -20899,7 +21289,10 @@ async function prepareInterruptedExecutors(opts, events, signal, now, parentId =
|
|
|
20899
21289
|
for (const node of interrupted) {
|
|
20900
21290
|
signal.throwIfAborted();
|
|
20901
21291
|
const owned = events.filter((event) => event.id === node.id);
|
|
20902
|
-
const
|
|
21292
|
+
const inputs = owned.filter((event) => event.kind === "execution-input");
|
|
21293
|
+
const latestInput = inputs.at(-1);
|
|
21294
|
+
const current = latestInput === void 0 || inputs.length === 1 ? owned : owned.slice(owned.indexOf(latestInput));
|
|
21295
|
+
const recorded = current.find((event) => event.kind === "execution-result");
|
|
20903
21296
|
if (recorded) {
|
|
20904
21297
|
await assertRecordedResult(node, recorded, owned, opts);
|
|
20905
21298
|
if (!rootBudget) throw new RuntimeRunStateError("retained results have no original root budget");
|
|
@@ -20921,10 +21314,9 @@ async function prepareInterruptedExecutors(opts, events, signal, now, parentId =
|
|
|
20921
21314
|
});
|
|
20922
21315
|
continue;
|
|
20923
21316
|
}
|
|
20924
|
-
const admissions =
|
|
21317
|
+
const admissions = current.flatMap((event) => event.kind === "execution-admitted" ? [event.admission] : []);
|
|
20925
21318
|
if (!opts.recoverExecutor || admissions.length === 0 && !node.ownedTreeRoot) continue;
|
|
20926
|
-
const
|
|
20927
|
-
const taskRef = input?.kind === "execution-input" ? input.taskRef : void 0;
|
|
21319
|
+
const taskRef = inputs[0]?.taskRef;
|
|
20928
21320
|
if (!node.profileRef || !taskRef || !node.identity?.profileDigest || !node.identity.taskDigest) throw new RuntimeRunStateError(`cannot recover '${node.id}' without exact profile and task bytes`);
|
|
20929
21321
|
const profile = agentProfileSchema.parse(await opts.blobs.get(node.profileRef));
|
|
20930
21322
|
const task = await opts.blobs.get(taskRef);
|
|
@@ -20943,12 +21335,14 @@ async function prepareInterruptedExecutors(opts, events, signal, now, parentId =
|
|
|
20943
21335
|
if (!prepared) continue;
|
|
20944
21336
|
}
|
|
20945
21337
|
const priorMaterialization = owned.find((event) => event.kind === "materialized")?.receipt;
|
|
21338
|
+
const continuation = latestInput !== void 0 && inputs.length > 1 && node.ownedTreeRoot === void 0 ? await leafContinuation(node, owned, latestInput, opts) : void 0;
|
|
20946
21339
|
recoveries.push({
|
|
20947
21340
|
...detachedSnapshot({
|
|
20948
21341
|
spawned: node,
|
|
20949
21342
|
task,
|
|
20950
21343
|
admissions,
|
|
20951
|
-
...priorMaterialization === void 0 ? {} : { priorMaterialization }
|
|
21344
|
+
...priorMaterialization === void 0 ? {} : { priorMaterialization },
|
|
21345
|
+
...continuation === void 0 ? {} : { continuation }
|
|
20952
21346
|
}, "retained child recovery"),
|
|
20953
21347
|
spec: prepared?.spec ?? {
|
|
20954
21348
|
profile,
|
|
@@ -20984,6 +21378,23 @@ async function prepareInterruptedExecutors(opts, events, signal, now, parentId =
|
|
|
20984
21378
|
recoveries
|
|
20985
21379
|
};
|
|
20986
21380
|
}
|
|
21381
|
+
/**
|
|
21382
|
+
* The later invocation a leaf was running when its process stopped: its own task, and the
|
|
21383
|
+
* environment admission of the invocation before it, which it continued in. The execution id is
|
|
21384
|
+
* derived from the input sequence exactly as the live scope derived it.
|
|
21385
|
+
*/
|
|
21386
|
+
async function leafContinuation(node, owned, latestInput, opts) {
|
|
21387
|
+
const priorSession = [...owned.slice(0, owned.indexOf(latestInput))].reverse().flatMap((event) => event.kind === "execution-admitted" && event.admission.phase === "environment" ? [event.admission] : [])[0];
|
|
21388
|
+
if (priorSession === void 0) throw new RuntimeRunStateError(`cannot recover '${node.id}': its continuation names no environment it continued in`);
|
|
21389
|
+
const task = await opts.blobs.get(latestInput.taskRef);
|
|
21390
|
+
if (task === void 0 || contentAddress(task) !== latestInput.taskRef) throw new RuntimeRunStateError(`continuation task for '${node.id}' is missing or corrupt`);
|
|
21391
|
+
return {
|
|
21392
|
+
task,
|
|
21393
|
+
inputSeq: latestInput.seq,
|
|
21394
|
+
executionId: leafContinuationExecutionId(node.id, latestInput.seq),
|
|
21395
|
+
priorSession
|
|
21396
|
+
};
|
|
21397
|
+
}
|
|
20987
21398
|
async function assertRecordedResult(node, result, events, opts) {
|
|
20988
21399
|
const before = events.slice(0, events.indexOf(result));
|
|
20989
21400
|
const receipt = before.find((event) => event.kind === "materialized")?.receipt;
|
|
@@ -20997,7 +21408,11 @@ async function assertRecordedResult(node, result, events, opts) {
|
|
|
20997
21408
|
* The keyed assignments a prior journal proves: every `spawned` event carrying a `key`, resolved
|
|
20998
21409
|
* against the replayed settlements. A key spawned more than once (a retry chain) resolves to its
|
|
20999
21410
|
* LATEST attempt — iterate in ordinal order so later spawns overwrite earlier ones. A spawned
|
|
21000
|
-
* event with no matching settlement is `in-doubt`: the process died with it in flight
|
|
21411
|
+
* event with no matching settlement is `in-doubt`: the process died with it in flight — UNLESS
|
|
21412
|
+
* the spawn's own runtime proves the execution cannot have outlived the process (`inline`, the
|
|
21413
|
+
* same predicate the budget layer charges no uncertain reservation for), in which case the resume
|
|
21414
|
+
* itself is the terminal evidence and the key resolves `down` with a derived reason, so a re-spawn
|
|
21415
|
+
* under the same key retries instead of wedging in-doubt.
|
|
21001
21416
|
*/
|
|
21002
21417
|
function keyedAssignments(events, settled) {
|
|
21003
21418
|
const byId = new Map(settled.map((s) => [s.handle.id, s]));
|
|
@@ -21007,7 +21422,13 @@ function keyedAssignments(events, settled) {
|
|
|
21007
21422
|
if (ev.key === void 0) continue;
|
|
21008
21423
|
if (ev.identity === void 0) throw new RuntimeRunStateError(`supervisor: keyed node '${ev.id}' has no durable execution identity`);
|
|
21009
21424
|
const s = byId.get(ev.id);
|
|
21010
|
-
keys.set(ev.key, s === void 0 ? {
|
|
21425
|
+
keys.set(ev.key, s === void 0 ? ev.runtime === "inline" ? {
|
|
21426
|
+
id: ev.id,
|
|
21427
|
+
label: ev.label,
|
|
21428
|
+
identity: ev.identity,
|
|
21429
|
+
state: "down",
|
|
21430
|
+
reason: `interrupted: the process died with '${ev.label}' in flight and its '${ev.runtime}' runtime cannot outlive that process, so the resume proves the attempt dead`
|
|
21431
|
+
} : {
|
|
21011
21432
|
id: ev.id,
|
|
21012
21433
|
label: ev.label,
|
|
21013
21434
|
identity: ev.identity,
|
|
@@ -22088,6 +22509,230 @@ function defaultSleep$1(ms, signal) {
|
|
|
22088
22509
|
return sleep(ms, signal);
|
|
22089
22510
|
}
|
|
22090
22511
|
//#endregion
|
|
22512
|
+
//#region src/runtime/supervise/worker-slots.ts
|
|
22513
|
+
/**
|
|
22514
|
+
* Worker slots: the one bound on how many spawned agents work at the same time.
|
|
22515
|
+
*
|
|
22516
|
+
* Every scope of a supervised tree draws from one allocator. A host that runs several trees in one
|
|
22517
|
+
* process can pass the same allocator to each of them, and the bound then holds across all of them.
|
|
22518
|
+
*
|
|
22519
|
+
* A spawn past the bound is never refused. It keeps its reserved budget slice and waits in a queue,
|
|
22520
|
+
* and it starts when a slot frees. The queue releases the deepest spawn first and then the oldest,
|
|
22521
|
+
* so a subtree that is already paid for finishes before the tree grows wider.
|
|
22522
|
+
*
|
|
22523
|
+
* A slot counts a working agent: a spawned agent that runs with no running child of its own. A
|
|
22524
|
+
* manager whose children are all running is waiting on them, so its first running child takes over
|
|
22525
|
+
* the manager's slot and the manager takes it back when its last running child ends. This makes
|
|
22526
|
+
* nested waits deadlock-free: a manager can always run one child on its own slot, so a queued subtree
|
|
22527
|
+
* always keeps a path to completion, whatever the bound and however deep the tree.
|
|
22528
|
+
*
|
|
22529
|
+
* The bound therefore limits working agents, not environments. The environments alive at one time
|
|
22530
|
+
* can exceed it by the number of managers that have a running child. The conserved budget pool and
|
|
22531
|
+
* the deadline bound the total work; this allocator only bounds how much of it runs at once.
|
|
22532
|
+
*
|
|
22533
|
+
* @experimental
|
|
22534
|
+
*/
|
|
22535
|
+
const allocatorStates = /* @__PURE__ */ new WeakMap();
|
|
22536
|
+
/**
|
|
22537
|
+
* Create a worker-slot allocator. `max` omitted, `0`, or negative leaves concurrency bounded by the
|
|
22538
|
+
* budget alone, and every spawn starts at once. Pass the returned allocator as `workerSlots` to each
|
|
22539
|
+
* run that should share one bound.
|
|
22540
|
+
*/
|
|
22541
|
+
function createWorkerSlots(max) {
|
|
22542
|
+
const bound = normalizeWorkerSlotBound(max);
|
|
22543
|
+
const state = {
|
|
22544
|
+
max: bound,
|
|
22545
|
+
working: 0,
|
|
22546
|
+
order: 0,
|
|
22547
|
+
waiters: []
|
|
22548
|
+
};
|
|
22549
|
+
const slots = Object.freeze({
|
|
22550
|
+
max: bound,
|
|
22551
|
+
get working() {
|
|
22552
|
+
return state.working;
|
|
22553
|
+
},
|
|
22554
|
+
get queued() {
|
|
22555
|
+
return state.waiters.length;
|
|
22556
|
+
}
|
|
22557
|
+
});
|
|
22558
|
+
allocatorStates.set(slots, state);
|
|
22559
|
+
return slots;
|
|
22560
|
+
}
|
|
22561
|
+
/** Accept a bound or an allocator; a number creates an allocator for one tree. */
|
|
22562
|
+
function resolveWorkerSlots(value) {
|
|
22563
|
+
if (value === void 0 || typeof value === "number") return createWorkerSlots(value);
|
|
22564
|
+
if (!allocatorStates.has(value)) throw new ValidationError$1("workerSlots must be a number or an allocator from createWorkerSlots");
|
|
22565
|
+
return value;
|
|
22566
|
+
}
|
|
22567
|
+
/** @internal Open the group of one scope's children. */
|
|
22568
|
+
function openSlotGroup(slots, owner) {
|
|
22569
|
+
const state = allocatorStates.get(slots);
|
|
22570
|
+
if (state === void 0) throw new ValidationError$1("openSlotGroup: allocator was not created by createWorkerSlots");
|
|
22571
|
+
const issued = owner === void 0 ? void 0 : deferredPermits.get(owner)?.() ?? owner;
|
|
22572
|
+
const ownerState = issued === void 0 ? void 0 : permitStates.get(issued);
|
|
22573
|
+
if (owner !== void 0 && ownerState === void 0) throw new ValidationError$1("openSlotGroup: owner permit was not issued by this allocator");
|
|
22574
|
+
const group = {
|
|
22575
|
+
owner: ownerState,
|
|
22576
|
+
running: 0,
|
|
22577
|
+
queued: 0
|
|
22578
|
+
};
|
|
22579
|
+
if (ownerState !== void 0) {
|
|
22580
|
+
if (ownerState.children !== void 0) throw new ValidationError$1("openSlotGroup: a permit owns at most one group of children");
|
|
22581
|
+
ownerState.children = group;
|
|
22582
|
+
}
|
|
22583
|
+
return {
|
|
22584
|
+
acquire: (depth, options) => acquire(state, group, depth, options?.force === true),
|
|
22585
|
+
get running() {
|
|
22586
|
+
return group.running;
|
|
22587
|
+
},
|
|
22588
|
+
get queued() {
|
|
22589
|
+
return group.queued;
|
|
22590
|
+
}
|
|
22591
|
+
};
|
|
22592
|
+
}
|
|
22593
|
+
const permitStates = /* @__PURE__ */ new WeakMap();
|
|
22594
|
+
/** Each deferred permit's reader of the permit it acquired, once it has acquired one. */
|
|
22595
|
+
const deferredPermits = /* @__PURE__ */ new WeakMap();
|
|
22596
|
+
/**
|
|
22597
|
+
* @internal A slot requested only once `after` resolves. A spawn that waits for budget takes no
|
|
22598
|
+
* slot while it waits, so it cannot hold a slot that the work it waits on needs. `ready` rejects
|
|
22599
|
+
* when `after` does; `release` before then withdraws the request.
|
|
22600
|
+
*/
|
|
22601
|
+
function deferSlot(after, acquire) {
|
|
22602
|
+
let inner;
|
|
22603
|
+
let released = false;
|
|
22604
|
+
const ready = after.then(() => {
|
|
22605
|
+
if (released) return;
|
|
22606
|
+
inner = acquire();
|
|
22607
|
+
return inner.ready;
|
|
22608
|
+
});
|
|
22609
|
+
ready.catch(() => void 0);
|
|
22610
|
+
const handle = {
|
|
22611
|
+
ready,
|
|
22612
|
+
get granted() {
|
|
22613
|
+
return inner?.granted ?? false;
|
|
22614
|
+
},
|
|
22615
|
+
release() {
|
|
22616
|
+
if (released) return;
|
|
22617
|
+
released = true;
|
|
22618
|
+
inner?.release();
|
|
22619
|
+
}
|
|
22620
|
+
};
|
|
22621
|
+
deferredPermits.set(handle, () => inner);
|
|
22622
|
+
return handle;
|
|
22623
|
+
}
|
|
22624
|
+
function acquire(state, group, depth, force) {
|
|
22625
|
+
if (!Number.isSafeInteger(depth) || depth < 0) throw new ValidationError$1(`worker slots: depth must be a non-negative integer, got ${depth}`);
|
|
22626
|
+
const permit = {
|
|
22627
|
+
group,
|
|
22628
|
+
granted: false,
|
|
22629
|
+
released: false,
|
|
22630
|
+
counted: false
|
|
22631
|
+
};
|
|
22632
|
+
let resolveReady;
|
|
22633
|
+
const ready = new Promise((resolve) => {
|
|
22634
|
+
resolveReady = resolve;
|
|
22635
|
+
});
|
|
22636
|
+
const grant = () => {
|
|
22637
|
+
grantPermit(state, permit);
|
|
22638
|
+
resolveReady();
|
|
22639
|
+
};
|
|
22640
|
+
if (force || costOf(group) === 0 || hasCapacity(state)) grant();
|
|
22641
|
+
else {
|
|
22642
|
+
group.queued += 1;
|
|
22643
|
+
const waiter = {
|
|
22644
|
+
permit,
|
|
22645
|
+
depth,
|
|
22646
|
+
order: state.order++,
|
|
22647
|
+
grant
|
|
22648
|
+
};
|
|
22649
|
+
state.waiters.push(waiter);
|
|
22650
|
+
}
|
|
22651
|
+
const handle = {
|
|
22652
|
+
ready,
|
|
22653
|
+
get granted() {
|
|
22654
|
+
return permit.granted;
|
|
22655
|
+
},
|
|
22656
|
+
release() {
|
|
22657
|
+
if (permit.released) return;
|
|
22658
|
+
permit.released = true;
|
|
22659
|
+
if (!permit.granted) {
|
|
22660
|
+
withdraw(state, permit);
|
|
22661
|
+
return;
|
|
22662
|
+
}
|
|
22663
|
+
releasePermit(state, permit);
|
|
22664
|
+
pump(state);
|
|
22665
|
+
}
|
|
22666
|
+
};
|
|
22667
|
+
permitStates.set(handle, permit);
|
|
22668
|
+
return handle;
|
|
22669
|
+
}
|
|
22670
|
+
/** A child of `group` adds a working unit unless its manager is working on its own slot now, in
|
|
22671
|
+
* which case the child takes that slot over. */
|
|
22672
|
+
function costOf(group) {
|
|
22673
|
+
const owner = group.owner;
|
|
22674
|
+
return owner?.granted && !owner.released && owner.counted ? 0 : 1;
|
|
22675
|
+
}
|
|
22676
|
+
function hasCapacity(state) {
|
|
22677
|
+
return state.max === void 0 || state.working < state.max;
|
|
22678
|
+
}
|
|
22679
|
+
function grantPermit(state, permit) {
|
|
22680
|
+
const group = permit.group;
|
|
22681
|
+
if (costOf(group) === 0) group.owner.counted = false;
|
|
22682
|
+
else state.working += 1;
|
|
22683
|
+
permit.granted = true;
|
|
22684
|
+
permit.counted = true;
|
|
22685
|
+
group.running += 1;
|
|
22686
|
+
}
|
|
22687
|
+
function releasePermit(state, permit) {
|
|
22688
|
+
if ((permit.children?.running ?? 0) > 0) return;
|
|
22689
|
+
const group = permit.group;
|
|
22690
|
+
group.running -= 1;
|
|
22691
|
+
if (group.running < 0) throw new ValidationError$1("worker slots: a group released more than it ran");
|
|
22692
|
+
if (!permit.counted) return;
|
|
22693
|
+
permit.counted = false;
|
|
22694
|
+
const owner = group.owner;
|
|
22695
|
+
if (group.running === 0 && owner !== void 0 && owner.granted) {
|
|
22696
|
+
owner.counted = true;
|
|
22697
|
+
if (owner.released) releasePermit(state, owner);
|
|
22698
|
+
return;
|
|
22699
|
+
}
|
|
22700
|
+
state.working -= 1;
|
|
22701
|
+
if (state.working < 0) throw new ValidationError$1("worker slots: released more than granted");
|
|
22702
|
+
}
|
|
22703
|
+
function withdraw(state, permit) {
|
|
22704
|
+
const index = state.waiters.findIndex((waiter) => waiter.permit === permit);
|
|
22705
|
+
if (index === -1) return;
|
|
22706
|
+
state.waiters.splice(index, 1);
|
|
22707
|
+
permit.group.queued -= 1;
|
|
22708
|
+
}
|
|
22709
|
+
/** Grant every waiter that can run: any waiter whose manager is working on its own slot, and then
|
|
22710
|
+
* the deepest and oldest waiters while capacity remains. */
|
|
22711
|
+
function pump(state) {
|
|
22712
|
+
for (;;) {
|
|
22713
|
+
let chosen;
|
|
22714
|
+
for (let index = 0; index < state.waiters.length; index++) {
|
|
22715
|
+
const waiter = state.waiters[index];
|
|
22716
|
+
if (costOf(waiter.permit.group) === 0) {
|
|
22717
|
+
chosen = index;
|
|
22718
|
+
break;
|
|
22719
|
+
}
|
|
22720
|
+
if (!hasCapacity(state)) continue;
|
|
22721
|
+
const best = chosen === void 0 ? void 0 : state.waiters[chosen];
|
|
22722
|
+
if (best === void 0 || waiter.depth > best.depth || waiter.depth === best.depth && waiter.order < best.order) chosen = index;
|
|
22723
|
+
}
|
|
22724
|
+
if (chosen === void 0) return;
|
|
22725
|
+
const [waiter] = state.waiters.splice(chosen, 1);
|
|
22726
|
+
waiter.permit.group.queued -= 1;
|
|
22727
|
+
waiter.grant();
|
|
22728
|
+
}
|
|
22729
|
+
}
|
|
22730
|
+
function normalizeWorkerSlotBound(value) {
|
|
22731
|
+
if (value === void 0 || value <= 0) return void 0;
|
|
22732
|
+
if (!Number.isSafeInteger(value)) throw new ValidationError$1(`workerSlots must be a positive safe integer, or 0 or less for no bound, got ${String(value)}`);
|
|
22733
|
+
return value;
|
|
22734
|
+
}
|
|
22735
|
+
//#endregion
|
|
22091
22736
|
//#region src/runtime/supervise/scope.ts
|
|
22092
22737
|
/**
|
|
22093
22738
|
*
|
|
@@ -22120,6 +22765,7 @@ const runtimeOwnedProviderMeters = /* @__PURE__ */ new WeakMap();
|
|
|
22120
22765
|
const pauseRecorders = /* @__PURE__ */ new WeakMap();
|
|
22121
22766
|
const recoveryStarters = /* @__PURE__ */ new WeakMap();
|
|
22122
22767
|
const retainedReleasers = /* @__PURE__ */ new WeakMap();
|
|
22768
|
+
const retainedSlotClosers = /* @__PURE__ */ new WeakMap();
|
|
22123
22769
|
const teardownRetriers = /* @__PURE__ */ new WeakMap();
|
|
22124
22770
|
const teardownRetriable = /* @__PURE__ */ new WeakMap();
|
|
22125
22771
|
/** The retained owner environments a scope's last release could not confirm destroyed. */
|
|
@@ -22130,6 +22776,17 @@ const nestedTeardownScopes = /* @__PURE__ */ new WeakMap();
|
|
|
22130
22776
|
* kept for a resume any longer. */
|
|
22131
22777
|
const releasingScopes = /* @__PURE__ */ new WeakSet();
|
|
22132
22778
|
const admissionSeals = /* @__PURE__ */ new WeakMap();
|
|
22779
|
+
/** Each scope's reader of its own team, for the settlement its owner writes one level up. */
|
|
22780
|
+
const subtreeReaders = /* @__PURE__ */ new WeakMap();
|
|
22781
|
+
/** Each scope's reader of its live team, for the progress its owner's lead reads mid-flight. */
|
|
22782
|
+
const liveTeamReaders = /* @__PURE__ */ new WeakMap();
|
|
22783
|
+
/** How many of a manager's direct children its summary lists; the counts always cover all. */
|
|
22784
|
+
const SUBTREE_RESULT_LIMIT = 8;
|
|
22785
|
+
/** @internal The bounded account of the children a scope spawned and everything below them.
|
|
22786
|
+
* `undefined` when the scope spawned no worker. */
|
|
22787
|
+
function subtreeSummaryOf(scope) {
|
|
22788
|
+
return subtreeReaders.get(scope)?.();
|
|
22789
|
+
}
|
|
22133
22790
|
/** Per-child bound on a retained release: one remote destroy, answered or abandoned. */
|
|
22134
22791
|
const RETAINED_RELEASE_TIMEOUT_MS = 3e4;
|
|
22135
22792
|
/**
|
|
@@ -22196,6 +22853,25 @@ async function releaseRetainedEnvironments(scope) {
|
|
|
22196
22853
|
return unconfirmedTeardowns(scope);
|
|
22197
22854
|
}
|
|
22198
22855
|
/**
|
|
22856
|
+
* @internal Close the cursor slot of every retained child this scope and its nested scopes still
|
|
22857
|
+
* hold open, once the settlement is final and the release and its retries are done.
|
|
22858
|
+
*
|
|
22859
|
+
* The release sweep closes a slot only when it confirms the environment destroyed, and nothing
|
|
22860
|
+
* else ever closed the rest: a refused delete, a teardown probe that failed after a destroy, or an
|
|
22861
|
+
* admission that timed out before `create` named an environment. Measured on the Discovery fleet
|
|
22862
|
+
* of 2026-09-23/24 (Runtime 0.249.1 to 0.261.0): 338 of 1,620 agents never settled, so they had no
|
|
22863
|
+
* terminal record, no spend but the reservation ceiling, and no transcript receipt (#1301). A
|
|
22864
|
+
* settlement under `retainedAtSettlement: 'release'` is final, so no process will recover these
|
|
22865
|
+
* executions. Each slot closes with the settlement the driver already received, under the seq it
|
|
22866
|
+
* saw, marked `retainedExecution: 'release-unconfirmed'`. The environment it may still hold stays
|
|
22867
|
+
* named by its `teardown-unconfirmed` record for a sweeper. Never call this on a run a later
|
|
22868
|
+
* process may resume.
|
|
22869
|
+
*/
|
|
22870
|
+
async function closeRetainedSlots(scope) {
|
|
22871
|
+
const close = retainedSlotClosers.get(scope);
|
|
22872
|
+
if (close) await close();
|
|
22873
|
+
}
|
|
22874
|
+
/**
|
|
22199
22875
|
* @internal Everything this scope cannot prove destroyed: its settled children whose teardown is
|
|
22200
22876
|
* unconfirmed and, after a release, the retained owner environment that release could not
|
|
22201
22877
|
* confirm. Read live, so a later retry that confirms a node removes it.
|
|
@@ -22257,13 +22933,11 @@ function hasRetriableTeardowns(scope, release) {
|
|
|
22257
22933
|
function closeScopeAdmission(scope) {
|
|
22258
22934
|
admissionSeals.get(scope)?.();
|
|
22259
22935
|
}
|
|
22260
|
-
/** Fail before execution when
|
|
22261
|
-
function assertRecursiveReservationPolicy(policy
|
|
22936
|
+
/** Fail before execution when a reservation policy is malformed. */
|
|
22937
|
+
function assertRecursiveReservationPolicy(policy) {
|
|
22262
22938
|
if (policy === void 0) return;
|
|
22263
22939
|
if (typeof policy !== "object" || policy === null || Object.keys(policy).some((key) => key !== "ownerShare")) throw new ValidationError$1("reservationPolicy accepts only ownerShare");
|
|
22264
22940
|
if (!Number.isFinite(policy.ownerShare) || policy.ownerShare <= 0 || policy.ownerShare >= 1) throw new ValidationError$1("reservationPolicy.ownerShare must be greater than 0 and less than 1");
|
|
22265
|
-
if (!Number.isSafeInteger(maxDepth) || maxDepth === void 0 || maxDepth < 2) throw new ValidationError$1("reservationPolicy requires maxDepth >= 2");
|
|
22266
|
-
if (!Number.isSafeInteger(maxLiveWorkers) || maxLiveWorkers === void 0 || maxLiveWorkers < maxDepth) throw new ValidationError$1("reservationPolicy requires maxLiveWorkers >= maxDepth to preserve a descendant path");
|
|
22267
22941
|
}
|
|
22268
22942
|
/**
|
|
22269
22943
|
* The recursion seam key. A `Scope` seeds a value of this on each child's
|
|
@@ -22272,7 +22946,7 @@ function assertRecursiveReservationPolicy(policy, maxDepth, maxLiveWorkers) {
|
|
|
22272
22946
|
* here so the scope and the driver-executor agree on the seam without a circular import.
|
|
22273
22947
|
*/
|
|
22274
22948
|
const nestedScopeSeamKey = "nested-scope";
|
|
22275
|
-
function makeNestedScopeSeam(args,
|
|
22949
|
+
function makeNestedScopeSeam(args, workerSlots, childSlot, childNodeId, childBudget, childDeadlineAtMs, deferredOwner) {
|
|
22276
22950
|
const now = args.now ?? Date.now;
|
|
22277
22951
|
const mountScope = (nestedRoot, signal, restored) => {
|
|
22278
22952
|
const mountedAtMs = now();
|
|
@@ -22291,7 +22965,8 @@ function makeNestedScopeSeam(args, liveWorkerCapacity, childNodeId, childBudget,
|
|
|
22291
22965
|
seams: args.seams,
|
|
22292
22966
|
depth: args.depth + 1,
|
|
22293
22967
|
...args.maxDepth !== void 0 ? { maxDepth: args.maxDepth } : {},
|
|
22294
|
-
|
|
22968
|
+
workerSlots,
|
|
22969
|
+
slotOwner: childSlot,
|
|
22295
22970
|
...args.reservationPolicy ? {
|
|
22296
22971
|
reservationPolicy: args.reservationPolicy,
|
|
22297
22972
|
ownerBudget: nestedBudget
|
|
@@ -22330,12 +23005,10 @@ function createScope(args) {
|
|
|
22330
23005
|
const settlementWrites = /* @__PURE__ */ new Set();
|
|
22331
23006
|
let admissionClosed = false;
|
|
22332
23007
|
const interactiveBindingDir = args.interactiveBindingDir;
|
|
22333
|
-
const
|
|
22334
|
-
|
|
22335
|
-
live: 0
|
|
22336
|
-
};
|
|
23008
|
+
const workerSlots = args.workerSlots ?? createWorkerSlots();
|
|
23009
|
+
const slotGroup = openSlotGroup(workerSlots, args.slotOwner);
|
|
22337
23010
|
const reservationPolicy = args.reservationPolicy;
|
|
22338
|
-
assertRecursiveReservationPolicy(reservationPolicy
|
|
23011
|
+
assertRecursiveReservationPolicy(reservationPolicy);
|
|
22339
23012
|
if (reservationPolicy && !args.ownerBudget) throw new ValidationError$1("reservationPolicy requires the owning scope budget");
|
|
22340
23013
|
const ownerBudget = args.ownerBudget;
|
|
22341
23014
|
const ownerFloor = reservationPolicy && ownerBudget ? {
|
|
@@ -22343,11 +23016,12 @@ function createScope(args) {
|
|
|
22343
23016
|
iterations: Math.ceil(ownerBudget.maxIterations * reservationPolicy.ownerShare),
|
|
22344
23017
|
...ownerBudget.maxUsd === void 0 ? {} : { usd: ownerBudget.maxUsd * reservationPolicy.ownerShare }
|
|
22345
23018
|
} : void 0;
|
|
22346
|
-
const admissionMax = reservationPolicy && liveWorkerCapacity.max !== void 0 && args.maxDepth !== void 0 ? liveWorkerCapacity.max - Math.max(0, args.maxDepth - (args.depth + 1)) : liveWorkerCapacity.max;
|
|
22347
23019
|
let spawnOrdinal = args.resumeFrom ? args.resumeFrom.maxSpawnOrdinal + 1 : 0;
|
|
22348
23020
|
let cursorSeq = args.resumeFrom ? args.resumeFrom.maxCursorSeq + 1 : 0;
|
|
22349
23021
|
let waitOrdinal = args.resumeFrom ? args.resumeFrom.maxWaitOrdinal + 1 : 0;
|
|
22350
23022
|
let meterSeq = 0;
|
|
23023
|
+
let ownerTurns = 0;
|
|
23024
|
+
let ownerLastTurnAt = 0;
|
|
22351
23025
|
let progressSeq = 0;
|
|
22352
23026
|
const now = args.now ?? Date.now;
|
|
22353
23027
|
const unclaimedWaits = new Map((args.resumeFrom?.waits ?? []).map((w) => [w.label, w]));
|
|
@@ -22368,6 +23042,14 @@ function createScope(args) {
|
|
|
22368
23042
|
identity: prior.identity,
|
|
22369
23043
|
reason: prior.settled.reason
|
|
22370
23044
|
});
|
|
23045
|
+
} else if (prior.state === "down" && prior.settled === void 0) {
|
|
23046
|
+
if (prior.identity === void 0) continue;
|
|
23047
|
+
keyed.set(key, {
|
|
23048
|
+
state: "down",
|
|
23049
|
+
id: prior.id,
|
|
23050
|
+
identity: prior.identity,
|
|
23051
|
+
reason: prior.reason ?? "interrupted: resume proved the prior attempt dead"
|
|
23052
|
+
});
|
|
22371
23053
|
} else if (prior.identity !== void 0) keyed.set(key, {
|
|
22372
23054
|
state: "in-doubt",
|
|
22373
23055
|
id: prior.id,
|
|
@@ -22464,54 +23146,21 @@ function createScope(args) {
|
|
|
22464
23146
|
reason: "depth-exceeded"
|
|
22465
23147
|
};
|
|
22466
23148
|
if (typeof agentOrFactory !== "function") prepared ??= prepare();
|
|
22467
|
-
if (!recovery && admissionMax !== void 0 && liveWorkerCapacity.live >= admissionMax) return {
|
|
22468
|
-
ok: false,
|
|
22469
|
-
reason: "max-live-workers"
|
|
22470
|
-
};
|
|
22471
|
-
if (!recovery && ownerFloor) {
|
|
22472
|
-
const available = args.pool.readout();
|
|
22473
|
-
const shortfalls = [];
|
|
22474
|
-
const short = (channel, requested, free) => {
|
|
22475
|
-
if (requested > free) shortfalls.push({
|
|
22476
|
-
channel,
|
|
22477
|
-
requested,
|
|
22478
|
-
free: Math.max(0, free)
|
|
22479
|
-
});
|
|
22480
|
-
};
|
|
22481
|
-
short("tokens", opts.budget.maxTokens, available.tokensLeft - ownerFloor.tokens);
|
|
22482
|
-
short("iterations", opts.budget.maxIterations, available.iterationsLeft - ownerFloor.iterations);
|
|
22483
|
-
if (ownerFloor.usd !== void 0) short("usd", opts.budget.maxUsd ?? 0, available.usdLeft - ownerFloor.usd);
|
|
22484
|
-
if (shortfalls.length > 0) return {
|
|
22485
|
-
ok: false,
|
|
22486
|
-
reason: "budget-exhausted",
|
|
22487
|
-
shortfalls
|
|
22488
|
-
};
|
|
22489
|
-
}
|
|
22490
|
-
const permit = acquireLiveWorker(liveWorkerCapacity);
|
|
22491
|
-
if (!permit.ok) return {
|
|
22492
|
-
ok: false,
|
|
22493
|
-
reason: "max-live-workers"
|
|
22494
|
-
};
|
|
22495
23149
|
const assignment = opts.assignmentId ?? opts.key;
|
|
22496
23150
|
let reservation;
|
|
22497
|
-
|
|
22498
|
-
|
|
22499
|
-
|
|
22500
|
-
|
|
22501
|
-
|
|
22502
|
-
|
|
22503
|
-
|
|
22504
|
-
|
|
22505
|
-
|
|
22506
|
-
|
|
22507
|
-
|
|
22508
|
-
|
|
22509
|
-
|
|
22510
|
-
ok: false,
|
|
22511
|
-
reason: reservation.reason,
|
|
22512
|
-
...reservation.shortfalls === void 0 ? {} : { shortfalls: reservation.shortfalls }
|
|
22513
|
-
};
|
|
22514
|
-
}
|
|
23151
|
+
reservation = args.pool.reserve(opts.budget, {
|
|
23152
|
+
...assignment === void 0 ? {} : { assignment },
|
|
23153
|
+
label: opts.label,
|
|
23154
|
+
stage: "admitted"
|
|
23155
|
+
}, recovery ? {} : {
|
|
23156
|
+
wait: true,
|
|
23157
|
+
...ownerFloor ? { keep: ownerFloor } : {}
|
|
23158
|
+
});
|
|
23159
|
+
if (!reservation.ok) return {
|
|
23160
|
+
ok: false,
|
|
23161
|
+
reason: reservation.reason,
|
|
23162
|
+
...reservation.shortfalls === void 0 ? {} : { shortfalls: reservation.shortfalls }
|
|
23163
|
+
};
|
|
22515
23164
|
let spec;
|
|
22516
23165
|
let resolved;
|
|
22517
23166
|
let identity;
|
|
@@ -22521,7 +23170,6 @@ function createScope(args) {
|
|
|
22521
23170
|
identity = prepared.identity;
|
|
22522
23171
|
if (opts.key !== void 0 && !isCompleteIdentity(identity)) {
|
|
22523
23172
|
args.pool.reconcile(reservation.ticket, withBudgetResources(zeroSpend(), opts.budget, true));
|
|
22524
|
-
permit.release();
|
|
22525
23173
|
return {
|
|
22526
23174
|
ok: false,
|
|
22527
23175
|
reason: "invalid-identity"
|
|
@@ -22535,9 +23183,10 @@ function createScope(args) {
|
|
|
22535
23183
|
resolved = outcome;
|
|
22536
23184
|
} catch (error) {
|
|
22537
23185
|
args.pool.reconcile(reservation.ticket, withBudgetResources(zeroSpend(), opts.budget, true));
|
|
22538
|
-
permit.release();
|
|
22539
23186
|
throw error;
|
|
22540
23187
|
}
|
|
23188
|
+
const budgetGranted = reservation.granted;
|
|
23189
|
+
const permit = budgetGranted === void 0 ? slotGroup.acquire(args.depth + 1, { force: recovery !== void 0 }) : deferSlot(budgetGranted, () => slotGroup.acquire(args.depth + 1));
|
|
22541
23190
|
let cascadeAbort;
|
|
22542
23191
|
let clearChildDeadline;
|
|
22543
23192
|
try {
|
|
@@ -22562,7 +23211,6 @@ function createScope(args) {
|
|
|
22562
23211
|
const recoveryReady = recovery?.spawned.ownedTreeRoot === void 0 ? void 0 : new Promise((resolve) => {
|
|
22563
23212
|
markRecoveryReady = resolve;
|
|
22564
23213
|
});
|
|
22565
|
-
const admissions = [...recovery?.admissions ?? []];
|
|
22566
23214
|
const retainedWrites = /* @__PURE__ */ new Set();
|
|
22567
23215
|
let retainedWritesClosed = false;
|
|
22568
23216
|
const retainedWrite = (write) => {
|
|
@@ -22577,6 +23225,99 @@ function createScope(args) {
|
|
|
22577
23225
|
retainedWritesClosed = true;
|
|
22578
23226
|
await Promise.allSettled([...retainedWrites]);
|
|
22579
23227
|
};
|
|
23228
|
+
let admissions = [...recovery?.admissions ?? []];
|
|
23229
|
+
let inputSeq = recovery?.continuation?.inputSeq ?? ordinal;
|
|
23230
|
+
let pauseSeq = 0;
|
|
23231
|
+
const onAdmission = (admission) => retainedWrite(async () => {
|
|
23232
|
+
controller.signal.throwIfAborted();
|
|
23233
|
+
const existing = admissions.find((record) => record.phase === admission.phase);
|
|
23234
|
+
if (existing) {
|
|
23235
|
+
if (contentAddress(existing) !== contentAddress(admission)) throw new ValidationError$1("scope retained admission conflicts with its committed phase");
|
|
23236
|
+
return;
|
|
23237
|
+
}
|
|
23238
|
+
if (identity?.taskDigest === void 0) throw new ValidationError$1("retained execution requires a canonical task identity");
|
|
23239
|
+
await args.journal.appendEvent(args.root, {
|
|
23240
|
+
kind: "execution-admitted",
|
|
23241
|
+
id,
|
|
23242
|
+
admission: detachedSnapshot(admission, "retained admission"),
|
|
23243
|
+
seq: ordinal,
|
|
23244
|
+
at: new Date(now()).toISOString()
|
|
23245
|
+
});
|
|
23246
|
+
admissions.push(detachedSnapshot(admission, "retained admission"));
|
|
23247
|
+
live.recoveryPending = true;
|
|
23248
|
+
controller.signal.throwIfAborted();
|
|
23249
|
+
});
|
|
23250
|
+
const onResult = (result) => retainedWrite(async () => {
|
|
23251
|
+
controller.signal.throwIfAborted();
|
|
23252
|
+
assertValidSpend(result.spent, "retained executor result");
|
|
23253
|
+
executorFailureReason(result);
|
|
23254
|
+
await pendingEvidence?.complete();
|
|
23255
|
+
const outRef = contentAddress(result.out);
|
|
23256
|
+
await args.blobs.put(outRef, result.out);
|
|
23257
|
+
controller.signal.throwIfAborted();
|
|
23258
|
+
await args.journal.appendEvent(args.root, {
|
|
23259
|
+
kind: "execution-result",
|
|
23260
|
+
id,
|
|
23261
|
+
outRef,
|
|
23262
|
+
spent: result.spent,
|
|
23263
|
+
...result.verdict ? { verdict: result.verdict } : {},
|
|
23264
|
+
...result.outcome ? { outcome: result.outcome } : {},
|
|
23265
|
+
seq: ordinal,
|
|
23266
|
+
at: new Date(now()).toISOString()
|
|
23267
|
+
});
|
|
23268
|
+
live.acceptedResult = detachedSnapshot({
|
|
23269
|
+
...result,
|
|
23270
|
+
outRef
|
|
23271
|
+
}, "accepted retained child result");
|
|
23272
|
+
live.recoveryPending = false;
|
|
23273
|
+
});
|
|
23274
|
+
const onPause = (pause) => retainedWrite(async () => {
|
|
23275
|
+
await args.journal.appendEvent(args.root, {
|
|
23276
|
+
kind: "paused",
|
|
23277
|
+
id,
|
|
23278
|
+
...pause,
|
|
23279
|
+
seq: pauseSeq++,
|
|
23280
|
+
at: new Date(now()).toISOString()
|
|
23281
|
+
});
|
|
23282
|
+
});
|
|
23283
|
+
const retainedContext = (continuing) => ({
|
|
23284
|
+
...continuing === void 0 ? {} : {
|
|
23285
|
+
executionId: continuing.executionId,
|
|
23286
|
+
priorSession: continuing.priorSession
|
|
23287
|
+
},
|
|
23288
|
+
admissions,
|
|
23289
|
+
...markRecoveryReady && continuing === void 0 ? { onReady: markRecoveryReady } : {},
|
|
23290
|
+
onAdmission,
|
|
23291
|
+
onResult,
|
|
23292
|
+
onPause,
|
|
23293
|
+
continueInvocation: async (nextTask) => {
|
|
23294
|
+
controller.signal.throwIfAborted();
|
|
23295
|
+
if (live.acceptedResult === void 0) throw new ValidationError$1("scope: a retained child continues only after its invocation committed a result");
|
|
23296
|
+
const environment = admissions.find((record) => record.phase === "environment");
|
|
23297
|
+
if (environment?.phase !== "environment") throw new ValidationError$1("scope: a retained child continues only in an environment it was admitted to");
|
|
23298
|
+
const snapshot = detachedSnapshot(nextTask, "retained child continuation");
|
|
23299
|
+
const taskRef = contentAddress(snapshot);
|
|
23300
|
+
const nextSeq = inputSeq + 1;
|
|
23301
|
+
await retainedWrite(async () => {
|
|
23302
|
+
await args.blobs.put(taskRef, snapshot);
|
|
23303
|
+
controller.signal.throwIfAborted();
|
|
23304
|
+
await args.journal.appendEvent(args.root, {
|
|
23305
|
+
kind: "execution-input",
|
|
23306
|
+
id,
|
|
23307
|
+
taskRef,
|
|
23308
|
+
seq: nextSeq,
|
|
23309
|
+
at: new Date(now()).toISOString()
|
|
23310
|
+
});
|
|
23311
|
+
});
|
|
23312
|
+
inputSeq = nextSeq;
|
|
23313
|
+
admissions = [];
|
|
23314
|
+
live.acceptedResult = void 0;
|
|
23315
|
+
return retainedContext({
|
|
23316
|
+
executionId: leafContinuationExecutionId(id, nextSeq),
|
|
23317
|
+
priorSession: environment
|
|
23318
|
+
});
|
|
23319
|
+
}
|
|
23320
|
+
});
|
|
22580
23321
|
const ctx = {
|
|
22581
23322
|
signal: controller.signal,
|
|
22582
23323
|
node: {
|
|
@@ -22589,54 +23330,11 @@ function createScope(args) {
|
|
|
22589
23330
|
},
|
|
22590
23331
|
seams: {
|
|
22591
23332
|
...args.seams,
|
|
22592
|
-
[retainedExecutorSeamKey]: {
|
|
22593
|
-
|
|
22594
|
-
|
|
22595
|
-
|
|
22596
|
-
|
|
22597
|
-
const existing = admissions.find((record) => record.phase === admission.phase);
|
|
22598
|
-
if (existing) {
|
|
22599
|
-
if (contentAddress(existing) !== contentAddress(admission)) throw new ValidationError$1("scope retained admission conflicts with its committed phase");
|
|
22600
|
-
return;
|
|
22601
|
-
}
|
|
22602
|
-
if (identity?.taskDigest === void 0) throw new ValidationError$1("retained execution requires a canonical task identity");
|
|
22603
|
-
await args.journal.appendEvent(args.root, {
|
|
22604
|
-
kind: "execution-admitted",
|
|
22605
|
-
id,
|
|
22606
|
-
admission: detachedSnapshot(admission, "retained admission"),
|
|
22607
|
-
seq: ordinal,
|
|
22608
|
-
at: new Date(now()).toISOString()
|
|
22609
|
-
});
|
|
22610
|
-
admissions.push(detachedSnapshot(admission, "retained admission"));
|
|
22611
|
-
live.recoveryPending = true;
|
|
22612
|
-
controller.signal.throwIfAborted();
|
|
22613
|
-
}),
|
|
22614
|
-
onResult: (result) => retainedWrite(async () => {
|
|
22615
|
-
controller.signal.throwIfAborted();
|
|
22616
|
-
assertValidSpend(result.spent, "retained executor result");
|
|
22617
|
-
executorFailureReason(result);
|
|
22618
|
-
await pendingEvidence?.complete();
|
|
22619
|
-
const outRef = contentAddress(result.out);
|
|
22620
|
-
await args.blobs.put(outRef, result.out);
|
|
22621
|
-
controller.signal.throwIfAborted();
|
|
22622
|
-
await args.journal.appendEvent(args.root, {
|
|
22623
|
-
kind: "execution-result",
|
|
22624
|
-
id,
|
|
22625
|
-
outRef,
|
|
22626
|
-
spent: result.spent,
|
|
22627
|
-
...result.verdict ? { verdict: result.verdict } : {},
|
|
22628
|
-
...result.outcome ? { outcome: result.outcome } : {},
|
|
22629
|
-
seq: ordinal,
|
|
22630
|
-
at: new Date(now()).toISOString()
|
|
22631
|
-
});
|
|
22632
|
-
live.acceptedResult = detachedSnapshot({
|
|
22633
|
-
...result,
|
|
22634
|
-
outRef
|
|
22635
|
-
}, "accepted retained child result");
|
|
22636
|
-
live.recoveryPending = false;
|
|
22637
|
-
})
|
|
22638
|
-
},
|
|
22639
|
-
[nestedScopeSeamKey]: makeNestedScopeSeam(args, liveWorkerCapacity, id, opts.budget, childDeadlineAtMs, deferredOwner),
|
|
23333
|
+
[retainedExecutorSeamKey]: retainedContext(recovery?.continuation === void 0 ? void 0 : {
|
|
23334
|
+
executionId: recovery.continuation.executionId,
|
|
23335
|
+
priorSession: recovery.continuation.priorSession
|
|
23336
|
+
}),
|
|
23337
|
+
[nestedScopeSeamKey]: makeNestedScopeSeam(args, workerSlots, permit, id, opts.budget, childDeadlineAtMs, deferredOwner),
|
|
22640
23338
|
...workerTrace ? { [workerTraceSeamKey]: workerTrace } : {},
|
|
22641
23339
|
...interactiveBindingDir ? { [interactiveAdmissionSeamKey]: (admission) => Promise.resolve(writeWorkerInteractiveAdmission(interactiveBindingDir, id, admission, now)).then(() => void 0) } : {}
|
|
22642
23340
|
}
|
|
@@ -22668,7 +23366,7 @@ function createScope(args) {
|
|
|
22668
23366
|
});
|
|
22669
23367
|
const live = {
|
|
22670
23368
|
id,
|
|
22671
|
-
status: "acquiring",
|
|
23369
|
+
status: permit.granted ? "acquiring" : "queued",
|
|
22672
23370
|
runtime: executor.runtime,
|
|
22673
23371
|
...ownedTreeRoot === void 0 ? {} : { ownedTreeRoot },
|
|
22674
23372
|
...identity ? { identity } : {},
|
|
@@ -22700,6 +23398,7 @@ function createScope(args) {
|
|
|
22700
23398
|
askTeardown,
|
|
22701
23399
|
...executor.heldEnvironments ? { readHeldEnvironments: executor.heldEnvironments.bind(executor) } : {},
|
|
22702
23400
|
...nestedTeardownScopes.has(executor) ? { readNestedScope: nestedTeardownScopes.get(executor) } : {},
|
|
23401
|
+
...executor.harnessTranscript ? { readTranscriptAtRelease: () => persistHarnessTranscript(readHarnessTranscript(executor), args.blobs) } : {},
|
|
22703
23402
|
teardownAttempts: 0,
|
|
22704
23403
|
abortChild: (reason) => controller.abort(reason)
|
|
22705
23404
|
};
|
|
@@ -22877,10 +23576,10 @@ function createScope(args) {
|
|
|
22877
23576
|
startedAt
|
|
22878
23577
|
}
|
|
22879
23578
|
}, { signal: args.signal });
|
|
22880
|
-
const childRun = runChild(live, executor, controller, task, opts, args.pool, reservation.ticket, args.blobs, now, args.journal, args.root, () => progressSeq++, materializationCommitted, {
|
|
23579
|
+
const childRun = runChild(live, executor, controller, recovery?.continuation?.task ?? task, opts, args.pool, reservation.ticket, args.blobs, now, args.journal, args.root, () => progressSeq++, materializationCommitted, {
|
|
22881
23580
|
complete: async () => pendingEvidence?.complete(),
|
|
22882
23581
|
fail: async () => pendingEvidence?.fail()
|
|
22883
|
-
}, childDeadlineAtMs, recovery !== void 0, closeRetainedWrites);
|
|
23582
|
+
}, childDeadlineAtMs, recovery !== void 0, closeRetainedWrites, budgetGranted, permit.ready);
|
|
22884
23583
|
args.pool.attribute(reservation.ticket, { stage: "executing" });
|
|
22885
23584
|
const interactiveBindingCommitted = spawnCommitted.then(async () => {
|
|
22886
23585
|
if (args.interactiveBindingDir === void 0) return;
|
|
@@ -23138,14 +23837,16 @@ function createScope(args) {
|
|
|
23138
23837
|
} catch {
|
|
23139
23838
|
fromExecutor = void 0;
|
|
23140
23839
|
}
|
|
23840
|
+
const team = liveTeamOf(child);
|
|
23141
23841
|
return readWorkerProgress({
|
|
23142
23842
|
id: child.id,
|
|
23143
23843
|
status: child.status,
|
|
23144
23844
|
steerable: child.deliver !== void 0 && !child.delivered,
|
|
23145
23845
|
startedAt: child.startedAt,
|
|
23146
|
-
lastActivityAt: child.lastActivityAt,
|
|
23846
|
+
lastActivityAt: Math.max(child.lastActivityAt, team?.lastActivityAt ?? 0),
|
|
23147
23847
|
...addResourceSpend(child.spent.resources),
|
|
23148
|
-
turns: child.spent.iterations,
|
|
23848
|
+
turns: Math.max(child.spent.iterations, team?.ownerTurns ?? 0),
|
|
23849
|
+
...team && team.agents > 0 ? { team: teamProgress(team) } : {},
|
|
23149
23850
|
tokens: child.spent.tokens,
|
|
23150
23851
|
...child.spent.tokensKnown === false ? { tokensKnown: false } : {},
|
|
23151
23852
|
usd: child.spent.usd,
|
|
@@ -23192,6 +23893,8 @@ function createScope(args) {
|
|
|
23192
23893
|
} catch (error) {
|
|
23193
23894
|
observeError = error;
|
|
23194
23895
|
}
|
|
23896
|
+
ownerLastTurnAt = now();
|
|
23897
|
+
if (!accountingOnly) ownerTurns += 1;
|
|
23195
23898
|
await args.journal.appendEvent(args.root, {
|
|
23196
23899
|
kind: "metered",
|
|
23197
23900
|
id: args.parentId,
|
|
@@ -23243,8 +23946,9 @@ function createScope(args) {
|
|
|
23243
23946
|
},
|
|
23244
23947
|
get workerCapacity() {
|
|
23245
23948
|
return {
|
|
23246
|
-
|
|
23247
|
-
|
|
23949
|
+
working: workerSlots.working,
|
|
23950
|
+
queued: workerSlots.queued,
|
|
23951
|
+
freeSlots: freeSlots(workerSlots.working, workerSlots.max),
|
|
23248
23952
|
unconfirmed: Object.freeze([...children.values()].filter((child) => isTerminalNodeStatus(child.status) && !child.cleanupConfirmed).map((child) => unconfirmedTeardownOf(child, releasingScopes.has(scope))))
|
|
23249
23953
|
};
|
|
23250
23954
|
}
|
|
@@ -23252,6 +23956,8 @@ function createScope(args) {
|
|
|
23252
23956
|
admissionSeals.set(scope, () => {
|
|
23253
23957
|
admissionClosed = true;
|
|
23254
23958
|
});
|
|
23959
|
+
subtreeReaders.set(scope, () => summarizeTeam(children.values()));
|
|
23960
|
+
liveTeamReaders.set(scope, () => readLiveTeam(children.values(), ownerTurns, ownerLastTurnAt));
|
|
23255
23961
|
runtimeOwnedProviderMeters.set(scope, async (spend, providerModel, detail, accountingOnly) => meterInternal(spend, detail, providerModel, accountingOnly));
|
|
23256
23962
|
let pauseSeq = 0;
|
|
23257
23963
|
pauseRecorders.set(scope, async (pause) => {
|
|
@@ -23384,24 +24090,44 @@ function createScope(args) {
|
|
|
23384
24090
|
const prior = released.length > 0 ? await args.journal.loadTree(args.root) ?? [] : [];
|
|
23385
24091
|
for (const { child, receipts } of released) {
|
|
23386
24092
|
for (const receipt of receipts) await appendEnvironmentTeardown(args.journal, args.root, child.id, receipt, new Date(now()).toISOString());
|
|
23387
|
-
if (child.cleanupConfirmed
|
|
23388
|
-
|
|
23389
|
-
|
|
23390
|
-
|
|
23391
|
-
|
|
23392
|
-
|
|
23393
|
-
|
|
23394
|
-
|
|
23395
|
-
|
|
23396
|
-
|
|
23397
|
-
|
|
23398
|
-
|
|
23399
|
-
|
|
23400
|
-
|
|
23401
|
-
payload: releasedChildPayload(child, child.resolved, settledAt, releasedAt)
|
|
23402
|
-
}, { signal: args.signal });
|
|
23403
|
-
}
|
|
24093
|
+
if (child.cleanupConfirmed) await closeRetainedSlot(child, "released", prior);
|
|
24094
|
+
}
|
|
24095
|
+
});
|
|
24096
|
+
const closeRetainedSlot = async (child, state, prior) => {
|
|
24097
|
+
if (child.recoveryPending !== true || child.resolved?.kind !== "down" || child.settledSeq === void 0 || child.settledAt === void 0 || child.retainedExecution === "released" || child.retainedExecution === "release-unconfirmed" || prior.some((event) => event.id === child.id && closesCursorSlot(event))) return;
|
|
24098
|
+
child.budgetViolation = child.retainedViolation;
|
|
24099
|
+
const settledAt = child.settledAt;
|
|
24100
|
+
let settlement = child.resolved;
|
|
24101
|
+
if (settlement.harnessTranscript?.status !== "available" && child.readTranscriptAtRelease) {
|
|
24102
|
+
const atRelease = await child.readTranscriptAtRelease();
|
|
24103
|
+
if (atRelease.status === "available") settlement = {
|
|
24104
|
+
...settlement,
|
|
24105
|
+
harnessTranscript: atRelease
|
|
24106
|
+
};
|
|
23404
24107
|
}
|
|
24108
|
+
await args.journal.appendEvent(args.root, terminalDownEvent(child, settlement, child.settledSeq, new Date(settledAt).toISOString(), state));
|
|
24109
|
+
child.retainedExecution = state;
|
|
24110
|
+
const closedAt = now();
|
|
24111
|
+
notifyRuntimeHookEvent(args.hooks, {
|
|
24112
|
+
id: `${child.id}:${state}`,
|
|
24113
|
+
runId: args.root,
|
|
24114
|
+
target: "agent.child",
|
|
24115
|
+
phase: "after",
|
|
24116
|
+
timestamp: closedAt,
|
|
24117
|
+
stepIndex: child.settledSeq,
|
|
24118
|
+
parentId: args.parentId,
|
|
24119
|
+
payload: releasedChildPayload(child, settlement, settledAt, closedAt, state)
|
|
24120
|
+
}, { signal: args.signal });
|
|
24121
|
+
};
|
|
24122
|
+
retainedSlotClosers.set(scope, async () => {
|
|
24123
|
+
for (const child of children.values()) {
|
|
24124
|
+
const nested = child.readNestedScope?.();
|
|
24125
|
+
if (nested !== void 0) await closeRetainedSlots(nested);
|
|
24126
|
+
}
|
|
24127
|
+
const open = [...children.values()].filter((child) => child.recoveryPending === true && !child.cleanupConfirmed && isTerminalNodeStatus(child.status) && child.retainedExecution !== "released" && child.retainedExecution !== "release-unconfirmed");
|
|
24128
|
+
if (open.length === 0) return;
|
|
24129
|
+
const prior = await args.journal.loadTree(args.root) ?? [];
|
|
24130
|
+
for (const child of open) await closeRetainedSlot(child, "release-unconfirmed", prior);
|
|
23405
24131
|
});
|
|
23406
24132
|
registerScopeRetainedOwner(scope, {
|
|
23407
24133
|
rootId: args.root,
|
|
@@ -23740,6 +24466,9 @@ async function appendAcknowledged(journal, root, event) {
|
|
|
23740
24466
|
async function finalizeSettlement(child, settlement, seq, args, now) {
|
|
23741
24467
|
const handle = frozenHandle(child);
|
|
23742
24468
|
if (child.wait) return finalizeWait(child, settlement, seq, args, now, handle);
|
|
24469
|
+
const nestedScope = child.readNestedScope?.();
|
|
24470
|
+
const subtree = nestedScope === void 0 ? void 0 : subtreeSummaryOf(nestedScope);
|
|
24471
|
+
if (subtree !== void 0) child.subtree = subtree;
|
|
23743
24472
|
const settledAt = now();
|
|
23744
24473
|
child.settledAt = settledAt;
|
|
23745
24474
|
child.settledSeq = seq;
|
|
@@ -23782,6 +24511,7 @@ async function finalizeSettlement(child, settlement, seq, args, now) {
|
|
|
23782
24511
|
...settlement.infra === void 0 ? {} : { infra: settlement.infra },
|
|
23783
24512
|
...retainedExecution,
|
|
23784
24513
|
spent: child.spent,
|
|
24514
|
+
...subtree === void 0 ? {} : { subtree },
|
|
23785
24515
|
...settledNodeEvidence(child, settlement, settledAt)
|
|
23786
24516
|
}
|
|
23787
24517
|
}, { signal: args.signal });
|
|
@@ -23796,6 +24526,7 @@ async function finalizeSettlement(child, settlement, seq, args, now) {
|
|
|
23796
24526
|
...child.budgetViolation ? { budgetViolation: child.budgetViolation } : {},
|
|
23797
24527
|
trace: settlement.trace,
|
|
23798
24528
|
...settlement.harnessTranscript ? { harnessTranscript: settlement.harnessTranscript } : {},
|
|
24529
|
+
...subtree === void 0 ? {} : { subtree },
|
|
23799
24530
|
settledAt,
|
|
23800
24531
|
seq
|
|
23801
24532
|
};
|
|
@@ -23816,6 +24547,7 @@ async function finalizeSettlement(child, settlement, seq, args, now) {
|
|
|
23816
24547
|
...child.budgetViolation ? { budgetViolation: child.budgetViolation } : {},
|
|
23817
24548
|
trace: settlement.trace,
|
|
23818
24549
|
...settlement.harnessTranscript ? { harnessTranscript: settlement.harnessTranscript } : {},
|
|
24550
|
+
...subtree === void 0 ? {} : { subtree },
|
|
23819
24551
|
seq,
|
|
23820
24552
|
at
|
|
23821
24553
|
});
|
|
@@ -23834,6 +24566,7 @@ async function finalizeSettlement(child, settlement, seq, args, now) {
|
|
|
23834
24566
|
score: settlement.verdict?.score,
|
|
23835
24567
|
valid: settlement.verdict?.valid,
|
|
23836
24568
|
spent: settlement.spent,
|
|
24569
|
+
...subtree === void 0 ? {} : { subtree },
|
|
23837
24570
|
...settledNodeEvidence(child, settlement, settledAt)
|
|
23838
24571
|
}
|
|
23839
24572
|
}, { signal: args.signal });
|
|
@@ -23848,6 +24581,7 @@ async function finalizeSettlement(child, settlement, seq, args, now) {
|
|
|
23848
24581
|
...child.budgetViolation ? { budgetViolation: child.budgetViolation } : {},
|
|
23849
24582
|
trace: settlement.trace,
|
|
23850
24583
|
...settlement.harnessTranscript ? { harnessTranscript: settlement.harnessTranscript } : {},
|
|
24584
|
+
...subtree === void 0 ? {} : { subtree },
|
|
23851
24585
|
settledAt,
|
|
23852
24586
|
seq
|
|
23853
24587
|
};
|
|
@@ -23925,7 +24659,7 @@ async function finalizeWait(child, settlement, seq, args, now, handle) {
|
|
|
23925
24659
|
* applied per child). The reservation is reconciled on EVERY path (success, abort, throw)
|
|
23926
24660
|
* so the conserved pool can never leak a reservation.
|
|
23927
24661
|
*/
|
|
23928
|
-
async function runChild(live, executor, childAbort, task, opts, pool, ticket, blobs, now, journal, journalRoot, nextProgressSeq, executionReady, executionEvidence, deadlineAtMs, recovering = false, closeRetainedWrites = async () => {}) {
|
|
24662
|
+
async function runChild(live, executor, childAbort, task, opts, pool, ticket, blobs, now, journal, journalRoot, nextProgressSeq, executionReady, executionEvidence, deadlineAtMs, recovering = false, closeRetainedWrites = async () => {}, budgetReady = void 0, slotReady = Promise.resolve()) {
|
|
23929
24663
|
let reconciled = false;
|
|
23930
24664
|
let reconciliationError;
|
|
23931
24665
|
let teardownFailure;
|
|
@@ -24010,6 +24744,8 @@ async function runChild(live, executor, childAbort, task, opts, pool, ticket, bl
|
|
|
24010
24744
|
try {
|
|
24011
24745
|
await executionReady;
|
|
24012
24746
|
if (childAbort.signal.aborted) throw abortError(childAbort.signal, "execution aborted");
|
|
24747
|
+
if (budgetReady !== void 0) await awaitAbortable(budgetReady, childAbort.signal);
|
|
24748
|
+
await awaitAbortable(slotReady, childAbort.signal);
|
|
24013
24749
|
live.status = "running";
|
|
24014
24750
|
started = true;
|
|
24015
24751
|
const ran = recovering ? executor.recover(task, childAbort.signal) : executor.execute(task, childAbort.signal);
|
|
@@ -24186,25 +24922,114 @@ function settledToIteration(settled) {
|
|
|
24186
24922
|
tokenUsage: cloneTokenUsage(settled.spent.tokens)
|
|
24187
24923
|
};
|
|
24188
24924
|
}
|
|
24189
|
-
|
|
24190
|
-
|
|
24191
|
-
|
|
24192
|
-
|
|
24925
|
+
/** Summarize one scope's children and everything below them, listing the best direct children
|
|
24926
|
+
* and counting the rest. Wait-states are not agents and are left out. */
|
|
24927
|
+
/** The live team a child manager leads, read through the nested scope it owns. */
|
|
24928
|
+
function liveTeamOf(child) {
|
|
24929
|
+
let nested;
|
|
24930
|
+
try {
|
|
24931
|
+
nested = child.readNestedScope?.();
|
|
24932
|
+
} catch {
|
|
24933
|
+
return;
|
|
24934
|
+
}
|
|
24935
|
+
return nested === void 0 ? void 0 : liveTeamReaders.get(nested)?.();
|
|
24193
24936
|
}
|
|
24194
|
-
function
|
|
24195
|
-
|
|
24196
|
-
|
|
24197
|
-
|
|
24198
|
-
|
|
24199
|
-
|
|
24200
|
-
|
|
24201
|
-
|
|
24202
|
-
|
|
24203
|
-
|
|
24204
|
-
|
|
24937
|
+
function teamProgress(team) {
|
|
24938
|
+
const { lastActivityAt: _lastActivityAt, ownerTurns: _ownerTurns, ...counts } = team;
|
|
24939
|
+
return Object.freeze(counts);
|
|
24940
|
+
}
|
|
24941
|
+
/** Latest activity an executor names, or 0. A throwing read is no evidence of activity. */
|
|
24942
|
+
function executorActivityAt(child) {
|
|
24943
|
+
try {
|
|
24944
|
+
const recent = child.readProgress?.()?.recentActivity;
|
|
24945
|
+
return recent?.length ? recent[recent.length - 1]?.at ?? 0 : 0;
|
|
24946
|
+
} catch {
|
|
24947
|
+
return 0;
|
|
24948
|
+
}
|
|
24949
|
+
}
|
|
24950
|
+
/** Count a scope's team at every depth and find its newest activity. Live members are read through
|
|
24951
|
+
* their own nested scopes; settled members contribute the summary their settlement carried. */
|
|
24952
|
+
function readLiveTeam(children, ownerTurns, ownerLastTurnAt) {
|
|
24953
|
+
let agents = 0;
|
|
24954
|
+
let depth = 0;
|
|
24955
|
+
let working = 0;
|
|
24956
|
+
let queued = 0;
|
|
24957
|
+
let done = 0;
|
|
24958
|
+
let down = 0;
|
|
24959
|
+
let lastActivityAt = ownerLastTurnAt;
|
|
24960
|
+
for (const child of children) {
|
|
24961
|
+
if (child.wait) continue;
|
|
24962
|
+
agents += 1;
|
|
24963
|
+
lastActivityAt = Math.max(lastActivityAt, child.lastActivityAt, child.settledAt ?? 0, executorActivityAt(child));
|
|
24964
|
+
if (child.status === "done") done += 1;
|
|
24965
|
+
else if (child.status === "failed" || child.status === "cancelled") down += 1;
|
|
24966
|
+
else if (child.status === "queued" || child.status === "pending") queued += 1;
|
|
24967
|
+
else working += 1;
|
|
24968
|
+
const below = isTerminalNodeStatus(child.status) ? child.subtree : liveTeamOf(child);
|
|
24969
|
+
if (below === void 0 || below.agents === 0) continue;
|
|
24970
|
+
agents += below.agents;
|
|
24971
|
+
depth = Math.max(depth, below.depth);
|
|
24972
|
+
done += below.done;
|
|
24973
|
+
down += below.down;
|
|
24974
|
+
if ("working" in below) {
|
|
24975
|
+
working += below.working;
|
|
24976
|
+
queued += below.queued;
|
|
24977
|
+
lastActivityAt = Math.max(lastActivityAt, below.lastActivityAt);
|
|
24205
24978
|
}
|
|
24979
|
+
}
|
|
24980
|
+
return {
|
|
24981
|
+
agents,
|
|
24982
|
+
depth: agents === 0 ? 0 : depth + 1,
|
|
24983
|
+
working,
|
|
24984
|
+
queued,
|
|
24985
|
+
done,
|
|
24986
|
+
down,
|
|
24987
|
+
lastActivityAt,
|
|
24988
|
+
ownerTurns
|
|
24206
24989
|
};
|
|
24207
24990
|
}
|
|
24991
|
+
function summarizeTeam(children) {
|
|
24992
|
+
let agents = 0;
|
|
24993
|
+
let depth = 0;
|
|
24994
|
+
let done = 0;
|
|
24995
|
+
let down = 0;
|
|
24996
|
+
let members = 0;
|
|
24997
|
+
const results = [];
|
|
24998
|
+
for (const child of children) {
|
|
24999
|
+
if (child.wait) continue;
|
|
25000
|
+
members += 1;
|
|
25001
|
+
const below = child.subtree;
|
|
25002
|
+
agents += 1 + (below?.agents ?? 0);
|
|
25003
|
+
depth = Math.max(depth, 1 + (below?.depth ?? 0));
|
|
25004
|
+
done += below?.done ?? 0;
|
|
25005
|
+
down += below?.down ?? 0;
|
|
25006
|
+
const status = child.status === "done" ? "done" : child.status === "failed" || child.status === "cancelled" ? "down" : void 0;
|
|
25007
|
+
if (status === void 0) continue;
|
|
25008
|
+
if (status === "done") done += 1;
|
|
25009
|
+
else down += 1;
|
|
25010
|
+
const score = child.resolved?.kind === "done" ? child.resolved.verdict?.score : void 0;
|
|
25011
|
+
results.push({
|
|
25012
|
+
id: child.id,
|
|
25013
|
+
label: child.label,
|
|
25014
|
+
status,
|
|
25015
|
+
...child.outRef === void 0 ? {} : { outRef: child.outRef },
|
|
25016
|
+
...score === void 0 ? {} : { score },
|
|
25017
|
+
agents: below?.agents ?? 0,
|
|
25018
|
+
order: child.settledSeq ?? Number.MAX_SAFE_INTEGER
|
|
25019
|
+
});
|
|
25020
|
+
}
|
|
25021
|
+
if (members === 0) return void 0;
|
|
25022
|
+
results.sort((a, b) => (a.status === b.status ? 0 : a.status === "done" ? -1 : 1) || (b.score ?? Number.NEGATIVE_INFINITY) - (a.score ?? Number.NEGATIVE_INFINITY) || a.order - b.order);
|
|
25023
|
+
const listed = results.slice(0, 8).map(({ order: _order, ...result }) => Object.freeze(result));
|
|
25024
|
+
return Object.freeze({
|
|
25025
|
+
agents,
|
|
25026
|
+
depth,
|
|
25027
|
+
done,
|
|
25028
|
+
down,
|
|
25029
|
+
results: Object.freeze(listed),
|
|
25030
|
+
omitted: members - listed.length
|
|
25031
|
+
});
|
|
25032
|
+
}
|
|
24208
25033
|
function makeTreeView(root, children) {
|
|
24209
25034
|
const nodes = [...children.values()].map((c) => ({
|
|
24210
25035
|
id: c.id,
|
|
@@ -24231,7 +25056,7 @@ function makeTreeView(root, children) {
|
|
|
24231
25056
|
return {
|
|
24232
25057
|
root,
|
|
24233
25058
|
nodes,
|
|
24234
|
-
inFlight: nodes.filter((n) => n.status === "running" || n.status === "acquiring").length,
|
|
25059
|
+
inFlight: nodes.filter((n) => n.status === "running" || n.status === "acquiring" || n.status === "queued").length,
|
|
24235
25060
|
waiting: nodes.filter((n) => n.status === "waiting").length
|
|
24236
25061
|
};
|
|
24237
25062
|
}
|
|
@@ -24954,8 +25779,6 @@ const DEFAULT_MAX_ATTEMPTS = 8;
|
|
|
24954
25779
|
const DEFAULT_MAX_BARREN_REPROMPTS = 2;
|
|
24955
25780
|
const DEFAULT_INITIAL_BACKOFF_MS = 2e3;
|
|
24956
25781
|
const DEFAULT_MAX_BACKOFF_MS = 3e4;
|
|
24957
|
-
const DEFAULT_UNAVAILABLE_PAUSE_MS = 15e3;
|
|
24958
|
-
const DEFAULT_MAX_UNAVAILABLE_PAUSE_MS = 3e5;
|
|
24959
25782
|
/**
|
|
24960
25783
|
* Bridge error classes the bridge itself never retries: a request that fails identically on
|
|
24961
25784
|
* every attempt, mapped below 5xx on its HTTP path (`parse_error` 400, the other two 501). On the
|
|
@@ -24991,39 +25814,6 @@ var HarnessTurnFailedError = class extends Error {
|
|
|
24991
25814
|
}
|
|
24992
25815
|
};
|
|
24993
25816
|
/**
|
|
24994
|
-
* Machine codes that say the upstream cannot serve now rather than that the request is wrong.
|
|
24995
|
-
* Each is a code an upstream publishes, not prose: the router's
|
|
24996
|
-
* (`lib/model-substitution.ts`, `lib/upstream-error-triage.ts`) and the OpenAI- and Anthropic-shaped
|
|
24997
|
-
* error types a relayed body carries.
|
|
24998
|
-
*/
|
|
24999
|
-
const UNAVAILABLE_CODES = /* @__PURE__ */ new Set([
|
|
25000
|
-
"provider_quota_exhausted",
|
|
25001
|
-
"provider_quota_exceeded",
|
|
25002
|
-
"provider_rate_limit",
|
|
25003
|
-
"upstream_unavailable",
|
|
25004
|
-
"provider_key_invalid",
|
|
25005
|
-
"rate_limit_exceeded",
|
|
25006
|
-
"rate_limit_error",
|
|
25007
|
-
"overloaded_error"
|
|
25008
|
-
]);
|
|
25009
|
-
/** 429 Too Many Requests, 503 Service Unavailable, and 529, which Anthropic uses for overload. */
|
|
25010
|
-
const UNAVAILABLE_STATUSES = /* @__PURE__ */ new Set([
|
|
25011
|
-
429,
|
|
25012
|
-
503,
|
|
25013
|
-
529
|
|
25014
|
-
]);
|
|
25015
|
-
/**
|
|
25016
|
-
* A harness that flattens the router's JSON body into its own failure text leaves the router's
|
|
25017
|
-
* code in that text: `opencode execution failed: No provider served model "..."
|
|
25018
|
-
* (provider_quota_exhausted)`. The code is matched as a whole token, and a status only in the
|
|
25019
|
-
* `status code <n>` framing a harness prints, so prose that merely mentions a quota does not match.
|
|
25020
|
-
*/
|
|
25021
|
-
function unavailableSignalInText(text) {
|
|
25022
|
-
for (const token of text.toLowerCase().matchAll(/[a-z_]+/gu)) if (UNAVAILABLE_CODES.has(token[0])) return token[0];
|
|
25023
|
-
const status = text.match(/\bstatus code (\d{3})\b/u)?.[1];
|
|
25024
|
-
return status !== void 0 && UNAVAILABLE_STATUSES.has(Number(status)) ? `http-${status}` : void 0;
|
|
25025
|
-
}
|
|
25026
|
-
/**
|
|
25027
25817
|
* The code or status that marks `error` as an upstream capacity refusal, or `undefined`.
|
|
25028
25818
|
*
|
|
25029
25819
|
* A structured field is read first: a turn outcome's `errorCode`, a transport error's
|
|
@@ -25032,11 +25822,10 @@ function unavailableSignalInText(text) {
|
|
|
25032
25822
|
*/
|
|
25033
25823
|
function upstreamUnavailableSignal(error) {
|
|
25034
25824
|
if (!(error instanceof Error)) return void 0;
|
|
25035
|
-
if (error instanceof HarnessTurnFailedError) {
|
|
25036
|
-
|
|
25037
|
-
|
|
25038
|
-
|
|
25039
|
-
}
|
|
25825
|
+
if (error instanceof HarnessTurnFailedError) return unavailableSignalOfFailure({
|
|
25826
|
+
error: error.message,
|
|
25827
|
+
...error.errorCode === void 0 ? {} : { errorCode: error.errorCode }
|
|
25828
|
+
});
|
|
25040
25829
|
if (error instanceof BackendTransportError) {
|
|
25041
25830
|
const code = error.upstreamCode?.toLowerCase();
|
|
25042
25831
|
if (code !== void 0 && UNAVAILABLE_CODES.has(code)) return code;
|
|
@@ -25182,8 +25971,6 @@ async function runDriverWithRetry(run) {
|
|
|
25182
25971
|
const maxAttempts = Math.max(1, policy.maxAttempts ?? DEFAULT_MAX_ATTEMPTS);
|
|
25183
25972
|
const initialBackoff = Math.max(0, policy.initialBackoffMs ?? DEFAULT_INITIAL_BACKOFF_MS);
|
|
25184
25973
|
const maxBackoff = Math.max(initialBackoff, policy.maxBackoffMs ?? DEFAULT_MAX_BACKOFF_MS);
|
|
25185
|
-
const unavailablePause = Math.max(0, policy.unavailablePauseMs ?? DEFAULT_UNAVAILABLE_PAUSE_MS);
|
|
25186
|
-
const maxUnavailablePause = Math.max(unavailablePause, policy.maxUnavailablePauseMs ?? DEFAULT_MAX_UNAVAILABLE_PAUSE_MS);
|
|
25187
25974
|
const maxReprompts = run.reprompt?.maxReprompts ?? 0;
|
|
25188
25975
|
if (maxReprompts === "until-complete") {
|
|
25189
25976
|
const deadline = run.budget().deadlineMs;
|
|
@@ -25272,7 +26059,7 @@ async function runDriverWithRetry(run) {
|
|
|
25272
26059
|
}
|
|
25273
26060
|
pauses += 1;
|
|
25274
26061
|
consecutivePauses = progressed ? 1 : consecutivePauses + 1;
|
|
25275
|
-
const pause =
|
|
26062
|
+
const pause = unavailablePauseMs(consecutivePauses, policy);
|
|
25276
26063
|
await emit({
|
|
25277
26064
|
attempt,
|
|
25278
26065
|
durationMs,
|
|
@@ -25553,12 +26340,8 @@ function assertResumeContract(events, opts) {
|
|
|
25553
26340
|
if (roots.length !== 1 || roots[0]?.id !== opts.runId) throw new RuntimeRunStateError(`supervisor: resumed run '${opts.runId}' must contain exactly one root identity event; found ${roots.length}`);
|
|
25554
26341
|
const recorded = roots[0];
|
|
25555
26342
|
if (contentAddress(recorded.budget) !== contentAddress(opts.budget)) throw new RuntimeRunStateError(`supervisor: resume budget mismatch for run '${opts.runId}'; use a new runId to change limits`);
|
|
25556
|
-
const expectedAdmission = opts.reservationPolicy ? {
|
|
25557
|
-
|
|
25558
|
-
maxDepth: opts.maxDepth ?? defaultMaxDepth,
|
|
25559
|
-
maxLiveWorkers: opts.maxLiveWorkers
|
|
25560
|
-
} : void 0;
|
|
25561
|
-
if (contentAddress(recorded.recursiveAdmission ?? null) !== contentAddress(expectedAdmission ?? null)) throw new RuntimeRunStateError(`supervisor: resume reservation policy or fleet limits mismatch for run '${opts.runId}'; use a new runId`);
|
|
26343
|
+
const expectedAdmission = opts.reservationPolicy ? { policy: opts.reservationPolicy } : void 0;
|
|
26344
|
+
if (contentAddress(recorded.recursiveAdmission ?? null) !== contentAddress(expectedAdmission ?? null)) throw new RuntimeRunStateError(`supervisor: resume reservation policy mismatch for run '${opts.runId}'; use a new runId`);
|
|
25562
26345
|
if (!sameOptionalIdentity(recorded.identity, opts.rootIdentity)) throw new RuntimeRunStateError(`supervisor: resume identity mismatch for run '${opts.runId}'; ${describeIdentityMismatch(recorded.identity, opts.rootIdentity)}`);
|
|
25563
26346
|
const receipts = events.filter((event) => event.kind === "materialized" && event.id === opts.runId);
|
|
25564
26347
|
const expectedReceipt = rootMaterializationReceipt(opts);
|
|
@@ -25655,9 +26438,11 @@ function rootStartedAtMs(root) {
|
|
|
25655
26438
|
if (!Number.isFinite(startedAt)) throw new RuntimeRunStateError(`supervisor: root event for '${root.id}' has an invalid timestamp '${root.at}'`);
|
|
25656
26439
|
return startedAt;
|
|
25657
26440
|
}
|
|
25658
|
-
/** The default
|
|
25659
|
-
*
|
|
25660
|
-
|
|
26441
|
+
/** The default recursion-depth ceiling. The conserved pool is what bounds a tree's depth: every
|
|
26442
|
+
* level draws its slice from the level above, so a tree ends when its slices run out. This
|
|
26443
|
+
* ceiling only stops a runaway recursion that keeps spawning tiny slices, so it sits well above
|
|
26444
|
+
* any depth a budget can usefully pay for. */
|
|
26445
|
+
const DEFAULT_MAX_DEPTH = 16;
|
|
25661
26446
|
/**
|
|
25662
26447
|
* Normalize an arbitrary rejection into the serializable triple the result carries. A driver may
|
|
25663
26448
|
* reject with anything (a string, a plain object, a null-prototype value), and the whole point of
|
|
@@ -25682,7 +26467,7 @@ function describeRejection(error) {
|
|
|
25682
26467
|
function createSupervisor() {
|
|
25683
26468
|
let attached;
|
|
25684
26469
|
async function run(root, task, opts) {
|
|
25685
|
-
const { budget, rootIdentity, rootMaterialization, runId, journal: journalStore, blobs: blobStore, executors: executorRegistry, recoverExecutor, probes, maxDepth,
|
|
26470
|
+
const { budget, rootIdentity, rootMaterialization, runId, journal: journalStore, blobs: blobStore, executors: executorRegistry, recoverExecutor, probes, maxDepth, workerSlots, reservationPolicy, maxRestarts, withinMs, childSettleGraceMs, teardownConfirmMs, resume, retainedAtSettlement: requestedRetainedAtSettlement, ownerWorkspaceRetention, now: suppliedNow, signal, hooks, workerTrace, workerTraceUnpropagated, interactiveBindingDir } = opts;
|
|
25686
26471
|
const input = detachedSnapshot({
|
|
25687
26472
|
task,
|
|
25688
26473
|
options: {
|
|
@@ -25691,7 +26476,7 @@ function createSupervisor() {
|
|
|
25691
26476
|
...rootIdentity === void 0 ? {} : { rootIdentity },
|
|
25692
26477
|
...rootMaterialization === void 0 ? {} : { rootMaterialization },
|
|
25693
26478
|
...maxDepth === void 0 ? {} : { maxDepth },
|
|
25694
|
-
...
|
|
26479
|
+
...typeof workerSlots === "number" ? { workerSlots } : {},
|
|
25695
26480
|
...reservationPolicy === void 0 ? {} : { reservationPolicy },
|
|
25696
26481
|
...maxRestarts === void 0 ? {} : { maxRestarts },
|
|
25697
26482
|
...withinMs === void 0 ? {} : { withinMs },
|
|
@@ -25703,7 +26488,8 @@ function createSupervisor() {
|
|
|
25703
26488
|
...interactiveBindingDir === void 0 ? {} : { interactiveBindingDir }
|
|
25704
26489
|
}
|
|
25705
26490
|
}, "supervisor.run");
|
|
25706
|
-
assertRecursiveReservationPolicy(reservationPolicy
|
|
26491
|
+
assertRecursiveReservationPolicy(reservationPolicy);
|
|
26492
|
+
const slots = resolveWorkerSlots(workerSlots);
|
|
25707
26493
|
if (teardownConfirmMs !== void 0 && (typeof teardownConfirmMs !== "number" || !Number.isFinite(teardownConfirmMs) || teardownConfirmMs < 0)) throw new ValidationError$1(`supervisor: teardownConfirmMs must be a nonnegative finite number of milliseconds, got ${JSON.stringify(teardownConfirmMs)}`);
|
|
25708
26494
|
if (requestedRetainedAtSettlement !== void 0 && requestedRetainedAtSettlement !== "release" && requestedRetainedAtSettlement !== "keep") throw new ValidationError$1(`supervisor: retainedAtSettlement must be 'release' or 'keep', got ${JSON.stringify(requestedRetainedAtSettlement)}`);
|
|
25709
26495
|
opts = Object.freeze({
|
|
@@ -25766,11 +26552,7 @@ function createSupervisor() {
|
|
|
25766
26552
|
label: "root",
|
|
25767
26553
|
budget: opts.budget,
|
|
25768
26554
|
runtime: rootRuntime,
|
|
25769
|
-
...opts.reservationPolicy ? { recursiveAdmission: {
|
|
25770
|
-
policy: opts.reservationPolicy,
|
|
25771
|
-
maxDepth: opts.maxDepth ?? defaultMaxDepth,
|
|
25772
|
-
maxLiveWorkers: opts.maxLiveWorkers
|
|
25773
|
-
} } : {},
|
|
26555
|
+
...opts.reservationPolicy ? { recursiveAdmission: { policy: opts.reservationPolicy } } : {},
|
|
25774
26556
|
...opts.rootIdentity ? { identity: opts.rootIdentity } : {},
|
|
25775
26557
|
seq: 0,
|
|
25776
26558
|
at: runStartedAt
|
|
@@ -25820,8 +26602,8 @@ function createSupervisor() {
|
|
|
25820
26602
|
executors: opts.executors,
|
|
25821
26603
|
seams: { [retainedOwnerWorkspaceRetentionSeamKey]: ownerWorkspaceRetention === true },
|
|
25822
26604
|
depth: 0,
|
|
25823
|
-
maxDepth: opts.maxDepth ??
|
|
25824
|
-
|
|
26605
|
+
maxDepth: opts.maxDepth ?? 16,
|
|
26606
|
+
workerSlots: slots,
|
|
25825
26607
|
...opts.reservationPolicy ? {
|
|
25826
26608
|
reservationPolicy: opts.reservationPolicy,
|
|
25827
26609
|
ownerBudget: opts.budget
|
|
@@ -25913,6 +26695,7 @@ function createSupervisor() {
|
|
|
25913
26695
|
seq: teardownSeq++,
|
|
25914
26696
|
at: new Date(now()).toISOString()
|
|
25915
26697
|
});
|
|
26698
|
+
if (releasing) await closeRetainedSlots(openScope);
|
|
25916
26699
|
} catch (error) {
|
|
25917
26700
|
if (actOutcome?.ok !== false) actOutcome = {
|
|
25918
26701
|
ok: false,
|
|
@@ -26231,13 +27014,14 @@ function pausable(ms) {
|
|
|
26231
27014
|
/** The unconfirmed set as one error clause: the count plus the node ids behind it. One integer is
|
|
26232
27015
|
* not actionable after a six-hour run. */
|
|
26233
27016
|
function describeUnconfirmed(scope) {
|
|
26234
|
-
const
|
|
27017
|
+
const unconfirmed = scope.workerCapacity.unconfirmed;
|
|
27018
|
+
const named = unconfirmed.map((node) => `${node.id} (${[
|
|
26235
27019
|
node.label,
|
|
26236
27020
|
node.runtime,
|
|
26237
27021
|
node.status,
|
|
26238
27022
|
...(node.environments ?? []).map((environment) => `${environment.provider}:${environment.environmentId}`)
|
|
26239
27023
|
].join(", ")})`).join(", ");
|
|
26240
|
-
return `${
|
|
27024
|
+
return `${unconfirmed.length} executor resource(s) not confirmed destroyed${named.length > 0 ? `: ${named}` : ""}`;
|
|
26241
27025
|
}
|
|
26242
27026
|
/** The settle window a driver's children actually get: the caller's grace, never past an explicit
|
|
26243
27027
|
* run deadline. `null` means no timer. `0` (the default) cascades immediately. */
|
|
@@ -26328,13 +27112,16 @@ function fleetYieldFromForest(forest) {
|
|
|
26328
27112
|
const spawned = /* @__PURE__ */ new Set();
|
|
26329
27113
|
const terminal = /* @__PURE__ */ new Map();
|
|
26330
27114
|
const released = /* @__PURE__ */ new Set();
|
|
27115
|
+
const unconfirmed = /* @__PURE__ */ new Set();
|
|
26331
27116
|
for (const { event } of forest.events) if (event.kind === "spawned") {
|
|
26332
27117
|
if (event.parent !== void 0) spawned.add(event.id);
|
|
26333
27118
|
} else if (event.kind === "settled" || event.kind === "cancelled") {
|
|
26334
27119
|
const bucket = event.kind === "cancelled" ? "cancelled" : event.status;
|
|
26335
27120
|
terminal.set(event.id, bucket);
|
|
26336
|
-
|
|
26337
|
-
|
|
27121
|
+
released.delete(event.id);
|
|
27122
|
+
unconfirmed.delete(event.id);
|
|
27123
|
+
if (bucket !== "done" && event.retainedExecution === "released") released.add(event.id);
|
|
27124
|
+
if (bucket !== "done" && event.retainedExecution === "release-unconfirmed") unconfirmed.add(event.id);
|
|
26338
27125
|
}
|
|
26339
27126
|
const count = (bucket) => [...terminal.values()].filter((value) => value === bucket).length;
|
|
26340
27127
|
const fleetYield = {
|
|
@@ -26343,10 +27130,11 @@ function fleetYieldFromForest(forest) {
|
|
|
26343
27130
|
down: count("down"),
|
|
26344
27131
|
cancelled: count("cancelled"),
|
|
26345
27132
|
neverSettled: forest.inDoubt.length,
|
|
26346
|
-
releasedUnrecovered: released.size
|
|
27133
|
+
releasedUnrecovered: released.size,
|
|
27134
|
+
releaseUnconfirmed: unconfirmed.size
|
|
26347
27135
|
};
|
|
26348
27136
|
if (fleetYield.done + fleetYield.down + fleetYield.cancelled + fleetYield.neverSettled !== fleetYield.spawned) throw new RuntimeRunStateError(`supervisor: fleet yield does not partition the spawned children of '${forest.root}' (${JSON.stringify(fleetYield)})`);
|
|
26349
|
-
if (fleetYield.releasedUnrecovered > fleetYield.down + fleetYield.cancelled) throw new RuntimeRunStateError(`supervisor: released children exceed down + cancelled for '${forest.root}' (${JSON.stringify(fleetYield)})`);
|
|
27137
|
+
if (fleetYield.releasedUnrecovered + fleetYield.releaseUnconfirmed > fleetYield.down + fleetYield.cancelled) throw new RuntimeRunStateError(`supervisor: released children exceed down + cancelled for '${forest.root}' (${JSON.stringify(fleetYield)})`);
|
|
26350
27138
|
return fleetYield;
|
|
26351
27139
|
}
|
|
26352
27140
|
/** The journaled nodes whose usage accounting is incomplete — one `SpendGap` per node+kind,
|
|
@@ -26411,6 +27199,6 @@ function isNonEmptySpend(s) {
|
|
|
26411
27199
|
return s.iterations > 0 || s.tokens.input > 0 || s.tokens.output > 0 || s.usd > 0 || s.ms > 0 || s.tokensKnown === false || s.usdKnown === false || Object.values(s.resources ?? {}).some((resource) => resource.amount > 0 || !resource.known);
|
|
26412
27200
|
}
|
|
26413
27201
|
//#endregion
|
|
26414
|
-
export {
|
|
27202
|
+
export { interactiveAdmissionSeamKey as $, sumSandboxUsage as $i, recoverRetainedRun as $n, controlProfileMaterialization as $r, taskToPrompt as $t, restoreScopeOwnerAcceptedExecution as A, runtimeOwnedExecutorMaterialization as Ai, acquireSandbox as An, padSpanId as Ar, insideCursorNamespace as At, bindScopeRetainedOwnerEnvironmentId as B, projectSandboxOutcome as Bi, providerAsExecutor as Bn, captureWorktreeDiff as Br, BudgetReconcileFault as Bt, beginScopeOwnerAttempt as C, teardownExecutor as Ca, knownExecutionBindingReceipt as Ci, TERMINAL_DECISIONS as Cn, buildRuntimeEventOtelSpans as Cr, FileResultBlobStore as Ct, meterRuntimeOwnedProviderAttempt as D, providerMessageText as Da, recordRuntimeOwnedDriveHarnessProviderEvidence as Di, runAgentRounds as Dn, flatOtelSpan as Dr, SpawnEventIndex as Dt, meterRuntimeOwnedAccounting as E, promptOptionsFromAgentTurnInput as Ea, providerAttemptEvidence as Ei, isTerminalDecision as En, exportEvalRuns as Er, InMemorySpawnJournal as Et, isWaitOutcome as F, unknownMaterializationReceipt as Fi, canonicalObservedModelParts as Fn, sanitizeAgentRuntimeEvent as Fr, WORKER_TOOL_TRACE_SCHEMA_VERSION as Ft, scopeRetainedOwnerContext as G, createSandboxUsageLedger as Gi, unavailableSignalOfFailure as Gn, harnessInvocation as Gr, newUsageTotals as Gt, consumeScopeRetainedOwnerResult as H, assertSandboxServedModel as Hi, resolveAgentEnvironmentProvider as Hn, removeWorktree as Hr, assertValidBudget as Ht, pollFor as I, executableAgentProfileSnapshot as Ii, observedModelMatchesDeclared as In, sanitizeKnowledgeReadinessReport as Ir, captureWorkerTraceEvidence as It, scopeRetainedOwnerResult as J, mapSandboxEvent as Ji, sandboxSessionTraceSource as Jn, parseCodexTokenUsage as Jr, WORKER_TRACE_PROPAGATION as Jt, scopeRetainedOwnerPriorSpend as K, extractLlmCallEvent as Ki, createPushTraceSource as Kn, harnessSupportsReasoningEffort as Kr, spendFromUsageEvents as Kt, timerAt as L, executableAgentSpecSnapshot as Li, DEFAULT_SANDBOX_IDLE_TIMEOUT_SECONDS as Ln, sanitizeRuntimeStreamEvent as Lr, isTraceAnalysisStore as Lt, settledToIteration as M, runtimeOwnedPendingExecutorMaterialization as Mi, readPromptOptions as Mn, toOtelAttributes as Mr, materializeTreeView as Mt, createWorkerSlots as N, runtimeOwnedScopeOwnerRuntime as Ni, routerBrain as Nn, createRuntimeEventCollector as Nr, pendingWaits as Nt, recordScopeOwnerMaterialization as O, runtimeOwnedDriveHarnessProviderEvidence as Oi, createSandboxLineage as On, generateSpanId as Or, assertContentAddress as Ot, createWaitProbes as P, unknownExecutionBindingReceipt as Pi, runBrainLoop as Pn, createRuntimeStreamEventCollector as Pr, replaySpawnTree as Pt, resolveSpawnResourcePaths as Q, sandboxProgressEvents as Qi, reconnectRetainedRun as Qn, assertProfileMaterialization as Qr, workerTraceSeamKey as Qt, validateWaitSpec as R, detachedFrozen as Ri, LEAF_CONTINUATION_TASK as Rn, runSettledCommand as Rr, parseWorkerToolTraceArtifact as Rt, assertRecursiveReservationPolicy as S, armDeadlineTimer as Sa, inheritRuntimeOwnedExecutorAttestation as Si, bridgeStopSignalKey as Sn, buildLoopSpanNodes as Sr, createInPlaceCliExecutor as St, deriveNodeExecutionIdentity as T, promptFromAgentTurnInput as Ta, newExecutionAttemptId as Ti, defaultSelectWinner as Tn, createOtelExporter as Tr, InMemoryResultBlobStore as Tt, prepareScopeRetainedOwnerTask as U, canonicalStreamEventFromSandboxEvent as Ui, sandboxClientAsProvider as Un, DEFAULT_LOCAL_HARNESS as Ur, createBudgetPool as Ut, bindScopeRetainedOwnerProvider as V, readSandboxOutcome as Vi, providerAsSandboxClient as Vn, createWorktree as Vr, ReservationWaitRefused as Vt, reconcileScopeRetainedOwnerEnvironment as W, createSandboxToolPartState as Wi, unavailablePauseMs as Wn, LOCAL_HARNESSES as Wr, meterUsageEvent as Wt, environmentReader as X, notifySandboxEventObserver as Xi, retainedExecutorSeamKey as Xn, terminateProcessTreeAndConfirm as Xr, workerTraceEnv as Xt, SPAWN_RESOURCE_PATH_MAX_BYTES as Y, mapSandboxToolEvent as Yi, registerRetainedExecutorPreparation as Yn, CodexExecutionDiagnosticError as Yr, readWorkerTraceContext as Yt, hostDirectoryReader as Z, sandboxEventServedBackend as Zi, errMessage as Zn, AGENT_PROFILE_MATERIALIZATION_AXES as Zr, workerTraceHeaders as Zt, driverChild as _, withBudgetResources as _a, writeAllBytes as _i, addHarnessUsage as _n, mergeTraceEnv as _r, createExecutorRegistry as _t, collectDelivered as a, isAbortError$2 as aa, promptOnlyProfileMaterialization as ai, DEFAULT_PEER_MAIL_LIMITS as an, RunCancellationReason as ar, workerInteractiveBindingsDir as at, withDriverExecutor as b, awaitAbortable$1 as ba, authoredProfileDigest as bi, readCodexRolloutSession as bn, INTELLIGENCE_WIRE_VERSION as br, DEFAULT_SANDBOX_STEERING_MAX_TURNS as bt, runTree as c, sleep as ca, renderUnsupported as ci, createPeerMailbox as cn, runAbortable as cr, startRetainedInteractiveRun as ct, HarnessTurnFailedError as d, throwIfAborted as da, validateProfileMaterialization as di, peerMailVerbNames as dn, resolveRedactor as dr, rollingDispatch as dt, addSpend as ea, defineProfileMaterializationContract as ei, DEFAULT_STALL_AFTER_MS as en, startRetainedRun as er, readWorkerInteractiveAdmissions as et, classifyDriverFailure as f, unmeteredSpend as fa, worktreeCliProfileMaterialization as fi, isLiveNodeStatus as fn, captureHarnessTranscript as fr, bindReusableExecutorExecutionId as ft, upstreamUnavailableSignal as g, resourceTelemetry as ga, readCommittedJsonLines as gi, bridgeModelRouteRefusal as gn, createPropagatingTraceEmitter as gr, createExecutor as gt, summarizeDriverAttempts as h, addResourceSpend as ha, prepareJsonlAppend as hi, bridgeAdmissionRefusal as hn, contentAddress as hr, cliWorktreeExecutor as ht, bestDelivered as i, hasCompleteCacheBreakdown as ia, promptModelProfileMaterialization as ii, AUTHORITY_MARKERS as in, environmentGone as ir, workerInteractiveBindingFile as it, scopeOwnerExecutorNodeContext as j, runtimeOwnedExecutorProviderEvidence as ji, assertBoxlessPromptOptions as jn, padTraceId as jr, loadSpawnForest as jt, recordScopeOwnerPause as k, runtimeOwnedExecutorExecutionBinding as ki, probeSandboxCapabilities as kn, loopEventToOtelSpan as kr, encodeResultBlob as kt, DEFAULT_MAX_BARREN_REPROMPTS as l, stringifySafe as la, sandboxActProfileMaterialization as li, isPeerMailEnvelope as ln, defaultRedactor as lr, freeSlots as lt, runDriverWithRetry as m, zeroSpend as ma, parseCommittedJsonLines as mi, executorFailure as mn, persistHarnessTranscript as mr, cliInPlaceExecutor as mt, createRootHandle as n, cloneSpend as na, profileMaterializationAxes$1 as ni, readWorkerProgress as nn, retainedCreateMaterial as nr, attachWorker as nt, pickBestDelivered as o, promptCacheTokenClasses as oa, promptResourceProfileMaterialization as oi, PEER_MAIL_WIRE_KEY as on, abortError as or, reconnectRetainedInteractiveRun as ot, defaultUnmetContractSteer as p, withTimeout as pa, isNoEntError as pi, isTerminalNodeStatus as pn, harnessTranscriptArtifact as pr, captureReusableExecutorConfig as pt, scopeRetainedOwnerResourceReader as q, isSandboxTerminalEvent as qi, decodeToolPart as qn, localHarnessExecutable as qr, spendFromUsageTotals as qt, createSupervisor as r, deleteBoxSafe as ra, promptControlProfileMaterialization as ri, createInbox as rn, destroyInteractiveEnvironment as rr, readWorkerInteractiveBinding as rt, runFinalizer as s, randomSuffix as sa, renderProfileMaterializationIssues as si, claimsAuthority as sn, linkAbort as sr, recoverRetainedInteractiveRun as st, DEFAULT_MAX_DEPTH as t, chargedTokens as ta, fullProfileMaterialization as ti, createActivityLog as tn, startRetainedRunInEnvironment as tr, workerInteractiveAdmissionFile as tt, DriverAttemptsExhaustedError as u, throwAbort as ua, unsupportedProfileDimensions as ui, peerMailTools as un, defaultRedactorIdentityMaterial as ur, queueOf as ut, driverExecutorFactory as v, decodeHarnessUsage as va, attestRuntimeOwnedPendingExecutor as vi, createCodexRolloutStoreReader as vn, readTraceContextFromEnv as vr, snapshotExecutorConfig as vt, createScope as w, teardownSurfaces as wa, knownMaterializationReceipt as wi, createSandboxForSpec as wn, createOpenInferenceFileExporter as wr, FileSpawnJournal as wt, SUBTREE_RESULT_LIMIT as x, DEFAULT_SUCCESSFUL_SHUTDOWN_MS as xa, finalizeRuntimeOwnedPendingExecutor as xi, bridgeRuntimeAttachmentsKey as xn, buildLoopOtelSpans as xr, createSteerableSandboxSession as xt, isDriverSpec as y, abortError$2 as ya, attestRuntimeOwnedScopeOwner as yi, harnessUsageIsEmpty as yn, traceContextToEnv as yr, createWorktreeCliExecutor as yt, waitUntil as z, detachedSnapshot as zi, createAgentEnvironmentProviderRegistry as zn, runWorktreeHarness as zr, workerTraceAnalysisStore as zt };
|
|
26415
27203
|
|
|
26416
|
-
//# sourceMappingURL=supervisor-
|
|
27204
|
+
//# sourceMappingURL=supervisor-Cvd5mf7x.js.map
|