@tangle-network/agent-runtime 0.300.0 → 0.301.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{activation-Bhp8dYV5.d.ts → activation-CRgEJutr.d.ts} +3 -3
- package/dist/{activation-DqWmWTyB.js → activation-V6aaOlZV.js} +2 -2
- package/dist/{activation-DqWmWTyB.js.map → activation-V6aaOlZV.js.map} +1 -1
- package/dist/agent.d.ts +2 -2
- package/dist/agent.js +2 -2
- package/dist/{coordination-driver-DzTvehd6.js → coordination-driver-vNrdQBSH.js} +317 -211
- package/dist/coordination-driver-vNrdQBSH.js.map +1 -0
- package/dist/{delegate-vJ2g76r6.js → delegate-B-jFJcbW.js} +5 -5
- package/dist/delegate-B-jFJcbW.js.map +1 -0
- package/dist/durable.d.ts +2 -2
- package/dist/durable.js +2 -2
- package/dist/{graph-BDQh3qry.js → graph-Dwv_hQB_.js} +5 -5
- package/dist/graph-Dwv_hQB_.js.map +1 -0
- package/dist/{improvement-cycle-D6cPLi6k.js → improvement-cycle-Cf37Kexd.js} +3 -3
- package/dist/{improvement-cycle-D6cPLi6k.js.map → improvement-cycle-Cf37Kexd.js.map} +1 -1
- package/dist/{index-sC6sl88d.d.ts → index-CxBVmgqL.d.ts} +149 -71
- package/dist/index.d.ts +6 -6
- package/dist/index.js +7 -7
- package/dist/intelligence.d.ts +3 -3
- package/dist/intelligence.js +4 -4
- package/dist/kernel.d.ts +4 -4
- package/dist/kernel.js +9 -9
- package/dist/{loop-runner-bin-DQIHnCL0.js → loop-runner-bin-C8MI8N1A.js} +3 -3
- package/dist/{loop-runner-bin-DQIHnCL0.js.map → loop-runner-bin-C8MI8N1A.js.map} +1 -1
- package/dist/{loop-runner-bin-DZglq4Cf.d.ts → loop-runner-bin-DiTNiJNs.d.ts} +3 -3
- package/dist/loop-runner-bin.d.ts +1 -1
- package/dist/loop-runner-bin.js +1 -1
- package/dist/mcp/bin.js +3 -3
- package/dist/mcp/index.d.ts +6 -6
- package/dist/mcp/index.js +5 -5
- package/dist/{openai-tools-DvR82zEo.d.ts → openai-tools-BeogJ6V8.d.ts} +2 -2
- package/dist/profiles.d.ts +2 -2
- package/dist/{provision-supervisor-IXBlLHE0.js → provision-supervisor-CS0fBIhp.js} +13 -24
- package/dist/provision-supervisor-CS0fBIhp.js.map +1 -0
- package/dist/{runtime-DpEt5v42.js → runtime-6VPSJVeg.js} +28 -36
- package/dist/runtime-6VPSJVeg.js.map +1 -0
- package/dist/{search-method-DvU-fc-f.js → search-method-wTTS7mGG.js} +2 -2
- package/dist/{search-method-DvU-fc-f.js.map → search-method-wTTS7mGG.js.map} +1 -1
- package/dist/{server-CIE8s4AN.js → server-BfHD5iLD.js} +3 -3
- package/dist/{server-CIE8s4AN.js.map → server-BfHD5iLD.js.map} +1 -1
- package/dist/{stream-agent-turn-I3bb0eqF.d.ts → stream-agent-turn-B1ipFtOc.d.ts} +4 -2
- package/dist/{substrate-UjsxukjZ.d.ts → substrate-DKBMd0Dl.d.ts} +2 -2
- package/dist/{supervise-C5mktzrh.js → supervise-BJrEC2Mh.js} +195 -59
- package/dist/supervise-BJrEC2Mh.js.map +1 -0
- package/dist/{supervisor-CQSu5XGl.js → supervisor-DA_nvF-h.js} +563 -181
- package/dist/supervisor-DA_nvF-h.js.map +1 -0
- package/dist/testing.d.ts +2 -2
- package/dist/testing.js +12 -12
- package/dist/tui/index.d.ts +1 -1
- package/dist/tui/index.js +1 -1
- package/dist/{types-C73caP0U.d.ts → types-jD9mNnJM.d.ts} +61 -11
- package/package.json +1 -1
- package/skills/profile-authoring/SKILL.md +3 -3
- package/skills/supervise/SKILL.md +2 -2
- package/dist/coordination-driver-DzTvehd6.js.map +0 -1
- package/dist/delegate-vJ2g76r6.js.map +0 -1
- package/dist/graph-BDQh3qry.js.map +0 -1
- package/dist/provision-supervisor-IXBlLHE0.js.map +0 -1
- package/dist/runtime-DpEt5v42.js.map +0 -1
- package/dist/supervise-C5mktzrh.js.map +0 -1
- package/dist/supervisor-CQSu5XGl.js.map +0 -1
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { $
|
|
1
|
+
import { $n as parseWorkerToolTraceArtifact, Bn as InMemoryResultBlobStore, Ca as promptCacheTokenClasses, E as failedItems, M as withDriverExecutor, Ma as addResourceSpend, Mi as prepareJsonlAppend, Mr as contentAddress, Ni as readCommittedJsonLines, Nt as createExecutorRegistry, Pa as withBudgetResources, Pi as writeAllBytes, Rn as FileResultBlobStore, Rt as createRouterTranscript, Tn as runBrainLoop, Ui as providerAttemptEvidence, Vn as InMemorySpawnJournal, Yt as createInbox, _ as CheckUnavailableError, an as isTerminalNodeStatus, c as runTree, en as createPeerMailbox, er as workerTraceAnalysisStore, fr as errMessage, i as bestDelivered, in as isLiveNodeStatus, ir as assertValidBudget, ka as unmeteredSpend, mt as resolveSpawnResourcePaths, o as pickBestDelivered, s as runFinalizer, st as scopeRetainedOwnerResourceReader, ta as detachedFrozen, va as chargedTokens, x as checkVerdictOf, z as meterRuntimeOwnedProviderAttempt, zn as FileSpawnJournal, zr as resolveRedactor } from "./supervisor-DA_nvF-h.js";
|
|
2
2
|
import { f as RuntimeRunStateError, m as ValidationError } from "./errors-DodWX-cb.js";
|
|
3
3
|
import { A as writeWorkerCancellation, M as writeWorkerSteerAcknowledgement, b as workerCancellationsDir, c as readWorkerCancelRequests, d as readWorkerSteerRequests, k as writeRunCancellation, l as readWorkerCancellation, o as readRunCancelRequest, r as claimWorkerSteerDelivery, s as readRunCancellation, u as readWorkerSteerAcknowledgement } from "./run-layout-DSCWdbX2.js";
|
|
4
4
|
import { agentProfileSchema, canonicalAgentProfileDigest, canonicalCandidateDigest } from "@tangle-network/agent-interface";
|
|
@@ -7,7 +7,6 @@ import { randomUUID } from "node:crypto";
|
|
|
7
7
|
import { mkdirSync, readFileSync, watch } from "node:fs";
|
|
8
8
|
import { join } from "node:path";
|
|
9
9
|
import { registry } from "zod";
|
|
10
|
-
import { AsyncLocalStorage } from "node:async_hooks";
|
|
11
10
|
//#region src/runtime/supervise/detector-monitor.ts
|
|
12
11
|
/**
|
|
13
12
|
*
|
|
@@ -57,6 +56,7 @@ function createEventBus(now = Date.now) {
|
|
|
57
56
|
const queue = [];
|
|
58
57
|
const log = [];
|
|
59
58
|
const subscribers = [];
|
|
59
|
+
const queuedHandlers = [];
|
|
60
60
|
const byKind = {};
|
|
61
61
|
const staged = /* @__PURE__ */ new WeakMap();
|
|
62
62
|
let seq = 0;
|
|
@@ -97,6 +97,7 @@ function createEventBus(now = Date.now) {
|
|
|
97
97
|
log.push(record);
|
|
98
98
|
published += 1;
|
|
99
99
|
byKind[event.type] = (byKind[event.type] ?? 0) + 1;
|
|
100
|
+
if (opts?.queue !== false) for (const handler of [...queuedHandlers]) handler(record);
|
|
100
101
|
return record;
|
|
101
102
|
},
|
|
102
103
|
pull(kinds) {
|
|
@@ -113,6 +114,13 @@ function createEventBus(now = Date.now) {
|
|
|
113
114
|
if (i >= 0) subscribers.splice(i, 1);
|
|
114
115
|
};
|
|
115
116
|
},
|
|
117
|
+
onQueued(handler) {
|
|
118
|
+
queuedHandlers.push(handler);
|
|
119
|
+
return () => {
|
|
120
|
+
const i = queuedHandlers.indexOf(handler);
|
|
121
|
+
if (i >= 0) queuedHandlers.splice(i, 1);
|
|
122
|
+
};
|
|
123
|
+
},
|
|
116
124
|
pending(kinds) {
|
|
117
125
|
return kinds ? queue.filter((r) => matches(r, kinds)).length : queue.length;
|
|
118
126
|
},
|
|
@@ -129,53 +137,6 @@ function createEventBus(now = Date.now) {
|
|
|
129
137
|
};
|
|
130
138
|
}
|
|
131
139
|
//#endregion
|
|
132
|
-
//#region src/mcp/tools/coordination-request-context.ts
|
|
133
|
-
/**
|
|
134
|
-
* The header a harness sends to a remote MCP server whose tool-call timeout it
|
|
135
|
-
* raised, carrying the applied timeout in milliseconds. The Tangle profile
|
|
136
|
-
* materializer writes it beside `MCP_TOOL_TIMEOUT` (Claude Code) and
|
|
137
|
-
* `tool_timeout_sec` (Codex) when an attachment declares
|
|
138
|
-
* `metadata.toolTimeoutMs`. Without it the server must assume the harness
|
|
139
|
-
* default, 60 s for Claude Code 2.1.287 and Codex (measured 2026-10-03).
|
|
140
|
-
*/
|
|
141
|
-
const MCP_TOOL_TIMEOUT_HEADER = "x-mcp-tool-timeout-ms";
|
|
142
|
-
/** The tool-call timeout Runtime asks harnesses to give its coordination server. */
|
|
143
|
-
const COORDINATION_CLIENT_TOOL_TIMEOUT_MS = 3e5;
|
|
144
|
-
/**
|
|
145
|
-
* The longest `await_event` holds for a caller that declared a raised timeout.
|
|
146
|
-
* A held response through the Sandbox edge was measured to 200 s on 2026-10-03.
|
|
147
|
-
*/
|
|
148
|
-
const MAX_DECLARED_AWAIT_FENCE_MS = 18e4;
|
|
149
|
-
const MIN_DECLARED_TOOL_TIMEOUT_MS = 1e3;
|
|
150
|
-
const MAX_DECLARED_TOOL_TIMEOUT_MS = 36e5;
|
|
151
|
-
const requestContext = new AsyncLocalStorage();
|
|
152
|
-
/**
|
|
153
|
-
* The tool-call timeout a request declared through {@link MCP_TOOL_TIMEOUT_HEADER},
|
|
154
|
-
* or `undefined` when it is absent or malformed. A malformed value is treated as
|
|
155
|
-
* absent: the caller then gets the default fence, which every harness tolerates.
|
|
156
|
-
*/
|
|
157
|
-
function declaredToolTimeoutMs(header) {
|
|
158
|
-
if (typeof header !== "string" || !/^[0-9]{1,8}$/.test(header)) return void 0;
|
|
159
|
-
const value = Number(header);
|
|
160
|
-
return value >= MIN_DECLARED_TOOL_TIMEOUT_MS && value <= MAX_DECLARED_TOOL_TIMEOUT_MS ? value : void 0;
|
|
161
|
-
}
|
|
162
|
-
/**
|
|
163
|
-
* How long `await_event` may hold for a caller that declared `declaredMs`: 60% of
|
|
164
|
-
* it, so the answer leaves well before the client gives up, and at most
|
|
165
|
-
* {@link MAX_DECLARED_AWAIT_FENCE_MS}.
|
|
166
|
-
*/
|
|
167
|
-
function declaredAwaitFenceMs(declaredMs) {
|
|
168
|
-
return Math.min(MAX_DECLARED_AWAIT_FENCE_MS, Math.floor(declaredMs * .6));
|
|
169
|
-
}
|
|
170
|
-
/** Run one coordination request with the fence its caller declared, if any. */
|
|
171
|
-
function withCoordinationAwaitFence(awaitFenceMs, run) {
|
|
172
|
-
return awaitFenceMs === void 0 ? run() : requestContext.run({ awaitFenceMs }, run);
|
|
173
|
-
}
|
|
174
|
-
/** The fence the current coordination request declared, or `undefined`. */
|
|
175
|
-
function coordinationRequestAwaitFenceMs() {
|
|
176
|
-
return requestContext.getStore()?.awaitFenceMs;
|
|
177
|
-
}
|
|
178
|
-
//#endregion
|
|
179
140
|
//#region src/mcp/tools/worker-output.ts
|
|
180
141
|
const WORKER_OUTPUT_PAGE_CHARS = 16384;
|
|
181
142
|
function workerOutputReadOptions(raw) {
|
|
@@ -532,13 +493,6 @@ function emptyPreflightCounts() {
|
|
|
532
493
|
"unmountable-tool": 0
|
|
533
494
|
};
|
|
534
495
|
}
|
|
535
|
-
/** Default ceiling for a single `await_event` block (ms). Every `pending` return costs the driver an
|
|
536
|
-
* inference turn, so the ceiling is as long as the harness clients allow: Claude Code 2.1.287 and
|
|
537
|
-
* Codex both time out an MCP tool call at 60 s by default (measured 2026-10-03: a 150 s HTTP MCP
|
|
538
|
-
* call failed at about 60 s and passed with `MCP_TOOL_TIMEOUT=300000`), and the Sandbox edge held
|
|
539
|
-
* a 200 s response. 45 s leaves 15 s of that minute for transfer and the driver's own latency. At
|
|
540
|
-
* 15 s a Discovery root director spent 303 of its 401 native tool calls (76%) re-polling. */
|
|
541
|
-
const DEFAULT_AWAIT_EVENT_TIMEOUT_MS = 45e3;
|
|
542
496
|
/** Default and maximum bounds for one {@link CoordinationTools} journal read. The defaults are
|
|
543
497
|
* sized for a driver turn (a page a model can read and still have room to act on); the maxima
|
|
544
498
|
* bound what a single tool result may cost even when the caller asks for everything. */
|
|
@@ -563,7 +517,9 @@ const journalEventKinds = [
|
|
|
563
517
|
"escalation",
|
|
564
518
|
"analyst-defined",
|
|
565
519
|
"analyst-assignment",
|
|
566
|
-
"acknowledgement"
|
|
520
|
+
"acknowledgement",
|
|
521
|
+
"tool-outcome",
|
|
522
|
+
"lead-message"
|
|
567
523
|
];
|
|
568
524
|
/** The reserved coordination verb names — the complete set `createCoordinationTools` can emit
|
|
569
525
|
* (the analyst pair is conditional but still reserved). A driver's extra WORK tools must not
|
|
@@ -582,7 +538,6 @@ const coordinationVerbNames = [
|
|
|
582
538
|
"cancel_worker",
|
|
583
539
|
"observe_agent",
|
|
584
540
|
"steer_agent",
|
|
585
|
-
"await_event",
|
|
586
541
|
"list_questions",
|
|
587
542
|
"answer_question",
|
|
588
543
|
"ask_parent",
|
|
@@ -596,22 +551,18 @@ const coordinationVerbNames = [
|
|
|
596
551
|
"define_analyst"
|
|
597
552
|
];
|
|
598
553
|
/**
|
|
599
|
-
* The `CoordinationEvent` kinds
|
|
600
|
-
*
|
|
601
|
-
*
|
|
602
|
-
*
|
|
603
|
-
*
|
|
604
|
-
* dispatch. Written twice, the two drift and the schema promises a kind the filter drops — a
|
|
605
|
-
* driver then blocks on a queue that already holds its event.
|
|
554
|
+
* The `CoordinationEvent` kinds that wake a waiting manager. The queue carries the UP-leg only:
|
|
555
|
+
* `steer` / `answer` / `instruction` / `delivery-attempt` are recorded `queue: false` (history and
|
|
556
|
+
* subscribers, never delivered back), and `mail` is delivered to its addressee's inbox. A
|
|
557
|
+
* `question` on this bus is the manager's own `ask_parent`, already escalated: it never wakes the
|
|
558
|
+
* asker, whose answer arrives as a `lead-message`.
|
|
606
559
|
*/
|
|
607
|
-
const
|
|
560
|
+
const wakeEventKinds = [
|
|
608
561
|
"settled",
|
|
609
|
-
"
|
|
610
|
-
"
|
|
562
|
+
"finding",
|
|
563
|
+
"tool-outcome",
|
|
564
|
+
"lead-message"
|
|
611
565
|
];
|
|
612
|
-
function isAwaitableEventKind(value) {
|
|
613
|
-
return awaitableEventKinds.includes(value);
|
|
614
|
-
}
|
|
615
566
|
const idArg = {
|
|
616
567
|
type: "string",
|
|
617
568
|
description: "The workerId returned by spawn_worker."
|
|
@@ -1007,6 +958,10 @@ function createCoordinationToolsForManager(opts, lifetime) {
|
|
|
1007
958
|
return readyInFlight;
|
|
1008
959
|
};
|
|
1009
960
|
const deliveries = [];
|
|
961
|
+
const openReceipts = /* @__PURE__ */ new Map();
|
|
962
|
+
let receiptSeq = 0;
|
|
963
|
+
let leadPublishing = 0;
|
|
964
|
+
let deadlineNoticeDelivered = false;
|
|
1010
965
|
let driverAttempt;
|
|
1011
966
|
let journalReadTo = 0;
|
|
1012
967
|
let lastRejection;
|
|
@@ -1338,7 +1293,16 @@ function createCoordinationToolsForManager(opts, lifetime) {
|
|
|
1338
1293
|
}, { analyst: route.kind });
|
|
1339
1294
|
} catch {}
|
|
1340
1295
|
};
|
|
1296
|
+
let flushing = 0;
|
|
1341
1297
|
const flushPendingSettlement = async () => {
|
|
1298
|
+
flushing += 1;
|
|
1299
|
+
try {
|
|
1300
|
+
return await flushPendingSettlementOnce();
|
|
1301
|
+
} finally {
|
|
1302
|
+
flushing -= 1;
|
|
1303
|
+
}
|
|
1304
|
+
};
|
|
1305
|
+
const flushPendingSettlementOnce = async () => {
|
|
1342
1306
|
const pending = pendingSettlement;
|
|
1343
1307
|
if (!pending) return false;
|
|
1344
1308
|
await bus.publish(pending.event);
|
|
@@ -1604,6 +1568,14 @@ function createCoordinationToolsForManager(opts, lifetime) {
|
|
|
1604
1568
|
type: "acknowledgement",
|
|
1605
1569
|
...ev.acknowledgement
|
|
1606
1570
|
};
|
|
1571
|
+
if (ev.type === "tool-outcome") return {
|
|
1572
|
+
type: "tool-outcome",
|
|
1573
|
+
...ev.outcome
|
|
1574
|
+
};
|
|
1575
|
+
if (ev.type === "lead-message") return {
|
|
1576
|
+
type: "lead-message",
|
|
1577
|
+
...ev.message
|
|
1578
|
+
};
|
|
1607
1579
|
return {
|
|
1608
1580
|
type: ev.type,
|
|
1609
1581
|
...ev.down
|
|
@@ -1908,19 +1880,24 @@ function createCoordinationToolsForManager(opts, lifetime) {
|
|
|
1908
1880
|
return {
|
|
1909
1881
|
running: nodes.filter((node) => isLiveNodeStatus(node.status) && node.status !== "waiting"),
|
|
1910
1882
|
unqueued: nodes.filter((node) => !isLiveNodeStatus(node.status) && !received.has(node.id) && !flushedAnalystRuns.has(node.id)),
|
|
1911
|
-
queued: bus.queued([
|
|
1883
|
+
queued: bus.queued([
|
|
1884
|
+
"settled",
|
|
1885
|
+
"finding",
|
|
1886
|
+
"lead-message"
|
|
1887
|
+
])
|
|
1912
1888
|
};
|
|
1913
1889
|
};
|
|
1914
1890
|
const openWorkRefusal = (verb) => {
|
|
1891
|
+
if (deadlineNoticeDelivered) return void 0;
|
|
1915
1892
|
const { running, unqueued, queued } = openWork();
|
|
1916
1893
|
if (running.length === 0 && unqueued.length === 0 && queued.length === 0) return void 0;
|
|
1917
1894
|
const parts = [];
|
|
1918
1895
|
if (running.length > 0) parts.push(`${running.length} worker${running.length === 1 ? " is" : "s are"} still running (${running.map((node) => node.id).join(", ")})`);
|
|
1919
1896
|
const waiting = unqueued.length + queued.length;
|
|
1920
|
-
if (waiting > 0) parts.push(`${waiting} event${waiting === 1 ? " is" : "s are"} waiting for you
|
|
1897
|
+
if (waiting > 0) parts.push(`${waiting} event${waiting === 1 ? " is" : "s are"} waiting for you`);
|
|
1921
1898
|
return {
|
|
1922
1899
|
error: "open-work",
|
|
1923
|
-
reason: `${parts.join(" and ")}.
|
|
1900
|
+
reason: `${parts.join(" and ")}. End your turn to wait: you will be woken with each result. Read each settled worker's output, and then call ${verb} again. Nothing was ${verb === "stop" ? "stopped" : "checked"}.`,
|
|
1924
1901
|
running: running.map((node) => ({
|
|
1925
1902
|
id: node.id,
|
|
1926
1903
|
label: node.label,
|
|
@@ -1994,7 +1971,6 @@ function createCoordinationToolsForManager(opts, lifetime) {
|
|
|
1994
1971
|
unsub();
|
|
1995
1972
|
} catch {}
|
|
1996
1973
|
};
|
|
1997
|
-
const awaitTimeoutMs = opts.awaitTimeoutMs ?? 45e3;
|
|
1998
1974
|
let inFlightDrain = null;
|
|
1999
1975
|
const ensureDrain = () => {
|
|
2000
1976
|
if (!inFlightDrain) inFlightDrain = drainSettlement().finally(() => {
|
|
@@ -2045,28 +2021,104 @@ function createCoordinationToolsForManager(opts, lifetime) {
|
|
|
2045
2021
|
});
|
|
2046
2022
|
return managerAnalysis;
|
|
2047
2023
|
};
|
|
2048
|
-
const
|
|
2049
|
-
|
|
2050
|
-
|
|
2024
|
+
const settledWithoutDrain = async () => {
|
|
2025
|
+
if (inFlightDrain) return;
|
|
2026
|
+
inFlightDrain = (async () => {
|
|
2027
|
+
let drained = false;
|
|
2028
|
+
while (await drainSettlement("nextResolved")) drained = true;
|
|
2029
|
+
return drained;
|
|
2030
|
+
})().finally(() => {
|
|
2031
|
+
inFlightDrain = null;
|
|
2032
|
+
});
|
|
2033
|
+
await inFlightDrain;
|
|
2034
|
+
};
|
|
2035
|
+
const hasOpenWork = () => {
|
|
2036
|
+
const { running, unqueued, queued } = openWork();
|
|
2037
|
+
return running.length > 0 || unqueued.length > 0 || queued.length > 0 || bus.pending(wakeEventKinds) > 0 || openReceipts.size > 0 || leadPublishing > 0 || pendingSettlement !== void 0 || flushing > 0;
|
|
2051
2038
|
};
|
|
2052
|
-
const
|
|
2053
|
-
const
|
|
2054
|
-
|
|
2039
|
+
const awaitWake = async (input) => {
|
|
2040
|
+
const startedAt = Date.now();
|
|
2041
|
+
await ready();
|
|
2042
|
+
await settledWithoutDrain();
|
|
2043
|
+
if (!hasOpenWork() || input.signal.aborted) return void 0;
|
|
2044
|
+
const heartbeatAt = startedAt + Math.max(0, input.heartbeatMs);
|
|
2045
|
+
const deadlineAt = input.deadlineAt ?? Number.POSITIVE_INFINITY;
|
|
2046
|
+
const wakeAt = Math.min(heartbeatAt, deadlineAt);
|
|
2047
|
+
let unsubscribe;
|
|
2055
2048
|
let timer;
|
|
2056
|
-
|
|
2057
|
-
|
|
2058
|
-
if (typeof timer?.unref === "function") timer.unref();
|
|
2059
|
-
});
|
|
2049
|
+
let onAbort;
|
|
2050
|
+
let outsideWake = false;
|
|
2060
2051
|
try {
|
|
2061
|
-
|
|
2052
|
+
const queuedEvent = new Promise((resolve) => {
|
|
2053
|
+
unsubscribe = bus.onQueued((record) => {
|
|
2054
|
+
if (wakeEventKinds.includes(record.event.type)) resolve();
|
|
2055
|
+
});
|
|
2056
|
+
});
|
|
2057
|
+
const elapsed = new Promise((resolve) => {
|
|
2058
|
+
timer = setTimeout(resolve, Math.max(0, wakeAt - Date.now()));
|
|
2059
|
+
});
|
|
2060
|
+
const aborted = new Promise((resolve) => {
|
|
2061
|
+
onAbort = () => resolve();
|
|
2062
|
+
input.signal.addEventListener("abort", onAbort, { once: true });
|
|
2063
|
+
});
|
|
2064
|
+
const outside = (input.alsoWakeOn ?? new Promise(() => void 0)).then(() => {
|
|
2065
|
+
outsideWake = true;
|
|
2066
|
+
});
|
|
2067
|
+
let cursorIdle = false;
|
|
2068
|
+
while (bus.pending(wakeEventKinds) === 0 && !outsideWake && !input.signal.aborted && Date.now() < wakeAt) {
|
|
2069
|
+
const drained = !cursorIdle && openWork().running.length > 0 ? ensureDrain() : new Promise(() => void 0);
|
|
2070
|
+
if (await Promise.race([
|
|
2071
|
+
drained.then((value) => value ? "drained" : "idle"),
|
|
2072
|
+
queuedEvent.then(() => "queued"),
|
|
2073
|
+
elapsed.then(() => "elapsed"),
|
|
2074
|
+
aborted.then(() => "aborted"),
|
|
2075
|
+
outside.then(() => "outside")
|
|
2076
|
+
]) === "idle") {
|
|
2077
|
+
cursorIdle = true;
|
|
2078
|
+
if (!hasOpenWork()) break;
|
|
2079
|
+
}
|
|
2080
|
+
}
|
|
2081
|
+
if (input.signal.aborted) return void 0;
|
|
2082
|
+
if (flushing > 0 && inFlightDrain !== null) await inFlightDrain.catch(() => false);
|
|
2083
|
+
if (bus.pending(wakeEventKinds) > 0 && input.debounceMs > 0 && openWork().running.length > 0) {
|
|
2084
|
+
await Promise.race([new Promise((resolve) => {
|
|
2085
|
+
setTimeout(resolve, input.debounceMs);
|
|
2086
|
+
}), aborted]);
|
|
2087
|
+
if (input.signal.aborted) return void 0;
|
|
2088
|
+
await settledWithoutDrain();
|
|
2089
|
+
}
|
|
2062
2090
|
} finally {
|
|
2091
|
+
unsubscribe?.();
|
|
2063
2092
|
if (timer) clearTimeout(timer);
|
|
2093
|
+
if (onAbort) input.signal.removeEventListener("abort", onAbort);
|
|
2064
2094
|
}
|
|
2095
|
+
const events = [];
|
|
2096
|
+
for (let next = bus.pullRecord(wakeEventKinds); next; next = bus.pullRecord(wakeEventKinds)) {
|
|
2097
|
+
deliveries.push({
|
|
2098
|
+
record: next,
|
|
2099
|
+
attempt: input.attempt ?? driverAttempt,
|
|
2100
|
+
acknowledged: false
|
|
2101
|
+
});
|
|
2102
|
+
events.push({
|
|
2103
|
+
...projectEvent(next.event),
|
|
2104
|
+
eventSeq: next.seq
|
|
2105
|
+
});
|
|
2106
|
+
}
|
|
2107
|
+
if (events.length === 0 && !outsideWake && !hasOpenWork()) return void 0;
|
|
2108
|
+
const reason = events.length > 0 || outsideWake ? "events" : Date.now() >= deadlineAt ? "deadline" : "heartbeat";
|
|
2109
|
+
if (reason === "deadline") deadlineNoticeDelivered = true;
|
|
2110
|
+
return detachedFrozen({
|
|
2111
|
+
reason,
|
|
2112
|
+
events,
|
|
2113
|
+
live: liveSnapshot(),
|
|
2114
|
+
freeSlots: freeWorkerSlots(),
|
|
2115
|
+
idleMs: Date.now() - startedAt
|
|
2116
|
+
});
|
|
2065
2117
|
};
|
|
2066
2118
|
const tools = [
|
|
2067
2119
|
{
|
|
2068
2120
|
name: "spawn_worker",
|
|
2069
|
-
description: "Start a worker the driver will drive. `profile` is the worker or another driver; `task` is what it should do. Reserves budget from the conserved pool and fails closed. Pass an optional `budget` (per-field) to give a sub-task more or less than the default — it merges over the per-worker default; the conserved pool is still the hard fence. Each worker reserves its whole budget when it starts, so the pool divided by the per-worker budget is how many run at once: to run a wide team at once, give each worker less. When every worker slot is busy, or the pool cannot cover the budget until your running workers return what they hold, the worker is admitted with `status: \"queued\"` and starts on its own when a slot and its budget free — it is never refused for concurrency, so spawn all the work you want done
|
|
2121
|
+
description: "Start a worker the driver will drive. `profile` is the worker or another driver; `task` is what it should do. Reserves budget from the conserved pool and fails closed. Pass an optional `budget` (per-field) to give a sub-task more or less than the default — it merges over the per-worker default; the conserved pool is still the hard fence. Each worker reserves its whole budget when it starts, so the pool divided by the per-worker budget is how many run at once: to run a wide team at once, give each worker less. When every worker slot is busy, or the pool cannot cover the budget until your running workers return what they hold, the worker is admitted with `status: \"queued\"` and starts on its own when a slot and its budget free — it is never refused for concurrency, so spawn all the work you want done, then end your turn: you are woken with each result. A queued worker whose budget never frees settles down as budget-exhausted. Pass a `key` naming the assignment to make it run-once ACROSS restarts: a key that already completed returns the finished result (`resumed: \"completed\"` — no work re-runs, nothing is spent), a key whose prior attempt failed (`down`) spawns fresh and says so (`resumed: \"retried\"`), and a key the process died with IN FLIGHT retries under the same key when the executor provably died with the process (an inline worker — the same `resumed: \"retried\"`, naming the interruption). A key with no terminal receipt whose execution may still exist elsewhere (sandbox, CLI bridge, router) is refused (`error: \"in-doubt\"`) until its exact prior execution is recovered. A key still running is refused (`error: \"duplicate-key\"`). Returns `freeSlots`: how many MORE workers start at once (`null` = no slot bound), and `queued`: how many of your workers wait for a slot. Parallel workers finish the run sooner than one at a time." + (profileTable === void 0 ? "" : " This run has a profiles table: pass `profile` as one of the table's names to run that exact profile, or write a new profile under a name the table does not hold."),
|
|
2070
2122
|
inputSchema: {
|
|
2071
2123
|
type: "object",
|
|
2072
2124
|
properties: {
|
|
@@ -2087,7 +2139,7 @@ function createCoordinationToolsForManager(opts, lifetime) {
|
|
|
2087
2139
|
},
|
|
2088
2140
|
successorOf: {
|
|
2089
2141
|
type: "string",
|
|
2090
|
-
description: "The workerId of a SETTLED worker you spawned that this new worker replaces — for example one that failed, stalled, or finished with a weak result, restarted with a changed profile or task. The run record then names the predecessor on the new worker. Refused for an id you did not spawn (`error: \"successor-unknown\"`) and for a worker still live (`error: \"successor-live\"` —
|
|
2142
|
+
description: "The workerId of a SETTLED worker you spawned that this new worker replaces — for example one that failed, stalled, or finished with a weak result, restarted with a changed profile or task. The run record then names the predecessor on the new worker. Refused for an id you did not spawn (`error: \"successor-unknown\"`) and for a worker still live (`error: \"successor-live\"` — end your turn until it settles). Put what the successor needs from its predecessor in `task`."
|
|
2091
2143
|
},
|
|
2092
2144
|
budget: {
|
|
2093
2145
|
type: "object",
|
|
@@ -2187,7 +2239,7 @@ function createCoordinationToolsForManager(opts, lifetime) {
|
|
|
2187
2239
|
};
|
|
2188
2240
|
if (isLiveNodeStatus(predecessor.status)) return {
|
|
2189
2241
|
error: "successor-live",
|
|
2190
|
-
reason: `worker '${successorOf}' has not settled —
|
|
2242
|
+
reason: `worker '${successorOf}' has not settled — end your turn until it does, then spawn its successor (steer_agent redirects a live worker instead)`,
|
|
2191
2243
|
live: liveWorkerCount(),
|
|
2192
2244
|
freeSlots: freeWorkerSlots()
|
|
2193
2245
|
};
|
|
@@ -2293,7 +2345,7 @@ function createCoordinationToolsForManager(opts, lifetime) {
|
|
|
2293
2345
|
},
|
|
2294
2346
|
{
|
|
2295
2347
|
name: "observe_agent",
|
|
2296
|
-
description: "Inspect a worker WHILE IT RUNS, not only after it finishes: status, spend so far, and `progress` — how long since it last did anything (`idleMs`), whether that counts as stalled, how many turns it has taken, the last tools/files it touched (`recentActivity`), what its executor CHANGED about the profile you gave it (`derived` — an MCP config it materialized, an extension it had to add), whether a steer can even reach it (`steerable`), and how many steers it has not yet read (`pendingMessages`). For a worker that leads its own team, `progress.team` counts the agents below it (working, queued, done, down) and its idle clock reads the newest activity anywhere in that team, so a lead whose workers are busy is not stalled. A queued worker waits for a worker slot and is never stalled. A worker whose executor has FINISHED but whose settlement you have not yet drained reports `settlementPending` with its terminal kind; its `status` still reads running until
|
|
2348
|
+
description: "Inspect a worker WHILE IT RUNS, not only after it finishes: status, spend so far, and `progress` — how long since it last did anything (`idleMs`), whether that counts as stalled, how many turns it has taken, the last tools/files it touched (`recentActivity`), what its executor CHANGED about the profile you gave it (`derived` — an MCP config it materialized, an extension it had to add), whether a steer can even reach it (`steerable`), and how many steers it has not yet read (`pendingMessages`). For a worker that leads its own team, `progress.team` counts the agents below it (working, queued, done, down) and its idle clock reads the newest activity anywhere in that team, so a lead whose workers are busy is not stalled. A queued worker waits for a worker slot and is never stalled. A worker whose executor has FINISHED but whose settlement you have not yet drained reports `settlementPending` with its terminal kind; its `status` still reads running until your next wake delivers it, so end your turn rather than observing again. The settled output artifact is returned once drained. Select a field with `outputPath`, for example [\"content\"] to read a provider result without its event history. Omit it to read the complete artifact. Large outputs return a bounded JSON `outputPage`; repeat with `outputOffset: outputPage.nextOffset` until it is null, concatenate page text, then parse the JSON. Offsets and limits count UTF-16 characters. The full artifact stays retained; an outRef is a content address, not a workspace path. Use this BEFORE steer_agent: a steer is only worth sending when the progress says the worker is on the wrong path or has stopped making any.",
|
|
2297
2349
|
inputSchema: {
|
|
2298
2350
|
type: "object",
|
|
2299
2351
|
properties: {
|
|
@@ -2358,7 +2410,7 @@ function createCoordinationToolsForManager(opts, lifetime) {
|
|
|
2358
2410
|
progress: progress ?? null,
|
|
2359
2411
|
...pending ? {
|
|
2360
2412
|
settlementPending: pending,
|
|
2361
|
-
hint: `this worker has finished (${pending.kind}); its settlement is queued —
|
|
2413
|
+
hint: `this worker has finished (${pending.kind}); its settlement is queued for your next wake — end your turn to receive it, observing again will not change this`
|
|
2362
2414
|
} : {}
|
|
2363
2415
|
};
|
|
2364
2416
|
}
|
|
@@ -2400,7 +2452,7 @@ function createCoordinationToolsForManager(opts, lifetime) {
|
|
|
2400
2452
|
},
|
|
2401
2453
|
{
|
|
2402
2454
|
name: "cancel_worker",
|
|
2403
|
-
description: "Cancel one of YOUR live workers — running, or queued for a worker slot — and every worker it leads. Its unspent budget returns to your pool when it settles, so this is how you take back what a stalled or no-longer-needed worker holds (for example when spawn_worker is refused budget-exhausted while workers you no longer need hold the budget). The cancelled worker still settles `down
|
|
2455
|
+
description: "Cancel one of YOUR live workers — running, or queued for a worker slot — and every worker it leads. Its unspent budget returns to your pool when it settles, so this is how you take back what a stalled or no-longer-needed worker holds (for example when spawn_worker is refused budget-exhausted while workers you no longer need hold the budget). The cancelled worker still settles `down`, delivered in your next wake with what it spent. Refused (`error: \"not-live\"`) for a worker that already settled or is not yours.",
|
|
2404
2456
|
inputSchema: {
|
|
2405
2457
|
type: "object",
|
|
2406
2458
|
properties: {
|
|
@@ -2432,117 +2484,6 @@ function createCoordinationToolsForManager(opts, lifetime) {
|
|
|
2432
2484
|
};
|
|
2433
2485
|
}
|
|
2434
2486
|
},
|
|
2435
|
-
{
|
|
2436
|
-
name: "await_event",
|
|
2437
|
-
description: "Wait for and pull the next message a worker, sub-driver, or analyst sent up — the unified inbox. Read a settled artifact with the returned `outputRead` call; the receipt carries its reference, not its bytes. An event is one of: a settled worker output ('settled'), a question needing your answer ('question', from ask_parent / the worker's ask-user), or a trace-analyst finding ('finding', from analyze-on-settle). Pass kinds:['settled'] for just the next finished worker; omit `kinds` to also receive questions and findings. Returns { idle: true } when nothing is queued and no workers are live. If a worker is still running when the wait elapses, returns { pending: true, live: [...] } (the workers still in flight) instead of blocking indefinitely — call await_event again to keep waiting; the settlement is not lost. Every reply carries `freeSlots`: how many more workers you can start right now (`null` = uncapped). A settled worker frees its slot, so `freeSlots > 0` means capacity is sitting idle — spawn into it before waiting again. Each event carries `eventSeq`; pass the ones you have processed in `acknowledge` on a later call. An event delivered to a turn that ends in failure is named again when you are re-entered, until it is acknowledged. Pass `max` above 1 to read a whole team at once: the reply is `{ events: [...] }` with the first event it waited for plus every other event already waiting, up to `max`, so a lead of hundreds of workers reads their receipts in a few turns instead of one each.",
|
|
2438
|
-
inputSchema: {
|
|
2439
|
-
type: "object",
|
|
2440
|
-
properties: {
|
|
2441
|
-
kinds: {
|
|
2442
|
-
type: "array",
|
|
2443
|
-
items: {
|
|
2444
|
-
type: "string",
|
|
2445
|
-
enum: [...awaitableEventKinds]
|
|
2446
|
-
},
|
|
2447
|
-
description: "Restrict to these event kinds (any if omitted)."
|
|
2448
|
-
},
|
|
2449
|
-
acknowledge: {
|
|
2450
|
-
type: "array",
|
|
2451
|
-
items: {
|
|
2452
|
-
type: "integer",
|
|
2453
|
-
minimum: 0
|
|
2454
|
-
},
|
|
2455
|
-
description: "The eventSeq of each earlier event you have now processed."
|
|
2456
|
-
},
|
|
2457
|
-
max: {
|
|
2458
|
-
type: "integer",
|
|
2459
|
-
minimum: 1,
|
|
2460
|
-
description: "Most events to return in one reply. Omit or 1 for one event; above 1 returns `{ events }`."
|
|
2461
|
-
}
|
|
2462
|
-
}
|
|
2463
|
-
},
|
|
2464
|
-
handler: async (raw) => {
|
|
2465
|
-
const args = raw === void 0 ? {} : obj(raw);
|
|
2466
|
-
const max = args.max === void 0 ? 1 : args.max;
|
|
2467
|
-
if (typeof max !== "number" || !Number.isSafeInteger(max) || max < 1) throw new Error("coordination tools: \"max\" must be a positive integer");
|
|
2468
|
-
const k = args.kinds;
|
|
2469
|
-
const kinds = Array.isArray(k) ? k.filter(isAwaitableEventKind) : void 0;
|
|
2470
|
-
if (Array.isArray(args.acknowledge)) {
|
|
2471
|
-
const named = new Set(args.acknowledge.filter((value) => typeof value === "number" && Number.isInteger(value)));
|
|
2472
|
-
await acknowledge(deliveries.filter((delivery) => named.has(delivery.record.seq)), "manager");
|
|
2473
|
-
}
|
|
2474
|
-
const deliver = (record) => {
|
|
2475
|
-
deliveries.push({
|
|
2476
|
-
record,
|
|
2477
|
-
attempt: driverAttempt,
|
|
2478
|
-
acknowledged: false
|
|
2479
|
-
});
|
|
2480
|
-
return {
|
|
2481
|
-
...projectEvent(record.event),
|
|
2482
|
-
eventSeq: record.seq
|
|
2483
|
-
};
|
|
2484
|
-
};
|
|
2485
|
-
if (max > 1) {
|
|
2486
|
-
const fenceMs = awaitFenceMs();
|
|
2487
|
-
const fenceAt = fenceMs > 0 ? Date.now() + fenceMs / 2 : Infinity;
|
|
2488
|
-
if (!inFlightDrain) {
|
|
2489
|
-
inFlightDrain = (async () => {
|
|
2490
|
-
let drained = false;
|
|
2491
|
-
while (bus.pending(kinds) < max && Date.now() < fenceAt) {
|
|
2492
|
-
if (!await drainSettlement("nextResolved")) break;
|
|
2493
|
-
drained = true;
|
|
2494
|
-
}
|
|
2495
|
-
return drained;
|
|
2496
|
-
})().finally(() => {
|
|
2497
|
-
inFlightDrain = null;
|
|
2498
|
-
});
|
|
2499
|
-
await inFlightDrain;
|
|
2500
|
-
}
|
|
2501
|
-
const events = [];
|
|
2502
|
-
const takeQueued = () => {
|
|
2503
|
-
for (let next = bus.pullRecord(kinds); next; next = bus.pullRecord(kinds)) {
|
|
2504
|
-
events.push(deliver(next));
|
|
2505
|
-
if (events.length >= max) return;
|
|
2506
|
-
}
|
|
2507
|
-
};
|
|
2508
|
-
takeQueued();
|
|
2509
|
-
if (events.length === 0) {
|
|
2510
|
-
const first = await pullOne();
|
|
2511
|
-
if (!("eventSeq" in first)) return first;
|
|
2512
|
-
const { freeSlots: _slots, ...head } = first;
|
|
2513
|
-
events.push(head);
|
|
2514
|
-
takeQueued();
|
|
2515
|
-
}
|
|
2516
|
-
return {
|
|
2517
|
-
events,
|
|
2518
|
-
freeSlots: freeWorkerSlots()
|
|
2519
|
-
};
|
|
2520
|
-
}
|
|
2521
|
-
return pullOne();
|
|
2522
|
-
async function pullOne() {
|
|
2523
|
-
let ev = bus.pullRecord(kinds);
|
|
2524
|
-
if (ev) return {
|
|
2525
|
-
...deliver(ev),
|
|
2526
|
-
freeSlots: freeWorkerSlots()
|
|
2527
|
-
};
|
|
2528
|
-
const raced = await raceDrainWithTimeout(ensureDrain());
|
|
2529
|
-
if (raced === void 0) return {
|
|
2530
|
-
pending: true,
|
|
2531
|
-
live: liveSnapshot(),
|
|
2532
|
-
freeSlots: freeWorkerSlots()
|
|
2533
|
-
};
|
|
2534
|
-
ev = bus.pullRecord(kinds);
|
|
2535
|
-
if (!ev) return {
|
|
2536
|
-
idle: !raced.drained,
|
|
2537
|
-
freeSlots: freeWorkerSlots()
|
|
2538
|
-
};
|
|
2539
|
-
return {
|
|
2540
|
-
...deliver(ev),
|
|
2541
|
-
freeSlots: freeWorkerSlots()
|
|
2542
|
-
};
|
|
2543
|
-
}
|
|
2544
|
-
}
|
|
2545
|
-
},
|
|
2546
2487
|
{
|
|
2547
2488
|
name: "list_questions",
|
|
2548
2489
|
description: "List questions raised by workers, drivers, or analysts. Blocking stop behavior follows questionPolicy.",
|
|
@@ -3010,7 +2951,7 @@ function createCoordinationToolsForManager(opts, lifetime) {
|
|
|
3010
2951
|
};
|
|
3011
2952
|
if (isLiveNodeStatus(node.status)) return {
|
|
3012
2953
|
error: "worker-not-settled",
|
|
3013
|
-
reason: `worker ${JSON.stringify(id)} has not settled, so it has no trace to analyze yet —
|
|
2954
|
+
reason: `worker ${JSON.stringify(id)} has not settled, so it has no trace to analyze yet — end your turn until it settles, then run the lens`
|
|
3014
2955
|
};
|
|
3015
2956
|
const trace = ledger.find((worker) => worker.id === id)?.trace ?? node.trace ?? {
|
|
3016
2957
|
status: "unavailable",
|
|
@@ -3163,6 +3104,69 @@ function createCoordinationToolsForManager(opts, lifetime) {
|
|
|
3163
3104
|
const attempt = driverAttempt;
|
|
3164
3105
|
if (outcome === "completed") await acknowledge(deliveries.filter((delivery) => delivery.attempt === attempt), "turn-completed");
|
|
3165
3106
|
},
|
|
3107
|
+
hasOpenWork,
|
|
3108
|
+
awaitWake,
|
|
3109
|
+
awaitSettlement: async (workerIds, signal) => {
|
|
3110
|
+
const wanted = new Set(workerIds);
|
|
3111
|
+
const settledIds = () => ledger.filter((worker) => wanted.has(worker.id)).map((w) => w.id);
|
|
3112
|
+
await ready();
|
|
3113
|
+
for (;;) {
|
|
3114
|
+
await settledWithoutDrain();
|
|
3115
|
+
const hit = settledIds();
|
|
3116
|
+
if (hit.length > 0) return hit;
|
|
3117
|
+
signal?.throwIfAborted();
|
|
3118
|
+
if (!(opts.scope.view?.nodes ?? []).some((node) => wanted.has(node.id) && isLiveNodeStatus(node.status))) return [];
|
|
3119
|
+
if (!await (signal === void 0 ? ensureDrain() : Promise.race([ensureDrain(), new Promise((_resolve, reject) => {
|
|
3120
|
+
signal.addEventListener("abort", () => reject(signal.reason), { once: true });
|
|
3121
|
+
})])) && settledIds().length === 0) return [];
|
|
3122
|
+
}
|
|
3123
|
+
},
|
|
3124
|
+
receiveLeadMessage: (message) => {
|
|
3125
|
+
const parsed = leadMessageOf(message);
|
|
3126
|
+
if (parsed === void 0) return false;
|
|
3127
|
+
leadPublishing += 1;
|
|
3128
|
+
bus.publish({
|
|
3129
|
+
type: "lead-message",
|
|
3130
|
+
message: parsed
|
|
3131
|
+
}).catch((error) => {
|
|
3132
|
+
console.error("Runtime could not queue a lead message for its manager", {
|
|
3133
|
+
manager: opts.scope.view.root,
|
|
3134
|
+
error: error instanceof Error ? error.message : String(error)
|
|
3135
|
+
});
|
|
3136
|
+
}).finally(() => {
|
|
3137
|
+
leadPublishing -= 1;
|
|
3138
|
+
});
|
|
3139
|
+
return true;
|
|
3140
|
+
},
|
|
3141
|
+
openToolReceipt: (tool) => {
|
|
3142
|
+
receiptSeq += 1;
|
|
3143
|
+
const receipt = `${tool}#${receiptSeq}`;
|
|
3144
|
+
openReceipts.set(receipt, {
|
|
3145
|
+
tool,
|
|
3146
|
+
startedAt: Date.now()
|
|
3147
|
+
});
|
|
3148
|
+
return receipt;
|
|
3149
|
+
},
|
|
3150
|
+
settleToolReceipt: async (receipt, outcome) => {
|
|
3151
|
+
const open = openReceipts.get(receipt);
|
|
3152
|
+
if (open === void 0) return;
|
|
3153
|
+
openReceipts.delete(receipt);
|
|
3154
|
+
await bus.publish({
|
|
3155
|
+
type: "tool-outcome",
|
|
3156
|
+
outcome: detachedFrozen({
|
|
3157
|
+
receipt,
|
|
3158
|
+
tool: open.tool,
|
|
3159
|
+
...outcome.ok ? {
|
|
3160
|
+
ok: true,
|
|
3161
|
+
value: outcome.value
|
|
3162
|
+
} : {
|
|
3163
|
+
ok: false,
|
|
3164
|
+
error: outcome.error instanceof Error ? outcome.error.message : String(outcome.error)
|
|
3165
|
+
},
|
|
3166
|
+
elapsedMs: Date.now() - open.startedAt
|
|
3167
|
+
})
|
|
3168
|
+
});
|
|
3169
|
+
},
|
|
3166
3170
|
reentryState: () => {
|
|
3167
3171
|
const { unqueued, queued } = openWork();
|
|
3168
3172
|
const deliveredSettled = new Set(deliveries.flatMap((delivery) => delivery.record.event.type === "settled" ? [delivery.record.event.worker.id] : []));
|
|
@@ -3272,6 +3276,22 @@ function safeJsonText(value) {
|
|
|
3272
3276
|
return String(value);
|
|
3273
3277
|
}
|
|
3274
3278
|
}
|
|
3279
|
+
function leadMessageOf(message) {
|
|
3280
|
+
if (message === null || typeof message !== "object") return void 0;
|
|
3281
|
+
const m = message;
|
|
3282
|
+
const interrupt = m.interrupt === true;
|
|
3283
|
+
if (typeof m.steer === "string") return {
|
|
3284
|
+
kind: "steer",
|
|
3285
|
+
text: m.steer,
|
|
3286
|
+
interrupt
|
|
3287
|
+
};
|
|
3288
|
+
if (typeof m.answer === "string") return {
|
|
3289
|
+
kind: "answer",
|
|
3290
|
+
text: m.answer,
|
|
3291
|
+
...typeof m.questionId === "string" ? { questionId: m.questionId } : {},
|
|
3292
|
+
interrupt
|
|
3293
|
+
};
|
|
3294
|
+
}
|
|
3275
3295
|
//#endregion
|
|
3276
3296
|
//#region src/runtime/supervise/run-cancellation.ts
|
|
3277
3297
|
/** Apply a durable request through the run's existing abort controller. */
|
|
@@ -3919,6 +3939,56 @@ function progressStop(tracker, rule, ledger, scope, now, stallAfterMs) {
|
|
|
3919
3939
|
return tracker.evaluate(rule, scope, stallAfterMs !== void 0 ? { stallAfterMs } : void 0);
|
|
3920
3940
|
}
|
|
3921
3941
|
//#endregion
|
|
3942
|
+
//#region src/runtime/supervise/wake.ts
|
|
3943
|
+
/** How long a waiting manager hears nothing before Runtime wakes it to reconsider: 15 min. */
|
|
3944
|
+
const DEFAULT_WAKE_HEARTBEAT_MS = 15 * 6e4;
|
|
3945
|
+
/** How long a wake waits for more events after its first, while other workers still run: 2 s. */
|
|
3946
|
+
const DEFAULT_WAKE_DEBOUNCE_MS = 2e3;
|
|
3947
|
+
/** How long before its deadline a manager is woken once to submit its best result: 15 min. */
|
|
3948
|
+
const DEFAULT_DEADLINE_WARNING_MS = 15 * 6e4;
|
|
3949
|
+
/** The instruction every manager receives about waiting. */
|
|
3950
|
+
const WAIT_BY_ENDING_TURN = "When you are waiting for workers, end your turn. You will be woken with what happened: a worker settled, a finding, a message or answer from your lead, or the deadline approaching.";
|
|
3951
|
+
function resolveWakePolicy(policy) {
|
|
3952
|
+
const pick = (value, fallback, name) => {
|
|
3953
|
+
const resolved = value ?? fallback;
|
|
3954
|
+
if (!Number.isSafeInteger(resolved) || resolved < 0) throw new TypeError(`wake.${name} must be a nonnegative safe integer of milliseconds`);
|
|
3955
|
+
return resolved;
|
|
3956
|
+
};
|
|
3957
|
+
return {
|
|
3958
|
+
heartbeatMs: pick(policy?.heartbeatMs, DEFAULT_WAKE_HEARTBEAT_MS, "heartbeatMs"),
|
|
3959
|
+
debounceMs: pick(policy?.debounceMs, DEFAULT_WAKE_DEBOUNCE_MS, "debounceMs"),
|
|
3960
|
+
deadlineWarningMs: pick(policy?.deadlineWarningMs, DEFAULT_DEADLINE_WARNING_MS, "deadlineWarningMs")
|
|
3961
|
+
};
|
|
3962
|
+
}
|
|
3963
|
+
/**
|
|
3964
|
+
* The deadline warning, at most once per manager: the epoch ms to wake at, or undefined. The lead
|
|
3965
|
+
* is at most a quarter of the manager's whole time to its deadline, so a short run is not warned
|
|
3966
|
+
* the moment it starts.
|
|
3967
|
+
*/
|
|
3968
|
+
function deadlineWarningAt(deadlineMs, policy, warned, startedAtMs) {
|
|
3969
|
+
if (warned || deadlineMs <= 0 || policy.deadlineWarningMs <= 0) return void 0;
|
|
3970
|
+
return deadlineMs - Math.min(policy.deadlineWarningMs, Math.max(0, deadlineMs - startedAtMs) / 4);
|
|
3971
|
+
}
|
|
3972
|
+
/** The input a woken manager's next turn receives. */
|
|
3973
|
+
function composeWakeInput(wake, context) {
|
|
3974
|
+
const span = (ms) => ms < 12e4 ? `${Math.max(0, Math.round(ms / 1e3))} s` : `${Math.round(ms / 6e4)} min`;
|
|
3975
|
+
const lines = [];
|
|
3976
|
+
const waited = `You waited ${span(wake.idleMs)} without spending a turn.`;
|
|
3977
|
+
if (wake.reason === "deadline") {
|
|
3978
|
+
const left = context.deadlineMs === void 0 ? void 0 : context.deadlineMs - context.now;
|
|
3979
|
+
lines.push(`The run's deadline is ${left === void 0 ? "near" : `in ${span(left)}`}. ${waited}`, context.tools?.includes("submit_result") === true ? "Submit your best result now with submit_result. A run that reaches its deadline without an accepted result delivers nothing." : "Finish now with your best result.");
|
|
3980
|
+
} else if (wake.reason === "heartbeat") lines.push(`Nothing reached your inbox. ${waited} Reconsider whether the running workers still serve the objective.`);
|
|
3981
|
+
else lines.push(`${wake.events.length} event${wake.events.length === 1 ? "" : "s"} arrived. ${waited}`);
|
|
3982
|
+
if (wake.events.length > 0) {
|
|
3983
|
+
lines.push("", "## Events", "");
|
|
3984
|
+
for (const event of wake.events) lines.push(`- ${JSON.stringify(event)}`);
|
|
3985
|
+
}
|
|
3986
|
+
lines.push("", wake.live.length === 0 ? "Workers running: none." : `Workers running: ${wake.live.map((worker) => String(worker.id)).join(", ")}.`);
|
|
3987
|
+
if (wake.freeSlots !== null) lines.push(`Free worker slots: ${wake.freeSlots}.`);
|
|
3988
|
+
lines.push("", WAIT_BY_ENDING_TURN);
|
|
3989
|
+
return lines.join("\n");
|
|
3990
|
+
}
|
|
3991
|
+
//#endregion
|
|
3922
3992
|
//#region src/runtime/supervise/coordination-driver.ts
|
|
3923
3993
|
/**
|
|
3924
3994
|
*
|
|
@@ -3987,12 +4057,14 @@ function poolStarved(scope, perWorker, preserveOwnerTurns = false) {
|
|
|
3987
4057
|
/** The number of driver turns a manager whose spend Runtime cannot measure may take when no
|
|
3988
4058
|
* deadline or dollar cap bounds it. */
|
|
3989
4059
|
const UNMETERED_DRIVER_TURNS = 16;
|
|
4060
|
+
/** How often a waiting driver applies operator cancels and steers between its turns. */
|
|
4061
|
+
const WAITING_CONTROL_POLL_MS = 5e3;
|
|
3990
4062
|
/**
|
|
3991
4063
|
* Safety guards for a manager's own turns when no deadline bounds them.
|
|
3992
4064
|
*
|
|
3993
4065
|
* `poolStarved` lets a manager keep taking turns while its workers run, because those workers hold
|
|
3994
4066
|
* the pool its turns would otherwise be refused from. With no turn cap and no deadline, a manager
|
|
3995
|
-
* waiting on a worker that never settles would take a metered turn per
|
|
4067
|
+
* waiting on a worker that never settles would take a metered turn per heartbeat wake forever.
|
|
3996
4068
|
* Money bounds that: a manager that has overdrawn its pool by as much again as the whole pool stops.
|
|
3997
4069
|
* A brain that reports no usage cannot be bounded by money, so without a deadline or dollar cap it
|
|
3998
4070
|
* keeps the historical 16-turn bound. A caller's explicit `maxTurns` or a deadline replaces both.
|
|
@@ -4416,7 +4488,6 @@ function driverAgent(opts) {
|
|
|
4416
4488
|
...opts.sourceEvidenceRefs ? { sourceEvidenceRefs: opts.sourceEvidenceRefs } : {},
|
|
4417
4489
|
...opts.watchWorkers ? { watchWorkers: opts.watchWorkers } : {},
|
|
4418
4490
|
...opts.stallAfterMs !== void 0 ? { stallAfterMs: opts.stallAfterMs } : {},
|
|
4419
|
-
...opts.awaitTimeoutMs !== void 0 ? { awaitTimeoutMs: opts.awaitTimeoutMs } : {},
|
|
4420
4491
|
...opts.continuityByProfile ? { continuityByProfile: opts.continuityByProfile } : {},
|
|
4421
4492
|
...opts.preflightSpawn ? { preflightSpawn: opts.preflightSpawn } : {},
|
|
4422
4493
|
...opts.resolveSpawnProfile ? { resolveSpawnProfile: opts.resolveSpawnProfile } : {},
|
|
@@ -4453,7 +4524,7 @@ function driverAgent(opts) {
|
|
|
4453
4524
|
const modelTools = selectedTools.flatMap((selected) => selected.kind === "descriptor" ? [selected.descriptor] : []);
|
|
4454
4525
|
probeableByName = new Map(modelTools.map((tool) => [tool.name, tool]));
|
|
4455
4526
|
const selectedExtraNames = new Set(selectedTools.flatMap((selected) => selected.kind === "extra" ? [selected.extra.name] : []));
|
|
4456
|
-
opts.onCoordinationTools?.(modelTools);
|
|
4527
|
+
opts.onCoordinationTools?.(modelTools, coord);
|
|
4457
4528
|
const acknowledger = opts.controlDir === void 0 ? void 0 : createCancelAcknowledger({
|
|
4458
4529
|
dir: opts.controlDir,
|
|
4459
4530
|
coord,
|
|
@@ -4496,6 +4567,9 @@ function driverAgent(opts) {
|
|
|
4496
4567
|
const poolTokens = scope.budget.tokensLeft + scope.budget.reservedTokens;
|
|
4497
4568
|
let driverTurns = 0;
|
|
4498
4569
|
let progressStopReason;
|
|
4570
|
+
const wakePolicy = resolveWakePolicy(opts.wake);
|
|
4571
|
+
let deadlineWarned = false;
|
|
4572
|
+
const startedAt = now();
|
|
4499
4573
|
let driverTurn = 0;
|
|
4500
4574
|
let driverCall = 0;
|
|
4501
4575
|
const meteredBrain = async (messages, tools, detail, conversationTurn) => {
|
|
@@ -4645,6 +4719,38 @@ function driverAgent(opts) {
|
|
|
4645
4719
|
});
|
|
4646
4720
|
transcript.observe(messages);
|
|
4647
4721
|
},
|
|
4722
|
+
afterFinal: async () => {
|
|
4723
|
+
if (coord.isStopped() || coord.submittedResult() !== void 0 || scope.signal.aborted) return;
|
|
4724
|
+
if (inbox.pending() === 0 && !coord.hasOpenWork()) return void 0;
|
|
4725
|
+
const deadlineMs = scope.budget.deadlineMs;
|
|
4726
|
+
const warnAt = deadlineWarningAt(deadlineMs, wakePolicy, deadlineWarned, startedAt);
|
|
4727
|
+
const controlClock = acknowledger === void 0 && steerAcknowledger === void 0 ? void 0 : setInterval(() => {
|
|
4728
|
+
(async () => {
|
|
4729
|
+
await steerAcknowledger?.pass("turn");
|
|
4730
|
+
acknowledger?.pass("turn");
|
|
4731
|
+
})().catch(() => void 0);
|
|
4732
|
+
}, WAITING_CONTROL_POLL_MS);
|
|
4733
|
+
controlClock?.unref?.();
|
|
4734
|
+
let wake;
|
|
4735
|
+
try {
|
|
4736
|
+
wake = await coord.awaitWake({
|
|
4737
|
+
signal: scope.signal,
|
|
4738
|
+
heartbeatMs: wakePolicy.heartbeatMs,
|
|
4739
|
+
debounceMs: wakePolicy.debounceMs,
|
|
4740
|
+
...warnAt === void 0 ? {} : { deadlineAt: warnAt },
|
|
4741
|
+
alsoWakeOn: inbox.pending() > 0 ? Promise.resolve() : inbox.nextDelivery()
|
|
4742
|
+
});
|
|
4743
|
+
} finally {
|
|
4744
|
+
if (controlClock !== void 0) clearInterval(controlClock);
|
|
4745
|
+
}
|
|
4746
|
+
if (wake === void 0) return void 0;
|
|
4747
|
+
if (wake.reason === "deadline") deadlineWarned = true;
|
|
4748
|
+
return composeWakeInput(wake, {
|
|
4749
|
+
tools: selectedTools.map((selected) => selected.kind === "descriptor" ? selected.descriptor.name : selected.extra.name),
|
|
4750
|
+
deadlineMs,
|
|
4751
|
+
now: now()
|
|
4752
|
+
});
|
|
4753
|
+
},
|
|
4648
4754
|
stopBefore: () => {
|
|
4649
4755
|
if (coord.isStopped() || scope.signal.aborted || poolStarved(scope, opts.perWorker, opts.preserveOwnerTurns) || deadlinePassed(scope, now) || ownTurnsUnbounded(scope, poolTokens, driverTurns, opts.maxTurns)) return true;
|
|
4650
4756
|
if (!opts.stopRule || !tracker) return false;
|
|
@@ -4710,10 +4816,10 @@ function resumeBrief(resume, current, prior) {
|
|
|
4710
4816
|
const inDoubt = byState("in-doubt").filter(([, value]) => !currentIds.has(value.id));
|
|
4711
4817
|
const failed = byState("down");
|
|
4712
4818
|
if (completed.length > 0) lines.push("", "COMPLETED keys — spawn_worker with the same key returns the finished result, spending nothing:", ...completed.map(([k, v]) => `- ${k} → ${v.id} (${v.label})`));
|
|
4713
|
-
if (recovered.length > 0) lines.push("", "Recovered keys are attached to this scope.
|
|
4819
|
+
if (recovered.length > 0) lines.push("", "Recovered keys are attached to this scope. Their results arrive when you are woken; coordinate with their original workers:", ...recovered.map(([key, value]) => `- ${key} → ${value.id} (${value.label})`));
|
|
4714
4820
|
if (inDoubt.length > 0) lines.push("", "Keys IN DOUBT — a prior process recorded them as started but never recorded a terminal receipt. Do NOT spawn a replacement under these keys; inspect or recover each exact prior execution first:", ...inDoubt.map(([k, v]) => `- ${k} (prior attempt ${v.id}, ${v.label})`));
|
|
4715
4821
|
if (failed.length > 0) lines.push("", "Keys whose prior attempt FAILED (settled down) — spawn_worker with the same key retries:", ...failed.map(([k, v]) => `- ${k} (prior attempt ${v.id}, ${v.label})`));
|
|
4716
|
-
if (resume.waits.length > 0) lines.push("", "Pending waits RE-ARMED on their original deadlines (they settle
|
|
4822
|
+
if (resume.waits.length > 0) lines.push("", "Pending waits RE-ARMED on their original deadlines (they settle into your next wake):", ...resume.waits.map((w) => `- ${w.label} (${w.spec.kind})`));
|
|
4717
4823
|
appendPriorCoordination(lines, prior);
|
|
4718
4824
|
const spent = resume.priorSpend;
|
|
4719
4825
|
lines.push("", "Budget the run ALREADY spent before this process (it counts toward the run total):", `- child work: tokens charged=${chargedTokens(spent.childWork.tokens)} (in=${spent.childWork.tokens.input} out=${spent.childWork.tokens.output}), usd=${spent.childWork.usd}, iterations=${spent.childWork.iterations}`, `- driver inference: tokens charged=${chargedTokens(spent.driverInference.tokens)} (in=${spent.driverInference.tokens.input} out=${spent.driverInference.tokens.output}), usd=${spent.driverInference.usd}`);
|
|
@@ -4784,6 +4890,6 @@ function safeJson(v) {
|
|
|
4784
4890
|
}
|
|
4785
4891
|
}
|
|
4786
4892
|
//#endregion
|
|
4787
|
-
export {
|
|
4893
|
+
export { watchRunCancellation as A, questionEscalationTargets as B, renderAnytimeTable as C, FileCoordinationLog as D, withRunContext as E, createCoordinationTools as F, defaultToolDetectors as H, createCoordinationToolsForManager as I, downMessageRefusalReasons as L, analystToolGroupNames as M, canonicalFindingEvent as N, foldCoordinationRecords as O, coordinationVerbNames as P, normalizeAnalyzeOnSettle as R, plateauLength as S, createInMemoryRunContext as T, watchTrace as U, createEventBus as V, progressStop as _, DEFAULT_DEADLINE_WARNING_MS as a, areaUnderCurve as b, composeWakeInput as c, allOf as d, allWorkersStalled as f, plateau as g, noProgressFor as h, finalizeBestDelivered as i, ANALYST_DEFINITION_BOUNDS as j, applyRunCancellation as k, deadlineWarningAt as l, createProgressTracker as m, createSteerAcknowledger as n, DEFAULT_WAKE_DEBOUNCE_MS as o, anyOf as p, driverAgent as r, DEFAULT_WAKE_HEARTBEAT_MS as s, createCancelAcknowledger as t, resolveWakePolicy as u, sampleFromSettled as v, createFileRunContext as w, bestSoFar as x, anytimeReport as y, parseAuthoredAnalystDefinition as z };
|
|
4788
4894
|
|
|
4789
|
-
//# sourceMappingURL=coordination-driver-
|
|
4895
|
+
//# sourceMappingURL=coordination-driver-vNrdQBSH.js.map
|