@tangle-network/agent-runtime 0.174.1 → 0.177.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -0
- package/dist/{activation-CsdJWRXX.js → activation-XVI_z1g3.js} +3 -3
- package/dist/{activation-CsdJWRXX.js.map → activation-XVI_z1g3.js.map} +1 -1
- package/dist/agent.d.ts +1 -1
- package/dist/agent.js +2 -2
- package/dist/{authoring-DsLNInAz.js → authoring-bXmPNfCI.js} +2 -2
- package/dist/{authoring-DsLNInAz.js.map → authoring-bXmPNfCI.js.map} +1 -1
- package/dist/candidate-execution/index.js +4 -4
- package/dist/{candidate-execution-BbMbyhgC.js → candidate-execution-fBNUyWyE.js} +10 -4
- package/dist/candidate-execution-fBNUyWyE.js.map +1 -0
- package/dist/{conversation-7XGKoDM6.js → conversation-BREx2hi-.js} +3 -3
- package/dist/{conversation-7XGKoDM6.js.map → conversation-BREx2hi-.js.map} +1 -1
- package/dist/conversation.d.ts +1 -1
- package/dist/conversation.js +1 -1
- package/dist/coordination-driver-xO1tkxy-.js +3063 -0
- package/dist/coordination-driver-xO1tkxy-.js.map +1 -0
- package/dist/durable-file-g3YdfEh4.js +131 -0
- package/dist/durable-file-g3YdfEh4.js.map +1 -0
- package/dist/durable.d.ts +2 -2
- package/dist/durable.js +1 -1
- package/dist/{environment-provider-DWjbGH8W.d.ts → environment-provider-5E82esWE.d.ts} +6 -3
- package/dist/{environment-provider-Bn3652YU.js → environment-provider-DKyMWQJd.js} +121 -17
- package/dist/environment-provider-DKyMWQJd.js.map +1 -0
- package/dist/environment-provider.d.ts +1 -1
- package/dist/environment-provider.js +1 -1
- package/dist/{graph-BbeHgLXH.js → graph-DwJXa1Ph.js} +3 -3
- package/dist/graph-DwJXa1Ph.js.map +1 -0
- package/dist/graph.d.ts +11 -3
- package/dist/graph.js +35 -4
- package/dist/graph.js.map +1 -1
- package/dist/{improvement-cycle-DKGDCukL.js → improvement-cycle-eiSv6aH4.js} +5 -5
- package/dist/{improvement-cycle-DKGDCukL.js.map → improvement-cycle-eiSv6aH4.js.map} +1 -1
- package/dist/{index-CIhkH7mq.d.ts → index-BnAiQVLD.d.ts} +290 -29
- package/dist/{index-CFMB9ux3.d.ts → index-Bs0uhf-y.d.ts} +2 -2
- package/dist/{index-Dk5X9eKg.d.ts → index-C9XTLVP1.d.ts} +4 -4
- package/dist/index.d.ts +7 -7
- package/dist/index.js +13 -13
- package/dist/intelligence.d.ts +4 -4
- package/dist/intelligence.js +6 -6
- package/dist/kernel.d.ts +6 -6
- package/dist/kernel.js +13 -11
- package/dist/{knowledge-BaAX0HVu.js → knowledge-Bmtj_7u7.js} +5 -5
- package/dist/{knowledge-BaAX0HVu.js.map → knowledge-Bmtj_7u7.js.map} +1 -1
- package/dist/knowledge.d.ts +1 -1
- package/dist/knowledge.js +1 -1
- package/dist/{loop-runner-bin-DRYApMiu.d.ts → loop-runner-bin-ChS5deJ0.d.ts} +3 -3
- package/dist/{loop-runner-bin-D63Tytdk.js → loop-runner-bin-GTN5wqCt.js} +3 -3
- package/dist/{loop-runner-bin-D63Tytdk.js.map → loop-runner-bin-GTN5wqCt.js.map} +1 -1
- package/dist/loop-runner-bin.d.ts +1 -1
- package/dist/loop-runner-bin.js +1 -1
- package/dist/{materialization-Ct3I4wu3.js → materialization-CekWK6OO.js} +5 -3
- package/dist/materialization-CekWK6OO.js.map +1 -0
- package/dist/mcp/bin.js +3 -3
- package/dist/mcp/index.d.ts +3 -3
- package/dist/mcp/index.js +5 -4
- package/dist/mcp/index.js.map +1 -1
- package/dist/{openai-tools-BBgjWYNw.js → openai-tools-BBj1QGJp.js} +2 -2
- package/dist/{openai-tools-BBgjWYNw.js.map → openai-tools-BBj1QGJp.js.map} +1 -1
- package/dist/{prepare-BAyaNoZu.js → prepare-Tip-2ZoZ.js} +2 -2
- package/dist/{prepare-BAyaNoZu.js.map → prepare-Tip-2ZoZ.js.map} +1 -1
- package/dist/primeintellect/index.d.ts +1 -1
- package/dist/{protected-model-port-T2UWrDNQ.js → protected-model-port-Dd3a1SRm.js} +2 -2
- package/dist/{protected-model-port-T2UWrDNQ.js.map → protected-model-port-Dd3a1SRm.js.map} +1 -1
- package/dist/provision-supervisor-zA__ayUX.js +1020 -0
- package/dist/provision-supervisor-zA__ayUX.js.map +1 -0
- package/dist/{redact-DvLf4x1m.d.ts → redact-DkRLcSiE.d.ts} +3 -3
- package/dist/run-layout-Cd_l2XWV.js +645 -0
- package/dist/run-layout-Cd_l2XWV.js.map +1 -0
- package/dist/{runtime-D5u1M6T5.d.ts → runtime-B2eZ76kG.d.ts} +3 -3
- package/dist/{runtime-CojgP-hr.js → runtime-CJNlF5wM.js} +16 -1095
- package/dist/runtime-CJNlF5wM.js.map +1 -0
- package/dist/{spawn-journal-Tgvp5VS5.js → spawn-journal--N8Ten1q.js} +3 -1
- package/dist/{spawn-journal-Tgvp5VS5.js.map → spawn-journal--N8Ten1q.js.map} +1 -1
- package/dist/{stream-agent-turn-DxQ3GOuR.d.ts → stream-agent-turn-ByY8po9W.d.ts} +2 -2
- package/dist/{stream-agent-turn-CKIpCbSq.js → stream-agent-turn-CPj9SqVm.js} +2 -2
- package/dist/{stream-agent-turn-CKIpCbSq.js.map → stream-agent-turn-CPj9SqVm.js.map} +1 -1
- package/dist/{structural-rollout-BUYA61iV.js → structural-rollout-BeAUK7Ta.js} +4 -4
- package/dist/{structural-rollout-BUYA61iV.js.map → structural-rollout-BeAUK7Ta.js.map} +1 -1
- package/dist/{supervise-Ci0RfWQF.js → supervise-HsRKOlKY.js} +405 -3374
- package/dist/supervise-HsRKOlKY.js.map +1 -0
- package/dist/{supervisor-Bt1XLCVq.js → supervisor-ChLgoYhG.js} +1620 -33
- package/dist/supervisor-ChLgoYhG.js.map +1 -0
- package/dist/testing.d.ts +2 -2
- package/dist/testing.js +13 -12
- package/dist/testing.js.map +1 -1
- package/dist/{top-app-3rYfFPTW.js → top-app-DIBApkt1.js} +25 -9
- package/dist/top-app-DIBApkt1.js.map +1 -0
- package/dist/tui/bin.js +1 -1
- package/dist/tui/index.d.ts +9 -1
- package/dist/tui/index.js +3 -2
- package/dist/{types-DjVO-p7R.d.ts → types-a2ZZUl80.d.ts} +25 -2
- package/dist/{workspace-archive-B0Hqp2eJ.js → workspace-archive-DOZAeeKU.js} +2 -2
- package/dist/{workspace-archive-B0Hqp2eJ.js.map → workspace-archive-DOZAeeKU.js.map} +1 -1
- package/package.json +5 -4
- package/dist/candidate-execution-BbMbyhgC.js.map +0 -1
- package/dist/environment-provider-Bn3652YU.js.map +0 -1
- package/dist/graph-BbeHgLXH.js.map +0 -1
- package/dist/materialization-Ct3I4wu3.js.map +0 -1
- package/dist/run-layout-C2jgwsCq.js +0 -393
- package/dist/run-layout-C2jgwsCq.js.map +0 -1
- package/dist/runtime-CojgP-hr.js.map +0 -1
- package/dist/supervise-Ci0RfWQF.js.map +0 -1
- package/dist/supervisor-Bt1XLCVq.js.map +0 -1
- package/dist/top-app-3rYfFPTW.js.map +0 -1
|
@@ -0,0 +1,3063 @@
|
|
|
1
|
+
import { d as providerAttemptEvidence } from "./materialization-CekWK6OO.js";
|
|
2
|
+
import { f as RuntimeRunStateError, m as ValidationError } from "./errors-CDZ8XsVj.js";
|
|
3
|
+
import { t as detachedFrozen } from "./snapshot-CTAf4uuA.js";
|
|
4
|
+
import { b as workerTraceAnalysisStore, i as InMemorySpawnJournal, n as FileSpawnJournal, r as InMemoryResultBlobStore, t as FileResultBlobStore, y as parseWorkerToolTraceArtifact } from "./spawn-journal--N8Ten1q.js";
|
|
5
|
+
import { _ as unmeteredSpend, r as chargedTokens, u as promptCacheTokenClasses } from "./util-D6ZEuBMi.js";
|
|
6
|
+
import { i as writeAllBytes, n as parseCommittedJsonLines, r as prepareJsonlAppend, t as isNoEntError } from "./jsonl-file-CDfsCI5s.js";
|
|
7
|
+
import { At as isLiveNodeStatus, Et as createPeerMailbox, H as freeSlots, Ut as runBrainLoop, at as createExecutorRegistry, c as pickBestDelivered, f as withDriverExecutor, g as meterRuntimeOwnedProviderAttempt, jt as isTerminalNodeStatus, l as runFinalizer, o as bestDelivered, q as assertValidBudget, u as runTree, xt as createInbox } from "./supervisor-ChLgoYhG.js";
|
|
8
|
+
import { A as writeWorkerCancellation, M as writeWorkerSteerAcknowledgement, c as readWorkerCancelRequests, d as readWorkerSteerRequests, k as writeRunCancellation, l as readWorkerCancellation, o as readRunCancelRequest, r as claimWorkerSteerDelivery, s as readRunCancellation, u as readWorkerSteerAcknowledgement } from "./run-layout-Cd_l2XWV.js";
|
|
9
|
+
import { agentProfileSchema, canonicalCandidateDigest } from "@tangle-network/agent-interface";
|
|
10
|
+
import { argHash, errorStreakDetector, observeAll, repeatedActionDetector } from "@tangle-network/agent-eval";
|
|
11
|
+
import { randomUUID } from "node:crypto";
|
|
12
|
+
import { join } from "node:path";
|
|
13
|
+
import { readFileSync } from "node:fs";
|
|
14
|
+
//#region src/runtime/supervise/detector-monitor.ts
|
|
15
|
+
/**
|
|
16
|
+
*
|
|
17
|
+
* The ONLINE analyst: watch a `TraceSource` and fold each tool span through agent-eval's published
|
|
18
|
+
* streaming detector kernel (`repeatedActionDetector`/`errorStreakDetector` — the SAME kernel the
|
|
19
|
+
* control loop folds), firing `onSignal` the moment a worker loops or error-storms. Substrate-
|
|
20
|
+
* agnostic: it consumes spans from any source (owned router/bridge loop OR a sandbox box session),
|
|
21
|
+
* never the raw tool seam. Detection logic + the failure taxonomy live in agent-eval; not reimplemented.
|
|
22
|
+
*
|
|
23
|
+
* @experimental
|
|
24
|
+
*/
|
|
25
|
+
/** The default online panel for a tool-call pipe: a worker repeating the same call, or hammering
|
|
26
|
+
* consecutive errors. (No-progress needs a domain progress-probe, so it is opt-in, not default.)
|
|
27
|
+
*
|
|
28
|
+
* Coverage note: `repeated-action` works for EVERY harness (it needs only tool name + args, which
|
|
29
|
+
* every adapter provides). `error-streak` needs per-call status — opencode carries it inline
|
|
30
|
+
* (`state.status`, VALIDATED live), but claude-code/codex tool-call parts do NOT (their errors live
|
|
31
|
+
* in separate result blocks not yet decoded), so error-streak is silent for those until result-block
|
|
32
|
+
* decoding is added + live-validated. It is in the panel because it is correct where status exists. */
|
|
33
|
+
function defaultToolDetectors() {
|
|
34
|
+
return [repeatedActionDetector({ maxRepeated: 3 }), errorStreakDetector({ maxErrors: 3 })];
|
|
35
|
+
}
|
|
36
|
+
/** Subscribe to a `TraceSource` and run the streaming detectors over its live spans. Returns an
|
|
37
|
+
* unsubscribe. A defensive `argHash` failure (circular args) never throws out of the side-channel. */
|
|
38
|
+
function watchTrace(source, opts = {}) {
|
|
39
|
+
const detectors = opts.detectors ?? defaultToolDetectors();
|
|
40
|
+
return source.onSpan((span) => {
|
|
41
|
+
let fingerprint;
|
|
42
|
+
try {
|
|
43
|
+
fingerprint = `${span.toolName}|${argHash(span.args)}`;
|
|
44
|
+
} catch {
|
|
45
|
+
fingerprint = `${span.toolName}|<unhashable>`;
|
|
46
|
+
}
|
|
47
|
+
const signals = observeAll(detectors, {
|
|
48
|
+
actionFingerprint: fingerprint,
|
|
49
|
+
...span.status ? { status: span.status } : {},
|
|
50
|
+
label: span.toolName
|
|
51
|
+
});
|
|
52
|
+
for (const s of signals) opts.onSignal?.(s, span);
|
|
53
|
+
});
|
|
54
|
+
}
|
|
55
|
+
//#endregion
|
|
56
|
+
//#region src/runtime/supervise/event-bus.ts
|
|
57
|
+
/** Create the child→parent coordination bus: one typed pipe for settled outputs, questions, and analyst findings, with a priority-ordered pull queue and a pass-through subscribe lane.
|
|
58
|
+
* @experimental In-process queue; durability is a transport swap that does not exist yet. */
|
|
59
|
+
function createEventBus(now = Date.now) {
|
|
60
|
+
const queue = [];
|
|
61
|
+
const log = [];
|
|
62
|
+
const subscribers = [];
|
|
63
|
+
const byKind = {};
|
|
64
|
+
const staged = /* @__PURE__ */ new WeakMap();
|
|
65
|
+
let seq = 0;
|
|
66
|
+
let published = 0;
|
|
67
|
+
let pulled = 0;
|
|
68
|
+
const matches = (r, kinds) => !kinds || kinds.includes(r.event.type);
|
|
69
|
+
const bestIndex = (kinds) => {
|
|
70
|
+
let best = -1;
|
|
71
|
+
let bestPriority = Number.NEGATIVE_INFINITY;
|
|
72
|
+
for (let i = 0; i < queue.length; i++) {
|
|
73
|
+
const r = queue[i];
|
|
74
|
+
if (!r || !matches(r, kinds)) continue;
|
|
75
|
+
if (r.priority > bestPriority) {
|
|
76
|
+
best = i;
|
|
77
|
+
bestPriority = r.priority;
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
return best;
|
|
81
|
+
};
|
|
82
|
+
return {
|
|
83
|
+
async publish(event, opts) {
|
|
84
|
+
const record = staged.get(event) ?? {
|
|
85
|
+
seq: seq++,
|
|
86
|
+
at: now(),
|
|
87
|
+
priority: opts?.priority ?? 0,
|
|
88
|
+
event
|
|
89
|
+
};
|
|
90
|
+
staged.set(event, record);
|
|
91
|
+
for (const handler of subscribers) await handler(record);
|
|
92
|
+
staged.delete(event);
|
|
93
|
+
if (opts?.queue !== false) queue.push(record);
|
|
94
|
+
log.push(record);
|
|
95
|
+
published += 1;
|
|
96
|
+
byKind[event.type] = (byKind[event.type] ?? 0) + 1;
|
|
97
|
+
return record;
|
|
98
|
+
},
|
|
99
|
+
pull(kinds) {
|
|
100
|
+
const i = bestIndex(kinds);
|
|
101
|
+
if (i < 0) return void 0;
|
|
102
|
+
pulled++;
|
|
103
|
+
return queue.splice(i, 1)[0]?.event;
|
|
104
|
+
},
|
|
105
|
+
subscribe(handler) {
|
|
106
|
+
subscribers.push(handler);
|
|
107
|
+
return () => {
|
|
108
|
+
const i = subscribers.indexOf(handler);
|
|
109
|
+
if (i >= 0) subscribers.splice(i, 1);
|
|
110
|
+
};
|
|
111
|
+
},
|
|
112
|
+
pending(kinds) {
|
|
113
|
+
return kinds ? queue.filter((r) => matches(r, kinds)).length : queue.length;
|
|
114
|
+
},
|
|
115
|
+
history() {
|
|
116
|
+
return log;
|
|
117
|
+
},
|
|
118
|
+
stats() {
|
|
119
|
+
return {
|
|
120
|
+
published,
|
|
121
|
+
pulled,
|
|
122
|
+
byKind: { ...byKind }
|
|
123
|
+
};
|
|
124
|
+
}
|
|
125
|
+
};
|
|
126
|
+
}
|
|
127
|
+
//#endregion
|
|
128
|
+
//#region src/mcp/tools/coordination.ts
|
|
129
|
+
/**
|
|
130
|
+
*
|
|
131
|
+
* MCP binding for a live `Scope`. A sandbox driver gets the same small verbs
|
|
132
|
+
* the in-process driver has: spawn, observe, await, steer, ask/answer, analyze,
|
|
133
|
+
* and stop. Settled outputs remain Scope artifacts; product code can project
|
|
134
|
+
* them into any UI/report envelope it needs.
|
|
135
|
+
*
|
|
136
|
+
* @experimental
|
|
137
|
+
*/
|
|
138
|
+
/** Where a question this driver cannot answer goes next. `answer_question` accepts these and
|
|
139
|
+
* nothing else, so the decision type states them and nothing else. */
|
|
140
|
+
const questionEscalationTargets = ["parent", "user"];
|
|
141
|
+
const isQuestionEscalationTarget = (value) => questionEscalationTargets.includes(value);
|
|
142
|
+
/** Producer-side cleanliness for the `finding` event. The findings payload is arbitrary analyst
|
|
143
|
+
* output, the digest a subscriber computes (RFC 8785) throws on ANY `undefined` value — nested
|
|
144
|
+
* included — and a throwing subscriber leaves the event invisible to EVERY subscriber. The
|
|
145
|
+
* producer, not the digest, owns keeping the event canonical: an `undefined` payload is stripped
|
|
146
|
+
* to key-absence, everything else is JSON round-tripped (nested `undefined` object values drop,
|
|
147
|
+
* `undefined` array slots become `null`), and a payload JSON cannot represent at all (cycle,
|
|
148
|
+
* BigInt, bare function) becomes a record OF that fact — degraded findings beat a vanished
|
|
149
|
+
* event. */
|
|
150
|
+
function canonicalFindingEvent(finding) {
|
|
151
|
+
if (finding.findings === void 0) {
|
|
152
|
+
const { findings: _absent, ...present } = finding;
|
|
153
|
+
return present;
|
|
154
|
+
}
|
|
155
|
+
try {
|
|
156
|
+
return {
|
|
157
|
+
...finding,
|
|
158
|
+
findings: JSON.parse(JSON.stringify(finding.findings))
|
|
159
|
+
};
|
|
160
|
+
} catch (error) {
|
|
161
|
+
return {
|
|
162
|
+
...finding,
|
|
163
|
+
findings: { nonCanonicalFindings: error instanceof Error ? error.message : String(error) }
|
|
164
|
+
};
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
/** Normalize the two spellings of an analyst-on-settle entry to the route form. */
|
|
168
|
+
function normalizeAnalyzeOnSettle(entry) {
|
|
169
|
+
return typeof entry === "string" ? { kind: entry } : entry;
|
|
170
|
+
}
|
|
171
|
+
/** Every cause at zero — a pre-flight publishes its whole ledger from the first read. */
|
|
172
|
+
function emptyPreflightCounts() {
|
|
173
|
+
return {
|
|
174
|
+
"model-route": 0,
|
|
175
|
+
"bridge-full": 0,
|
|
176
|
+
"unmountable-tool": 0
|
|
177
|
+
};
|
|
178
|
+
}
|
|
179
|
+
/** Default ceiling for a single `await_event` block (ms). Chosen well under any reasonable remote
|
|
180
|
+
* MCP client request timeout so the call returns a `pending` liveness snapshot instead of erroring;
|
|
181
|
+
* the supervisor re-polls until the worker settles. */
|
|
182
|
+
const DEFAULT_AWAIT_EVENT_TIMEOUT_MS = 15e3;
|
|
183
|
+
/** The reserved coordination verb names — the complete set `createCoordinationTools` can emit
|
|
184
|
+
* (the analyst pair is conditional but still reserved). A driver's extra WORK tools must not
|
|
185
|
+
* collide with any of these, or it could no longer coordinate; callers validate eagerly against
|
|
186
|
+
* this set so the conflict fails loud at construction, not buried in a swallowed `act()` throw. */
|
|
187
|
+
const coordinationVerbNames = [
|
|
188
|
+
"spawn_agent",
|
|
189
|
+
"observe_agent",
|
|
190
|
+
"steer_agent",
|
|
191
|
+
"await_event",
|
|
192
|
+
"list_questions",
|
|
193
|
+
"answer_question",
|
|
194
|
+
"ask_parent",
|
|
195
|
+
"submit_result",
|
|
196
|
+
"stop",
|
|
197
|
+
"list_analysts",
|
|
198
|
+
"run_analyst"
|
|
199
|
+
];
|
|
200
|
+
/**
|
|
201
|
+
* The `CoordinationEvent` kinds a driver may name in `await_event`. The pull queue carries the
|
|
202
|
+
* UP-leg only: `steer` / `answer` / `instruction` / `delivery-attempt` are recorded `queue: false`
|
|
203
|
+
* (history and subscribers, never pulled back), and `mail` is delivered to its addressee's inbox.
|
|
204
|
+
*
|
|
205
|
+
* Declared once because the tool advertises this list in its JSON Schema AND filters on it at
|
|
206
|
+
* dispatch. Written twice, the two drift and the schema promises a kind the filter drops — a
|
|
207
|
+
* driver then blocks on a queue that already holds its event.
|
|
208
|
+
*/
|
|
209
|
+
const awaitableEventKinds = [
|
|
210
|
+
"settled",
|
|
211
|
+
"question",
|
|
212
|
+
"finding"
|
|
213
|
+
];
|
|
214
|
+
function isAwaitableEventKind(value) {
|
|
215
|
+
return awaitableEventKinds.includes(value);
|
|
216
|
+
}
|
|
217
|
+
const idArg = {
|
|
218
|
+
type: "string",
|
|
219
|
+
description: "The workerId returned by spawn_agent."
|
|
220
|
+
};
|
|
221
|
+
/**
|
|
222
|
+
* Strip zod's object-KEY CODEC artifact from a derived JSON Schema, at every depth.
|
|
223
|
+
*
|
|
224
|
+
* `z.record(z.string(), …)` converts to `propertyNames: { type: 'string', pattern:
|
|
225
|
+
* '^u(?:[0-9a-f]{4})*$' }` — a marker of the lossy key round-trip, not a constraint anything
|
|
226
|
+
* enforces: `agentProfileSchema.safeParse({ name: 'r', tools: { bash: true, Read: false } })`
|
|
227
|
+
* succeeds with those plain keys. Published verbatim it reads to a model as "every key must be a
|
|
228
|
+
* run of hex quads", and the model either emits hex-encoded garbage keys or drops the field —
|
|
229
|
+
* on `tools`, `permissions`, `metadata`, `mcp`, `mcp.*.env`, and `model.metadata`, which is most
|
|
230
|
+
* of what a parent actually configures.
|
|
231
|
+
*
|
|
232
|
+
* Every `propertyNames` in the canonical profile schema is this artifact (25 of 25 at Interface
|
|
233
|
+
* 0.40), and the canonical schema constrains no key by pattern, so dropping the keyword outright
|
|
234
|
+
* loses nothing real and cannot be defeated by zod changing the encoding's exact regex.
|
|
235
|
+
*
|
|
236
|
+
* Rebuilds rather than mutates: the canonical conversion output must stay untouched for callers
|
|
237
|
+
* that compare against it.
|
|
238
|
+
*/
|
|
239
|
+
const stripKeyCodecArtifacts = (node) => {
|
|
240
|
+
if (Array.isArray(node)) return node.map(stripKeyCodecArtifacts);
|
|
241
|
+
if (!node || typeof node !== "object") return node;
|
|
242
|
+
return Object.fromEntries(Object.entries(node).filter(([key]) => key !== "propertyNames").map(([key, value]) => [key, stripKeyCodecArtifacts(value)]));
|
|
243
|
+
};
|
|
244
|
+
/** The canonical profile fields a spawning parent actually sets on a child, in the order a reader
|
|
245
|
+
* needs them, each with the description published alongside it. Everything else stays legal to
|
|
246
|
+
* pass — see {@link deriveSpawnProfileArg}.
|
|
247
|
+
*
|
|
248
|
+
* Why these eleven, with the measured numbers (serialized JSON bytes, Interface 0.40). The
|
|
249
|
+
* canonical `properties` map is 10629 bytes, against 2424 bytes for every argument of every other
|
|
250
|
+
* coordination tool COMBINED — publishing it whole makes one parameter four times the rest of the
|
|
251
|
+
* surface a driver re-reads each turn. Of the seven omitted fields (tags, connections, subagents,
|
|
252
|
+
* hooks, modes, confidential, extensions) none is something a parent hands a worker; `permissions`
|
|
253
|
+
* (321 bytes) IS, so it is published — a child that must not touch the network or the filesystem
|
|
254
|
+
* is fenced there and nowhere else. `mcp` (2663 bytes) and `resources` (3122 bytes) are the two a
|
|
255
|
+
* parent is least likely to author inline and were together 85% of the published cost, so they
|
|
256
|
+
* carry a brief shape plus a description naming the full form instead of the canonical sub-tree. */
|
|
257
|
+
const spawnProfileFields = [
|
|
258
|
+
{
|
|
259
|
+
name: "name",
|
|
260
|
+
description: "Short identifier for this child, e.g. \"researcher\" or \"patch-writer\". A child normally sets it to its role; it shows up in traces and worker labels."
|
|
261
|
+
},
|
|
262
|
+
{
|
|
263
|
+
name: "description",
|
|
264
|
+
description: "One line saying what this child is for. Read by humans and by a parent listing its workers."
|
|
265
|
+
},
|
|
266
|
+
{
|
|
267
|
+
name: "version",
|
|
268
|
+
description: "Optional version string for this profile, so two revisions of the same role are distinguishable in evidence. A child usually omits it."
|
|
269
|
+
},
|
|
270
|
+
{
|
|
271
|
+
name: "harness",
|
|
272
|
+
description: "Which coding backend runs the child. Omit to inherit the run default; set it only when this child needs a specific backend (e.g. a long refactor on \"claude-code\")."
|
|
273
|
+
},
|
|
274
|
+
{
|
|
275
|
+
name: "model",
|
|
276
|
+
description: "Model routing: `default` model id, optional `small` for cheap sub-calls, `provider`, and `reasoningEffort`. A child typically sets `default` and `reasoningEffort` — raise effort for a hard reasoning task, lower it for bulk mechanical work."
|
|
277
|
+
},
|
|
278
|
+
{
|
|
279
|
+
name: "prompt",
|
|
280
|
+
description: "The child's standing instructions: `systemPrompt` (who it is and how it works) and `instructions` (durable rules). This is the role; the separate `task` argument carries what to do right now. A child with no systemPrompt has no role — always set one."
|
|
281
|
+
},
|
|
282
|
+
{
|
|
283
|
+
name: "tools",
|
|
284
|
+
description: "Per-tool on/off map keyed by tool name, e.g. `{ \"bash\": true, \"webfetch\": false }`. Omit to inherit the backend's default tool set; set it to narrow a child to the tools its task needs. Keys are plain tool names."
|
|
285
|
+
},
|
|
286
|
+
{
|
|
287
|
+
name: "permissions",
|
|
288
|
+
description: "Per-tool permission decisions: each tool name maps to \"allow\", \"deny\", or \"ask\", or to a nested map of finer-grained rules. This is where a parent fences a child it does not fully trust — a read-only child denies write and network tools here, not in `tools`."
|
|
289
|
+
},
|
|
290
|
+
{
|
|
291
|
+
name: "mcp",
|
|
292
|
+
description: "MCP tool servers to mount for the child, keyed by server name. Brief form: each value is either `{ transport: \"stdio\", command, args?, env?, cwd? }` or `{ transport: \"sse\" | \"http\", url, headers? }`. A child normally sets this only when it needs a tool server the task requires; the canonical AgentProfile schema carries the full form (secret-ref values for env/headers, per-server metadata) and governs validation.",
|
|
293
|
+
brief: {
|
|
294
|
+
type: "object",
|
|
295
|
+
additionalProperties: { type: "object" }
|
|
296
|
+
}
|
|
297
|
+
},
|
|
298
|
+
{
|
|
299
|
+
name: "resources",
|
|
300
|
+
description: "Files, tools, skills, and agents materialized into the child workspace before it starts. Brief form: `{ files?, tools?, skills?, agents? }`, each an array whose entries are `{ kind: \"inline\", name, content }` or `{ kind: \"github\", repository?, path, ref? }` (`files` entries wrap that as `{ path, resource, executable? }`). A child typically gets `files` for seed inputs and `skills` for a procedure it must follow; the canonical AgentProfile schema carries the full form and governs validation.",
|
|
301
|
+
brief: {
|
|
302
|
+
type: "object",
|
|
303
|
+
properties: {
|
|
304
|
+
files: {
|
|
305
|
+
type: "array",
|
|
306
|
+
items: { type: "object" }
|
|
307
|
+
},
|
|
308
|
+
tools: {
|
|
309
|
+
type: "array",
|
|
310
|
+
items: { type: "object" }
|
|
311
|
+
},
|
|
312
|
+
skills: {
|
|
313
|
+
type: "array",
|
|
314
|
+
items: { type: "object" }
|
|
315
|
+
},
|
|
316
|
+
agents: {
|
|
317
|
+
type: "array",
|
|
318
|
+
items: { type: "object" }
|
|
319
|
+
}
|
|
320
|
+
},
|
|
321
|
+
additionalProperties: true
|
|
322
|
+
}
|
|
323
|
+
},
|
|
324
|
+
{
|
|
325
|
+
name: "metadata",
|
|
326
|
+
description: "Free-form key/value bag carried with the profile (plain string keys). Use it for run bookkeeping a reader will want later; it does not change how the child executes."
|
|
327
|
+
}
|
|
328
|
+
];
|
|
329
|
+
/**
|
|
330
|
+
* Build the published shape of `spawn_agent`'s `profile` argument from the canonical
|
|
331
|
+
* `agentProfileSchema` conversion's `properties` map, so it cannot drift from the profile the
|
|
332
|
+
* runtime materializes.
|
|
333
|
+
*
|
|
334
|
+
* DEGRADES, never throws. A canonical field that is absent — renamed or removed upstream — is
|
|
335
|
+
* simply omitted from the published shape, and a canonical schema that is no longer an object
|
|
336
|
+
* publishes no properties at all. This function is reached from a statically-imported module, so a
|
|
337
|
+
* throw here bricks `import '@tangle-network/agent-runtime/kernel'` for every consumer over an
|
|
338
|
+
* upstream rename that costs them, at worst, one advisory field. The drift itself is still caught
|
|
339
|
+
* loudly — as a test assertion in `tests/kernel/coordination.test.ts`, in CI, where it is our
|
|
340
|
+
* problem rather than at a consumer's import, where it is theirs.
|
|
341
|
+
*
|
|
342
|
+
* Permissive on purpose: `additionalProperties: true` with no `required` list, so every canonical
|
|
343
|
+
* field this shape omits stays legal to pass. This tool layer performs no profile validation.
|
|
344
|
+
*
|
|
345
|
+
* @internal exported for the drift and degradation tests; not part of the package's public API.
|
|
346
|
+
*/
|
|
347
|
+
function deriveSpawnProfileArg(canonicalProperties) {
|
|
348
|
+
const published = [];
|
|
349
|
+
for (const field of spawnProfileFields) {
|
|
350
|
+
const canonical = canonicalProperties?.[field.name];
|
|
351
|
+
if (canonical === void 0) continue;
|
|
352
|
+
const shape = field.brief ?? stripKeyCodecArtifacts(canonical);
|
|
353
|
+
published.push([field.name, {
|
|
354
|
+
...shape,
|
|
355
|
+
description: field.description
|
|
356
|
+
}]);
|
|
357
|
+
}
|
|
358
|
+
return {
|
|
359
|
+
type: "object",
|
|
360
|
+
description: "The child agent profile to run — this is the shape the DEFAULT worker seam accepts; a run wired with a custom makeWorkerAgent may accept a different one. The properties below are derived from the canonical AgentProfile schema and reduced to what a spawning parent sets (`mcp` and `resources` in a brief form); every other canonical field — tags, connections, subagents, hooks, modes, confidential, extensions — may still be passed. This tool does not validate the profile: the canonical AgentProfile schema governs validation downstream.",
|
|
361
|
+
properties: Object.fromEntries(published),
|
|
362
|
+
additionalProperties: true
|
|
363
|
+
};
|
|
364
|
+
}
|
|
365
|
+
spawnProfileFields.map((f) => f.name);
|
|
366
|
+
let spawnProfileArgCache;
|
|
367
|
+
/** The published `profile` shape, computed on FIRST tool-definition access and memoized — not at
|
|
368
|
+
* module load. The conversion walks the whole canonical profile tree, and the strip walk rebuilds
|
|
369
|
+
* it: 3.5ms on the first `createCoordinationTools`, 0.017ms on every later one (measured, Interface
|
|
370
|
+
* 0.40, node 24). `src/runtime/index.ts` imports this module statically, so paying that at import
|
|
371
|
+
* taxes every consumer of the kernel entrypoint — including the ones that never build a
|
|
372
|
+
* coordination toolbox. The memo keeps it at once per process for the ones that do.
|
|
373
|
+
*
|
|
374
|
+
* Conversion choices. `io: 'input'` is the CALLER's view — pre-default, pre-transform — which is
|
|
375
|
+
* what a spawning parent may pass, not what the runtime ends up holding. `unrepresentable: 'any'`
|
|
376
|
+
* keeps the conversion total: the canonical schema contains transforms with no JSON Schema form,
|
|
377
|
+
* and zod's default is to throw on them, which would leave the tool with no published shape. */
|
|
378
|
+
function spawnProfileArg() {
|
|
379
|
+
if (!spawnProfileArgCache) spawnProfileArgCache = detachedFrozen(deriveSpawnProfileArg(agentProfileSchema.toJSONSchema({
|
|
380
|
+
io: "input",
|
|
381
|
+
target: "draft-07",
|
|
382
|
+
unrepresentable: "any"
|
|
383
|
+
}).properties));
|
|
384
|
+
return spawnProfileArgCache;
|
|
385
|
+
}
|
|
386
|
+
/** Build the driver's MCP tools over a live scope. */
|
|
387
|
+
function createCoordinationTools(opts) {
|
|
388
|
+
const deliverable = opts.deliverable;
|
|
389
|
+
let stopped = false;
|
|
390
|
+
let reason;
|
|
391
|
+
let stopNotified = false;
|
|
392
|
+
let submitted;
|
|
393
|
+
let questionSeq = 0;
|
|
394
|
+
const ledger = [];
|
|
395
|
+
const questions = [...opts.priorQuestions ?? []];
|
|
396
|
+
const questionPolicy = opts.questionPolicy ?? "auto";
|
|
397
|
+
const notifyStop = () => {
|
|
398
|
+
if (stopNotified) return;
|
|
399
|
+
stopNotified = true;
|
|
400
|
+
opts.onStop?.(reason);
|
|
401
|
+
};
|
|
402
|
+
const completedKeys = /* @__PURE__ */ new Set();
|
|
403
|
+
const keyByWorker = /* @__PURE__ */ new Map();
|
|
404
|
+
const profileNameByWorker = /* @__PURE__ */ new Map();
|
|
405
|
+
const liveHandles = /* @__PURE__ */ new Map();
|
|
406
|
+
let unkeyedAssignmentOrdinal = nextUnkeyedAssignmentOrdinal(opts.scope);
|
|
407
|
+
const preflightCounts = emptyPreflightCounts();
|
|
408
|
+
for (const [key, prior] of opts.scope.resume?.keys ?? []) if (prior.state === "completed") completedKeys.add(key);
|
|
409
|
+
const nodeForWorker = (id) => opts.scope.view.nodes.find((node) => node.id === id) ?? opts.scope.resume?.view.nodes.find((node) => node.id === id);
|
|
410
|
+
const projectSettled = (settled, resumed = false) => {
|
|
411
|
+
const node = nodeForWorker(settled.handle.id);
|
|
412
|
+
const assignmentId = settled.handle.assignmentId ?? node?.assignmentId;
|
|
413
|
+
const identity = settled.handle.identity ?? node?.identity;
|
|
414
|
+
const materialization = settled.handle.materialization ?? node?.materialization;
|
|
415
|
+
const executionBindings = settled.handle.executionBindings ?? node?.executionBindings;
|
|
416
|
+
const settledAt = settled.settledAt ?? node?.settledAt;
|
|
417
|
+
const trace = settled.trace ?? node?.trace ?? {
|
|
418
|
+
status: "unavailable",
|
|
419
|
+
reason: "legacy-settlement-without-trace-evidence"
|
|
420
|
+
};
|
|
421
|
+
const common = {
|
|
422
|
+
id: settled.handle.id,
|
|
423
|
+
...assignmentId === void 0 ? {} : { assignmentId },
|
|
424
|
+
...identity === void 0 ? {} : { identity },
|
|
425
|
+
...materialization === void 0 ? {} : { materialization },
|
|
426
|
+
...executionBindings === void 0 ? {} : { executionBindings },
|
|
427
|
+
...settledAt === void 0 ? {} : { settledAt },
|
|
428
|
+
trace,
|
|
429
|
+
...resumed ? { resumed: true } : {}
|
|
430
|
+
};
|
|
431
|
+
return detachedFrozen(settled.kind === "done" ? {
|
|
432
|
+
...common,
|
|
433
|
+
status: "done",
|
|
434
|
+
spent: settled.spent,
|
|
435
|
+
...settled.verdict?.score === void 0 ? {} : { score: settled.verdict.score },
|
|
436
|
+
...settled.verdict?.valid === void 0 ? {} : { valid: settled.verdict.valid },
|
|
437
|
+
outRef: settled.outRef
|
|
438
|
+
} : {
|
|
439
|
+
...common,
|
|
440
|
+
status: "down",
|
|
441
|
+
...node?.spent === void 0 ? {} : { spent: node.spent },
|
|
442
|
+
reason: settled.reason
|
|
443
|
+
});
|
|
444
|
+
};
|
|
445
|
+
const resumedWorkers = [];
|
|
446
|
+
for (const s of opts.scope.resume?.settled ?? []) {
|
|
447
|
+
const worker = projectSettled(s, true);
|
|
448
|
+
resumedWorkers.push(worker);
|
|
449
|
+
ledger.push(worker);
|
|
450
|
+
}
|
|
451
|
+
const bus = createEventBus();
|
|
452
|
+
if (opts.onEvent) {
|
|
453
|
+
const cb = opts.onEvent;
|
|
454
|
+
bus.subscribe((rec) => cb(rec.event, rec));
|
|
455
|
+
}
|
|
456
|
+
const resumeEvents = opts.replaySettlements ? resumedWorkers.map((worker) => detachedFrozen({
|
|
457
|
+
type: "settled",
|
|
458
|
+
worker
|
|
459
|
+
})) : [];
|
|
460
|
+
let resumeEventIndex = 0;
|
|
461
|
+
let readyInFlight;
|
|
462
|
+
const ready = () => {
|
|
463
|
+
if (resumeEventIndex >= resumeEvents.length) return Promise.resolve();
|
|
464
|
+
if (readyInFlight) return readyInFlight;
|
|
465
|
+
readyInFlight = (async () => {
|
|
466
|
+
while (resumeEventIndex < resumeEvents.length) {
|
|
467
|
+
const event = resumeEvents[resumeEventIndex];
|
|
468
|
+
if (!event) break;
|
|
469
|
+
await bus.publish(event);
|
|
470
|
+
resumeEventIndex += 1;
|
|
471
|
+
}
|
|
472
|
+
})().finally(() => {
|
|
473
|
+
readyInFlight = void 0;
|
|
474
|
+
});
|
|
475
|
+
return readyInFlight;
|
|
476
|
+
};
|
|
477
|
+
const urgencyPriority = (u) => u === "blocks-run" ? 20 : u === "blocks-step" ? 10 : 0;
|
|
478
|
+
const str = (v, field) => {
|
|
479
|
+
if (typeof v !== "string" || v.length === 0) throw new Error(`coordination tools: "${field}" must be a non-empty string`);
|
|
480
|
+
return v;
|
|
481
|
+
};
|
|
482
|
+
const obj = (raw) => {
|
|
483
|
+
if (!raw || typeof raw !== "object") throw new Error("coordination tools: arguments must be an object");
|
|
484
|
+
return raw;
|
|
485
|
+
};
|
|
486
|
+
const mergeBudget = (base, raw) => {
|
|
487
|
+
if (!raw || typeof raw !== "object" || Array.isArray(raw)) throw new Error("coordination tools: \"budget\" must be an object");
|
|
488
|
+
const o = raw;
|
|
489
|
+
const field = (name) => {
|
|
490
|
+
const v = o[name];
|
|
491
|
+
if (v === void 0) return void 0;
|
|
492
|
+
if (typeof v !== "number" || !Number.isFinite(v)) throw new Error(`coordination tools: "budget.${name}" must be a finite number`);
|
|
493
|
+
return v;
|
|
494
|
+
};
|
|
495
|
+
const maxIterations = field("maxIterations");
|
|
496
|
+
const maxTokens = field("maxTokens");
|
|
497
|
+
const maxUsd = field("maxUsd");
|
|
498
|
+
const deadlineMs = field("deadlineMs");
|
|
499
|
+
const merged = {
|
|
500
|
+
maxIterations: maxIterations ?? base.maxIterations,
|
|
501
|
+
maxTokens: maxTokens ?? base.maxTokens,
|
|
502
|
+
...(maxUsd ?? base.maxUsd) === void 0 ? {} : { maxUsd: maxUsd ?? base.maxUsd },
|
|
503
|
+
...(deadlineMs ?? base.deadlineMs) === void 0 ? {} : { deadlineMs: deadlineMs ?? base.deadlineMs }
|
|
504
|
+
};
|
|
505
|
+
assertValidBudget(merged, "coordination tools: budget");
|
|
506
|
+
return merged;
|
|
507
|
+
};
|
|
508
|
+
const level = (v) => {
|
|
509
|
+
if (v === "worker" || v === "driver" || v === "loop") return v;
|
|
510
|
+
throw new Error("coordination tools: \"level\" must be worker, driver, or loop");
|
|
511
|
+
};
|
|
512
|
+
const urgency = (v) => {
|
|
513
|
+
if (v === "continue-without" || v === "blocks-step" || v === "blocks-run") return v;
|
|
514
|
+
throw new Error("coordination tools: \"urgency\" must be continue-without, blocks-step, or blocks-run");
|
|
515
|
+
};
|
|
516
|
+
const commitSettled = (s, w) => {
|
|
517
|
+
const settledKey = keyByWorker.get(s.handle.id);
|
|
518
|
+
if (settledKey !== void 0 && s.kind === "done") completedKeys.add(settledKey);
|
|
519
|
+
ledger.push(w);
|
|
520
|
+
unwatchWorker(w.id);
|
|
521
|
+
};
|
|
522
|
+
let pendingSettlement;
|
|
523
|
+
const analystRuns = /* @__PURE__ */ new Map();
|
|
524
|
+
let analystRunOrdinal = 0;
|
|
525
|
+
/** The `finding` event an analyst-agent settlement becomes: its settle OUTPUT is the findings
|
|
526
|
+
* (a failed run publishes the failure as findings — degraded beats vanished). */
|
|
527
|
+
const analystRunFinding = (run, settled) => detachedFrozen({
|
|
528
|
+
type: "finding",
|
|
529
|
+
finding: canonicalFindingEvent({
|
|
530
|
+
fromWorker: run.sourceWorker,
|
|
531
|
+
analyst: run.route.kind,
|
|
532
|
+
findings: settled.kind === "done" ? settled.out : { analystRunFailed: settled.reason }
|
|
533
|
+
})
|
|
534
|
+
});
|
|
535
|
+
/**
|
|
536
|
+
* Spawn one analyst-AGENT run over a settled worker's evidence, through the SAME spawn
|
|
537
|
+
* machinery a driver spawn uses (`scope.spawn` + `makeWorkerAgent`): the analyst's spend
|
|
538
|
+
* reserves from the conserved pool, its node is journaled/traced like any worker, and a
|
|
539
|
+
* node-pinning seam sees `context.analyst`. Its task is the route directive plus the settled
|
|
540
|
+
* worker's persisted tool-trace spans. A refused spawn publishes a finding RECORDING the
|
|
541
|
+
* refusal — observable, never silent — and must never take down the settlement path.
|
|
542
|
+
*/
|
|
543
|
+
const spawnAnalystRun = async (route, worker) => {
|
|
544
|
+
let spansText = "";
|
|
545
|
+
let spanCount = 0;
|
|
546
|
+
if (worker.trace.status === "available") try {
|
|
547
|
+
const artifact = parseWorkerToolTraceArtifact(await opts.blobs.get(worker.trace.traceRef), worker.trace.traceRef);
|
|
548
|
+
spanCount = artifact.spans.length;
|
|
549
|
+
spansText = safeJsonText(artifact.spans);
|
|
550
|
+
} catch {
|
|
551
|
+
spansText = "";
|
|
552
|
+
}
|
|
553
|
+
const task = [
|
|
554
|
+
...route.directive === void 0 || route.directive.length === 0 ? [] : [route.directive],
|
|
555
|
+
`Evidence — settled worker '${worker.id}' tool trace (${spanCount} spans):`,
|
|
556
|
+
spansText.length === 0 ? "(no tool spans available)" : spansText
|
|
557
|
+
].join("\n\n");
|
|
558
|
+
const assignmentId = `analyst:${route.kind}:o${analystRunOrdinal++}`;
|
|
559
|
+
const label = `analyst:${route.kind}`;
|
|
560
|
+
const context = Object.freeze({
|
|
561
|
+
assignmentId,
|
|
562
|
+
parentNodeId: opts.scope.view.root,
|
|
563
|
+
budget: opts.perWorker,
|
|
564
|
+
task,
|
|
565
|
+
label,
|
|
566
|
+
analyst: route.kind,
|
|
567
|
+
continuity: "fresh"
|
|
568
|
+
});
|
|
569
|
+
let refusal;
|
|
570
|
+
let spawnedId;
|
|
571
|
+
try {
|
|
572
|
+
const res = opts.scope.spawn(() => opts.makeWorkerAgent(route.agent, context), task, {
|
|
573
|
+
budget: opts.perWorker,
|
|
574
|
+
label,
|
|
575
|
+
assignmentId
|
|
576
|
+
});
|
|
577
|
+
if (res.ok) spawnedId = res.handle.id;
|
|
578
|
+
else refusal = String(res.reason);
|
|
579
|
+
} catch (cause) {
|
|
580
|
+
refusal = cause instanceof Error ? cause.message : String(cause);
|
|
581
|
+
}
|
|
582
|
+
if (spawnedId === void 0) {
|
|
583
|
+
await bus.publish({
|
|
584
|
+
type: "finding",
|
|
585
|
+
finding: canonicalFindingEvent({
|
|
586
|
+
fromWorker: worker.id,
|
|
587
|
+
analyst: route.kind,
|
|
588
|
+
findings: { analystSpawnRefused: refusal ?? "unknown" }
|
|
589
|
+
})
|
|
590
|
+
});
|
|
591
|
+
return;
|
|
592
|
+
}
|
|
593
|
+
analystRuns.set(spawnedId, {
|
|
594
|
+
route,
|
|
595
|
+
sourceWorker: worker.id
|
|
596
|
+
});
|
|
597
|
+
watchWorker(spawnedId);
|
|
598
|
+
};
|
|
599
|
+
const workerRouteNames = (workerId) => {
|
|
600
|
+
const names = /* @__PURE__ */ new Set();
|
|
601
|
+
const profileName = profileNameByWorker.get(workerId);
|
|
602
|
+
if (profileName !== void 0) names.add(profileName);
|
|
603
|
+
const label = nodeForWorker(workerId)?.label;
|
|
604
|
+
if (label !== void 0) names.add(label);
|
|
605
|
+
return names;
|
|
606
|
+
};
|
|
607
|
+
/** The LIVE worker a route destination names, by profile name first, label second. */
|
|
608
|
+
const liveWorkerIdNamed = (destination) => {
|
|
609
|
+
const live = opts.scope.view.nodes.filter((node) => isLiveNodeStatus(node.status));
|
|
610
|
+
return live.find((node) => profileNameByWorker.get(node.id) === destination)?.id ?? live.find((node) => node.label === destination)?.id;
|
|
611
|
+
};
|
|
612
|
+
const liveWorkerForNode = (name) => opts.scope.view.nodes.find((node) => isLiveNodeStatus(node.status) && profileNameByWorker.get(node.id) === name)?.id;
|
|
613
|
+
const latestSettledWorkerForNode = (name) => {
|
|
614
|
+
for (let i = ledger.length - 1; i >= 0; i -= 1) {
|
|
615
|
+
const worker = ledger[i];
|
|
616
|
+
if (profileNameByWorker.get(worker.id) === name) return worker.id;
|
|
617
|
+
}
|
|
618
|
+
};
|
|
619
|
+
const nodeSpawnCount = (name) => {
|
|
620
|
+
let count = 0;
|
|
621
|
+
for (const profileName of profileNameByWorker.values()) if (profileName === name) count += 1;
|
|
622
|
+
return count;
|
|
623
|
+
};
|
|
624
|
+
const parseContinuity = (v) => {
|
|
625
|
+
if (v === void 0) return void 0;
|
|
626
|
+
if (v === "fresh" || v === "resume") return v;
|
|
627
|
+
throw new Error("coordination tools: \"continuity\" must be \"fresh\" or \"resume\"");
|
|
628
|
+
};
|
|
629
|
+
/**
|
|
630
|
+
* Resolve the EFFECTIVE continuity of one spawn: the per-call request wins, else the profile
|
|
631
|
+
* name's declared default, else `'fresh'`. Every refusal is loud and actionable:
|
|
632
|
+
* - an EXPLICIT `'resume'` with no settled prior worker refuses (`resume-no-prior`) — the
|
|
633
|
+
* DECLARED default degrades to `'fresh'` instead, so a resume edge's first traversal is
|
|
634
|
+
* simply the first spawn;
|
|
635
|
+
* - resume while a prior worker of the node is still LIVE refuses (`resume-while-live`) —
|
|
636
|
+
* that is what steer is for, and the error says so;
|
|
637
|
+
* - resume under a semantic `key` refuses (`resume-with-key`) — a key makes an assignment
|
|
638
|
+
* run-once, resume explicitly runs the node again.
|
|
639
|
+
*/
|
|
640
|
+
const resolveContinuity = (requested, profileName, key) => {
|
|
641
|
+
const declared = profileName === void 0 ? "fresh" : opts.continuityByProfile?.[profileName] ?? "fresh";
|
|
642
|
+
if (!(requested === "resume" || requested === void 0 && declared === "resume")) return { continuity: "fresh" };
|
|
643
|
+
if (profileName === void 0 || profileName.length === 0) return {
|
|
644
|
+
error: "resume-unnamed-profile",
|
|
645
|
+
hint: "Resume targets a node by profile.name — the stable node identity — and this profile has none. Name the profile, or spawn fresh."
|
|
646
|
+
};
|
|
647
|
+
if (key !== void 0) return {
|
|
648
|
+
error: "resume-with-key",
|
|
649
|
+
hint: "A semantic key makes an assignment run-once (a completed key returns its committed result instead of running again); resume explicitly runs the node AGAIN. Drop the key to resume, or keep the key and spawn fresh."
|
|
650
|
+
};
|
|
651
|
+
const live = liveWorkerForNode(profileName);
|
|
652
|
+
if (live !== void 0) return {
|
|
653
|
+
error: "resume-while-live",
|
|
654
|
+
hint: `Worker '${live}' on node '${profileName}' is still LIVE — resume re-attaches to a SETTLED session. To redirect the live worker, use steer_agent (that is the live-worker channel); to run a parallel sibling instead, pass continuity: 'fresh'.`
|
|
655
|
+
};
|
|
656
|
+
const prior = latestSettledWorkerForNode(profileName);
|
|
657
|
+
if (prior === void 0) {
|
|
658
|
+
if (requested === void 0) return { continuity: "fresh" };
|
|
659
|
+
return {
|
|
660
|
+
error: "resume-no-prior",
|
|
661
|
+
hint: `Node '${profileName}' has no settled prior worker in this process to resume — resume continues a FINISHED session. Spawn the node fresh first (omit continuity or pass 'fresh').`
|
|
662
|
+
};
|
|
663
|
+
}
|
|
664
|
+
return {
|
|
665
|
+
continuity: "resume",
|
|
666
|
+
resume: {
|
|
667
|
+
ofWorker: prior,
|
|
668
|
+
sequence: nodeSpawnCount(profileName) + 1
|
|
669
|
+
}
|
|
670
|
+
};
|
|
671
|
+
};
|
|
672
|
+
/**
|
|
673
|
+
* Deliver one routed analyst finding to its destination worker through the SAME authorized
|
|
674
|
+
* steer machinery a driver steer uses, so the delivery is recorded (`steer` event carrying
|
|
675
|
+
* `analyst`) and its outcome is a fact. No live destination ⇒ a record-only failed steer —
|
|
676
|
+
* observable, never a silent drop. A throw here must not kill the settle path: failures are
|
|
677
|
+
* recorded on the bus (a `steer` with `delivered: false`) before being swallowed, with ONE
|
|
678
|
+
* narrow exception — a bus that refuses the `delivery-attempt` record itself leaves only the
|
|
679
|
+
* `instruction` receipt (an attempt with no outcome = explicitly unknown, per
|
|
680
|
+
* recordDeliveryAttempt's own contract).
|
|
681
|
+
*/
|
|
682
|
+
const deliverRoutedFinding = async (route, findings) => {
|
|
683
|
+
const destination = route.to;
|
|
684
|
+
const text = route.directive === void 0 || route.directive.length === 0 ? safeJsonText(findings) : `${route.directive}\n\n${safeJsonText(findings)}`;
|
|
685
|
+
const targetId = liveWorkerIdNamed(destination);
|
|
686
|
+
if (targetId === void 0) {
|
|
687
|
+
await bus.publish({
|
|
688
|
+
type: "steer",
|
|
689
|
+
down: detachedFrozen({
|
|
690
|
+
receiptId: randomUUID(),
|
|
691
|
+
toWorker: destination,
|
|
692
|
+
instruction: text,
|
|
693
|
+
instructionDigest: canonicalCandidateDigest(text),
|
|
694
|
+
delivered: false,
|
|
695
|
+
outcome: "unknown-worker"
|
|
696
|
+
}),
|
|
697
|
+
analyst: route.kind
|
|
698
|
+
}, { queue: false });
|
|
699
|
+
return;
|
|
700
|
+
}
|
|
701
|
+
let instruction;
|
|
702
|
+
try {
|
|
703
|
+
instruction = authorizeInstruction("steer", targetId, text, false);
|
|
704
|
+
await recordInstruction(instruction);
|
|
705
|
+
} catch (cause) {
|
|
706
|
+
try {
|
|
707
|
+
await sendDown("steer", detachedFrozen({
|
|
708
|
+
receiptId: randomUUID(),
|
|
709
|
+
toWorker: targetId,
|
|
710
|
+
instruction: text,
|
|
711
|
+
instructionDigest: canonicalCandidateDigest(text),
|
|
712
|
+
delivered: false,
|
|
713
|
+
outcome: "runtime-error",
|
|
714
|
+
error: cause instanceof Error ? cause.message : String(cause)
|
|
715
|
+
}), route.kind);
|
|
716
|
+
} catch {}
|
|
717
|
+
return;
|
|
718
|
+
}
|
|
719
|
+
try {
|
|
720
|
+
await attemptDelivery(instruction, {
|
|
721
|
+
steer: instruction.instruction,
|
|
722
|
+
interrupt: false
|
|
723
|
+
}, { analyst: route.kind });
|
|
724
|
+
} catch {}
|
|
725
|
+
};
|
|
726
|
+
const flushPendingSettlement = async () => {
|
|
727
|
+
const pending = pendingSettlement;
|
|
728
|
+
if (!pending) return false;
|
|
729
|
+
await bus.publish(pending.event);
|
|
730
|
+
if (pending.analystRun) {
|
|
731
|
+
analystRuns.delete(pending.worker.id);
|
|
732
|
+
unwatchWorker(pending.worker.id);
|
|
733
|
+
pendingSettlement = void 0;
|
|
734
|
+
const { route } = pending.analystRun;
|
|
735
|
+
if (route.to !== void 0 && pending.event.type === "finding") await deliverRoutedFinding({
|
|
736
|
+
kind: route.kind,
|
|
737
|
+
to: route.to
|
|
738
|
+
}, pending.event.finding.findings);
|
|
739
|
+
return true;
|
|
740
|
+
}
|
|
741
|
+
commitSettled(pending.settled, pending.worker);
|
|
742
|
+
pendingSettlement = void 0;
|
|
743
|
+
if (pending.analyze && pending.worker.status === "done" && pending.worker.trace.status === "available" && opts.analyzeOnSettle?.length) {
|
|
744
|
+
const routes = opts.analyzeOnSettle.map(normalizeAnalyzeOnSettle);
|
|
745
|
+
const sourceNames = workerRouteNames(pending.worker.id);
|
|
746
|
+
const applicable = routes.filter((route) => route.over === void 0 || route.over.some((name) => sourceNames.has(name)));
|
|
747
|
+
const lensRoutes = applicable.filter((route) => route.agent === void 0);
|
|
748
|
+
const agentRoutes = applicable.filter((route) => route.agent !== void 0);
|
|
749
|
+
if (lensRoutes.length > 0 && opts.analysts) {
|
|
750
|
+
const trace = await workerTraceAnalysisStore(pending.worker.trace, opts.blobs);
|
|
751
|
+
for (const route of lensRoutes) {
|
|
752
|
+
const findings = await opts.analysts.run(route.kind, trace);
|
|
753
|
+
await bus.publish({
|
|
754
|
+
type: "finding",
|
|
755
|
+
finding: canonicalFindingEvent({
|
|
756
|
+
fromWorker: pending.worker.id,
|
|
757
|
+
analyst: route.kind,
|
|
758
|
+
findings
|
|
759
|
+
})
|
|
760
|
+
});
|
|
761
|
+
if (route.to !== void 0) await deliverRoutedFinding(route, findings);
|
|
762
|
+
}
|
|
763
|
+
}
|
|
764
|
+
for (const route of agentRoutes) await spawnAnalystRun(route, pending.worker);
|
|
765
|
+
}
|
|
766
|
+
return true;
|
|
767
|
+
};
|
|
768
|
+
const drainSettlement = async () => {
|
|
769
|
+
if (!pendingSettlement) {
|
|
770
|
+
const settled = await opts.scope.next();
|
|
771
|
+
if (!settled) return false;
|
|
772
|
+
const worker = projectSettled(settled);
|
|
773
|
+
const run = analystRuns.get(settled.handle.id);
|
|
774
|
+
pendingSettlement = run ? {
|
|
775
|
+
settled,
|
|
776
|
+
worker,
|
|
777
|
+
event: analystRunFinding(run, settled),
|
|
778
|
+
analyze: false,
|
|
779
|
+
analystRun: run
|
|
780
|
+
} : {
|
|
781
|
+
settled,
|
|
782
|
+
worker,
|
|
783
|
+
event: detachedFrozen({
|
|
784
|
+
type: "settled",
|
|
785
|
+
worker
|
|
786
|
+
}),
|
|
787
|
+
analyze: true
|
|
788
|
+
};
|
|
789
|
+
}
|
|
790
|
+
return flushPendingSettlement();
|
|
791
|
+
};
|
|
792
|
+
const drainResolved = async () => {
|
|
793
|
+
let drained = 0;
|
|
794
|
+
for (;;) {
|
|
795
|
+
if (!pendingSettlement) {
|
|
796
|
+
const settled = await opts.scope.nextResolved();
|
|
797
|
+
if (!settled) return drained;
|
|
798
|
+
const worker = projectSettled(settled);
|
|
799
|
+
const run = analystRuns.get(settled.handle.id);
|
|
800
|
+
pendingSettlement = run ? {
|
|
801
|
+
settled,
|
|
802
|
+
worker,
|
|
803
|
+
event: analystRunFinding(run, settled),
|
|
804
|
+
analyze: false,
|
|
805
|
+
analystRun: run
|
|
806
|
+
} : {
|
|
807
|
+
settled,
|
|
808
|
+
worker,
|
|
809
|
+
event: detachedFrozen({
|
|
810
|
+
type: "settled",
|
|
811
|
+
worker
|
|
812
|
+
}),
|
|
813
|
+
analyze: false
|
|
814
|
+
};
|
|
815
|
+
}
|
|
816
|
+
await flushPendingSettlement();
|
|
817
|
+
drained += 1;
|
|
818
|
+
}
|
|
819
|
+
};
|
|
820
|
+
async function sendDown(type, down, questionIdOrAnalyst) {
|
|
821
|
+
await bus.publish(type === "answer" ? {
|
|
822
|
+
type,
|
|
823
|
+
down,
|
|
824
|
+
questionId: str(questionIdOrAnalyst, "questionId")
|
|
825
|
+
} : questionIdOrAnalyst !== void 0 ? {
|
|
826
|
+
type,
|
|
827
|
+
down,
|
|
828
|
+
analyst: questionIdOrAnalyst
|
|
829
|
+
} : {
|
|
830
|
+
type,
|
|
831
|
+
down
|
|
832
|
+
}, { queue: false });
|
|
833
|
+
}
|
|
834
|
+
const authorizeInstruction = (kind, workerId, instruction, interrupt, questionId) => {
|
|
835
|
+
const workerIdentity = opts.scope.view.nodes.find((node) => node.id === workerId)?.identity;
|
|
836
|
+
let authorizedInstruction = instruction;
|
|
837
|
+
if (opts.authorizeDownMessage) {
|
|
838
|
+
if (workerIdentity === void 0) throw new Error(`coordination tools: cannot authorize ${kind} for worker ${JSON.stringify(workerId)} without durable identity`);
|
|
839
|
+
const decision = detachedFrozen(opts.authorizeDownMessage(detachedFrozen({
|
|
840
|
+
kind,
|
|
841
|
+
workerId,
|
|
842
|
+
workerIdentity,
|
|
843
|
+
instruction,
|
|
844
|
+
interrupt,
|
|
845
|
+
...questionId !== void 0 ? { questionId } : {}
|
|
846
|
+
})));
|
|
847
|
+
if (typeof decision !== "object" || decision === null || Array.isArray(decision) || typeof decision.instruction !== "string" || decision.instruction.length === 0) throw new Error("coordination tools: authorizeDownMessage must return an instruction");
|
|
848
|
+
authorizedInstruction = decision.instruction;
|
|
849
|
+
}
|
|
850
|
+
return detachedFrozen({
|
|
851
|
+
receiptId: randomUUID(),
|
|
852
|
+
kind,
|
|
853
|
+
toWorker: workerId,
|
|
854
|
+
instruction: authorizedInstruction,
|
|
855
|
+
instructionDigest: canonicalCandidateDigest(authorizedInstruction),
|
|
856
|
+
...workerIdentity !== void 0 ? { workerIdentity } : {},
|
|
857
|
+
interrupt,
|
|
858
|
+
...questionId !== void 0 ? { questionId } : {}
|
|
859
|
+
});
|
|
860
|
+
};
|
|
861
|
+
/** Publish before `scope.send`: an awaited durable subscriber therefore commits the exact bytes
|
|
862
|
+
* before the worker can observe them. */
|
|
863
|
+
const recordInstruction = async (instruction) => {
|
|
864
|
+
await bus.publish({
|
|
865
|
+
type: "instruction",
|
|
866
|
+
instruction
|
|
867
|
+
}, { queue: false });
|
|
868
|
+
};
|
|
869
|
+
/** Commit delivery intent after the authorization receipt and before `Scope.send`. An attempt with
|
|
870
|
+
* no matching outcome after a crash is explicitly unknown and must never be replayed. */
|
|
871
|
+
const recordDeliveryAttempt = async (instruction) => {
|
|
872
|
+
const attempt = detachedFrozen({
|
|
873
|
+
receiptId: instruction.receiptId,
|
|
874
|
+
kind: instruction.kind,
|
|
875
|
+
toWorker: instruction.toWorker,
|
|
876
|
+
instructionDigest: instruction.instructionDigest,
|
|
877
|
+
interrupt: instruction.interrupt,
|
|
878
|
+
...instruction.questionId !== void 0 ? { questionId: instruction.questionId } : {}
|
|
879
|
+
});
|
|
880
|
+
await bus.publish({
|
|
881
|
+
type: "delivery-attempt",
|
|
882
|
+
attempt
|
|
883
|
+
}, { queue: false });
|
|
884
|
+
return attempt;
|
|
885
|
+
};
|
|
886
|
+
const deliveryOutcome = (workerId, delivered) => {
|
|
887
|
+
if (delivered) return "delivered";
|
|
888
|
+
if (opts.scope.signal.aborted) return "scope-stopped";
|
|
889
|
+
const node = opts.scope.view.nodes.find((candidate) => candidate.id === workerId);
|
|
890
|
+
if (!node) return "unknown-worker";
|
|
891
|
+
if (!isLiveNodeStatus(node.status)) return "already-settled";
|
|
892
|
+
return "runtime-has-no-inbox";
|
|
893
|
+
};
|
|
894
|
+
const attemptDelivery = async (instruction, message, origin) => {
|
|
895
|
+
await recordDeliveryAttempt(instruction);
|
|
896
|
+
let delivered = false;
|
|
897
|
+
let outcome;
|
|
898
|
+
let error;
|
|
899
|
+
try {
|
|
900
|
+
delivered = opts.scope.send(instruction.toWorker, message);
|
|
901
|
+
outcome = deliveryOutcome(instruction.toWorker, delivered);
|
|
902
|
+
} catch (cause) {
|
|
903
|
+
outcome = "runtime-error";
|
|
904
|
+
error = cause instanceof Error ? cause.message : String(cause);
|
|
905
|
+
}
|
|
906
|
+
const down = detachedFrozen({
|
|
907
|
+
receiptId: instruction.receiptId,
|
|
908
|
+
toWorker: instruction.toWorker,
|
|
909
|
+
instruction: instruction.instruction,
|
|
910
|
+
instructionDigest: instruction.instructionDigest,
|
|
911
|
+
delivered,
|
|
912
|
+
outcome,
|
|
913
|
+
...error !== void 0 ? { error } : {}
|
|
914
|
+
});
|
|
915
|
+
if (instruction.kind === "answer") await sendDown("answer", down, str(instruction.questionId, "questionId"));
|
|
916
|
+
else await sendDown("steer", down, origin?.analyst);
|
|
917
|
+
if (error !== void 0) throw new Error(`coordination tools: delivery failed: ${error}`);
|
|
918
|
+
return down;
|
|
919
|
+
};
|
|
920
|
+
const steerWorker = async (workerId, instruction, options = {}) => {
|
|
921
|
+
const interrupt = options.interrupt === true;
|
|
922
|
+
const authorized = authorizeInstruction("steer", workerId, instruction, interrupt);
|
|
923
|
+
await recordInstruction(authorized);
|
|
924
|
+
return await attemptDelivery(authorized, {
|
|
925
|
+
steer: authorized.instruction,
|
|
926
|
+
interrupt
|
|
927
|
+
});
|
|
928
|
+
};
|
|
929
|
+
const projectEvent = (ev) => {
|
|
930
|
+
if (ev.type === "settled") {
|
|
931
|
+
const { id, status, ...evidence } = ev.worker;
|
|
932
|
+
return {
|
|
933
|
+
type: "settled",
|
|
934
|
+
settled: id,
|
|
935
|
+
status,
|
|
936
|
+
...evidence
|
|
937
|
+
};
|
|
938
|
+
}
|
|
939
|
+
if (ev.type === "question") return {
|
|
940
|
+
type: "question",
|
|
941
|
+
question: ev.question
|
|
942
|
+
};
|
|
943
|
+
if (ev.type === "finding") return {
|
|
944
|
+
type: "finding",
|
|
945
|
+
...ev.finding
|
|
946
|
+
};
|
|
947
|
+
if (ev.type === "answer") return {
|
|
948
|
+
type: "answer",
|
|
949
|
+
...ev.down,
|
|
950
|
+
questionId: ev.questionId
|
|
951
|
+
};
|
|
952
|
+
if (ev.type === "instruction") return {
|
|
953
|
+
type: "instruction",
|
|
954
|
+
...ev.instruction
|
|
955
|
+
};
|
|
956
|
+
if (ev.type === "delivery-attempt") return {
|
|
957
|
+
type: "delivery-attempt",
|
|
958
|
+
...ev.attempt
|
|
959
|
+
};
|
|
960
|
+
if (ev.type === "mail") return {
|
|
961
|
+
type: "mail",
|
|
962
|
+
...ev.mail
|
|
963
|
+
};
|
|
964
|
+
return {
|
|
965
|
+
type: ev.type,
|
|
966
|
+
...ev.down
|
|
967
|
+
};
|
|
968
|
+
};
|
|
969
|
+
const peerMail = opts.peerMail ? createPeerMailbox({
|
|
970
|
+
scope: opts.scope,
|
|
971
|
+
publish: (mail) => bus.publish({
|
|
972
|
+
type: "mail",
|
|
973
|
+
mail
|
|
974
|
+
}, { queue: false }).then(() => void 0),
|
|
975
|
+
...opts.peerMail.limits ? { limits: opts.peerMail.limits } : {}
|
|
976
|
+
}) : void 0;
|
|
977
|
+
const nextQuestionId = (from) => {
|
|
978
|
+
for (;;) {
|
|
979
|
+
const id = `${from}:q${questionSeq++}`;
|
|
980
|
+
if (!questions.some((question) => question.id === id)) return id;
|
|
981
|
+
}
|
|
982
|
+
};
|
|
983
|
+
const normalizeQuestion = (q, fallbackFrom) => {
|
|
984
|
+
const from = str(q.from ?? fallbackFrom, "from");
|
|
985
|
+
return {
|
|
986
|
+
id: typeof q.id === "string" && q.id.length > 0 ? q.id : nextQuestionId(from),
|
|
987
|
+
from,
|
|
988
|
+
level: level(q.level),
|
|
989
|
+
question: str(q.question, "question"),
|
|
990
|
+
reason: str(q.reason, "reason"),
|
|
991
|
+
...q.options ? { options: q.options } : {},
|
|
992
|
+
urgency: urgency(q.urgency)
|
|
993
|
+
};
|
|
994
|
+
};
|
|
995
|
+
const addQuestion = (raw, fallbackFrom, decision) => {
|
|
996
|
+
const q = normalizeQuestion(raw, fallbackFrom);
|
|
997
|
+
const existing = questions.find((x) => x.id === q.id);
|
|
998
|
+
if (existing) return {
|
|
999
|
+
question: existing,
|
|
1000
|
+
added: false
|
|
1001
|
+
};
|
|
1002
|
+
const effectiveDecision = decision ?? (questionPolicy === "bubble" ? {
|
|
1003
|
+
kind: "escalate",
|
|
1004
|
+
to: "parent",
|
|
1005
|
+
reason: "question policy bubbled to parent"
|
|
1006
|
+
} : void 0);
|
|
1007
|
+
const status = effectiveDecision?.kind === "answer" ? "answered" : effectiveDecision?.kind === "defer" ? "deferred" : effectiveDecision?.kind === "escalate" ? "escalated" : "open";
|
|
1008
|
+
const record = {
|
|
1009
|
+
...q,
|
|
1010
|
+
status,
|
|
1011
|
+
openedAt: Date.now(),
|
|
1012
|
+
...effectiveDecision ? { decision: effectiveDecision } : {}
|
|
1013
|
+
};
|
|
1014
|
+
questions.push(record);
|
|
1015
|
+
return {
|
|
1016
|
+
question: record,
|
|
1017
|
+
added: true
|
|
1018
|
+
};
|
|
1019
|
+
};
|
|
1020
|
+
const emitNewQuestion = async (record) => {
|
|
1021
|
+
if (record.added) await bus.publish({
|
|
1022
|
+
type: "question",
|
|
1023
|
+
question: record.question
|
|
1024
|
+
}, { priority: urgencyPriority(record.question.urgency) });
|
|
1025
|
+
return record.question;
|
|
1026
|
+
};
|
|
1027
|
+
const decideQuestion = (questionId, decision) => {
|
|
1028
|
+
const idx = questions.findIndex((q) => q.id === questionId);
|
|
1029
|
+
if (idx < 0) throw new Error(`unknown questionId ${JSON.stringify(questionId)}`);
|
|
1030
|
+
const prior = questions[idx];
|
|
1031
|
+
const status = decision.kind === "answer" ? "answered" : decision.kind === "defer" ? "deferred" : "escalated";
|
|
1032
|
+
const next = {
|
|
1033
|
+
...prior,
|
|
1034
|
+
status,
|
|
1035
|
+
decision
|
|
1036
|
+
};
|
|
1037
|
+
questions[idx] = next;
|
|
1038
|
+
return next;
|
|
1039
|
+
};
|
|
1040
|
+
const blockingQuestionsForStop = () => {
|
|
1041
|
+
if (questionPolicy === "auto" || questionPolicy === "bubble") return [];
|
|
1042
|
+
return questions.filter((q) => {
|
|
1043
|
+
if (!(q.urgency === "blocks-step" || q.urgency === "blocks-run")) return false;
|
|
1044
|
+
if (questionPolicy === "mustDecide") return q.status === "open";
|
|
1045
|
+
return q.status !== "answered" && q.status !== "deferred";
|
|
1046
|
+
});
|
|
1047
|
+
};
|
|
1048
|
+
const maxLiveWorkers = opts.maxLiveWorkers;
|
|
1049
|
+
const localLiveWorkerCount = () => opts.scope.view.nodes.filter((n) => isLiveNodeStatus(n.status)).length;
|
|
1050
|
+
const sharedWorkerCapacity = () => {
|
|
1051
|
+
return opts.scope.workerCapacity;
|
|
1052
|
+
};
|
|
1053
|
+
const usesTreeWideLimit = () => {
|
|
1054
|
+
const capacity = sharedWorkerCapacity();
|
|
1055
|
+
return capacity !== void 0 && capacity.freeSlots !== null;
|
|
1056
|
+
};
|
|
1057
|
+
const liveWorkerCount = () => usesTreeWideLimit() ? sharedWorkerCapacity()?.live ?? localLiveWorkerCount() : localLiveWorkerCount();
|
|
1058
|
+
const projectNodeEvidence = (node, resumed = false) => ({
|
|
1059
|
+
id: node.id,
|
|
1060
|
+
status: node.status,
|
|
1061
|
+
...node.assignmentId === void 0 ? {} : { assignmentId: node.assignmentId },
|
|
1062
|
+
...node.identity === void 0 ? {} : { identity: node.identity },
|
|
1063
|
+
...node.materialization === void 0 ? {} : { materialization: node.materialization },
|
|
1064
|
+
...node.executionBindings === void 0 ? {} : { executionBindings: node.executionBindings },
|
|
1065
|
+
spent: node.spent,
|
|
1066
|
+
...node.settledAt === void 0 ? {} : { settledAt: node.settledAt },
|
|
1067
|
+
...node.outRef === void 0 ? {} : { outRef: node.outRef },
|
|
1068
|
+
...node.trace === void 0 ? {} : { trace: node.trace },
|
|
1069
|
+
...resumed ? { resumed: true } : {}
|
|
1070
|
+
});
|
|
1071
|
+
const liveSnapshot = () => opts.scope.view.nodes.filter((n) => isLiveNodeStatus(n.status)).map((n) => projectNodeEvidence(n));
|
|
1072
|
+
const freeWorkerSlots = () => usesTreeWideLimit() ? sharedWorkerCapacity()?.freeSlots ?? null : freeSlots(localLiveWorkerCount(), maxLiveWorkers);
|
|
1073
|
+
const readProgress = (id) => {
|
|
1074
|
+
const scope = opts.scope;
|
|
1075
|
+
if (typeof scope.progress !== "function") return void 0;
|
|
1076
|
+
try {
|
|
1077
|
+
return scope.progress(id, opts.stallAfterMs !== void 0 ? { stallAfterMs: opts.stallAfterMs } : {});
|
|
1078
|
+
} catch {
|
|
1079
|
+
return;
|
|
1080
|
+
}
|
|
1081
|
+
};
|
|
1082
|
+
const watchers = /* @__PURE__ */ new Map();
|
|
1083
|
+
const watchWorker = (id) => {
|
|
1084
|
+
const watch = opts.watchWorkers;
|
|
1085
|
+
if (!watch) return;
|
|
1086
|
+
const scope = opts.scope;
|
|
1087
|
+
if (typeof scope.traceSource !== "function") return;
|
|
1088
|
+
let source;
|
|
1089
|
+
try {
|
|
1090
|
+
source = scope.traceSource(id);
|
|
1091
|
+
} catch {
|
|
1092
|
+
return;
|
|
1093
|
+
}
|
|
1094
|
+
if (!source) return;
|
|
1095
|
+
const cap = watch.maxFindingsPerWorker ?? 3;
|
|
1096
|
+
let raised = 0;
|
|
1097
|
+
const unsub = watchTrace(source, {
|
|
1098
|
+
...watch.detectors ? { detectors: watch.detectors } : {},
|
|
1099
|
+
onSignal: async (signal, span) => {
|
|
1100
|
+
if (cap > 0 && raised >= cap) return;
|
|
1101
|
+
raised += 1;
|
|
1102
|
+
await bus.publish({
|
|
1103
|
+
type: "finding",
|
|
1104
|
+
finding: canonicalFindingEvent({
|
|
1105
|
+
fromWorker: id,
|
|
1106
|
+
analyst: `online:${signal.detector}`,
|
|
1107
|
+
findings: {
|
|
1108
|
+
detector: signal.detector,
|
|
1109
|
+
severity: signal.severity,
|
|
1110
|
+
reason: signal.reason,
|
|
1111
|
+
streak: signal.streak,
|
|
1112
|
+
...signal.failureClass ? { failureClass: signal.failureClass } : {},
|
|
1113
|
+
toolName: span.toolName,
|
|
1114
|
+
at: span.endedAt,
|
|
1115
|
+
progress: readProgress(id)
|
|
1116
|
+
}
|
|
1117
|
+
})
|
|
1118
|
+
});
|
|
1119
|
+
}
|
|
1120
|
+
});
|
|
1121
|
+
watchers.set(id, unsub);
|
|
1122
|
+
};
|
|
1123
|
+
const unwatchWorker = (id) => {
|
|
1124
|
+
const unsub = watchers.get(id);
|
|
1125
|
+
if (!unsub) return;
|
|
1126
|
+
watchers.delete(id);
|
|
1127
|
+
try {
|
|
1128
|
+
unsub();
|
|
1129
|
+
} catch {}
|
|
1130
|
+
};
|
|
1131
|
+
const awaitTimeoutMs = opts.awaitTimeoutMs ?? 15e3;
|
|
1132
|
+
let inFlightDrain = null;
|
|
1133
|
+
const ensureDrain = () => {
|
|
1134
|
+
if (!inFlightDrain) inFlightDrain = drainSettlement().finally(() => {
|
|
1135
|
+
inFlightDrain = null;
|
|
1136
|
+
});
|
|
1137
|
+
return inFlightDrain;
|
|
1138
|
+
};
|
|
1139
|
+
const raceDrainWithTimeout = async (drain) => {
|
|
1140
|
+
if (awaitTimeoutMs <= 0) return { drained: await drain };
|
|
1141
|
+
let timer;
|
|
1142
|
+
const timeout = new Promise((resolve) => {
|
|
1143
|
+
timer = setTimeout(() => resolve(void 0), awaitTimeoutMs);
|
|
1144
|
+
if (typeof timer?.unref === "function") timer.unref();
|
|
1145
|
+
});
|
|
1146
|
+
try {
|
|
1147
|
+
return await Promise.race([drain.then((drained) => ({ drained })), timeout]);
|
|
1148
|
+
} finally {
|
|
1149
|
+
if (timer) clearTimeout(timer);
|
|
1150
|
+
}
|
|
1151
|
+
};
|
|
1152
|
+
const tools = [
|
|
1153
|
+
{
|
|
1154
|
+
name: "spawn_agent",
|
|
1155
|
+
description: "Start a worker the driver will drive. `profile` is the worker or another driver; `task` is what it should do. Reserves budget from the conserved pool and fails closed. Pass an optional `budget` (per-field) to give a hard sub-task more than the default — it merges over the per-worker default; the conserved pool is still the hard fence. When a max-live-workers cap is set it also fails closed (`error: \"max-live-workers\"`) while that many workers are still in flight — settle or steer one before spawning another. Pass a `key` naming the assignment to make it run-once ACROSS restarts: a key that already completed returns the finished result (`resumed: \"completed\"` — no work re-runs, nothing is spent), a key whose prior attempt failed or was lost with a dead process spawns fresh and says so (`resumed: \"retried\" | \"lost\"`), and a key still running is refused (`error: \"duplicate-key\"`). Returns `freeSlots`: how many MORE workers you can start right now (`null` = uncapped). While `freeSlots > 0` there is idle capacity — call this again to fill it rather than waiting; parallel workers finish the run sooner than one at a time.",
|
|
1156
|
+
inputSchema: {
|
|
1157
|
+
type: "object",
|
|
1158
|
+
properties: {
|
|
1159
|
+
profile: spawnProfileArg(),
|
|
1160
|
+
task: { description: "The task the worker should perform." },
|
|
1161
|
+
label: {
|
|
1162
|
+
type: "string",
|
|
1163
|
+
description: "Optional trace label."
|
|
1164
|
+
},
|
|
1165
|
+
key: {
|
|
1166
|
+
type: "string",
|
|
1167
|
+
description: "Optional semantic name for this assignment (e.g. \"summarize-ch3\"). The same key never runs twice: completed keys return their committed result, even after a coordinator restart."
|
|
1168
|
+
},
|
|
1169
|
+
continuity: {
|
|
1170
|
+
type: "string",
|
|
1171
|
+
enum: ["fresh", "resume"],
|
|
1172
|
+
description: "How this spawn continues the node's prior work. \"fresh\" (the default) starts a brand-new session. \"resume\" re-attaches to the node's most recent SETTLED worker: a NEW live worker is spawned whose session continues where that worker stopped (the backend receives the prior workerId and the resume sequence). Resume fails closed when the node has no settled prior worker (`error: \"resume-no-prior\"` — spawn it fresh first), while a prior worker of the node is still live (`error: \"resume-while-live\"` — steer_agent is the live-worker channel), and under a `key` (`error: \"resume-with-key\"` — keys are run-once, resume runs again). Omit to use the run's declared default for this profile name."
|
|
1173
|
+
},
|
|
1174
|
+
budget: {
|
|
1175
|
+
type: "object",
|
|
1176
|
+
description: "Optional per-spawn budget that merges over the per-worker default (per field). Only set the ceilings this sub-task needs raised; the conserved pool still fences.",
|
|
1177
|
+
properties: {
|
|
1178
|
+
maxIterations: {
|
|
1179
|
+
type: "number",
|
|
1180
|
+
minimum: 0
|
|
1181
|
+
},
|
|
1182
|
+
maxTokens: {
|
|
1183
|
+
type: "number",
|
|
1184
|
+
minimum: 0
|
|
1185
|
+
},
|
|
1186
|
+
maxUsd: {
|
|
1187
|
+
type: "number",
|
|
1188
|
+
minimum: 0
|
|
1189
|
+
},
|
|
1190
|
+
deadlineMs: {
|
|
1191
|
+
type: "number",
|
|
1192
|
+
minimum: 0
|
|
1193
|
+
}
|
|
1194
|
+
}
|
|
1195
|
+
}
|
|
1196
|
+
},
|
|
1197
|
+
required: ["profile", "task"]
|
|
1198
|
+
},
|
|
1199
|
+
handler: async (raw) => {
|
|
1200
|
+
const a = obj(raw);
|
|
1201
|
+
const key = a.key === void 0 ? void 0 : str(a.key, "key");
|
|
1202
|
+
if (!(key !== void 0 && completedKeys.has(key)) && !usesTreeWideLimit() && maxLiveWorkers !== void 0 && maxLiveWorkers > 0 && liveWorkerCount() >= maxLiveWorkers) return Promise.resolve({
|
|
1203
|
+
error: "max-live-workers",
|
|
1204
|
+
live: liveWorkerCount(),
|
|
1205
|
+
freeSlots: freeWorkerSlots()
|
|
1206
|
+
});
|
|
1207
|
+
const parsedProfile = agentProfileSchema.safeParse(a.profile);
|
|
1208
|
+
if (!parsedProfile.success) return Promise.resolve({
|
|
1209
|
+
error: "invalid-profile",
|
|
1210
|
+
issues: parsedProfile.error.issues.map((issue) => ({
|
|
1211
|
+
path: issue.path.join("."),
|
|
1212
|
+
message: issue.message
|
|
1213
|
+
}))
|
|
1214
|
+
});
|
|
1215
|
+
const profile = detachedFrozen(parsedProfile.data);
|
|
1216
|
+
const continuity = resolveContinuity(parseContinuity(a.continuity), profile.name, key);
|
|
1217
|
+
if ("error" in continuity) return Promise.resolve({
|
|
1218
|
+
error: continuity.error,
|
|
1219
|
+
hint: continuity.hint,
|
|
1220
|
+
live: liveWorkerCount(),
|
|
1221
|
+
freeSlots: freeWorkerSlots()
|
|
1222
|
+
});
|
|
1223
|
+
const task = detachedFrozen(a.task);
|
|
1224
|
+
const label = typeof a.label === "string" ? a.label : "worker";
|
|
1225
|
+
if (opts.preflightSpawn) {
|
|
1226
|
+
const preflightProfile = opts.resolveSpawnProfile ? detachedFrozen(opts.resolveSpawnProfile(profile)) : profile;
|
|
1227
|
+
const refusal = await opts.preflightSpawn(preflightProfile, {
|
|
1228
|
+
label,
|
|
1229
|
+
...key !== void 0 ? { key } : {},
|
|
1230
|
+
task
|
|
1231
|
+
});
|
|
1232
|
+
if (refusal) {
|
|
1233
|
+
preflightCounts[refusal.cause] += 1;
|
|
1234
|
+
return {
|
|
1235
|
+
error: "preflight-refused",
|
|
1236
|
+
cause: refusal.cause,
|
|
1237
|
+
detail: refusal.detail,
|
|
1238
|
+
live: liveWorkerCount(),
|
|
1239
|
+
freeSlots: freeWorkerSlots()
|
|
1240
|
+
};
|
|
1241
|
+
}
|
|
1242
|
+
}
|
|
1243
|
+
const budget = Object.freeze(a.budget === void 0 ? opts.perWorker : mergeBudget(opts.perWorker, a.budget));
|
|
1244
|
+
const assignmentId = key !== void 0 ? `key:${key}` : `ordinal:${unkeyedAssignmentOrdinal++}`;
|
|
1245
|
+
const peerMailUrl = peerMail?.mintCapability(assignmentId);
|
|
1246
|
+
const context = Object.freeze({
|
|
1247
|
+
assignmentId,
|
|
1248
|
+
parentNodeId: opts.scope.view.root,
|
|
1249
|
+
budget,
|
|
1250
|
+
task,
|
|
1251
|
+
label,
|
|
1252
|
+
...key !== void 0 ? { key } : {},
|
|
1253
|
+
continuity: continuity.continuity,
|
|
1254
|
+
...continuity.continuity === "resume" ? { resume: continuity.resume } : {},
|
|
1255
|
+
...peerMailUrl !== void 0 ? { peerMailUrl } : {}
|
|
1256
|
+
});
|
|
1257
|
+
const res = opts.scope.spawn(() => opts.makeWorkerAgent(profile, context), task, {
|
|
1258
|
+
budget,
|
|
1259
|
+
label,
|
|
1260
|
+
assignmentId,
|
|
1261
|
+
...key !== void 0 ? { key } : {}
|
|
1262
|
+
});
|
|
1263
|
+
if (res.ok && res.prior?.state === "completed") {
|
|
1264
|
+
const s = res.prior.settled;
|
|
1265
|
+
if (key !== void 0) completedKeys.add(key);
|
|
1266
|
+
const { id, status, resumed: _resumed, ...evidence } = projectSettled(s);
|
|
1267
|
+
return Promise.resolve({
|
|
1268
|
+
workerId: id,
|
|
1269
|
+
resumed: "completed",
|
|
1270
|
+
status,
|
|
1271
|
+
...evidence,
|
|
1272
|
+
live: liveWorkerCount(),
|
|
1273
|
+
freeSlots: freeWorkerSlots()
|
|
1274
|
+
});
|
|
1275
|
+
}
|
|
1276
|
+
if (res.ok) {
|
|
1277
|
+
watchWorker(res.handle.id);
|
|
1278
|
+
liveHandles.set(res.handle.id, res.handle);
|
|
1279
|
+
peerMail?.bindCapability(assignmentId, res.handle.id);
|
|
1280
|
+
if (key !== void 0) keyByWorker.set(res.handle.id, key);
|
|
1281
|
+
if (typeof profile.name === "string" && profile.name.length > 0) profileNameByWorker.set(res.handle.id, profile.name);
|
|
1282
|
+
}
|
|
1283
|
+
const priorHistory = res.ok && res.prior !== void 0 && res.prior.state !== "completed" ? {
|
|
1284
|
+
resumed: res.prior.state,
|
|
1285
|
+
priorWorkerId: res.prior.priorId,
|
|
1286
|
+
...res.prior.state === "retried" ? { priorReason: res.prior.reason } : {}
|
|
1287
|
+
} : {};
|
|
1288
|
+
return Promise.resolve(res.ok ? {
|
|
1289
|
+
workerId: res.handle.id,
|
|
1290
|
+
assignmentId: res.handle.assignmentId ?? assignmentId,
|
|
1291
|
+
...res.handle.identity === void 0 ? {} : { identity: res.handle.identity },
|
|
1292
|
+
...res.handle.materialization === void 0 ? {} : { materialization: res.handle.materialization },
|
|
1293
|
+
...res.handle.executionBindings === void 0 ? {} : { executionBindings: res.handle.executionBindings },
|
|
1294
|
+
continuity: continuity.continuity,
|
|
1295
|
+
...continuity.continuity === "resume" ? { resume: continuity.resume } : {},
|
|
1296
|
+
live: liveWorkerCount(),
|
|
1297
|
+
freeSlots: freeWorkerSlots(),
|
|
1298
|
+
...priorHistory
|
|
1299
|
+
} : {
|
|
1300
|
+
error: res.reason,
|
|
1301
|
+
...res.reason === "usd-unbudgeted" ? { hint: "This run's root budget declares no maxUsd, so a child budget naming maxUsd can never be admitted — at any amount. Retrying with a smaller maxUsd will fail identically. Spawn with a budget that omits maxUsd, or ask the caller to give the run a root maxUsd." } : {},
|
|
1302
|
+
live: liveWorkerCount(),
|
|
1303
|
+
freeSlots: freeWorkerSlots()
|
|
1304
|
+
});
|
|
1305
|
+
}
|
|
1306
|
+
},
|
|
1307
|
+
{
|
|
1308
|
+
name: "observe_agent",
|
|
1309
|
+
description: "Inspect a worker WHILE IT RUNS, not only after it finishes: status, spend so far, and `progress` — how long since it last did anything (`idleMs`), whether that counts as stalled, how many turns it has taken, the last tools/files it touched (`recentActivity`), what its executor CHANGED about the profile you gave it (`derived` — an MCP config it materialized, an extension it had to add), whether a steer can even reach it (`steerable`), and how many steers it has not yet read (`pendingMessages`). Returns the settled output artifact once it exists. Use this BEFORE steer_agent: a steer is only worth sending when the progress says the worker is on the wrong path or has stopped making any.",
|
|
1310
|
+
inputSchema: {
|
|
1311
|
+
type: "object",
|
|
1312
|
+
properties: { workerId: idArg },
|
|
1313
|
+
required: ["workerId"]
|
|
1314
|
+
},
|
|
1315
|
+
handler: async (raw) => {
|
|
1316
|
+
const id = str(obj(raw).workerId, "workerId");
|
|
1317
|
+
const node = opts.scope.view.nodes.find((n) => n.id === id);
|
|
1318
|
+
if (!node) {
|
|
1319
|
+
const resumed = opts.scope.resume?.view.nodes.find((n) => n.id === id);
|
|
1320
|
+
if (!resumed) return { error: `unknown workerId ${JSON.stringify(id)}` };
|
|
1321
|
+
const output = resumed.outRef ? await opts.blobs.get(resumed.outRef) : void 0;
|
|
1322
|
+
return {
|
|
1323
|
+
...projectNodeEvidence(resumed, true),
|
|
1324
|
+
outRef: resumed.outRef ?? null,
|
|
1325
|
+
output: output ?? null,
|
|
1326
|
+
progress: null
|
|
1327
|
+
};
|
|
1328
|
+
}
|
|
1329
|
+
const output = node.outRef ? await opts.blobs.get(node.outRef) : void 0;
|
|
1330
|
+
const progress = readProgress(id);
|
|
1331
|
+
return {
|
|
1332
|
+
...projectNodeEvidence(node),
|
|
1333
|
+
outRef: node.outRef ?? null,
|
|
1334
|
+
output: output ?? null,
|
|
1335
|
+
progress: progress ?? null
|
|
1336
|
+
};
|
|
1337
|
+
}
|
|
1338
|
+
},
|
|
1339
|
+
{
|
|
1340
|
+
name: "steer_agent",
|
|
1341
|
+
description: "Send a message DOWN to a still-LIVE worker (parent→child): a new instruction, a course correction, or a continuation. The worker drains it at its next step boundary — and before it may settle, so it cannot finish while a message it never read is pending. A worker that already settled is gone (returns delivered:false) — spawn a fresh one instead.",
|
|
1342
|
+
inputSchema: {
|
|
1343
|
+
type: "object",
|
|
1344
|
+
properties: {
|
|
1345
|
+
workerId: idArg,
|
|
1346
|
+
instruction: {
|
|
1347
|
+
type: "string",
|
|
1348
|
+
description: "What the worker should do next."
|
|
1349
|
+
},
|
|
1350
|
+
interrupt: {
|
|
1351
|
+
type: "boolean",
|
|
1352
|
+
description: "true = forceful: abort the worker’s in-flight inference so it re-plans on the NEXT turn (a tool already mid-execution finishes first; only the owned tool-loop honors this). false/omitted = queued: it flushes at the next step boundary (and before it may settle)."
|
|
1353
|
+
}
|
|
1354
|
+
},
|
|
1355
|
+
required: ["workerId", "instruction"]
|
|
1356
|
+
},
|
|
1357
|
+
handler: async (raw) => {
|
|
1358
|
+
const a = obj(raw);
|
|
1359
|
+
const workerId = str(a.workerId, "workerId");
|
|
1360
|
+
const instruction = str(a.instruction, "instruction");
|
|
1361
|
+
const delivery = await steerWorker(workerId, instruction, { interrupt: a.interrupt === true });
|
|
1362
|
+
if (delivery.delivered) return {
|
|
1363
|
+
delivered: true,
|
|
1364
|
+
progress: readProgress(workerId) ?? null
|
|
1365
|
+
};
|
|
1366
|
+
return {
|
|
1367
|
+
delivered: false,
|
|
1368
|
+
reason: delivery.outcome,
|
|
1369
|
+
progress: readProgress(workerId) ?? null
|
|
1370
|
+
};
|
|
1371
|
+
}
|
|
1372
|
+
},
|
|
1373
|
+
{
|
|
1374
|
+
name: "await_event",
|
|
1375
|
+
description: "Wait for and pull the next message a worker, sub-driver, or analyst sent up — the unified inbox. An event is one of: a settled worker output ('settled'), a question needing your answer ('question', from ask_parent / the worker's ask-user), or a trace-analyst finding ('finding', from analyze-on-settle). Pass kinds:['settled'] for just the next finished worker; omit `kinds` to also receive questions and findings. Returns { idle: true } when nothing is queued and no workers are live. If a worker is still running when the wait elapses, returns { pending: true, live: [...] } (the workers still in flight) instead of blocking indefinitely — call await_event again to keep waiting; the settlement is not lost. Every reply carries `freeSlots`: how many more workers you can start right now (`null` = uncapped). A settled worker frees its slot, so `freeSlots > 0` means capacity is sitting idle — spawn into it before waiting again.",
|
|
1376
|
+
inputSchema: {
|
|
1377
|
+
type: "object",
|
|
1378
|
+
properties: { kinds: {
|
|
1379
|
+
type: "array",
|
|
1380
|
+
items: {
|
|
1381
|
+
type: "string",
|
|
1382
|
+
enum: [...awaitableEventKinds]
|
|
1383
|
+
},
|
|
1384
|
+
description: "Restrict to these event kinds (any if omitted)."
|
|
1385
|
+
} }
|
|
1386
|
+
},
|
|
1387
|
+
handler: async (raw) => {
|
|
1388
|
+
const k = obj(raw).kinds;
|
|
1389
|
+
const kinds = Array.isArray(k) ? k.filter(isAwaitableEventKind) : void 0;
|
|
1390
|
+
let ev = bus.pull(kinds);
|
|
1391
|
+
if (ev) return {
|
|
1392
|
+
...projectEvent(ev),
|
|
1393
|
+
freeSlots: freeWorkerSlots()
|
|
1394
|
+
};
|
|
1395
|
+
const raced = await raceDrainWithTimeout(ensureDrain());
|
|
1396
|
+
if (raced === void 0) return {
|
|
1397
|
+
pending: true,
|
|
1398
|
+
live: liveSnapshot(),
|
|
1399
|
+
freeSlots: freeWorkerSlots()
|
|
1400
|
+
};
|
|
1401
|
+
ev = bus.pull(kinds);
|
|
1402
|
+
if (!ev) return {
|
|
1403
|
+
idle: !raced.drained,
|
|
1404
|
+
freeSlots: freeWorkerSlots()
|
|
1405
|
+
};
|
|
1406
|
+
return {
|
|
1407
|
+
...projectEvent(ev),
|
|
1408
|
+
freeSlots: freeWorkerSlots()
|
|
1409
|
+
};
|
|
1410
|
+
}
|
|
1411
|
+
},
|
|
1412
|
+
{
|
|
1413
|
+
name: "list_questions",
|
|
1414
|
+
description: "List questions raised by workers, drivers, or analysts. Blocking stop behavior follows questionPolicy.",
|
|
1415
|
+
inputSchema: {
|
|
1416
|
+
type: "object",
|
|
1417
|
+
properties: {}
|
|
1418
|
+
},
|
|
1419
|
+
handler: () => Promise.resolve({ questions })
|
|
1420
|
+
},
|
|
1421
|
+
{
|
|
1422
|
+
name: "answer_question",
|
|
1423
|
+
description: "Record an answer, deferral, or escalation for a loop question.",
|
|
1424
|
+
inputSchema: {
|
|
1425
|
+
type: "object",
|
|
1426
|
+
properties: {
|
|
1427
|
+
questionId: { type: "string" },
|
|
1428
|
+
answer: { type: "string" },
|
|
1429
|
+
by: {
|
|
1430
|
+
type: "string",
|
|
1431
|
+
description: "Node id or \"user\"."
|
|
1432
|
+
},
|
|
1433
|
+
deferReason: { type: "string" },
|
|
1434
|
+
escalateTo: {
|
|
1435
|
+
type: "string",
|
|
1436
|
+
enum: questionEscalationTargets
|
|
1437
|
+
},
|
|
1438
|
+
escalateReason: { type: "string" }
|
|
1439
|
+
},
|
|
1440
|
+
required: ["questionId"]
|
|
1441
|
+
},
|
|
1442
|
+
handler: async (raw) => {
|
|
1443
|
+
const a = obj(raw);
|
|
1444
|
+
const questionId = str(a.questionId, "questionId");
|
|
1445
|
+
if (typeof a.answer === "string" && a.answer.length > 0) {
|
|
1446
|
+
const answer = a.answer;
|
|
1447
|
+
const pendingQuestion = questions.find((question) => question.id === questionId);
|
|
1448
|
+
if (pendingQuestion === void 0) throw new Error(`unknown questionId ${JSON.stringify(questionId)}`);
|
|
1449
|
+
const interrupt = pendingQuestion.urgency === "blocks-run" || pendingQuestion.urgency === "blocks-step";
|
|
1450
|
+
const authorized = authorizeInstruction("answer", pendingQuestion.from, answer, interrupt, questionId);
|
|
1451
|
+
await recordInstruction(authorized);
|
|
1452
|
+
const delivery = await attemptDelivery(authorized, {
|
|
1453
|
+
answer: authorized.instruction,
|
|
1454
|
+
questionId,
|
|
1455
|
+
interrupt
|
|
1456
|
+
});
|
|
1457
|
+
return {
|
|
1458
|
+
question: delivery.delivered ? decideQuestion(questionId, {
|
|
1459
|
+
kind: "answer",
|
|
1460
|
+
answer: authorized.instruction,
|
|
1461
|
+
by: typeof a.by === "string" && a.by.length > 0 ? a.by : "user"
|
|
1462
|
+
}) : pendingQuestion,
|
|
1463
|
+
delivered: delivery.delivered,
|
|
1464
|
+
...delivery.delivered ? {} : { reason: delivery.outcome }
|
|
1465
|
+
};
|
|
1466
|
+
}
|
|
1467
|
+
if (typeof a.deferReason === "string" && a.deferReason.length > 0) return Promise.resolve({ question: decideQuestion(questionId, {
|
|
1468
|
+
kind: "defer",
|
|
1469
|
+
reason: a.deferReason
|
|
1470
|
+
}) });
|
|
1471
|
+
if (typeof a.escalateTo === "string" && a.escalateTo.length > 0) {
|
|
1472
|
+
if (!isQuestionEscalationTarget(a.escalateTo)) throw new Error(`answer_question: escalateTo must be one of ${questionEscalationTargets.join(", ")}; received ${JSON.stringify(a.escalateTo)}`);
|
|
1473
|
+
const escalateReason = typeof a.escalateReason === "string" && a.escalateReason.length > 0 ? a.escalateReason : "driver escalated";
|
|
1474
|
+
return Promise.resolve({ question: decideQuestion(questionId, {
|
|
1475
|
+
kind: "escalate",
|
|
1476
|
+
to: a.escalateTo,
|
|
1477
|
+
reason: escalateReason
|
|
1478
|
+
}) });
|
|
1479
|
+
}
|
|
1480
|
+
throw new Error("answer_question: provide answer, deferReason, or escalateTo");
|
|
1481
|
+
}
|
|
1482
|
+
},
|
|
1483
|
+
{
|
|
1484
|
+
name: "ask_parent",
|
|
1485
|
+
description: "Raise a question to the parent driver/Pi/user when this driver cannot decide.",
|
|
1486
|
+
inputSchema: {
|
|
1487
|
+
type: "object",
|
|
1488
|
+
properties: {
|
|
1489
|
+
from: { type: "string" },
|
|
1490
|
+
level: {
|
|
1491
|
+
type: "string",
|
|
1492
|
+
enum: [
|
|
1493
|
+
"worker",
|
|
1494
|
+
"driver",
|
|
1495
|
+
"loop"
|
|
1496
|
+
]
|
|
1497
|
+
},
|
|
1498
|
+
question: { type: "string" },
|
|
1499
|
+
reason: { type: "string" },
|
|
1500
|
+
urgency: {
|
|
1501
|
+
type: "string",
|
|
1502
|
+
enum: [
|
|
1503
|
+
"continue-without",
|
|
1504
|
+
"blocks-step",
|
|
1505
|
+
"blocks-run"
|
|
1506
|
+
]
|
|
1507
|
+
}
|
|
1508
|
+
},
|
|
1509
|
+
required: [
|
|
1510
|
+
"from",
|
|
1511
|
+
"level",
|
|
1512
|
+
"question",
|
|
1513
|
+
"reason",
|
|
1514
|
+
"urgency"
|
|
1515
|
+
]
|
|
1516
|
+
},
|
|
1517
|
+
handler: async (raw) => {
|
|
1518
|
+
const a = obj(raw);
|
|
1519
|
+
const from = str(a.from, "from");
|
|
1520
|
+
return { question: await emitNewQuestion(addQuestion({
|
|
1521
|
+
from,
|
|
1522
|
+
level: level(a.level),
|
|
1523
|
+
question: str(a.question, "question"),
|
|
1524
|
+
reason: str(a.reason, "reason"),
|
|
1525
|
+
urgency: urgency(a.urgency)
|
|
1526
|
+
}, from, {
|
|
1527
|
+
kind: "escalate",
|
|
1528
|
+
to: "parent",
|
|
1529
|
+
reason: "asked parent"
|
|
1530
|
+
})) };
|
|
1531
|
+
}
|
|
1532
|
+
},
|
|
1533
|
+
...deliverable ? [{
|
|
1534
|
+
name: "submit_result",
|
|
1535
|
+
description: [
|
|
1536
|
+
"Submit the complete result to the injected independent check.",
|
|
1537
|
+
"The first passing result is retained; stop work when accepted.",
|
|
1538
|
+
...deliverable.describe ? [`Expected result: ${deliverable.describe}`] : []
|
|
1539
|
+
].join(" "),
|
|
1540
|
+
inputSchema: {
|
|
1541
|
+
type: "object",
|
|
1542
|
+
properties: { result: { description: "The complete result in the form requested by the task." } },
|
|
1543
|
+
required: ["result"],
|
|
1544
|
+
additionalProperties: false
|
|
1545
|
+
},
|
|
1546
|
+
handler: async (raw) => {
|
|
1547
|
+
if (submitted) return {
|
|
1548
|
+
accepted: true,
|
|
1549
|
+
retained: "earlier-passing-result",
|
|
1550
|
+
stop: true
|
|
1551
|
+
};
|
|
1552
|
+
const a = obj(raw);
|
|
1553
|
+
if (!Object.hasOwn(a, "result")) throw new Error("submit_result: \"result\" is required");
|
|
1554
|
+
const result = structuredClone(a.result);
|
|
1555
|
+
let accepted = false;
|
|
1556
|
+
try {
|
|
1557
|
+
accepted = await deliverable.check(result) === true;
|
|
1558
|
+
} catch {
|
|
1559
|
+
accepted = false;
|
|
1560
|
+
}
|
|
1561
|
+
if (!accepted) return {
|
|
1562
|
+
accepted: false,
|
|
1563
|
+
stop: false
|
|
1564
|
+
};
|
|
1565
|
+
if (submitted) return {
|
|
1566
|
+
accepted: true,
|
|
1567
|
+
retained: "earlier-passing-result",
|
|
1568
|
+
stop: true
|
|
1569
|
+
};
|
|
1570
|
+
submitted = Object.freeze({ result });
|
|
1571
|
+
stopped = true;
|
|
1572
|
+
reason = "result-accepted";
|
|
1573
|
+
notifyStop();
|
|
1574
|
+
return {
|
|
1575
|
+
accepted: true,
|
|
1576
|
+
retained: "this-result",
|
|
1577
|
+
stop: true
|
|
1578
|
+
};
|
|
1579
|
+
}
|
|
1580
|
+
}] : [],
|
|
1581
|
+
{
|
|
1582
|
+
name: "stop",
|
|
1583
|
+
description: "Declare the run complete.",
|
|
1584
|
+
inputSchema: {
|
|
1585
|
+
type: "object",
|
|
1586
|
+
properties: { reason: {
|
|
1587
|
+
type: "string",
|
|
1588
|
+
description: "Why you are stopping."
|
|
1589
|
+
} }
|
|
1590
|
+
},
|
|
1591
|
+
handler: (raw) => {
|
|
1592
|
+
const blocking = blockingQuestionsForStop();
|
|
1593
|
+
if (blocking.length) return Promise.resolve({
|
|
1594
|
+
stopped: false,
|
|
1595
|
+
error: "unresolved-blocking-questions",
|
|
1596
|
+
questions: blocking
|
|
1597
|
+
});
|
|
1598
|
+
stopped = true;
|
|
1599
|
+
const r = obj(raw).reason;
|
|
1600
|
+
reason = typeof r === "string" ? r : void 0;
|
|
1601
|
+
notifyStop();
|
|
1602
|
+
return Promise.resolve({ stopped: true });
|
|
1603
|
+
}
|
|
1604
|
+
}
|
|
1605
|
+
];
|
|
1606
|
+
if (opts.analysts) {
|
|
1607
|
+
tools.push({
|
|
1608
|
+
name: "list_analysts",
|
|
1609
|
+
description: "List trace-analyst lenses available to run over a settled worker.",
|
|
1610
|
+
inputSchema: {
|
|
1611
|
+
type: "object",
|
|
1612
|
+
properties: {}
|
|
1613
|
+
},
|
|
1614
|
+
handler: () => Promise.resolve({ analysts: opts.analysts?.kinds })
|
|
1615
|
+
});
|
|
1616
|
+
tools.push({
|
|
1617
|
+
name: "run_analyst",
|
|
1618
|
+
description: "Apply an analyst lens to a settled worker trace.",
|
|
1619
|
+
inputSchema: {
|
|
1620
|
+
type: "object",
|
|
1621
|
+
properties: {
|
|
1622
|
+
kind: {
|
|
1623
|
+
type: "string",
|
|
1624
|
+
description: "The analyst kind id."
|
|
1625
|
+
},
|
|
1626
|
+
workerId: idArg
|
|
1627
|
+
},
|
|
1628
|
+
required: ["kind", "workerId"]
|
|
1629
|
+
},
|
|
1630
|
+
handler: async (raw) => {
|
|
1631
|
+
const a = obj(raw);
|
|
1632
|
+
const id = str(a.workerId, "workerId");
|
|
1633
|
+
const node = nodeForWorker(id);
|
|
1634
|
+
if (!node) return { error: `unknown workerId ${JSON.stringify(id)}` };
|
|
1635
|
+
if (isLiveNodeStatus(node.status)) return { error: `worker ${JSON.stringify(id)} has not settled — no trace to analyze yet` };
|
|
1636
|
+
const trace = ledger.find((worker) => worker.id === id)?.trace ?? node.trace ?? {
|
|
1637
|
+
status: "unavailable",
|
|
1638
|
+
reason: "legacy-settlement-without-trace-evidence"
|
|
1639
|
+
};
|
|
1640
|
+
let store;
|
|
1641
|
+
try {
|
|
1642
|
+
store = await workerTraceAnalysisStore(trace, opts.blobs);
|
|
1643
|
+
} catch (error) {
|
|
1644
|
+
return {
|
|
1645
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1646
|
+
trace
|
|
1647
|
+
};
|
|
1648
|
+
}
|
|
1649
|
+
return { findings: await opts.analysts?.run(str(a.kind, "kind"), store) };
|
|
1650
|
+
}
|
|
1651
|
+
});
|
|
1652
|
+
}
|
|
1653
|
+
const abortWorker = (ref, reason) => {
|
|
1654
|
+
const live = opts.scope.view.nodes.filter((node) => isLiveNodeStatus(node.status));
|
|
1655
|
+
const target = live.find((node) => node.id === ref) ?? live.find((node) => profileNameByWorker.get(node.id) === ref) ?? live.find((node) => node.label === ref);
|
|
1656
|
+
if (target === void 0) return void 0;
|
|
1657
|
+
const handle = liveHandles.get(target.id);
|
|
1658
|
+
if (handle === void 0) return void 0;
|
|
1659
|
+
handle.abort(reason);
|
|
1660
|
+
return {
|
|
1661
|
+
id: target.id,
|
|
1662
|
+
label: target.label
|
|
1663
|
+
};
|
|
1664
|
+
};
|
|
1665
|
+
return {
|
|
1666
|
+
tools,
|
|
1667
|
+
ready,
|
|
1668
|
+
history: () => bus.history(),
|
|
1669
|
+
raiseFinding: (finding) => bus.publish({
|
|
1670
|
+
type: "finding",
|
|
1671
|
+
finding: canonicalFindingEvent(finding)
|
|
1672
|
+
}).then(() => void 0),
|
|
1673
|
+
steerWorker,
|
|
1674
|
+
stats: () => opts.preflightSpawn === void 0 ? bus.stats() : {
|
|
1675
|
+
...bus.stats(),
|
|
1676
|
+
preflight: { ...preflightCounts }
|
|
1677
|
+
},
|
|
1678
|
+
isStopped: () => stopped,
|
|
1679
|
+
stopReason: () => reason,
|
|
1680
|
+
submittedResult: () => submitted,
|
|
1681
|
+
settled: () => ledger,
|
|
1682
|
+
questions: () => questions,
|
|
1683
|
+
drainResolved,
|
|
1684
|
+
abortWorker,
|
|
1685
|
+
...peerMail ? { peerMail } : {}
|
|
1686
|
+
};
|
|
1687
|
+
}
|
|
1688
|
+
function nextUnkeyedAssignmentOrdinal(scope) {
|
|
1689
|
+
let next = 0;
|
|
1690
|
+
const views = [scope.resume?.view, scope.view];
|
|
1691
|
+
for (const view of views) {
|
|
1692
|
+
if (view === void 0) continue;
|
|
1693
|
+
for (const node of view.nodes) {
|
|
1694
|
+
const match = /^ordinal:(\d+)$/.exec(node.assignmentId ?? "");
|
|
1695
|
+
if (match === null) continue;
|
|
1696
|
+
const ordinal = Number(match[1]);
|
|
1697
|
+
if (!Number.isSafeInteger(ordinal)) throw new Error(`coordination: durable assignment id '${node.assignmentId}' exceeds the safe ordinal range`);
|
|
1698
|
+
next = Math.max(next, ordinal + 1);
|
|
1699
|
+
}
|
|
1700
|
+
}
|
|
1701
|
+
if (!Number.isSafeInteger(next)) throw new Error("coordination: durable assignment ordinal space is exhausted");
|
|
1702
|
+
return next;
|
|
1703
|
+
}
|
|
1704
|
+
/** Stringify a findings payload for a routed delivery; never throws (a cyclic payload degrades to
|
|
1705
|
+
* its String form rather than killing the settle path). */
|
|
1706
|
+
function safeJsonText(value) {
|
|
1707
|
+
if (typeof value === "string") return value;
|
|
1708
|
+
try {
|
|
1709
|
+
return JSON.stringify(value) ?? String(value);
|
|
1710
|
+
} catch {
|
|
1711
|
+
return String(value);
|
|
1712
|
+
}
|
|
1713
|
+
}
|
|
1714
|
+
//#endregion
|
|
1715
|
+
//#region src/runtime/supervise/coordination-log.ts
|
|
1716
|
+
/**
|
|
1717
|
+
* Durable side-log for coordination evidence the spawn journal does not own: questions, analyst
|
|
1718
|
+
* findings, answer decisions, authorized continuation receipts, delivery-attempt markers, and
|
|
1719
|
+
* delivery outcomes. A durable run
|
|
1720
|
+
* (`supervise({ runDir })`) appends them as they publish and loads them on resume, so a restarted
|
|
1721
|
+
* coordinator retains the exact evidence produced by prior processes.
|
|
1722
|
+
*
|
|
1723
|
+
* Answer down-events also fold status on load: a question answered before the crash reloads as
|
|
1724
|
+
* `answered`, not as a re-blocking `open`. Settled events are skipped (the spawn journal is their
|
|
1725
|
+
* ledger). A receipt followed by an attempt but no outcome proves the process died in the delivery
|
|
1726
|
+
* window; that outcome remains unknown and no prior instruction is auto-delivered.
|
|
1727
|
+
*
|
|
1728
|
+
* JSONL, one fsynced record per event, keyed by `runId` — several runs may share one log file
|
|
1729
|
+
* exactly as they share one spawn-journal file.
|
|
1730
|
+
*
|
|
1731
|
+
* @experimental
|
|
1732
|
+
*/
|
|
1733
|
+
/** Persist prior context plus exact continuation authorization, attempt, and result evidence.
|
|
1734
|
+
* Settlements have their own journal. */
|
|
1735
|
+
function persisted(event) {
|
|
1736
|
+
return event.type !== "settled";
|
|
1737
|
+
}
|
|
1738
|
+
/** FS-backed `CoordinationLog`: append-only JSONL, fsynced per record. */
|
|
1739
|
+
var FileCoordinationLog = class {
|
|
1740
|
+
path;
|
|
1741
|
+
appendTail = Promise.resolve();
|
|
1742
|
+
constructor(path) {
|
|
1743
|
+
this.path = path;
|
|
1744
|
+
}
|
|
1745
|
+
async append(runId, record, ownerId) {
|
|
1746
|
+
if (!persisted(record.event)) return;
|
|
1747
|
+
const append = this.appendTail.then(() => this.appendRecord(runId, record, ownerId));
|
|
1748
|
+
this.appendTail = append.catch(() => void 0);
|
|
1749
|
+
return append;
|
|
1750
|
+
}
|
|
1751
|
+
async appendRecord(runId, busRecord, ownerId) {
|
|
1752
|
+
const fs = await import("node:fs/promises");
|
|
1753
|
+
const path = await import("node:path");
|
|
1754
|
+
await fs.mkdir(path.dirname(this.path), { recursive: true });
|
|
1755
|
+
const record = {
|
|
1756
|
+
runId,
|
|
1757
|
+
...ownerId !== void 0 ? { ownerId } : {},
|
|
1758
|
+
...busRecord
|
|
1759
|
+
};
|
|
1760
|
+
const needsSeparator = await prepareJsonlAppend(this.path);
|
|
1761
|
+
const fh = await fs.open(this.path, "a");
|
|
1762
|
+
try {
|
|
1763
|
+
await writeAllBytes(fh, `${needsSeparator ? "\n" : ""}${JSON.stringify(record)}\n`);
|
|
1764
|
+
await fh.sync();
|
|
1765
|
+
} finally {
|
|
1766
|
+
await fh.close();
|
|
1767
|
+
}
|
|
1768
|
+
}
|
|
1769
|
+
async load(runId, ownerId) {
|
|
1770
|
+
const fs = await import("node:fs/promises");
|
|
1771
|
+
let text;
|
|
1772
|
+
try {
|
|
1773
|
+
text = await fs.readFile(this.path, "utf8");
|
|
1774
|
+
} catch (err) {
|
|
1775
|
+
if (isNoEntError(err)) return emptyPriorCoordination(ownerId);
|
|
1776
|
+
throw err;
|
|
1777
|
+
}
|
|
1778
|
+
const byId = /* @__PURE__ */ new Map();
|
|
1779
|
+
const findings = [];
|
|
1780
|
+
const continuations = [];
|
|
1781
|
+
const deliveryEvidence = [];
|
|
1782
|
+
const mail = [];
|
|
1783
|
+
const records = [];
|
|
1784
|
+
let legacySeq = 0;
|
|
1785
|
+
for (const stored of parseCommittedJsonLines(text, this.path)) {
|
|
1786
|
+
if (stored.runId !== runId) continue;
|
|
1787
|
+
if (ownerId !== void 0 && stored.ownerId !== ownerId) continue;
|
|
1788
|
+
const record = "seq" in stored ? {
|
|
1789
|
+
seq: stored.seq,
|
|
1790
|
+
at: stored.at,
|
|
1791
|
+
priority: stored.priority,
|
|
1792
|
+
event: stored.event
|
|
1793
|
+
} : {
|
|
1794
|
+
seq: legacySeq++,
|
|
1795
|
+
at: Date.parse(stored.at),
|
|
1796
|
+
priority: 0,
|
|
1797
|
+
event: stored.event
|
|
1798
|
+
};
|
|
1799
|
+
records.push(record);
|
|
1800
|
+
const ev = record.event;
|
|
1801
|
+
if (ev.type === "delivery-attempt" || ev.type === "steer" || ev.type === "answer") deliveryEvidence.push(ev);
|
|
1802
|
+
if (ev.type === "question") byId.set(ev.question.id, ev.question);
|
|
1803
|
+
else if (ev.type === "finding") findings.push(ev.finding);
|
|
1804
|
+
else if (ev.type === "answer") {
|
|
1805
|
+
const prior = byId.get(ev.questionId);
|
|
1806
|
+
if (prior && ev.down.delivered) byId.set(ev.questionId, {
|
|
1807
|
+
...prior,
|
|
1808
|
+
status: "answered",
|
|
1809
|
+
decision: {
|
|
1810
|
+
kind: "answer",
|
|
1811
|
+
answer: ev.down.instruction,
|
|
1812
|
+
by: "prior-run"
|
|
1813
|
+
}
|
|
1814
|
+
});
|
|
1815
|
+
} else if (ev.type === "instruction") continuations.push(ev.instruction);
|
|
1816
|
+
else if (ev.type === "mail") mail.push(ev.mail);
|
|
1817
|
+
}
|
|
1818
|
+
return {
|
|
1819
|
+
...ownerId !== void 0 ? { ownerId } : {},
|
|
1820
|
+
questions: [...byId.values()],
|
|
1821
|
+
findings,
|
|
1822
|
+
continuations,
|
|
1823
|
+
deliveryEvidence,
|
|
1824
|
+
mail,
|
|
1825
|
+
records
|
|
1826
|
+
};
|
|
1827
|
+
}
|
|
1828
|
+
};
|
|
1829
|
+
function emptyPriorCoordination(ownerId) {
|
|
1830
|
+
return {
|
|
1831
|
+
...ownerId !== void 0 ? { ownerId } : {},
|
|
1832
|
+
questions: [],
|
|
1833
|
+
findings: [],
|
|
1834
|
+
continuations: [],
|
|
1835
|
+
deliveryEvidence: [],
|
|
1836
|
+
mail: [],
|
|
1837
|
+
records: []
|
|
1838
|
+
};
|
|
1839
|
+
}
|
|
1840
|
+
//#endregion
|
|
1841
|
+
//#region src/runtime/supervise/run-context.ts
|
|
1842
|
+
/**
|
|
1843
|
+
*
|
|
1844
|
+
* `createInMemoryRunContext` — the one-call bundle of the in-memory stores a
|
|
1845
|
+
* `createSupervisor().run(root, task, opts)` needs: a fresh `InMemorySpawnJournal`
|
|
1846
|
+
* (the event-sourced spawn log), a fresh `InMemoryResultBlobStore` (the
|
|
1847
|
+
* content-addressed `outRef` payload store the driver's `observe`/`finalize` reads
|
|
1848
|
+
* settled outputs through), and a fresh `createExecutorRegistry()` (the open
|
|
1849
|
+
* `AgentSpec → Executor` resolver).
|
|
1850
|
+
*
|
|
1851
|
+
* It exists to kill the boilerplate every offline/local supervised run repeats by
|
|
1852
|
+
* hand — three constructors threaded into `SupervisorOpts` — and to single-source the
|
|
1853
|
+
* ONE wiring invariant that is easy to get wrong: when the root is the recursive
|
|
1854
|
+
* `driverAgent` LLM-driver brain AND it may spawn DRIVER children (agents
|
|
1855
|
+
* driving agents), the registry MUST be wrapped with `withDriverExecutor` so a
|
|
1856
|
+
* `role: 'driver'` child resolves to the nested-scope executor — and that SAME blob
|
|
1857
|
+
* store MUST be the one passed to `driverAgent({ blobs })`, or the driver
|
|
1858
|
+
* reads from a different store than the scope writes to. Pass `{ withDriver: true }`
|
|
1859
|
+
* and reuse the returned `blobs` for both.
|
|
1860
|
+
*
|
|
1861
|
+
* The spread shape matches `SupervisorOpts` exactly, so the call site reads:
|
|
1862
|
+
* const run = createInMemoryRunContext()
|
|
1863
|
+
* await createSupervisor().run(root, task, { budget, runId, ...run })
|
|
1864
|
+
*
|
|
1865
|
+
* @experimental
|
|
1866
|
+
*/
|
|
1867
|
+
/**
|
|
1868
|
+
* Build a fresh in-memory run context. Every call returns NEW stores (no shared global
|
|
1869
|
+
* state between runs), so two runs never cross-contaminate their journals/blobs.
|
|
1870
|
+
*/
|
|
1871
|
+
function createInMemoryRunContext(opts = {}) {
|
|
1872
|
+
const base = createExecutorRegistry();
|
|
1873
|
+
return {
|
|
1874
|
+
journal: new InMemorySpawnJournal(),
|
|
1875
|
+
blobs: new InMemoryResultBlobStore(),
|
|
1876
|
+
executors: opts.withDriver ? withDriverExecutor(base) : base
|
|
1877
|
+
};
|
|
1878
|
+
}
|
|
1879
|
+
/**
|
|
1880
|
+
* Build a DURABLE run context: the spawn journal and the result blobs are file-backed (fsynced
|
|
1881
|
+
* per append/write) under `dir`, and the context carries `resume: true` so spreading it into
|
|
1882
|
+
* `SupervisorOpts` makes the supervisor `loadTree`-first. A run that dies mid-flight therefore
|
|
1883
|
+
* resumes when it is re-run with the SAME `runId` and the SAME `dir`: the committed children come
|
|
1884
|
+
* back on `Scope.resume` (rehydrated by `replaySpawnTree`) instead of being re-executed.
|
|
1885
|
+
*
|
|
1886
|
+
* Layout: `${dir}/spawn-journal.jsonl` (one JSONL record per event), `${dir}/blobs/` (one
|
|
1887
|
+
* content-addressed JSON file per settled result), and `${dir}/coordination-log.jsonl`
|
|
1888
|
+
* (questions, findings, answer decisions, and authorized continuation receipts retained as
|
|
1889
|
+
* evidence). The directory is created on first write.
|
|
1890
|
+
*
|
|
1891
|
+
* Opt-in by construction — `createInMemoryRunContext()` is unchanged and stays the default, so no
|
|
1892
|
+
* existing consumer writes to disk or resumes unless it asks for this.
|
|
1893
|
+
*/
|
|
1894
|
+
function createFileRunContext(dir, opts = {}) {
|
|
1895
|
+
const base = createExecutorRegistry();
|
|
1896
|
+
return {
|
|
1897
|
+
journal: new FileSpawnJournal(`${dir}/spawn-journal.jsonl`),
|
|
1898
|
+
blobs: new FileResultBlobStore(`${dir}/blobs`),
|
|
1899
|
+
executors: opts.withDriver ? withDriverExecutor(base) : base,
|
|
1900
|
+
resume: true,
|
|
1901
|
+
coordinationLog: new FileCoordinationLog(`${dir}/coordination-log.jsonl`)
|
|
1902
|
+
};
|
|
1903
|
+
}
|
|
1904
|
+
//#endregion
|
|
1905
|
+
//#region src/runtime/anytime.ts
|
|
1906
|
+
/**
|
|
1907
|
+
* The best-so-far fold — the ONE definition of "how good was the run after k results", shared by
|
|
1908
|
+
* the post-run anytime report below and by the LIVE progress-based stop rules
|
|
1909
|
+
* (`supervise/stop-rules.ts`). Given the observed objective per settled result in order, it returns
|
|
1910
|
+
* the running maximum. A result with no objective (`undefined` — it failed, or it was never
|
|
1911
|
+
* scored) carries the previous best forward rather than resetting it.
|
|
1912
|
+
*
|
|
1913
|
+
* It is extracted rather than duplicated on purpose: a stop rule that decides a run has plateaued
|
|
1914
|
+
* must agree, number for number, with the report that later says whether stopping was right.
|
|
1915
|
+
*/
|
|
1916
|
+
function bestSoFar(values) {
|
|
1917
|
+
const out = [];
|
|
1918
|
+
let best = 0;
|
|
1919
|
+
for (const v of values) {
|
|
1920
|
+
if (typeof v === "number" && v > best) best = v;
|
|
1921
|
+
out.push(best);
|
|
1922
|
+
}
|
|
1923
|
+
return out;
|
|
1924
|
+
}
|
|
1925
|
+
/** Mean of a best-so-far curve — the anytime AUC when the curve is normalized to [0,1]. Higher =
|
|
1926
|
+
* the run climbed earlier. Shared with the stop rules so "improving" means one thing. */
|
|
1927
|
+
function areaUnderCurve(curve) {
|
|
1928
|
+
if (curve.length === 0) return 0;
|
|
1929
|
+
return curve.reduce((s, v) => s + v, 0) / curve.length;
|
|
1930
|
+
}
|
|
1931
|
+
/**
|
|
1932
|
+
* How many trailing entries of a best-so-far curve are within `minDelta` of the curve's value
|
|
1933
|
+
* `window` steps back — i.e. the length of the current PLATEAU, in settles. `0` means the most
|
|
1934
|
+
* recent settle improved the best by more than `minDelta`.
|
|
1935
|
+
*
|
|
1936
|
+
* The plateau math the live stop rules read. Defined here, beside the report that measures whether
|
|
1937
|
+
* the plateau was real, so there is exactly one notion of "not improving".
|
|
1938
|
+
*/
|
|
1939
|
+
function plateauLength(curve, minDelta) {
|
|
1940
|
+
if (curve.length === 0) return 0;
|
|
1941
|
+
const last = curve[curve.length - 1];
|
|
1942
|
+
let i = curve.length - 1;
|
|
1943
|
+
while (i > 0 && last - curve[i - 1] <= minDelta) i -= 1;
|
|
1944
|
+
return curve.length - 1 - i;
|
|
1945
|
+
}
|
|
1946
|
+
const median = (xs) => {
|
|
1947
|
+
if (xs.length === 0) return null;
|
|
1948
|
+
const s = [...xs].sort((a, b) => a - b);
|
|
1949
|
+
const mid = Math.floor(s.length / 2);
|
|
1950
|
+
return s.length % 2 === 1 ? s[mid] : (s[mid - 1] + s[mid]) / 2;
|
|
1951
|
+
};
|
|
1952
|
+
/** Derive anytime metrics from waterfall spans. `targets` are the satisficing score
|
|
1953
|
+
* bars (default [1] = fully resolved; COCO-style multi-target: [0.5, 0.8, 1]);
|
|
1954
|
+
* `targetFor` overrides the bar per task (task-specific satisfaction) — when set, the
|
|
1955
|
+
* per-task bar replaces every entry of `targets` for that task. */
|
|
1956
|
+
function anytimeReport(spans, opts) {
|
|
1957
|
+
const targets = opts?.targets ?? [1];
|
|
1958
|
+
const byRun = /* @__PURE__ */ new Map();
|
|
1959
|
+
for (const s of spans) {
|
|
1960
|
+
if (!s.label.startsWith("shot:")) continue;
|
|
1961
|
+
const list = byRun.get(s.runId) ?? [];
|
|
1962
|
+
list.push(s);
|
|
1963
|
+
byRun.set(s.runId, list);
|
|
1964
|
+
}
|
|
1965
|
+
const perTask = [];
|
|
1966
|
+
for (const [runId, shots] of byRun) {
|
|
1967
|
+
const m = runId.match(/^agentic:(.+):(.+)$/);
|
|
1968
|
+
const strategy = m?.[1] ?? runId;
|
|
1969
|
+
const taskId = m?.[2] ?? runId;
|
|
1970
|
+
const ordered = [...shots].sort((a, b) => (a.endMs ?? a.startMs) - (b.endMs ?? b.startMs));
|
|
1971
|
+
const t0 = Math.min(...ordered.map((s) => s.startMs));
|
|
1972
|
+
const taskTargets = opts?.targetFor ? [opts.targetFor(taskId)] : targets;
|
|
1973
|
+
const bests = bestSoFar(ordered.map((s) => typeof s.score === "number" ? s.score : void 0));
|
|
1974
|
+
let cumUsd = 0;
|
|
1975
|
+
const points = [];
|
|
1976
|
+
const hits = {};
|
|
1977
|
+
for (const t of taskTargets) hits[String(t)] = null;
|
|
1978
|
+
for (const [i, s] of ordered.entries()) {
|
|
1979
|
+
cumUsd += s.usd;
|
|
1980
|
+
const best = bests[i];
|
|
1981
|
+
const elapsedMs = (s.endMs ?? s.startMs) - t0;
|
|
1982
|
+
points.push({
|
|
1983
|
+
elapsedMs,
|
|
1984
|
+
cumUsd,
|
|
1985
|
+
best
|
|
1986
|
+
});
|
|
1987
|
+
for (const t of taskTargets) if (hits[String(t)] === null && best >= t) hits[String(t)] = {
|
|
1988
|
+
ms: elapsedMs,
|
|
1989
|
+
shots: points.length,
|
|
1990
|
+
usd: cumUsd
|
|
1991
|
+
};
|
|
1992
|
+
}
|
|
1993
|
+
perTask.push({
|
|
1994
|
+
taskId,
|
|
1995
|
+
strategy,
|
|
1996
|
+
points,
|
|
1997
|
+
hits
|
|
1998
|
+
});
|
|
1999
|
+
}
|
|
2000
|
+
const byStrategy = /* @__PURE__ */ new Map();
|
|
2001
|
+
for (const t of perTask) {
|
|
2002
|
+
const list = byStrategy.get(t.strategy) ?? [];
|
|
2003
|
+
list.push(t);
|
|
2004
|
+
byStrategy.set(t.strategy, list);
|
|
2005
|
+
}
|
|
2006
|
+
const perStrategy = [];
|
|
2007
|
+
for (const [strategy, tasks] of byStrategy) {
|
|
2008
|
+
const totalMs = tasks.reduce((s, t) => s + (t.points[t.points.length - 1]?.elapsedMs ?? 0), 0);
|
|
2009
|
+
const totalUsd = tasks.reduce((s, t) => s + (t.points[t.points.length - 1]?.cumUsd ?? 0), 0);
|
|
2010
|
+
const maxShots = Math.max(0, ...tasks.map((t) => t.points.length));
|
|
2011
|
+
const curveByShot = [];
|
|
2012
|
+
for (let i = 0; i < maxShots; i += 1) {
|
|
2013
|
+
const vals = tasks.map((t) => t.points[Math.min(i, t.points.length - 1)].best);
|
|
2014
|
+
curveByShot.push(vals.reduce((s, v) => s + v, 0) / vals.length);
|
|
2015
|
+
}
|
|
2016
|
+
const auc = areaUnderCurve(curveByShot);
|
|
2017
|
+
const summaryTargets = opts?.targetFor ? [NaN] : targets;
|
|
2018
|
+
for (const t of summaryTargets) {
|
|
2019
|
+
const key = (taskCurve) => opts?.targetFor ? Object.values(taskCurve.hits)[0] ?? null : taskCurve.hits[String(t)] ?? null;
|
|
2020
|
+
const reached = tasks.filter((x) => key(x) !== null);
|
|
2021
|
+
perStrategy.push({
|
|
2022
|
+
strategy,
|
|
2023
|
+
target: t,
|
|
2024
|
+
tasks: tasks.length,
|
|
2025
|
+
reachedTarget: reached.length,
|
|
2026
|
+
medianTttMs: median(reached.map((x) => key(x).ms)),
|
|
2027
|
+
medianShotsToTarget: median(reached.map((x) => key(x).shots)),
|
|
2028
|
+
ertMs: reached.length > 0 ? totalMs / reached.length : null,
|
|
2029
|
+
erUsd: reached.length > 0 ? totalUsd / reached.length : null,
|
|
2030
|
+
curveByShot,
|
|
2031
|
+
auc
|
|
2032
|
+
});
|
|
2033
|
+
}
|
|
2034
|
+
}
|
|
2035
|
+
perStrategy.sort((a, b) => a.strategy.localeCompare(b.strategy) || a.target - b.target);
|
|
2036
|
+
return {
|
|
2037
|
+
targets,
|
|
2038
|
+
perTask,
|
|
2039
|
+
perStrategy
|
|
2040
|
+
};
|
|
2041
|
+
}
|
|
2042
|
+
/** One row per (strategy, satisficing target): the shareable time-to-satisfactory table. */
|
|
2043
|
+
function renderAnytimeTable(report) {
|
|
2044
|
+
const lines = [`anytime metrics · satisficing targets [${report.targets.join(", ")}] · ERT = Σ all wall-time / #successes (COCO)`, "strategy ≥tgt reach med-TTT med-shots ERT(all-in) $/success AUC curve"];
|
|
2045
|
+
for (const s of report.perStrategy) {
|
|
2046
|
+
const curve = s.curveByShot.map((v) => "▁▂▃▄▅▆▇█"[Math.min(7, Math.floor(v * 8))]).join("");
|
|
2047
|
+
const tgt = Number.isNaN(s.target) ? "task" : s.target.toFixed(2);
|
|
2048
|
+
lines.push(`${s.strategy.padEnd(19)} ${tgt.padStart(4)} ${String(s.reachedTarget).padStart(4)}/${String(s.tasks).padEnd(3)} ${s.medianTttMs === null ? " —" : `${(s.medianTttMs / 1e3).toFixed(1).padStart(6)}s`} ${s.medianShotsToTarget === null ? " —" : String(s.medianShotsToTarget).padStart(5)} ${s.ertMs === null ? " —" : `${(s.ertMs / 1e3).toFixed(1).padStart(9)}s`} ${s.erUsd === null ? " —" : `$${s.erUsd.toFixed(4)}`} ${s.auc.toFixed(2)} ${curve}`);
|
|
2049
|
+
}
|
|
2050
|
+
return lines.join("\n");
|
|
2051
|
+
}
|
|
2052
|
+
//#endregion
|
|
2053
|
+
//#region src/runtime/supervise/stop-rules.ts
|
|
2054
|
+
/**
|
|
2055
|
+
*
|
|
2056
|
+
* PROGRESS-BASED STOP RULES — end a long-horizon run for the right reason.
|
|
2057
|
+
*
|
|
2058
|
+
* Every existing bound is a CEILING: iterations, tokens, dollars, an absolute deadline, a turn
|
|
2059
|
+
* cap. A ceiling answers "may this run continue?" and never "is this run still getting anywhere?".
|
|
2060
|
+
* So a supervision tree that stopped learning at settle 4 keeps buying workers until it hits a
|
|
2061
|
+
* wall — the run ends on exhaustion, and the operator cannot tell a run that finished from a run
|
|
2062
|
+
* that ran out.
|
|
2063
|
+
*
|
|
2064
|
+
* A stop rule reads the run's own PROGRESS and decides. Three signals feed it:
|
|
2065
|
+
* - the objective curve over settled work (best-so-far, from `anytime.ts` — see below),
|
|
2066
|
+
* - the LIVE worker feed (`WorkerProgress`: `idleMs`, `stalled`, `turns`, `tokens`),
|
|
2067
|
+
* - tree-level shape (how many are in flight, how many are waiting, when the last settle landed).
|
|
2068
|
+
*
|
|
2069
|
+
* ── Two boundaries this module holds deliberately ───────────────────────────────────────────────
|
|
2070
|
+
*
|
|
2071
|
+
* ENFORCEMENT lives here; THRESHOLDS do not. "Stop after 5 settles with no improvement" is a
|
|
2072
|
+
* judgment about a domain — how noisy its scores are, how expensive a worker is, how much a late
|
|
2073
|
+
* breakthrough is worth. That belongs to the caller (a loop, a bench, a product). Every rule below
|
|
2074
|
+
* takes its thresholds as required options with no hidden defaults for the numbers that decide;
|
|
2075
|
+
* the module ships the MECHANISM and refuses to ship the judgment.
|
|
2076
|
+
*
|
|
2077
|
+
* A stop rule can only ADD a stop, never remove one. The driver evaluates the hard ceilings
|
|
2078
|
+
* (`poolStarved`, `deadlinePassed`, abort, the driver's own stop) FIRST and independently; the
|
|
2079
|
+
* rule is consulted only when they all say "continue". So no rule can talk a run past its budget.
|
|
2080
|
+
*
|
|
2081
|
+
* ── What is reused, not re-derived ──────────────────────────────────────────────────────────────
|
|
2082
|
+
*
|
|
2083
|
+
* `anytime.ts` already computed best-so-far curves, their area, and plateau detection — but only
|
|
2084
|
+
* after a run was over, from waterfall spans. Rather than write a second copy for the live path,
|
|
2085
|
+
* `bestSoFar` / `areaUnderCurve` / `plateauLength` were extracted there and are imported here. A
|
|
2086
|
+
* stop rule that calls a run plateaued therefore agrees, number for number, with the report that
|
|
2087
|
+
* later judges whether stopping was right.
|
|
2088
|
+
*
|
|
2089
|
+
* @experimental
|
|
2090
|
+
*/
|
|
2091
|
+
const CONTINUE = { stop: false };
|
|
2092
|
+
/** Build the settled-work ledger a `StopRule` decides from: record each settlement (idempotent by
|
|
2093
|
+
* id) and materialize a `ProgressView` combining the best-so-far curve with the live worker feed. */
|
|
2094
|
+
function createProgressTracker(opts = {}) {
|
|
2095
|
+
const now = opts.now ?? Date.now;
|
|
2096
|
+
const requireDelivered = opts.requireDelivered ?? true;
|
|
2097
|
+
const minImprovement = opts.minImprovement ?? 0;
|
|
2098
|
+
const seen = /* @__PURE__ */ new Set();
|
|
2099
|
+
const recorded = [];
|
|
2100
|
+
return {
|
|
2101
|
+
record(sample) {
|
|
2102
|
+
if (seen.has(sample.id)) return false;
|
|
2103
|
+
seen.add(sample.id);
|
|
2104
|
+
recorded.push(sample);
|
|
2105
|
+
return true;
|
|
2106
|
+
},
|
|
2107
|
+
samples: () => [...recorded],
|
|
2108
|
+
view(scope, viewOpts) {
|
|
2109
|
+
const curve = bestSoFar(recorded.map((s) => requireDelivered && !s.delivered ? void 0 : s.objective));
|
|
2110
|
+
const best = curve.length > 0 ? curve[curve.length - 1] : 0;
|
|
2111
|
+
let lastImprovementAt = 0;
|
|
2112
|
+
let lastImprovementIdx = -1;
|
|
2113
|
+
let prev = 0;
|
|
2114
|
+
for (const [i, value] of curve.entries()) {
|
|
2115
|
+
if (value - prev > minImprovement) {
|
|
2116
|
+
lastImprovementAt = recorded[i].at;
|
|
2117
|
+
lastImprovementIdx = i;
|
|
2118
|
+
}
|
|
2119
|
+
prev = value;
|
|
2120
|
+
}
|
|
2121
|
+
const treeView = scope?.view;
|
|
2122
|
+
const workers = [];
|
|
2123
|
+
if (scope && treeView) for (const node of treeView.nodes) {
|
|
2124
|
+
if (isTerminalNodeStatus(node.status)) continue;
|
|
2125
|
+
if (node.status === "waiting") continue;
|
|
2126
|
+
const p = scope.progress(node.id, viewOpts?.stallAfterMs !== void 0 ? {
|
|
2127
|
+
now: now(),
|
|
2128
|
+
stallAfterMs: viewOpts.stallAfterMs
|
|
2129
|
+
} : { now: now() });
|
|
2130
|
+
if (p) workers.push(p);
|
|
2131
|
+
}
|
|
2132
|
+
return {
|
|
2133
|
+
now: now(),
|
|
2134
|
+
settles: recorded.length,
|
|
2135
|
+
delivered: recorded.filter((s) => s.delivered).length,
|
|
2136
|
+
curve,
|
|
2137
|
+
best,
|
|
2138
|
+
auc: areaUnderCurve(curve),
|
|
2139
|
+
lastSettleAt: recorded.length > 0 ? recorded[recorded.length - 1].at : 0,
|
|
2140
|
+
lastImprovementAt,
|
|
2141
|
+
settlesSinceImprovement: lastImprovementIdx < 0 ? recorded.length : recorded.length - 1 - lastImprovementIdx,
|
|
2142
|
+
workers,
|
|
2143
|
+
inFlight: treeView?.inFlight ?? 0,
|
|
2144
|
+
waiting: treeView?.waiting ?? 0
|
|
2145
|
+
};
|
|
2146
|
+
},
|
|
2147
|
+
evaluate(rule, scope, viewOpts) {
|
|
2148
|
+
return rule(this.view(scope, viewOpts));
|
|
2149
|
+
}
|
|
2150
|
+
};
|
|
2151
|
+
}
|
|
2152
|
+
/** Build a `ProgressSample` from a scope settlement. The objective is the verdict score and
|
|
2153
|
+
* `delivered` is the verdict's `valid` — the SAME single delivery signal `finalizeBestDelivered`
|
|
2154
|
+
* and `defaultSelectWinner` use, so "progress" and "winner" cannot disagree. */
|
|
2155
|
+
function sampleFromSettled(settled, at) {
|
|
2156
|
+
if (settled.kind === "down") return {
|
|
2157
|
+
id: settled.handle.id,
|
|
2158
|
+
at,
|
|
2159
|
+
delivered: false
|
|
2160
|
+
};
|
|
2161
|
+
return {
|
|
2162
|
+
id: settled.handle.id,
|
|
2163
|
+
at,
|
|
2164
|
+
...settled.verdict?.score !== void 0 ? { objective: settled.verdict.score } : {},
|
|
2165
|
+
delivered: settled.verdict?.valid === true
|
|
2166
|
+
};
|
|
2167
|
+
}
|
|
2168
|
+
/**
|
|
2169
|
+
* "Nothing new has happened." Fires when the run has produced no new settled work for `ms`, or no
|
|
2170
|
+
* IMPROVEMENT over the last `settles` settlements.
|
|
2171
|
+
*
|
|
2172
|
+
* A tree whose only remaining nodes are armed WAITS is exempt from the time bound: a run waiting
|
|
2173
|
+
* on CI is not a run that stopped making progress, and killing it there would defeat mechanic C.
|
|
2174
|
+
*/
|
|
2175
|
+
function noProgressFor(opts) {
|
|
2176
|
+
if (opts.ms === void 0 && opts.settles === void 0) throw new ValidationError("noProgressFor: set at least one of { ms, settles }");
|
|
2177
|
+
if (opts.ms !== void 0 && opts.ms <= 0) throw new ValidationError("noProgressFor: ms must be > 0");
|
|
2178
|
+
if (opts.settles !== void 0 && opts.settles < 1) throw new ValidationError("noProgressFor: settles must be >= 1");
|
|
2179
|
+
const minSettles = opts.minSettles ?? 1;
|
|
2180
|
+
return (v) => {
|
|
2181
|
+
if (v.settles < minSettles) return CONTINUE;
|
|
2182
|
+
if (opts.settles !== void 0 && v.settlesSinceImprovement >= opts.settles) return {
|
|
2183
|
+
stop: true,
|
|
2184
|
+
reason: `no-progress: ${v.settlesSinceImprovement} settles with no improvement (limit ${opts.settles}), best=${v.best}`
|
|
2185
|
+
};
|
|
2186
|
+
if (opts.ms !== void 0 && v.waiting === 0 && v.lastSettleAt > 0) {
|
|
2187
|
+
const idle = v.now - v.lastSettleAt;
|
|
2188
|
+
if (idle >= opts.ms) return {
|
|
2189
|
+
stop: true,
|
|
2190
|
+
reason: `no-progress: ${idle}ms since the last settlement (limit ${opts.ms}ms)`
|
|
2191
|
+
};
|
|
2192
|
+
}
|
|
2193
|
+
return CONTINUE;
|
|
2194
|
+
};
|
|
2195
|
+
}
|
|
2196
|
+
/**
|
|
2197
|
+
* "The objective has stopped climbing." Fires when the best-so-far curve has risen by no more than
|
|
2198
|
+
* `minDelta` across the last `window` settlements.
|
|
2199
|
+
*
|
|
2200
|
+
* Built on `anytime.plateauLength` — the same plateau math the post-run anytime report uses, so a
|
|
2201
|
+
* rule that stops a run and a report that grades the decision cannot disagree about whether the
|
|
2202
|
+
* run was flat.
|
|
2203
|
+
*/
|
|
2204
|
+
function plateau(opts) {
|
|
2205
|
+
if (!Number.isInteger(opts.window) || opts.window < 1) throw new ValidationError("plateau: window must be a positive integer");
|
|
2206
|
+
if (!Number.isFinite(opts.minDelta) || opts.minDelta < 0) throw new ValidationError("plateau: minDelta must be >= 0");
|
|
2207
|
+
const minSettles = opts.minSettles ?? opts.window;
|
|
2208
|
+
return (v) => {
|
|
2209
|
+
if (v.settles < minSettles) return CONTINUE;
|
|
2210
|
+
const flat = plateauLength(v.curve, opts.minDelta);
|
|
2211
|
+
if (flat >= opts.window) return {
|
|
2212
|
+
stop: true,
|
|
2213
|
+
reason: `plateau: best-so-far rose <= ${opts.minDelta} over the last ${flat} settles (window ${opts.window}), best=${v.best}, auc=${v.auc.toFixed(3)}`
|
|
2214
|
+
};
|
|
2215
|
+
return CONTINUE;
|
|
2216
|
+
};
|
|
2217
|
+
}
|
|
2218
|
+
/**
|
|
2219
|
+
* "Everyone is stuck." Fires when every live worker reads `stalled` — no metered activity for
|
|
2220
|
+
* longer than the stall threshold — and none of the tree is merely waiting.
|
|
2221
|
+
*
|
|
2222
|
+
* `stalled` is a derived read at observation time, never a background watchdog; this rule only
|
|
2223
|
+
* reads it. A tree with armed waits never fires: waiting is not stalling.
|
|
2224
|
+
*/
|
|
2225
|
+
function allWorkersStalled(opts = {}) {
|
|
2226
|
+
const minWorkers = opts.minWorkers ?? 1;
|
|
2227
|
+
return (v) => {
|
|
2228
|
+
if (v.waiting > 0) return CONTINUE;
|
|
2229
|
+
if (v.workers.length < minWorkers) return CONTINUE;
|
|
2230
|
+
if (!v.workers.every((w) => w.stalled)) return CONTINUE;
|
|
2231
|
+
const worst = Math.max(...v.workers.map((w) => w.idleMs));
|
|
2232
|
+
return {
|
|
2233
|
+
stop: true,
|
|
2234
|
+
reason: `all-stalled: ${v.workers.length} live workers idle, worst ${worst}ms`
|
|
2235
|
+
};
|
|
2236
|
+
};
|
|
2237
|
+
}
|
|
2238
|
+
/** Stop when ANY rule stops — the ordinary composition (each rule is a separate reason to end). */
|
|
2239
|
+
function anyOf(...rules) {
|
|
2240
|
+
return (v) => {
|
|
2241
|
+
for (const rule of rules) {
|
|
2242
|
+
const d = rule(v);
|
|
2243
|
+
if (d.stop) return d;
|
|
2244
|
+
}
|
|
2245
|
+
return CONTINUE;
|
|
2246
|
+
};
|
|
2247
|
+
}
|
|
2248
|
+
/** Stop only when EVERY rule stops — for a conservative gate that needs corroboration. */
|
|
2249
|
+
function allOf(...rules) {
|
|
2250
|
+
if (rules.length === 0) throw new ValidationError("allOf: needs at least one rule");
|
|
2251
|
+
return (v) => {
|
|
2252
|
+
const reasons = [];
|
|
2253
|
+
for (const rule of rules) {
|
|
2254
|
+
const d = rule(v);
|
|
2255
|
+
if (!d.stop) return CONTINUE;
|
|
2256
|
+
reasons.push(d.reason);
|
|
2257
|
+
}
|
|
2258
|
+
return {
|
|
2259
|
+
stop: true,
|
|
2260
|
+
reason: reasons.join(" AND ")
|
|
2261
|
+
};
|
|
2262
|
+
};
|
|
2263
|
+
}
|
|
2264
|
+
/**
|
|
2265
|
+
* Evaluate a rule against the run's settled work — the ONE evaluator both supervisor arms call.
|
|
2266
|
+
*
|
|
2267
|
+
* The router arm calls it before each driver inference turn; the harness arm calls it on each
|
|
2268
|
+
* worker settle. Ordering is the contract in both: the hard ceilings (`poolStarved`,
|
|
2269
|
+
* `deadlinePassed`, abort, the driver's own stop) are checked first and independently, so a stop
|
|
2270
|
+
* rule can only ever ADD a stop — it can never keep a run alive past a budget it has exhausted.
|
|
2271
|
+
*
|
|
2272
|
+
* Folding the whole roster each call is idempotent by worker id, so it costs O(settled) and never
|
|
2273
|
+
* double-counts. `settledAt` carries the instant the ledger recorded a settlement; `now()` is the
|
|
2274
|
+
* fallback resolution a per-turn guard has.
|
|
2275
|
+
*/
|
|
2276
|
+
function progressStop(tracker, rule, ledger, scope, now, stallAfterMs) {
|
|
2277
|
+
for (const w of ledger.settled()) tracker.record({
|
|
2278
|
+
id: w.id,
|
|
2279
|
+
at: w.settledAt ?? now(),
|
|
2280
|
+
...w.score !== void 0 ? { objective: w.score } : {},
|
|
2281
|
+
delivered: w.status === "done" && w.valid === true
|
|
2282
|
+
});
|
|
2283
|
+
return tracker.evaluate(rule, scope, stallAfterMs !== void 0 ? { stallAfterMs } : void 0);
|
|
2284
|
+
}
|
|
2285
|
+
//#endregion
|
|
2286
|
+
//#region src/runtime/supervise/coordination-driver.ts
|
|
2287
|
+
/**
|
|
2288
|
+
*
|
|
2289
|
+
* `driverAgent` — the driver's BRAIN.
|
|
2290
|
+
*
|
|
2291
|
+
* The recursive driver-executor (`driver-executor.ts`) runs a driver `Agent.act` inside a
|
|
2292
|
+
* nested `Scope`; this is the intelligent `act`: it mounts the coordination MCP verbs
|
|
2293
|
+
* (`createCoordinationTools`) over that scope and runs an LLM tool-loop, so the driver
|
|
2294
|
+
* REASONS — spawn / observe / steer / await / stop — about how to drive its children,
|
|
2295
|
+
* instead of running a fixed script. Each turn: ask the driver LLM for tool calls, run them
|
|
2296
|
+
* against the live scope, fold the results back, repeat until the driver stops (no tool
|
|
2297
|
+
* calls) or the turn cap forces a keep-best finalize.
|
|
2298
|
+
*
|
|
2299
|
+
* Recursion composes through `makeWorkerAgent`: `spawn_agent` resolves a `profile` to a
|
|
2300
|
+
* worker LEAF or — when the profile is a driver — a `driverChild` wrapping ANOTHER
|
|
2301
|
+
* `driverAgent` over its own nested scope (see `driver-executor.ts`). So an agent
|
|
2302
|
+
* drives an agent that drives an agent, each an LLM tool-loop, all on one conserved-budget
|
|
2303
|
+
* tree.
|
|
2304
|
+
*
|
|
2305
|
+
* Two seams are INJECTED so the loop runs offline with no creds and stays decoupled:
|
|
2306
|
+
* - `brain` (`ToolLoopChat`) — one driver-LLM turn over the canonical tool-loop seam; a test
|
|
2307
|
+
* drives a scripted mock, production passes the router's tool-calling (`routerBrain`), a
|
|
2308
|
+
* sandboxed harness drives the verbs as MCP tools. The same seam every tool-loop uses.
|
|
2309
|
+
* - `systemPrompt` — the driver's stance (the agent-eval worker-driver prompt / the prompt
|
|
2310
|
+
* generator). Injected, never hardcoded — the prompt is a pluggable role.
|
|
2311
|
+
*
|
|
2312
|
+
* @experimental
|
|
2313
|
+
*/
|
|
2314
|
+
/** The default chapter-close prompt: the brain summarizes its OWN progress for its future self before
|
|
2315
|
+
* the detailed history is dropped. Emphasis on PENDING work — the part a too-eager chapter-close
|
|
2316
|
+
* loses (the coding-burn counter-finding: closing after one fix leaves integration bugs uncircled). */
|
|
2317
|
+
const distillInstruction = "CONTEXT COMPACTION. Your detailed turn-by-turn history is about to be discarded to free your context window. Write a COMPLETE, compact handoff note for your future self so you can keep going without it. Cover: (1) what you have accomplished; (2) every worker you spawned and its current status/result; (3) what subtasks remain unfinished, failing, or unverified — be specific and exhaustive here, this is the part you must not lose; (4) your immediate next action. Do not call any tools; respond with the note only.";
|
|
2318
|
+
/** Factual ground truth for the digest — the live worker roster from Scope plus the delivered-result
|
|
2319
|
+
* ledger, independent of whatever the brain's prose summary captures. */
|
|
2320
|
+
function summarizeRoster(view, settled) {
|
|
2321
|
+
if (view.nodes.length === 0) return "Workers in current live scope: none yet.";
|
|
2322
|
+
const settledById = new Map(settled.map((w) => [w.id, w]));
|
|
2323
|
+
const lines = view.nodes.map((node) => formatRosterNode(node, settledById.get(node.id)));
|
|
2324
|
+
return `Workers in current live scope (ground truth from the run, ${view.nodes.length} total, ${view.inFlight} in flight):\n${lines.join("\n")}`;
|
|
2325
|
+
}
|
|
2326
|
+
function formatRosterNode(node, settled) {
|
|
2327
|
+
const result = settled?.status === "done" ? `, delivered=${settled.valid ?? false}${settled.score !== void 0 ? `, score=${settled.score}` : ""}${settled.outRef ? `, outRef=${settled.outRef}` : ""}` : settled?.status === "down" ? `, reason=${settled.reason ?? "unknown"}` : node.outRef ? `, outRef=${node.outRef}` : "";
|
|
2328
|
+
return `- ${node.id}: ${node.status}, label=${node.label}, runtime=${node.runtime}${result}`;
|
|
2329
|
+
}
|
|
2330
|
+
/** Spawn-progress is impossible: the pool can't afford another worker AND nothing is in flight to
|
|
2331
|
+
* await. A long-horizon driver bounded by the conserved pool stops here instead of spinning (the
|
|
2332
|
+
* in-loop budget guard the turn cap alone never provided). Checks BOTH conserved channels: tokens
|
|
2333
|
+
* (can't afford a worker) and usd (a usd-capped pool whose ceiling the driver's own metered
|
|
2334
|
+
* inference has drained — `meter` debits usd, so without this a huge-token/small-usd pool would
|
|
2335
|
+
* overspend usd up to the turn tripwire). */
|
|
2336
|
+
function poolStarved(scope, perWorker) {
|
|
2337
|
+
const b = scope.budget;
|
|
2338
|
+
if (scope.view.inFlight > 0 || scope.view.waiting > 0) return false;
|
|
2339
|
+
const tokenStarved = b.tokensLeft < perWorker.maxTokens;
|
|
2340
|
+
const iterationStarved = b.iterationsLeft <= 0;
|
|
2341
|
+
const usdStarved = b.usdCapped && (b.usdLeft <= 0 || perWorker.maxUsd !== void 0 && b.usdLeft < perWorker.maxUsd);
|
|
2342
|
+
return tokenStarved || iterationStarved || usdStarved;
|
|
2343
|
+
}
|
|
2344
|
+
/** The absolute wall-clock deadline (when the root set one) has passed. */
|
|
2345
|
+
function deadlinePassed(scope, now) {
|
|
2346
|
+
const b = scope.budget;
|
|
2347
|
+
return b.deadlineMs > 0 && now() >= b.deadlineMs;
|
|
2348
|
+
}
|
|
2349
|
+
/** The USD-denominated members of {@link PromptCacheUsage} — the schema, not a guess. Every
|
|
2350
|
+
* other known member (`readTokens`, `writeTokens`, `missTokens`) is a token COUNT. */
|
|
2351
|
+
const PROMPT_CACHE_USD_FIELDS = /* @__PURE__ */ new Set(["readSavingsUsd"]);
|
|
2352
|
+
/** Dollar amounts above this are provider nonsense, not evidence. The old all-integer rule
|
|
2353
|
+
* rejected them as a side effect; keeping an explicit ceiling preserves that protection
|
|
2354
|
+
* without pretending a dollar amount is an integer. */
|
|
2355
|
+
const MAX_PROMPT_CACHE_USD = 1e6;
|
|
2356
|
+
/**
|
|
2357
|
+
* Validate provider-reported prompt-cache evidence.
|
|
2358
|
+
*
|
|
2359
|
+
* Prompt-cache carries two kinds of number and they obey different rules: token COUNTS are
|
|
2360
|
+
* integers, and USD amounts are fractional by nature. Applying the count rule to a dollar
|
|
2361
|
+
* field refuses every provider that reports cache savings in dollars — a healthy router
|
|
2362
|
+
* response carrying `readSavingsUsd: 0.0034` failed the driver outright before this split.
|
|
2363
|
+
*
|
|
2364
|
+
* Classification is schema-first: a field named in {@link PromptCacheUsage} is validated by
|
|
2365
|
+
* what that member IS. `promptCache` is an open record (the sandbox path forwards provider
|
|
2366
|
+
* fields verbatim), so an unknown field falls back to the `usd` name-suffix convention —
|
|
2367
|
+
* documented here as the contract a provider must follow to report dollars.
|
|
2368
|
+
*
|
|
2369
|
+
* Returns the refusal, or `undefined` when the evidence is acceptable.
|
|
2370
|
+
*/
|
|
2371
|
+
function validateDriverPromptCache(promptCache) {
|
|
2372
|
+
for (const [field, value] of Object.entries(promptCache ?? {})) {
|
|
2373
|
+
if (typeof value !== "number") continue;
|
|
2374
|
+
if (PROMPT_CACHE_USD_FIELDS.has(field) || /usd$/i.test(field)) {
|
|
2375
|
+
if (!Number.isFinite(value) || value < 0 || value > MAX_PROMPT_CACHE_USD) return new ValidationError(`driverAgent: prompt-cache field ${JSON.stringify(field)} must be a non-negative finite number of dollars`);
|
|
2376
|
+
continue;
|
|
2377
|
+
}
|
|
2378
|
+
if (!Number.isSafeInteger(value) || value < 0) return new ValidationError(`driverAgent: prompt-cache field ${JSON.stringify(field)} must be a non-negative safe integer`);
|
|
2379
|
+
}
|
|
2380
|
+
}
|
|
2381
|
+
/** The journal file `createFileRunContext` writes inside the run directory. The acknowledger
|
|
2382
|
+
* reads it as EVIDENCE for the terminated-descendants set — the nested trees of a cancelled
|
|
2383
|
+
* lead journal their terminal records there before the lead settles into this scope. */
|
|
2384
|
+
const SPAWN_JOURNAL_FILE = "spawn-journal.jsonl";
|
|
2385
|
+
/**
|
|
2386
|
+
* Apply externally admitted steers once from the owning manager's turn loop.
|
|
2387
|
+
*
|
|
2388
|
+
* The `unknown` acknowledgement lands before authorization or delivery. A crash after that write
|
|
2389
|
+
* can lose this steer, but a restarted manager never delivers it again. This is the same
|
|
2390
|
+
* at-most-once crash boundary as coordination instruction receipts: no duplicate instruction is
|
|
2391
|
+
* safer than replaying a mutation whose first delivery may already have succeeded.
|
|
2392
|
+
*/
|
|
2393
|
+
function createSteerAcknowledger(deps) {
|
|
2394
|
+
const iso = () => new Date(deps.now()).toISOString();
|
|
2395
|
+
const directChildId = (ref) => ref.startsWith(`${deps.ownerId}:s`) && /^s\d+$/.test(ref.slice(deps.ownerId.length + 1));
|
|
2396
|
+
const base = (request) => ({
|
|
2397
|
+
schemaVersion: 1,
|
|
2398
|
+
operationId: request.operationId,
|
|
2399
|
+
requestDigest: request.requestDigest,
|
|
2400
|
+
worker: request.worker,
|
|
2401
|
+
requestedAt: request.at
|
|
2402
|
+
});
|
|
2403
|
+
return { async pass(phase) {
|
|
2404
|
+
for (const request of readWorkerSteerRequests(deps.dir)) {
|
|
2405
|
+
if (!directChildId(request.worker)) continue;
|
|
2406
|
+
if (readWorkerSteerAcknowledgement(deps.dir, request.operationId) !== void 0) continue;
|
|
2407
|
+
if (phase === "final") {
|
|
2408
|
+
writeWorkerSteerAcknowledgement(deps.dir, {
|
|
2409
|
+
...base(request),
|
|
2410
|
+
effect: "not_live",
|
|
2411
|
+
observedAt: iso(),
|
|
2412
|
+
detail: "run ended before the steer was applied"
|
|
2413
|
+
});
|
|
2414
|
+
continue;
|
|
2415
|
+
}
|
|
2416
|
+
if (!claimWorkerSteerDelivery(deps.dir, {
|
|
2417
|
+
...base(request),
|
|
2418
|
+
effect: "unknown",
|
|
2419
|
+
observedAt: iso(),
|
|
2420
|
+
detail: "delivery admitted; outcome not yet known"
|
|
2421
|
+
})) continue;
|
|
2422
|
+
try {
|
|
2423
|
+
const outcome = await deps.coord.steerWorker(request.worker, request.message, { interrupt: request.interrupt });
|
|
2424
|
+
const effect = outcome.delivered ? "delivered" : outcome.outcome === "runtime-has-no-inbox" ? "unsupported" : outcome.outcome === "unknown-worker" || outcome.outcome === "already-settled" || outcome.outcome === "scope-stopped" ? "not_live" : "unknown";
|
|
2425
|
+
writeWorkerSteerAcknowledgement(deps.dir, {
|
|
2426
|
+
...base(request),
|
|
2427
|
+
effect,
|
|
2428
|
+
observedAt: iso(),
|
|
2429
|
+
detail: steerAcknowledgementDetail(outcome)
|
|
2430
|
+
});
|
|
2431
|
+
} catch (error) {
|
|
2432
|
+
writeWorkerSteerAcknowledgement(deps.dir, {
|
|
2433
|
+
...base(request),
|
|
2434
|
+
effect: "unknown",
|
|
2435
|
+
observedAt: iso(),
|
|
2436
|
+
detail: "delivery outcome is unknown after a runtime error"
|
|
2437
|
+
});
|
|
2438
|
+
}
|
|
2439
|
+
}
|
|
2440
|
+
} };
|
|
2441
|
+
}
|
|
2442
|
+
function steerAcknowledgementDetail(outcome) {
|
|
2443
|
+
switch (outcome.outcome) {
|
|
2444
|
+
case "delivered": return "the owning manager delivered the steer to the exact live worker";
|
|
2445
|
+
case "runtime-has-no-inbox": return "the exact worker does not expose a steer inbox";
|
|
2446
|
+
case "unknown-worker": return "the owning manager does not know the exact worker";
|
|
2447
|
+
case "already-settled": return "the exact worker settled before delivery";
|
|
2448
|
+
case "scope-stopped": return "the owning manager stopped before delivery";
|
|
2449
|
+
case "runtime-error": return "delivery outcome is unknown after a runtime error";
|
|
2450
|
+
}
|
|
2451
|
+
}
|
|
2452
|
+
/**
|
|
2453
|
+
* The worker-cancel ACKNOWLEDGER — the runtime-side half of `run-layout`'s `cancelWorker`
|
|
2454
|
+
* contract, run from the coordination driver's turn loop (one cancellation-inbox read per turn,
|
|
2455
|
+
* no new process, no poller, no extra lifetime). Every manager with a `controlDir` mounts one;
|
|
2456
|
+
* OWNERSHIP keeps them from colliding: a request naming a node id is owned by the manager whose
|
|
2457
|
+
* own id is that node's parent, and a label/profile-name reference is owned by the `'run'`-scoped
|
|
2458
|
+
* (root) manager only — so exactly one acknowledger can ever apply one operation.
|
|
2459
|
+
*
|
|
2460
|
+
* Two-phase, honestly reported: `cancel_requested` is written the moment a live worker's abort is
|
|
2461
|
+
* issued (through the per-child abort chain the scope already owns, so siblings are untouched);
|
|
2462
|
+
* `cancelled` is written only when that worker's settlement is DELIVERED on the settle path with
|
|
2463
|
+
* a terminal `down`, and then the record names every subtree node id proven terminated. A worker
|
|
2464
|
+
* that already settled — or that settles `done` despite the abort — records `not_live`; a
|
|
2465
|
+
* reference matching nothing this manager owns stays pending (`cancelWorker` reports it
|
|
2466
|
+
* `unknown`). No path reports success for a missing worker.
|
|
2467
|
+
*
|
|
2468
|
+
* Expiry is run end, not a clock: `finish()` (after the final post-drain pass) writes `not_live`
|
|
2469
|
+
* for every owned request never applied and `unknown` for an issued abort whose settle the run
|
|
2470
|
+
* ended too soon to observe. A pending request can therefore never outlive its run and abort a
|
|
2471
|
+
* future spawn that happens to reuse a label.
|
|
2472
|
+
*
|
|
2473
|
+
* Idempotency is a lookup, in-process and across processes: an operation with a durable
|
|
2474
|
+
* acknowledgement is returned as-is and never re-applied.
|
|
2475
|
+
*/
|
|
2476
|
+
function createCancelAcknowledger(deps) {
|
|
2477
|
+
const tracked = /* @__PURE__ */ new Map();
|
|
2478
|
+
let runTracked;
|
|
2479
|
+
const abortIssuedAt = /* @__PURE__ */ new Map();
|
|
2480
|
+
const iso = () => new Date(deps.now()).toISOString();
|
|
2481
|
+
const write = (record) => {
|
|
2482
|
+
writeWorkerCancellation(deps.dir, record);
|
|
2483
|
+
tracked.set(record.operationId, record);
|
|
2484
|
+
};
|
|
2485
|
+
/** `ref` is exactly one of THIS manager's direct-child node ids (`${ownerId}:s<seq>`). */
|
|
2486
|
+
const directChildId = (ref) => ref.startsWith(`${deps.ownerId}:s`) && /^s\d+$/.test(ref.slice(deps.ownerId.length + 1));
|
|
2487
|
+
/** Whether this acknowledger owns `ref`. A node id deeper in this subtree belongs to the nested
|
|
2488
|
+
* manager that parents it; anything that is not a node id under this manager is a
|
|
2489
|
+
* label/profile-name reference, owned by the `'run'`-scoped manager alone. */
|
|
2490
|
+
const owned = (ref) => {
|
|
2491
|
+
if (directChildId(ref)) return true;
|
|
2492
|
+
if (deps.controlScope !== "run") return false;
|
|
2493
|
+
return !ref.startsWith(`${deps.ownerId}:`) && ref !== deps.ownerId;
|
|
2494
|
+
};
|
|
2495
|
+
const deliveredTerminal = (id) => {
|
|
2496
|
+
const row = deps.coord.settled().find((w) => w.id === id);
|
|
2497
|
+
if (row !== void 0) return row.status;
|
|
2498
|
+
const node = deps.scope.view.nodes.find((n) => n.id === id);
|
|
2499
|
+
if (node === void 0) return void 0;
|
|
2500
|
+
if (node.status === "done") return "done";
|
|
2501
|
+
if (node.status === "failed" || node.status === "cancelled") return "down";
|
|
2502
|
+
};
|
|
2503
|
+
const apply = (request) => {
|
|
2504
|
+
const aborted = deps.coord.abortWorker(request.worker, request.reason ?? "cancel requested");
|
|
2505
|
+
const base = {
|
|
2506
|
+
operationId: request.operationId,
|
|
2507
|
+
worker: request.worker,
|
|
2508
|
+
requestedAt: request.at,
|
|
2509
|
+
observedAt: iso(),
|
|
2510
|
+
...request.reason === void 0 ? {} : { reason: request.reason }
|
|
2511
|
+
};
|
|
2512
|
+
if (aborted !== void 0) {
|
|
2513
|
+
abortIssuedAt.set(request.operationId, base.observedAt);
|
|
2514
|
+
write({
|
|
2515
|
+
...base,
|
|
2516
|
+
effect: "cancel_requested",
|
|
2517
|
+
workerId: aborted.id,
|
|
2518
|
+
detail: `abort issued to live worker '${aborted.label}' (${aborted.id}); termination not yet proven`,
|
|
2519
|
+
terminated: []
|
|
2520
|
+
});
|
|
2521
|
+
return;
|
|
2522
|
+
}
|
|
2523
|
+
const goneId = deps.scope.view.nodes.find((n) => (n.id === request.worker || n.label === request.worker) && isTerminalNodeStatus(n.status))?.id ?? deps.coord.settled().find((w) => w.id === request.worker)?.id;
|
|
2524
|
+
if (goneId !== void 0) write({
|
|
2525
|
+
...base,
|
|
2526
|
+
effect: "not_live",
|
|
2527
|
+
workerId: goneId,
|
|
2528
|
+
detail: `worker '${goneId}' had already settled before this operation was applied`,
|
|
2529
|
+
terminated: []
|
|
2530
|
+
});
|
|
2531
|
+
};
|
|
2532
|
+
/** The proven-terminated set for one record: the worker plus every subtree id with a terminal
|
|
2533
|
+
* journal record at/after the abort was issued. Union with what the record already names, so
|
|
2534
|
+
* the set only ever grows (a late teardown journal adds; nothing removes). */
|
|
2535
|
+
const provenTerminated = (record, workerId) => {
|
|
2536
|
+
const since = abortIssuedAt.get(record.operationId) ?? record.observedAt;
|
|
2537
|
+
return [.../* @__PURE__ */ new Set([
|
|
2538
|
+
...record.terminated,
|
|
2539
|
+
workerId,
|
|
2540
|
+
...terminatedDescendants(deps.dir, workerId, since)
|
|
2541
|
+
])].sort();
|
|
2542
|
+
};
|
|
2543
|
+
const reconcile = (record) => {
|
|
2544
|
+
const workerId = record.workerId;
|
|
2545
|
+
if (workerId === void 0) return;
|
|
2546
|
+
const terminal = deliveredTerminal(workerId);
|
|
2547
|
+
if (terminal === void 0) return;
|
|
2548
|
+
if (terminal === "down") {
|
|
2549
|
+
write({
|
|
2550
|
+
...record,
|
|
2551
|
+
effect: "cancelled",
|
|
2552
|
+
observedAt: iso(),
|
|
2553
|
+
terminated: provenTerminated(record, workerId),
|
|
2554
|
+
detail: `worker '${workerId}' reached a terminal down state on the settle path`
|
|
2555
|
+
});
|
|
2556
|
+
return;
|
|
2557
|
+
}
|
|
2558
|
+
write({
|
|
2559
|
+
...record,
|
|
2560
|
+
effect: "not_live",
|
|
2561
|
+
observedAt: iso(),
|
|
2562
|
+
terminated: [],
|
|
2563
|
+
detail: `worker '${workerId}' settled done despite the abort request; nothing was terminated`
|
|
2564
|
+
});
|
|
2565
|
+
};
|
|
2566
|
+
/** Re-scan a `cancelled` record while the manager still turns: a descendant whose teardown
|
|
2567
|
+
* journals after the lead's settle joins the set on a later pass instead of being lost. Only
|
|
2568
|
+
* a grown set is re-written; the window needs the in-process abort instant, so a record a
|
|
2569
|
+
* PRIOR process closed stays as that process proved it. */
|
|
2570
|
+
const regrow = (record) => {
|
|
2571
|
+
const workerId = record.workerId;
|
|
2572
|
+
if (workerId === void 0 || !abortIssuedAt.has(record.operationId)) return;
|
|
2573
|
+
const terminated = provenTerminated(record, workerId);
|
|
2574
|
+
if (terminated.length > record.terminated.length) write({
|
|
2575
|
+
...record,
|
|
2576
|
+
observedAt: iso(),
|
|
2577
|
+
terminated
|
|
2578
|
+
});
|
|
2579
|
+
};
|
|
2580
|
+
/**
|
|
2581
|
+
* The RUN-scoped request: seen once, `cancel_requested` written the moment the run's cascading
|
|
2582
|
+
* abort is issued through the one controller the run already has. The `supervise()` settle path
|
|
2583
|
+
* records what the run then actually did — this manager cannot observe its own tree's terminal
|
|
2584
|
+
* state from inside `act`.
|
|
2585
|
+
*
|
|
2586
|
+
* Applied only at a TURN boundary, never on the final post-drain pass: by then the driver has
|
|
2587
|
+
* finished and drained, so a root abort could only void work that is already delivered. A
|
|
2588
|
+
* request that arrives that late expires in `finish()` instead — it terminated nothing.
|
|
2589
|
+
*/
|
|
2590
|
+
const passRun = () => {
|
|
2591
|
+
if (deps.controlScope !== "run" || deps.abortRun === void 0) return;
|
|
2592
|
+
const request = readRunCancelRequest(deps.dir);
|
|
2593
|
+
if (request === void 0) return;
|
|
2594
|
+
if (runTracked !== void 0) return;
|
|
2595
|
+
const prior = readRunCancellation(deps.dir, request.operationId);
|
|
2596
|
+
if (prior !== void 0) {
|
|
2597
|
+
runTracked = prior;
|
|
2598
|
+
return;
|
|
2599
|
+
}
|
|
2600
|
+
const record = {
|
|
2601
|
+
operationId: request.operationId,
|
|
2602
|
+
effect: "cancel_requested",
|
|
2603
|
+
requestedAt: request.at,
|
|
2604
|
+
observedAt: iso(),
|
|
2605
|
+
...request.reason === void 0 ? {} : { reason: request.reason },
|
|
2606
|
+
detail: "root abort issued to the whole run; termination not yet proven"
|
|
2607
|
+
};
|
|
2608
|
+
writeRunCancellation(deps.dir, record);
|
|
2609
|
+
runTracked = record;
|
|
2610
|
+
deps.abortRun(request.reason ?? "run cancel requested");
|
|
2611
|
+
};
|
|
2612
|
+
const pass = (phase) => {
|
|
2613
|
+
if (phase === "turn") passRun();
|
|
2614
|
+
for (const request of readWorkerCancelRequests(deps.dir)) {
|
|
2615
|
+
if (!owned(request.worker)) continue;
|
|
2616
|
+
let record = tracked.get(request.operationId);
|
|
2617
|
+
if (record === void 0) {
|
|
2618
|
+
record = readWorkerCancellation(deps.dir, request.operationId);
|
|
2619
|
+
if (record !== void 0) tracked.set(request.operationId, record);
|
|
2620
|
+
}
|
|
2621
|
+
if (record === void 0) {
|
|
2622
|
+
apply(request);
|
|
2623
|
+
continue;
|
|
2624
|
+
}
|
|
2625
|
+
if (record.effect === "cancel_requested") reconcile(record);
|
|
2626
|
+
else if (record.effect === "cancelled") regrow(record);
|
|
2627
|
+
}
|
|
2628
|
+
};
|
|
2629
|
+
return {
|
|
2630
|
+
pass,
|
|
2631
|
+
finish() {
|
|
2632
|
+
const runRequest = deps.controlScope === "run" && deps.abortRun !== void 0 ? readRunCancelRequest(deps.dir) : void 0;
|
|
2633
|
+
if (runRequest !== void 0 && readRunCancellation(deps.dir, runRequest.operationId) === void 0) writeRunCancellation(deps.dir, {
|
|
2634
|
+
operationId: runRequest.operationId,
|
|
2635
|
+
effect: "not_live",
|
|
2636
|
+
requestedAt: runRequest.at,
|
|
2637
|
+
observedAt: iso(),
|
|
2638
|
+
...runRequest.reason === void 0 ? {} : { reason: runRequest.reason },
|
|
2639
|
+
detail: "run ended before the request was applied"
|
|
2640
|
+
});
|
|
2641
|
+
for (const request of readWorkerCancelRequests(deps.dir)) {
|
|
2642
|
+
if (!owned(request.worker)) continue;
|
|
2643
|
+
const record = tracked.get(request.operationId) ?? readWorkerCancellation(deps.dir, request.operationId);
|
|
2644
|
+
if (record === void 0) {
|
|
2645
|
+
write({
|
|
2646
|
+
operationId: request.operationId,
|
|
2647
|
+
worker: request.worker,
|
|
2648
|
+
effect: "not_live",
|
|
2649
|
+
requestedAt: request.at,
|
|
2650
|
+
observedAt: iso(),
|
|
2651
|
+
...request.reason === void 0 ? {} : { reason: request.reason },
|
|
2652
|
+
detail: "run ended before the request was applied",
|
|
2653
|
+
terminated: []
|
|
2654
|
+
});
|
|
2655
|
+
continue;
|
|
2656
|
+
}
|
|
2657
|
+
if (record.effect === "cancel_requested") write({
|
|
2658
|
+
...record,
|
|
2659
|
+
effect: "unknown",
|
|
2660
|
+
observedAt: iso(),
|
|
2661
|
+
detail: "abort issued; run ended before termination was observed"
|
|
2662
|
+
});
|
|
2663
|
+
}
|
|
2664
|
+
}
|
|
2665
|
+
};
|
|
2666
|
+
}
|
|
2667
|
+
/**
|
|
2668
|
+
* Subtree node ids with a terminal `down`/`cancelled` journal record at or after `sinceIso` —
|
|
2669
|
+
* the abort-issue instant (the acknowledger's own `observedAt` on the `cancel_requested` record,
|
|
2670
|
+
* runtime clock), never the client's `requestedAt` — read from the durable spawn journal beside
|
|
2671
|
+
* the run layout. The set is proven at acknowledgement time and is approximate about post-abort
|
|
2672
|
+
* causation: a descendant that died of its OWN cause after the abort was issued is
|
|
2673
|
+
* indistinguishable from the cascade and may be included; one whose teardown journals late joins
|
|
2674
|
+
* on a later acknowledger pass; a teardown journal still absent when the run ends is absent from
|
|
2675
|
+
* the set. Ids are hierarchical (`parent:sN`), so `${nodeId}:` prefixes exactly the subtree.
|
|
2676
|
+
* Tolerant of a missing or partially-written journal: evidence that cannot be read names fewer
|
|
2677
|
+
* nodes, never wrong ones.
|
|
2678
|
+
*/
|
|
2679
|
+
function terminatedDescendants(dir, nodeId, sinceIso) {
|
|
2680
|
+
let raw;
|
|
2681
|
+
try {
|
|
2682
|
+
raw = readFileSync(join(dir, SPAWN_JOURNAL_FILE), "utf8");
|
|
2683
|
+
} catch {
|
|
2684
|
+
return [];
|
|
2685
|
+
}
|
|
2686
|
+
const prefix = `${nodeId}:`;
|
|
2687
|
+
const ids = /* @__PURE__ */ new Set();
|
|
2688
|
+
for (const line of raw.split("\n")) {
|
|
2689
|
+
const trimmed = line.trim();
|
|
2690
|
+
if (!trimmed) continue;
|
|
2691
|
+
let parsed;
|
|
2692
|
+
try {
|
|
2693
|
+
parsed = JSON.parse(trimmed);
|
|
2694
|
+
} catch {
|
|
2695
|
+
continue;
|
|
2696
|
+
}
|
|
2697
|
+
if (parsed.kind !== "event" || parsed.event === void 0) continue;
|
|
2698
|
+
const event = parsed.event;
|
|
2699
|
+
if (!(event.kind === "settled" && event.status === "down" || event.kind === "cancelled")) continue;
|
|
2700
|
+
if (typeof event.id !== "string" || !event.id.startsWith(prefix)) continue;
|
|
2701
|
+
if (typeof event.at !== "string" || event.at < sinceIso) continue;
|
|
2702
|
+
ids.add(event.id);
|
|
2703
|
+
}
|
|
2704
|
+
return [...ids].sort();
|
|
2705
|
+
}
|
|
2706
|
+
/**
|
|
2707
|
+
* Build the intelligent recursive driver. Its `act` is the LLM tool-loop; spawn it as a
|
|
2708
|
+
* `driverChild` (`driver-executor.ts`) to run it inside a nested scope, recursively.
|
|
2709
|
+
*/
|
|
2710
|
+
function driverAgent(opts) {
|
|
2711
|
+
if (typeof opts.brain !== "function") throw new ValidationError("driverAgent: opts.brain must be a function");
|
|
2712
|
+
if ((opts.extraTools?.length ?? 0) > 0 && typeof opts.executeExtraTool !== "function") throw new ValidationError("driverAgent: extraTools requires executeExtraTool (how to run a work-tool call)");
|
|
2713
|
+
if ((opts.analyzeOnSettle ?? []).map(normalizeAnalyzeOnSettle).some((route) => route.agent === void 0) && !opts.analysts) throw new ValidationError("driverAgent: analyzeOnSettle requires analysts (the lens registry the kinds resolve against)");
|
|
2714
|
+
const reserved = new Set(coordinationVerbNames);
|
|
2715
|
+
for (const tool of opts.nodeTools ?? []) {
|
|
2716
|
+
if (reserved.has(tool.name)) throw new ValidationError(`driverAgent: node tool "${tool.name}" collides with a coordination verb or another node tool`);
|
|
2717
|
+
reserved.add(tool.name);
|
|
2718
|
+
}
|
|
2719
|
+
for (const t of opts.extraTools ?? []) {
|
|
2720
|
+
if (reserved.has(t.name)) throw new ValidationError(`driverAgent: extra work tool "${t.name}" collides with a coordination verb or node tool`);
|
|
2721
|
+
reserved.add(t.name);
|
|
2722
|
+
}
|
|
2723
|
+
if (opts.maxTurns !== void 0 && opts.maxTurns < 0) throw new ValidationError("driverAgent: maxTurns must be >= 0 (0 lifts the turn cap; bounds become the conserved pool + deadline + abort)");
|
|
2724
|
+
const maxTurns = opts.maxTurns ?? 16;
|
|
2725
|
+
const now = opts.now ?? Date.now;
|
|
2726
|
+
const inbox = opts.inbox ?? createInbox();
|
|
2727
|
+
return {
|
|
2728
|
+
name: opts.name,
|
|
2729
|
+
deliver(message) {
|
|
2730
|
+
return inbox.deliver(message);
|
|
2731
|
+
},
|
|
2732
|
+
async act(task, scope) {
|
|
2733
|
+
const coord = createCoordinationTools({
|
|
2734
|
+
scope,
|
|
2735
|
+
blobs: opts.blobs,
|
|
2736
|
+
makeWorkerAgent: opts.makeWorkerAgent,
|
|
2737
|
+
...opts.authorizeDownMessage ? { authorizeDownMessage: opts.authorizeDownMessage } : {},
|
|
2738
|
+
perWorker: opts.perWorker,
|
|
2739
|
+
...opts.deliverable ? { deliverable: opts.deliverable } : {},
|
|
2740
|
+
...opts.maxLiveWorkers !== void 0 ? { maxLiveWorkers: opts.maxLiveWorkers } : {},
|
|
2741
|
+
...opts.analysts ? { analysts: opts.analysts } : {},
|
|
2742
|
+
...opts.analyzeOnSettle ? { analyzeOnSettle: opts.analyzeOnSettle } : {},
|
|
2743
|
+
...opts.watchWorkers ? { watchWorkers: opts.watchWorkers } : {},
|
|
2744
|
+
...opts.stallAfterMs !== void 0 ? { stallAfterMs: opts.stallAfterMs } : {},
|
|
2745
|
+
...opts.continuityByProfile ? { continuityByProfile: opts.continuityByProfile } : {},
|
|
2746
|
+
...opts.preflightSpawn ? { preflightSpawn: opts.preflightSpawn } : {},
|
|
2747
|
+
...opts.resolveSpawnProfile ? { resolveSpawnProfile: opts.resolveSpawnProfile } : {},
|
|
2748
|
+
...opts.onEvent ? { onEvent: opts.onEvent } : {},
|
|
2749
|
+
...opts.replaySettlements ? { replaySettlements: true } : {},
|
|
2750
|
+
...opts.priorCoordination?.questions.length ? { priorQuestions: opts.priorCoordination.questions } : {}
|
|
2751
|
+
});
|
|
2752
|
+
await coord.ready();
|
|
2753
|
+
opts.onCoordinationTools?.(coord.tools);
|
|
2754
|
+
const acknowledger = opts.controlDir === void 0 ? void 0 : createCancelAcknowledger({
|
|
2755
|
+
dir: opts.controlDir,
|
|
2756
|
+
coord,
|
|
2757
|
+
scope,
|
|
2758
|
+
now,
|
|
2759
|
+
ownerId: scope.view.root,
|
|
2760
|
+
controlScope: opts.controlScope ?? "run",
|
|
2761
|
+
...opts.abortRun ? { abortRun: opts.abortRun } : {}
|
|
2762
|
+
});
|
|
2763
|
+
const steerAcknowledger = opts.controlDir === void 0 ? void 0 : createSteerAcknowledger({
|
|
2764
|
+
dir: opts.controlDir,
|
|
2765
|
+
coord,
|
|
2766
|
+
now,
|
|
2767
|
+
ownerId: scope.view.root
|
|
2768
|
+
});
|
|
2769
|
+
for (const w of scope.resume?.waits ?? []) {
|
|
2770
|
+
const rearmed = scope.wait(w.spec, { label: w.label });
|
|
2771
|
+
if (!rearmed.ok) throw new RuntimeRunStateError(`driverAgent: cannot re-arm resumed wait '${w.label}' (${rearmed.reason})`);
|
|
2772
|
+
}
|
|
2773
|
+
const byName = new Map([...coord.tools, ...opts.nodeTools ?? []].map((t) => [t.name, t]));
|
|
2774
|
+
const toolSpecs = [
|
|
2775
|
+
...coord.tools.map((t) => ({
|
|
2776
|
+
type: "function",
|
|
2777
|
+
function: {
|
|
2778
|
+
name: t.name,
|
|
2779
|
+
description: t.description,
|
|
2780
|
+
parameters: t.inputSchema
|
|
2781
|
+
}
|
|
2782
|
+
})),
|
|
2783
|
+
...(opts.nodeTools ?? []).map((t) => ({
|
|
2784
|
+
type: "function",
|
|
2785
|
+
function: {
|
|
2786
|
+
name: t.name,
|
|
2787
|
+
description: t.description,
|
|
2788
|
+
parameters: t.inputSchema
|
|
2789
|
+
}
|
|
2790
|
+
})),
|
|
2791
|
+
...(opts.extraTools ?? []).map((t) => ({
|
|
2792
|
+
type: "function",
|
|
2793
|
+
function: {
|
|
2794
|
+
name: t.name,
|
|
2795
|
+
description: t.description,
|
|
2796
|
+
parameters: t.parameters
|
|
2797
|
+
}
|
|
2798
|
+
}))
|
|
2799
|
+
];
|
|
2800
|
+
const system = typeof opts.systemPrompt === "function" ? opts.systemPrompt(task) : opts.systemPrompt;
|
|
2801
|
+
const tracker = opts.stopRule ? createProgressTracker({ now }) : void 0;
|
|
2802
|
+
let progressStopReason;
|
|
2803
|
+
let driverTurn = 0;
|
|
2804
|
+
let driverCall = 0;
|
|
2805
|
+
const meteredBrain = async (messages, tools, detail) => {
|
|
2806
|
+
let res;
|
|
2807
|
+
const call = driverCall;
|
|
2808
|
+
driverCall += 1;
|
|
2809
|
+
const callContext = Object.freeze({
|
|
2810
|
+
signal: scope.signal,
|
|
2811
|
+
callId: `${scope.view.root}:brain:${crypto.randomUUID()}`,
|
|
2812
|
+
correlationId: scope.view.root
|
|
2813
|
+
});
|
|
2814
|
+
try {
|
|
2815
|
+
res = await opts.brain(messages, tools, callContext);
|
|
2816
|
+
} catch (error) {
|
|
2817
|
+
opts.onProviderModel?.(void 0);
|
|
2818
|
+
await meterRuntimeOwnedProviderAttempt(scope, unmeteredSpend(0), providerAttemptEvidence(void 0), {
|
|
2819
|
+
driver: opts.name,
|
|
2820
|
+
inferenceFailed: true,
|
|
2821
|
+
call,
|
|
2822
|
+
callId: callContext.callId,
|
|
2823
|
+
correlationId: callContext.correlationId,
|
|
2824
|
+
...detail
|
|
2825
|
+
});
|
|
2826
|
+
throw error;
|
|
2827
|
+
}
|
|
2828
|
+
let evidenceError;
|
|
2829
|
+
opts.onProviderModel?.(res.model);
|
|
2830
|
+
if (opts.expectedModel !== void 0) {
|
|
2831
|
+
if (res.model === void 0) evidenceError = new ValidationError(`driverAgent: Router response omitted model identity; expected ${JSON.stringify(opts.expectedModel)}`);
|
|
2832
|
+
else if (res.model !== opts.expectedModel) evidenceError = new ValidationError(`driverAgent: Router response reported model ${JSON.stringify(res.model)}; expected ${JSON.stringify(opts.expectedModel)}`);
|
|
2833
|
+
}
|
|
2834
|
+
if (res.transportAttempts !== void 0 && (!Number.isSafeInteger(res.transportAttempts) || res.transportAttempts < 1)) evidenceError = new ValidationError("driverAgent: transportAttempts must be a positive safe integer when reported");
|
|
2835
|
+
evidenceError = validateDriverPromptCache(res.promptCache) ?? evidenceError;
|
|
2836
|
+
const trustedCost = res.costProvenance === "provider-receipt" || res.costProvenance === "billing-receipt";
|
|
2837
|
+
const cacheUsage = promptCacheTokenClasses(res.usage?.input, res.promptCache);
|
|
2838
|
+
await meterRuntimeOwnedProviderAttempt(scope, {
|
|
2839
|
+
iterations: 0,
|
|
2840
|
+
tokens: {
|
|
2841
|
+
input: res.usage?.input ?? 0,
|
|
2842
|
+
output: res.usage?.output ?? 0,
|
|
2843
|
+
...cacheUsage
|
|
2844
|
+
},
|
|
2845
|
+
...res.usage === void 0 ? { tokensKnown: false } : {},
|
|
2846
|
+
usd: trustedCost ? res.costUsd ?? 0 : 0,
|
|
2847
|
+
...trustedCost && res.costUsd !== void 0 ? {} : { usdKnown: false },
|
|
2848
|
+
ms: 0
|
|
2849
|
+
}, providerAttemptEvidence(res.model), {
|
|
2850
|
+
driver: opts.name,
|
|
2851
|
+
call,
|
|
2852
|
+
callId: callContext.callId,
|
|
2853
|
+
correlationId: callContext.correlationId,
|
|
2854
|
+
toolCalls: (res.toolCalls ?? []).map((c) => c.name),
|
|
2855
|
+
...res.model !== void 0 ? { model: res.model } : {},
|
|
2856
|
+
...res.transportAttempts !== void 0 ? { transportAttempts: res.transportAttempts } : {},
|
|
2857
|
+
...res.usage?.reasoning !== void 0 ? { reasoningTokens: res.usage.reasoning } : {},
|
|
2858
|
+
...res.promptCache !== void 0 ? { promptCache: res.promptCache } : {},
|
|
2859
|
+
...res.usageUnknown === true ? { streamUsageMissing: true } : {},
|
|
2860
|
+
...res.costProvenance === "catalog-estimate" ? { estimatedCostUsd: res.costUsd } : {},
|
|
2861
|
+
...detail
|
|
2862
|
+
});
|
|
2863
|
+
if (evidenceError !== void 0) throw evidenceError;
|
|
2864
|
+
return res;
|
|
2865
|
+
};
|
|
2866
|
+
const chat = async (messages, tools) => {
|
|
2867
|
+
const res = await meteredBrain(messages, tools, {
|
|
2868
|
+
kind: "driver-inference",
|
|
2869
|
+
turn: driverTurn
|
|
2870
|
+
});
|
|
2871
|
+
driverTurn += 1;
|
|
2872
|
+
return res;
|
|
2873
|
+
};
|
|
2874
|
+
const compaction = opts.compaction ? {
|
|
2875
|
+
thresholdTokens: opts.compaction.thresholdTokens,
|
|
2876
|
+
distill: opts.compaction.distill ?? (async (msgs) => {
|
|
2877
|
+
const roster = summarizeRoster(scope.view, coord.settled());
|
|
2878
|
+
try {
|
|
2879
|
+
const narrative = ((await meteredBrain([...msgs, {
|
|
2880
|
+
role: "user",
|
|
2881
|
+
content: distillInstruction
|
|
2882
|
+
}], [], {
|
|
2883
|
+
kind: "driver-compaction",
|
|
2884
|
+
compactingTurn: driverTurn
|
|
2885
|
+
})).content ?? "").trim();
|
|
2886
|
+
return narrative ? `${roster}\n\n## Progress notes\n${narrative}` : roster;
|
|
2887
|
+
} catch (e) {
|
|
2888
|
+
return `${roster}\n\n## Progress notes\nSummary unavailable: ${errMessage(e)}`;
|
|
2889
|
+
}
|
|
2890
|
+
}),
|
|
2891
|
+
...opts.compaction.onCompact ? { onCompact: opts.compaction.onCompact } : {},
|
|
2892
|
+
...opts.compaction.preserveHead !== void 0 ? { preserveHead: opts.compaction.preserveHead } : {},
|
|
2893
|
+
...opts.compaction.estimateTokens ? { estimateTokens: opts.compaction.estimateTokens } : {}
|
|
2894
|
+
} : void 0;
|
|
2895
|
+
await runBrainLoop({
|
|
2896
|
+
chat,
|
|
2897
|
+
tools: toolSpecs,
|
|
2898
|
+
...compaction ? { compaction } : {},
|
|
2899
|
+
execute: async (name, args) => {
|
|
2900
|
+
if (opts.executeExtraTool) {
|
|
2901
|
+
const worked = await runExtraTool(opts.executeExtraTool, name, args);
|
|
2902
|
+
if (worked !== null && worked !== void 0) return worked;
|
|
2903
|
+
}
|
|
2904
|
+
const tool = byName.get(name);
|
|
2905
|
+
return safeJson(tool ? await runTool(tool, args) : { error: `unknown tool: ${name}` });
|
|
2906
|
+
},
|
|
2907
|
+
initialMessages: [
|
|
2908
|
+
{
|
|
2909
|
+
role: "system",
|
|
2910
|
+
content: system
|
|
2911
|
+
},
|
|
2912
|
+
{
|
|
2913
|
+
role: "user",
|
|
2914
|
+
content: stringifyTask(task)
|
|
2915
|
+
},
|
|
2916
|
+
...scope.resume ? [{
|
|
2917
|
+
role: "user",
|
|
2918
|
+
content: resumeBrief(scope.resume, opts.priorCoordination)
|
|
2919
|
+
}] : hasPriorCoordination(opts.priorCoordination) ? [{
|
|
2920
|
+
role: "user",
|
|
2921
|
+
content: priorCoordinationBrief(opts.priorCoordination)
|
|
2922
|
+
}] : []
|
|
2923
|
+
],
|
|
2924
|
+
maxTurns,
|
|
2925
|
+
hooks: {
|
|
2926
|
+
beforeTurn: async (_turn, messages) => {
|
|
2927
|
+
await steerAcknowledger?.pass("turn");
|
|
2928
|
+
acknowledger?.pass("turn");
|
|
2929
|
+
const pending = inbox.drain();
|
|
2930
|
+
if (pending.length > 0) messages.push({
|
|
2931
|
+
role: "user",
|
|
2932
|
+
content: inbox.fold(pending)
|
|
2933
|
+
});
|
|
2934
|
+
},
|
|
2935
|
+
stopBefore: () => {
|
|
2936
|
+
if (coord.isStopped() || scope.signal.aborted || poolStarved(scope, opts.perWorker) || deadlinePassed(scope, now)) return true;
|
|
2937
|
+
if (!opts.stopRule || !tracker) return false;
|
|
2938
|
+
const decision = progressStop(tracker, opts.stopRule, coord, scope, now, opts.stallAfterMs);
|
|
2939
|
+
if (!decision.stop) return false;
|
|
2940
|
+
if (progressStopReason === void 0) {
|
|
2941
|
+
progressStopReason = decision.reason;
|
|
2942
|
+
opts.onProgressStop?.(decision.reason);
|
|
2943
|
+
}
|
|
2944
|
+
return true;
|
|
2945
|
+
}
|
|
2946
|
+
}
|
|
2947
|
+
});
|
|
2948
|
+
await coord.drainResolved();
|
|
2949
|
+
await steerAcknowledger?.pass("final");
|
|
2950
|
+
acknowledger?.pass("final");
|
|
2951
|
+
acknowledger?.finish();
|
|
2952
|
+
const submitted = coord.submittedResult();
|
|
2953
|
+
if (submitted) return submitted.result;
|
|
2954
|
+
return runFinalizer(opts.finalizer ?? bestDelivered, {
|
|
2955
|
+
settled: coord.settled(),
|
|
2956
|
+
blobs: opts.blobs,
|
|
2957
|
+
tree: runTree(scope),
|
|
2958
|
+
budget: scope.budget
|
|
2959
|
+
});
|
|
2960
|
+
}
|
|
2961
|
+
};
|
|
2962
|
+
}
|
|
2963
|
+
/**
|
|
2964
|
+
* The factual context a resumed driver starts from — everything the durable stores prove about
|
|
2965
|
+
* the prior process(es): committed settlements, per-key states (completed / lost / failed),
|
|
2966
|
+
* re-armed waits, carried-over questions/findings/continuation receipts, and spend already paid.
|
|
2967
|
+
* Injected as the brain's first user-context on a resumed run so it continues from unresolved work;
|
|
2968
|
+
* old continuation receipts are evidence and are never auto-delivered.
|
|
2969
|
+
*/
|
|
2970
|
+
function resumeBrief(resume, prior) {
|
|
2971
|
+
const lines = [
|
|
2972
|
+
"RESUME: this run continues a prior coordinator process. Its committed work is restored",
|
|
2973
|
+
"below and already counts toward the deliverable — do NOT redo it. Continue from the",
|
|
2974
|
+
"unresolved work only.",
|
|
2975
|
+
"",
|
|
2976
|
+
`Committed workers (${resume.settled.length}):`
|
|
2977
|
+
];
|
|
2978
|
+
if (resume.settled.length === 0) lines.push("- none");
|
|
2979
|
+
for (const s of resume.settled) lines.push(s.kind === "done" ? `- ${s.handle.id} (${s.handle.label}): done, score=${s.verdict?.score ?? 0}, valid=${s.verdict?.valid ?? false}, outRef=${s.outRef}` : `- ${s.handle.id} (${s.handle.label}): down, reason=${s.reason}`);
|
|
2980
|
+
const byState = (state) => [...resume.keys].filter(([, v]) => v.state === state);
|
|
2981
|
+
const completed = byState("completed");
|
|
2982
|
+
const lost = byState("in-doubt");
|
|
2983
|
+
const failed = byState("down");
|
|
2984
|
+
if (completed.length > 0) lines.push("", "COMPLETED keys — spawn_agent with the same key returns the finished result, spending nothing:", ...completed.map(([k, v]) => `- ${k} → ${v.id} (${v.label})`));
|
|
2985
|
+
if (lost.length > 0) lines.push("", "Keys LOST in flight with the prior process — this is the unresolved work; spawn_agent with the same key starts a fresh attempt:", ...lost.map(([k, v]) => `- ${k} (prior attempt ${v.id}, ${v.label})`));
|
|
2986
|
+
if (failed.length > 0) lines.push("", "Keys whose prior attempt FAILED (settled down) — spawn_agent with the same key retries:", ...failed.map(([k, v]) => `- ${k} (prior attempt ${v.id}, ${v.label})`));
|
|
2987
|
+
if (resume.waits.length > 0) lines.push("", "Pending waits RE-ARMED on their original deadlines (they settle through await_event):", ...resume.waits.map((w) => `- ${w.label} (${w.spec.kind})`));
|
|
2988
|
+
appendPriorCoordination(lines, prior);
|
|
2989
|
+
const spent = resume.priorSpend;
|
|
2990
|
+
lines.push("", "Budget the run ALREADY spent before this process (it counts toward the run total):", `- child work: tokens charged=${chargedTokens(spent.childWork.tokens)} (in=${spent.childWork.tokens.input} out=${spent.childWork.tokens.output}), usd=${spent.childWork.usd}, iterations=${spent.childWork.iterations}`, `- driver inference: tokens charged=${chargedTokens(spent.driverInference.tokens)} (in=${spent.driverInference.tokens.input} out=${spent.driverInference.tokens.output}), usd=${spent.driverInference.usd}`);
|
|
2991
|
+
return lines.join("\n");
|
|
2992
|
+
}
|
|
2993
|
+
function hasPriorCoordination(prior) {
|
|
2994
|
+
return prior !== void 0 && (prior.questions.length > 0 || prior.findings.length > 0 || prior.continuations.length > 0 || prior.deliveryEvidence.length > 0);
|
|
2995
|
+
}
|
|
2996
|
+
function priorCoordinationBrief(prior) {
|
|
2997
|
+
const lines = [
|
|
2998
|
+
"PRIOR COORDINATION EVIDENCE: this logical supervisor ran in an earlier process.",
|
|
2999
|
+
"Use the evidence below as context. Never auto-deliver an old continuation; issue a new",
|
|
3000
|
+
"authorized instruction only when current live state still warrants it."
|
|
3001
|
+
];
|
|
3002
|
+
appendPriorCoordination(lines, prior);
|
|
3003
|
+
return lines.join("\n");
|
|
3004
|
+
}
|
|
3005
|
+
function appendPriorCoordination(lines, prior) {
|
|
3006
|
+
const openQuestions = (prior?.questions ?? []).filter((q) => q.status === "open" || q.status === "escalated");
|
|
3007
|
+
if (openQuestions.length > 0) lines.push("", "Questions carried over, still undecided (answer_question decides them; list_questions shows all):", ...openQuestions.map((q) => `- [${q.id}] from=${q.from}, urgency=${q.urgency}: ${q.question}`));
|
|
3008
|
+
if ((prior?.findings.length ?? 0) > 0) lines.push("", "Analyst findings from the prior process:", ...(prior?.findings ?? []).map((f) => `- ${f.analyst} on ${f.fromWorker}: ${safeJson(f.findings)}`));
|
|
3009
|
+
if ((prior?.continuations.length ?? 0) > 0) {
|
|
3010
|
+
const attempts = new Set((prior?.deliveryEvidence ?? []).filter((event) => event.type === "delivery-attempt").map((event) => event.attempt.receiptId));
|
|
3011
|
+
const outcomes = new Map((prior?.deliveryEvidence ?? []).filter((event) => event.type === "steer" || event.type === "answer").map((event) => [event.down.receiptId, event.down.outcome]));
|
|
3012
|
+
lines.push("", "Authorized continuations committed by the prior process (evidence only; never replayed automatically):", ...(prior?.continuations ?? []).map((continuation) => {
|
|
3013
|
+
const delivery = outcomes.get(continuation.receiptId) ?? (attempts.has(continuation.receiptId) ? "unknown-after-crash" : "not-attempted-before-crash");
|
|
3014
|
+
return `- receipt=${continuation.receiptId}, ${continuation.kind} → ${continuation.toWorker}, instruction=${continuation.instructionDigest}, delivery=${delivery}`;
|
|
3015
|
+
}));
|
|
3016
|
+
}
|
|
3017
|
+
}
|
|
3018
|
+
/** Run a work tool. A throw is data to the driver (it can recover next turn), not a crash — fold
|
|
3019
|
+
* the error back as a string result. null/undefined passes through (the caller treats it as "not
|
|
3020
|
+
* handled" and falls to the coordination dispatch). */
|
|
3021
|
+
async function runExtraTool(execute, name, args) {
|
|
3022
|
+
try {
|
|
3023
|
+
return await execute(name, args);
|
|
3024
|
+
} catch (e) {
|
|
3025
|
+
return `error: ${e instanceof Error ? e.message : String(e)}`;
|
|
3026
|
+
}
|
|
3027
|
+
}
|
|
3028
|
+
async function runTool(tool, args) {
|
|
3029
|
+
try {
|
|
3030
|
+
return await tool.handler(args);
|
|
3031
|
+
} catch (e) {
|
|
3032
|
+
return { error: e instanceof Error ? e.message : String(e) };
|
|
3033
|
+
}
|
|
3034
|
+
}
|
|
3035
|
+
/** Keep-best finalize under the completion-oracle: return the highest-scoring DELIVERED child's
|
|
3036
|
+
* output (settled `done` AND `valid` — its deliverable check passed). Returns undefined when no
|
|
3037
|
+
* child delivered — an honest "the driver produced nothing", never a high-scoring result that
|
|
3038
|
+
* ran without passing its check (Foreman's 0/18 lesson). `valid` is the single delivery signal,
|
|
3039
|
+
* matching `defaultSelectWinner`'s valid-first rule; the oracle just doesn't fall back to an
|
|
3040
|
+
* unchecked best-effort. The same argmax as the `bestDelivered` finalizer (`pickBestDelivered`);
|
|
3041
|
+
* this direct form serves callers that hold a bare ledger + blob store. */
|
|
3042
|
+
async function finalizeBestDelivered(settled, blobs) {
|
|
3043
|
+
const best = pickBestDelivered(settled.filter((w) => w.status === "done" && w.valid === true));
|
|
3044
|
+
if (best === void 0) return void 0;
|
|
3045
|
+
return best.outRef ? await blobs.get(best.outRef) : void 0;
|
|
3046
|
+
}
|
|
3047
|
+
function stringifyTask(task) {
|
|
3048
|
+
return typeof task === "string" ? task : safeJson(task);
|
|
3049
|
+
}
|
|
3050
|
+
function safeJson(v) {
|
|
3051
|
+
try {
|
|
3052
|
+
return JSON.stringify(v) ?? String(v);
|
|
3053
|
+
} catch {
|
|
3054
|
+
return String(v);
|
|
3055
|
+
}
|
|
3056
|
+
}
|
|
3057
|
+
function errMessage(e) {
|
|
3058
|
+
return e instanceof Error ? e.message : String(e);
|
|
3059
|
+
}
|
|
3060
|
+
//#endregion
|
|
3061
|
+
export { coordinationVerbNames as C, createEventBus as D, questionEscalationTargets as E, defaultToolDetectors as O, canonicalFindingEvent as S, normalizeAnalyzeOnSettle as T, renderAnytimeTable as _, allOf as a, FileCoordinationLog as b, createProgressTracker as c, progressStop as d, sampleFromSettled as f, plateauLength as g, bestSoFar as h, finalizeBestDelivered as i, watchTrace as k, noProgressFor as l, areaUnderCurve as m, createSteerAcknowledger as n, allWorkersStalled as o, anytimeReport as p, driverAgent as r, anyOf as s, createCancelAcknowledger as t, plateau as u, createFileRunContext as v, createCoordinationTools as w, DEFAULT_AWAIT_EVENT_TIMEOUT_MS as x, createInMemoryRunContext as y };
|
|
3062
|
+
|
|
3063
|
+
//# sourceMappingURL=coordination-driver-xO1tkxy-.js.map
|