@tangle-network/agent-runtime 0.219.0 → 0.220.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{activation-C-8VqFSN.js → activation-D3yrSA_b.js} +2 -2
- package/dist/{activation-C-8VqFSN.js.map → activation-D3yrSA_b.js.map} +1 -1
- package/dist/agent.d.ts +1 -1
- package/dist/agent.js +2 -2
- package/dist/{coordination-driver-CvinSUI-.js → coordination-driver-BxGh-Tnj.js} +599 -355
- package/dist/coordination-driver-BxGh-Tnj.js.map +1 -0
- package/dist/{delegate-BH8r8hkQ.js → delegate-CiGfF17V.js} +2 -2
- package/dist/{delegate-BH8r8hkQ.js.map → delegate-CiGfF17V.js.map} +1 -1
- package/dist/durable.d.ts +1 -1
- package/dist/durable.js +2 -2
- package/dist/{graph-5N4gLnOt.js → graph-D9tSg6Vd.js} +3 -3
- package/dist/{graph-5N4gLnOt.js.map → graph-D9tSg6Vd.js.map} +1 -1
- package/dist/{improvement-cycle-BIAKVZ3B.js → improvement-cycle-BZ_KfBap.js} +3 -3
- package/dist/{improvement-cycle-BIAKVZ3B.js.map → improvement-cycle-BZ_KfBap.js.map} +1 -1
- package/dist/{index-CLDpu-bs.d.ts → index-B9_U4zYI.d.ts} +16 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.js +7 -7
- package/dist/intelligence.js +3 -3
- package/dist/kernel.d.ts +1 -1
- package/dist/kernel.js +8 -8
- package/dist/{loop-runner-bin-Cn9DBrLv.d.ts → loop-runner-bin-BaeOlnpi.d.ts} +2 -2
- package/dist/{loop-runner-bin-CTEJAcac.js → loop-runner-bin-DkU6vtuc.js} +3 -3
- package/dist/{loop-runner-bin-CTEJAcac.js.map → loop-runner-bin-DkU6vtuc.js.map} +1 -1
- package/dist/loop-runner-bin.d.ts +1 -1
- package/dist/loop-runner-bin.js +1 -1
- package/dist/mcp/bin.js +3 -3
- package/dist/mcp/index.d.ts +1 -1
- package/dist/mcp/index.js +4 -4
- package/dist/{provision-supervisor-40EOGn14.js → provision-supervisor-Cg3fETK-.js} +3 -3
- package/dist/{provision-supervisor-40EOGn14.js.map → provision-supervisor-Cg3fETK-.js.map} +1 -1
- package/dist/{runtime-BDK_GOAd.js → runtime-DZcPHLwa.js} +8 -8
- package/dist/{runtime-BDK_GOAd.js.map → runtime-DZcPHLwa.js.map} +1 -1
- package/dist/{server-DuvbSAVA.js → server-BgPZIHRd.js} +3 -3
- package/dist/{server-DuvbSAVA.js.map → server-BgPZIHRd.js.map} +1 -1
- package/dist/{structural-rollout-BAr61WjX.js → structural-rollout-iCePfrcK.js} +2 -2
- package/dist/{structural-rollout-BAr61WjX.js.map → structural-rollout-iCePfrcK.js.map} +1 -1
- package/dist/{supervise-W_mtzswa.js → supervise-CSYu0BeH.js} +440 -726
- package/dist/supervise-CSYu0BeH.js.map +1 -0
- package/dist/{supervisor-WRrw0nlk.js → supervisor-n8DDNGli.js} +368 -31
- package/dist/supervisor-n8DDNGli.js.map +1 -0
- package/dist/testing.d.ts +1 -1
- package/dist/testing.js +12 -12
- package/dist/tui/index.d.ts +1 -1
- package/dist/tui/index.js +1 -1
- package/package.json +1 -1
- package/dist/coordination-driver-CvinSUI-.js.map +0 -1
- package/dist/supervise-W_mtzswa.js.map +0 -1
- package/dist/supervisor-WRrw0nlk.js.map +0 -1
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
import { $ as
|
|
2
|
-
import {
|
|
1
|
+
import { $ as bindReusableExecutorExecutionId, Br as fullProfileMaterialization, C as scopeOwnerExecutorNodeContext, Ci as detachedSnapshot, Dt as spendFromUsageEvents, F as scopeRetainedOwnerResult, Fn as withTimeout, Gn as RunCancellationReason, Hr as promptControlProfileMaterialization, In as zeroSpend, Jn as runAbortable, Jt as bridgeAdmissionRefusal, Ln as addResourceSpend, Lr as assertProfileMaterialization, Lt as createInbox, M as prepareScopeRetainedOwnerTask, N as scopeRetainedOwnerContext, Ot as WORKER_TRACE_PROPAGATION, P as scopeRetainedOwnerPriorSpend, Pn as unmeteredSpend, Q as teardownExecutor, Rn as resourceTelemetry, Rr as controlProfileMaterialization, S as restoreScopeOwnerAcceptedExecution, Sn as addSpend, Tt as assertValidBudget, Un as registerRetainedExecutorPreparation, Ur as promptModelProfileMaterialization, Vr as profileMaterializationAxes$1, Wn as retainedExecutorSeamKey, Yr as unsupportedProfileDimensions, Yt as bridgeModelRouteRefusal, Z as DEFAULT_SUCCESSFUL_SHUTDOWN_MS, Zr as worktreeCliProfileMaterialization, _i as runtimeOwnedScopeOwnerRuntime, at as snapshotExecutorConfig, b as meterRuntimeOwnedProviderAttempt, bn as resolveAgentEnvironmentProvider, d as runDriverWithRetry, di as recordRuntimeOwnedDriveHarnessProviderEvidence, en as bridgeRuntimeAttachmentsKey, et as captureReusableExecutorConfig, f as driverChild, fi as runtimeOwnedDriveHarnessProviderEvidence, fn as routerBrain, fr as createOtelExporter, g as beginScopeOwnerAttempt, gi as runtimeOwnedPendingExecutorMaterialization, hi as runtimeOwnedExecutorProviderEvidence, hr as generateSpanId, j as consumeScopeRetainedOwnerResult, m as isDriverSpec, mi as runtimeOwnedExecutorMaterialization, n as createSupervisor, o as runFinalizer, oi as inheritRuntimeOwnedExecutorAttestation, p as driverExecutorFactory, pi as runtimeOwnedExecutorExecutionBinding, qr as renderUnsupported, r as bestDelivered, ri as attestRuntimeOwnedScopeOwner, rt as createExecutor, s as runTree, t as createRootHandle, tn as bridgeStopSignalKey, u as defaultUnmetContractSteer, ui as providerAttemptEvidence, v as deriveNodeExecutionIdentity, wt as contentAddress, x as recordScopeOwnerMaterialization, y as meterRuntimeOwnedAccounting, yr as toOtelAttributes, zn as withBudgetResources, zr as defineProfileMaterializationContract } from "./supervisor-n8DDNGli.js";
|
|
2
|
+
import { i as ConfigError, m as ValidationError } from "./errors-DodWX-cb.js";
|
|
3
3
|
import { a as concreteProfileModel, c as profileModelExecutionSettings, d as agentHarness, f as harnessRunsAgent, n as assertModelAllowed, o as enforceTokenLimits, r as assertProfileModelsAllowed, s as profileBridgeWireModel, t as assertExecutableAgentProfile } from "./model-policy-DKDyr-fc.js";
|
|
4
4
|
import { t as composeRuntimeHooks } from "./runtime-hooks-tXpAarhW.js";
|
|
5
|
-
import { k as writeRunCancellation, o as readRunCancelRequest, s as readRunCancellation } from "./run-layout-hZREkcfu.js";
|
|
6
|
-
import { C as createCoordinationTools, F as watchRunCancellation, M as createInMemoryRunContext, P as applyRunCancellation, S as coordinationVerbNames, c as createProgressTracker, d as progressStop, j as createFileRunContext, r as driverAgent, y as DEFAULT_AWAIT_EVENT_TIMEOUT_MS } from "./coordination-driver-CvinSUI-.js";
|
|
7
5
|
import { t as createStdioToolServer } from "./tool-server-BJbCPhoW.js";
|
|
6
|
+
import { D as coordinationVerbNames, O as createCoordinationTools, S as watchRunCancellation, c as createProgressTracker, d as progressStop, r as driverAgent, v as createFileRunContext, w as DEFAULT_AWAIT_EVENT_TIMEOUT_MS, x as applyRunCancellation, y as createInMemoryRunContext } from "./coordination-driver-BxGh-Tnj.js";
|
|
7
|
+
import { k as writeRunCancellation, o as readRunCancelRequest, s as readRunCancellation } from "./run-layout-hZREkcfu.js";
|
|
8
8
|
import { agentProfileSchema, canonicalAgentProfileDigest, canonicalCandidateDigest, canonicalCandidateJson, validateAgentProfileSecurity } from "@tangle-network/agent-interface";
|
|
9
9
|
import { createHash, createHmac, randomBytes, randomUUID, timingSafeEqual } from "node:crypto";
|
|
10
10
|
import { resolve } from "node:path";
|
|
@@ -154,357 +154,43 @@ function isAsyncIterable$2(v) {
|
|
|
154
154
|
return v != null && typeof v[Symbol.asyncIterator] === "function";
|
|
155
155
|
}
|
|
156
156
|
//#endregion
|
|
157
|
-
//#region src/runtime/supervise/
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
* parallelism) needs its own bespoke reader. A span carrying `parent_span_id` IS a tree, and any
|
|
164
|
-
* system can emit one — so emitting spans makes the supervisor readable by the same viewer as
|
|
165
|
-
* everything else, with no per-system reader.
|
|
166
|
-
*
|
|
167
|
-
* WHAT IT IS NOT. This is telemetry, never the record of truth. The spawn journal remains the sole
|
|
168
|
-
* durable ledger for replay/resume and for cost; nothing here is read back, and a run whose export
|
|
169
|
-
* fails is unaffected in every observable way. The two data models are deliberately separate.
|
|
170
|
-
*
|
|
171
|
-
* HOW IT ATTACHES. This is a pure `RuntimeHooks` observer over the lifecycle events `Scope` ALREADY
|
|
172
|
-
* emits — `agent.spawn` (a node opened), `agent.child` (a node settled), `agent.turn` (a driver
|
|
173
|
-
* inference turn was metered). It adds no event, mutates no journal, and changes no result. Because
|
|
174
|
-
* `Scope` re-seeds the same hooks into every nested scope (`makeNestedScopeSeam`), one observer sees
|
|
175
|
-
* the WHOLE recursion at arbitrary depth.
|
|
176
|
-
*
|
|
177
|
-
* SPAN SHAPE. One span per supervised node, opened at spawn and closed at settle, parented to its
|
|
178
|
-
* parent node's span; the run root is the trace. Driver inference rides as an `LLM` child span under
|
|
179
|
-
* the node that metered it. Attributes reuse the vocabulary the rest of the stack already reads
|
|
180
|
-
* (`openinference.span.kind`, `agent.name`, `llm.token_count.*`, `llm.cost_usd`, `tangle.cost.usd`)
|
|
181
|
-
* — see `@tangle-network/agent-eval`'s `src/trace/attribute-vocabulary.ts`, which is the consumer.
|
|
182
|
-
*
|
|
183
|
-
* UNKNOWN IS NEVER ZERO. `Spend.tokensKnown === false` / `usdKnown === false` mark work that
|
|
184
|
-
* HAPPENED with an unreported count. Those spans OMIT the token/cost attribute entirely and set
|
|
185
|
-
* `tangle.supervise.tokens_known` / `tangle.supervise.cost_known` to `false`, so a reader can never
|
|
186
|
-
* mistake an unmeasured turn for a free one.
|
|
187
|
-
*/
|
|
188
|
-
/** OTEL status codes (`UNSET` / `OK` / `ERROR`) — the numeric wire values `OtelSpan.status` carries. */
|
|
189
|
-
const STATUS_UNSET = 0;
|
|
190
|
-
const STATUS_OK = 1;
|
|
191
|
-
const STATUS_ERROR = 2;
|
|
192
|
-
/** Longest string attribute value written from free-form detail, so an oversized turn payload
|
|
193
|
-
* cannot inflate a span. Identity/label attributes we control are never truncated. */
|
|
194
|
-
const MAX_DETAIL_CHARS = 256;
|
|
195
|
-
/**
|
|
196
|
-
* Build the span recorder for one supervised run, or `undefined` when no exporter resolves — the
|
|
197
|
-
* off-by-default path. A run that passes no `exporter` and no `exportConfig` never reaches this
|
|
198
|
-
* function at all; one that passes an `exportConfig` with no endpoint (and no env endpoint) gets
|
|
199
|
-
* `undefined` here, so "configured but unreachable" also costs nothing.
|
|
200
|
-
*/
|
|
201
|
-
function createSupervisorSpanRecorder(opts) {
|
|
202
|
-
const exporter = opts.exporter ?? createOtelExporter(opts.exportConfig);
|
|
203
|
-
if (!exporter) return void 0;
|
|
204
|
-
const ownsExporter = opts.exporter === void 0;
|
|
205
|
-
const now = opts.now ?? Date.now;
|
|
206
|
-
const traceId = normalizeTraceId(opts.traceId, opts.runId);
|
|
207
|
-
const rootSpanId = generateSpanId();
|
|
208
|
-
const rootStartMs = now();
|
|
209
|
-
const base = {
|
|
210
|
-
"tangle.run.id": opts.runId,
|
|
211
|
-
"tangle.sessionId": opts.runId,
|
|
212
|
-
...opts.attributes ?? {}
|
|
213
|
-
};
|
|
214
|
-
/** Node id → its open span. Seeded with the run id ⇒ the root span, because a depth-0 spawn's
|
|
215
|
-
* `parentId` is the run id itself and every deeper spawn's is a real node id. */
|
|
216
|
-
const open = /* @__PURE__ */ new Map();
|
|
217
|
-
const spanIdOf = /* @__PURE__ */ new Map([[opts.runId, rootSpanId]]);
|
|
218
|
-
let finished = false;
|
|
219
|
-
/** Every export is best-effort: a throwing exporter must never reach the run. */
|
|
220
|
-
const emit = (span) => {
|
|
221
|
-
try {
|
|
222
|
-
exporter.exportSpan(span);
|
|
223
|
-
} catch {}
|
|
157
|
+
//#region src/runtime/supervise/coordination-http.ts
|
|
158
|
+
function coordinationHttpLimits(options) {
|
|
159
|
+
const positive = (name, value, fallback) => {
|
|
160
|
+
const resolved = value ?? fallback;
|
|
161
|
+
if (!Number.isSafeInteger(resolved) || resolved <= 0) throw new ConfigError(`coordination ${name} must be a positive safe integer`);
|
|
162
|
+
return resolved;
|
|
224
163
|
};
|
|
225
|
-
const
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
name,
|
|
230
|
-
kind: 1,
|
|
231
|
-
startTimeUnixNano: msToNano(startMs),
|
|
232
|
-
endTimeUnixNano: msToNano(Math.max(startMs, endMs)),
|
|
233
|
-
attributes: toOtelAttributes(attrs),
|
|
234
|
-
status: {
|
|
235
|
-
code: status,
|
|
236
|
-
...message ? { message } : {}
|
|
237
|
-
}
|
|
238
|
-
});
|
|
239
|
-
function onSpawn(event) {
|
|
240
|
-
const p = record(event.payload);
|
|
241
|
-
const childId = str(p.childId);
|
|
242
|
-
if (!childId) return;
|
|
243
|
-
const label = str(p.label) ?? "node";
|
|
244
|
-
const runtime = str(p.runtime);
|
|
245
|
-
const isWait = runtime === "wait";
|
|
246
|
-
const attrs = {
|
|
247
|
-
...base,
|
|
248
|
-
"openinference.span.kind": isWait ? "CHAIN" : "AGENT",
|
|
249
|
-
"agent.name": label,
|
|
250
|
-
"tangle.supervise.node.id": childId,
|
|
251
|
-
"tangle.supervise.node.label": label,
|
|
252
|
-
"tangle.supervise.node.kind": isWait ? "wait" : "agent",
|
|
253
|
-
"tangle.supervise.tree.root": event.runId
|
|
254
|
-
};
|
|
255
|
-
if (event.parentId) attrs["tangle.supervise.node.parent_id"] = event.parentId;
|
|
256
|
-
if (runtime) attrs["tangle.supervise.node.runtime"] = runtime;
|
|
257
|
-
if (typeof p.depth === "number") attrs["tangle.supervise.node.depth"] = p.depth;
|
|
258
|
-
if (typeof event.stepIndex === "number") attrs["tangle.supervise.node.ordinal"] = event.stepIndex;
|
|
259
|
-
if (p.resumed === true) attrs["tangle.supervise.node.resumed"] = true;
|
|
260
|
-
assignBudget(attrs, p.budget);
|
|
261
|
-
const spanId = generateSpanId();
|
|
262
|
-
spanIdOf.set(childId, spanId);
|
|
263
|
-
open.set(childId, {
|
|
264
|
-
spanId,
|
|
265
|
-
parentSpanId: event.parentId && spanIdOf.get(event.parentId) || rootSpanId,
|
|
266
|
-
name: label,
|
|
267
|
-
startMs: event.timestamp,
|
|
268
|
-
attrs
|
|
269
|
-
});
|
|
270
|
-
}
|
|
271
|
-
function onSettled(event) {
|
|
272
|
-
const p = record(event.payload);
|
|
273
|
-
const childId = str(p.childId);
|
|
274
|
-
if (!childId) return;
|
|
275
|
-
const node = open.get(childId);
|
|
276
|
-
if (!node) return;
|
|
277
|
-
open.delete(childId);
|
|
278
|
-
const status = str(p.status);
|
|
279
|
-
const down = status === "down";
|
|
280
|
-
const attrs = {
|
|
281
|
-
...node.attrs,
|
|
282
|
-
"tangle.supervise.node.status": status ?? "done"
|
|
283
|
-
};
|
|
284
|
-
if (typeof event.stepIndex === "number") attrs["tangle.supervise.node.seq"] = event.stepIndex;
|
|
285
|
-
if (typeof p.outRef === "string") attrs["tangle.supervise.node.out_ref"] = p.outRef;
|
|
286
|
-
if (typeof p.valid === "boolean") attrs["tangle.supervise.verdict.valid"] = p.valid;
|
|
287
|
-
if (typeof p.score === "number") attrs["tangle.supervise.verdict.score"] = p.score;
|
|
288
|
-
if (down) {
|
|
289
|
-
attrs["error.type"] = p.infra === true ? "infra" : "child-down";
|
|
290
|
-
const reason = str(p.reason);
|
|
291
|
-
if (reason) attrs["error.message"] = truncate(reason);
|
|
292
|
-
if (typeof p.infra === "boolean") attrs["tangle.supervise.node.infra"] = p.infra;
|
|
293
|
-
}
|
|
294
|
-
const wokeBy = str(record(p.wait).settled);
|
|
295
|
-
if (wokeBy) attrs["tangle.supervise.wait.settled"] = wokeBy;
|
|
296
|
-
assignSpend(attrs, p.spent);
|
|
297
|
-
emit(span(node.spanId, node.parentSpanId, node.name, node.startMs, event.timestamp, attrs, down ? STATUS_ERROR : STATUS_OK, down ? str(p.reason) ?? "child down" : void 0));
|
|
298
|
-
}
|
|
299
|
-
function onTurn(event) {
|
|
300
|
-
const p = record(event.payload);
|
|
301
|
-
const parentId = event.parentId;
|
|
302
|
-
const parentSpanId = parentId && spanIdOf.get(parentId) || rootSpanId;
|
|
303
|
-
const attrs = {
|
|
304
|
-
...base,
|
|
305
|
-
"openinference.span.kind": "LLM",
|
|
306
|
-
"inference.observation_kind": "LLM",
|
|
307
|
-
"tangle.supervise.node.kind": "inference"
|
|
308
|
-
};
|
|
309
|
-
if (parentId) attrs["tangle.supervise.node.id"] = parentId;
|
|
310
|
-
for (const [key, value] of Object.entries(p)) {
|
|
311
|
-
if (key === "spend") continue;
|
|
312
|
-
if (key === "driver" && typeof value === "string") {
|
|
313
|
-
attrs["agent.name"] = value;
|
|
314
|
-
attrs["inference.agent_name"] = value;
|
|
315
|
-
continue;
|
|
316
|
-
}
|
|
317
|
-
if (key === "model" && typeof value === "string") {
|
|
318
|
-
attrs["llm.model_name"] = value;
|
|
319
|
-
continue;
|
|
320
|
-
}
|
|
321
|
-
if (Array.isArray(value)) {
|
|
322
|
-
const scalars = value.filter((v) => typeof v === "string" || typeof v === "number");
|
|
323
|
-
if (scalars.length > 0) attrs[`tangle.supervise.turn.${key}`] = truncate(scalars.join(","));
|
|
324
|
-
continue;
|
|
325
|
-
}
|
|
326
|
-
if (typeof value === "string") attrs[`tangle.supervise.turn.${key}`] = truncate(value);
|
|
327
|
-
else if (typeof value === "number" || typeof value === "boolean") attrs[`tangle.supervise.turn.${key}`] = value;
|
|
328
|
-
}
|
|
329
|
-
const spend = assignSpend(attrs, p.spend);
|
|
330
|
-
const endMs = event.timestamp + (spend?.ms ?? 0);
|
|
331
|
-
emit(span(generateSpanId(), parentSpanId, "gen_ai.client.inference", event.timestamp, endMs, attrs, STATUS_OK));
|
|
164
|
+
const origins = new Set(options.allowedOrigins ?? []);
|
|
165
|
+
for (const origin of origins) {
|
|
166
|
+
const parsed = new URL(origin);
|
|
167
|
+
if (parsed.origin !== origin || !["http:", "https:"].includes(parsed.protocol)) throw new ConfigError("coordination allowedOrigins must contain canonical HTTP origins");
|
|
332
168
|
}
|
|
169
|
+
const requestTimeoutMs = positive("requestTimeoutMs", options.requestTimeoutMs, 3e4);
|
|
170
|
+
if (requestTimeoutMs > 2147483647) throw new ConfigError("coordination requestTimeoutMs exceeds the timer limit");
|
|
333
171
|
return {
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
else if (event.target === "agent.child") onSettled(event);
|
|
340
|
-
else if (event.target === "agent.turn") onTurn(event);
|
|
341
|
-
} catch {}
|
|
342
|
-
} },
|
|
343
|
-
traceId,
|
|
344
|
-
rootSpanId,
|
|
345
|
-
workerTrace(spawningNodeId) {
|
|
346
|
-
return {
|
|
347
|
-
traceId,
|
|
348
|
-
parentSpanId: spanIdOf.get(spawningNodeId) ?? rootSpanId
|
|
349
|
-
};
|
|
350
|
-
},
|
|
351
|
-
async finish(outcome) {
|
|
352
|
-
if (finished) return;
|
|
353
|
-
finished = true;
|
|
354
|
-
const endMs = now();
|
|
355
|
-
try {
|
|
356
|
-
for (const [nodeId, node] of open) emit(span(node.spanId, node.parentSpanId, node.name, node.startMs, endMs, {
|
|
357
|
-
...node.attrs,
|
|
358
|
-
"tangle.supervise.node.status": "unsettled",
|
|
359
|
-
"tangle.supervise.node.settled": false,
|
|
360
|
-
"tangle.supervise.node.id": nodeId
|
|
361
|
-
}, STATUS_UNSET));
|
|
362
|
-
open.clear();
|
|
363
|
-
emit(span(rootSpanId, opts.parentSpanId, "supervisor.run", rootStartMs, endMs, rootAttrs(base, opts.agentName ?? "supervisor", outcome), rootStatus(outcome), rootMessage(outcome)));
|
|
364
|
-
await exporter.flush();
|
|
365
|
-
if (ownsExporter) await exporter.shutdown();
|
|
366
|
-
} catch {}
|
|
367
|
-
}
|
|
172
|
+
maxRequestBytes: positive("maxRequestBytes", options.maxRequestBytes, 1024 * 1024),
|
|
173
|
+
requestTimeoutMs,
|
|
174
|
+
maxConcurrentRequests: positive("maxConcurrentRequests", options.maxConcurrentRequests, 32),
|
|
175
|
+
requestsPerMinute: positive("requestsPerMinute", options.requestsPerMinute, 600),
|
|
176
|
+
origins
|
|
368
177
|
};
|
|
369
178
|
}
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
if (result.reason === "driver-failed") {
|
|
386
|
-
attrs["error.type"] = result.error.name;
|
|
387
|
-
attrs["error.message"] = truncate(result.error.message);
|
|
388
|
-
}
|
|
389
|
-
}
|
|
390
|
-
assignSpend(attrs, result.spentTotal);
|
|
391
|
-
}
|
|
392
|
-
if (outcome?.error !== void 0) {
|
|
393
|
-
attrs["tangle.supervise.result"] = "error";
|
|
394
|
-
const err = outcome.error;
|
|
395
|
-
attrs["error.type"] = err instanceof Error ? err.name : typeof err;
|
|
396
|
-
attrs["error.message"] = truncate(err instanceof Error ? err.message : String(err));
|
|
397
|
-
}
|
|
398
|
-
return attrs;
|
|
399
|
-
}
|
|
400
|
-
function rootStatus(outcome) {
|
|
401
|
-
if (outcome?.error !== void 0) return STATUS_ERROR;
|
|
402
|
-
if (!outcome?.result) return STATUS_UNSET;
|
|
403
|
-
return outcome.result.kind === "winner" ? STATUS_OK : STATUS_ERROR;
|
|
404
|
-
}
|
|
405
|
-
function rootMessage(outcome) {
|
|
406
|
-
if (outcome?.error !== void 0) return truncate(outcome.error instanceof Error ? outcome.error.message : String(outcome.error));
|
|
407
|
-
const result = outcome?.result;
|
|
408
|
-
return result && result.kind === "no-winner" ? result.reason : void 0;
|
|
409
|
-
}
|
|
410
|
-
/**
|
|
411
|
-
* Write a `Spend` onto a span, honouring the "a missing measurement is never zero" invariant: a
|
|
412
|
-
* channel marked NOT known contributes no number at all and instead flags itself, so nothing
|
|
413
|
-
* downstream can sum an unmeasured turn as free. Returns the spend it read (for the duration).
|
|
414
|
-
*/
|
|
415
|
-
function assignSpend(attrs, value) {
|
|
416
|
-
if (!isRecord(value)) return void 0;
|
|
417
|
-
const spend = value;
|
|
418
|
-
const tokens = isRecord(spend.tokens) ? spend.tokens : void 0;
|
|
419
|
-
if (spend.tokensKnown === false) attrs["tangle.supervise.tokens_known"] = false;
|
|
420
|
-
else if (tokens) {
|
|
421
|
-
if (typeof tokens.input === "number") attrs["llm.token_count.prompt"] = tokens.input;
|
|
422
|
-
if (typeof tokens.output === "number") attrs["llm.token_count.completion"] = tokens.output;
|
|
423
|
-
}
|
|
424
|
-
if (spend.usdKnown === false) attrs["tangle.supervise.cost_known"] = false;
|
|
425
|
-
else if (typeof spend.usd === "number" && Number.isFinite(spend.usd)) {
|
|
426
|
-
attrs["llm.cost_usd"] = spend.usd;
|
|
427
|
-
attrs["tangle.cost.usd"] = spend.usd;
|
|
428
|
-
}
|
|
429
|
-
if (typeof spend.iterations === "number") attrs["tangle.supervise.iterations"] = spend.iterations;
|
|
430
|
-
if (typeof spend.ms === "number" && spend.ms > 0) attrs["tangle.supervise.duration_ms"] = spend.ms;
|
|
431
|
-
if (typeof spend.boxMinutes === "number" && Number.isFinite(spend.boxMinutes)) attrs["tangle.platform.box_minutes"] = spend.boxMinutes;
|
|
432
|
-
if (spend.boxMinutesProvenance !== void 0) {
|
|
433
|
-
attrs["tangle.platform.box_minutes_provenance"] = spend.boxMinutesProvenance;
|
|
434
|
-
attrs["tangle.platform.box_minutes_known"] = spend.boxMinutesKnown === true;
|
|
435
|
-
}
|
|
436
|
-
if (spend.tokensProvenance !== void 0) attrs["tangle.supervise.tokens_provenance"] = spend.tokensProvenance;
|
|
437
|
-
return spend;
|
|
438
|
-
}
|
|
439
|
-
function assignBudget(attrs, value) {
|
|
440
|
-
if (!isRecord(value)) return;
|
|
441
|
-
const budget = value;
|
|
442
|
-
if (typeof budget.maxTokens === "number") attrs["tangle.supervise.budget.max_tokens"] = budget.maxTokens;
|
|
443
|
-
if (typeof budget.maxIterations === "number") attrs["tangle.supervise.budget.max_iterations"] = budget.maxIterations;
|
|
444
|
-
if (typeof budget.maxUsd === "number") attrs["tangle.supervise.budget.max_usd"] = budget.maxUsd;
|
|
445
|
-
}
|
|
446
|
-
/**
|
|
447
|
-
* A trace id must be 32 hex characters. A caller-supplied one is used verbatim when it already is;
|
|
448
|
-
* anything else (and the default) is DERIVED from the run id by content address — deterministic, so
|
|
449
|
-
* a resumed run rejoins the trace its first process opened rather than forking a new one.
|
|
450
|
-
*/
|
|
451
|
-
function normalizeTraceId(traceId, runId) {
|
|
452
|
-
if (traceId && /^[0-9a-f]{32}$/i.test(traceId)) return traceId.toLowerCase();
|
|
453
|
-
return contentAddress(traceId ?? runId).slice(7, 39);
|
|
454
|
-
}
|
|
455
|
-
function msToNano(ms) {
|
|
456
|
-
return (BigInt(Number.isFinite(ms) ? Math.floor(ms) : 0) * 1000000n).toString();
|
|
457
|
-
}
|
|
458
|
-
function isRecord(value) {
|
|
459
|
-
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
460
|
-
}
|
|
461
|
-
function record(value) {
|
|
462
|
-
return isRecord(value) ? value : {};
|
|
463
|
-
}
|
|
464
|
-
function str(value) {
|
|
465
|
-
return typeof value === "string" && value.length > 0 ? value : void 0;
|
|
466
|
-
}
|
|
467
|
-
function truncate(value) {
|
|
468
|
-
return value.length > MAX_DETAIL_CHARS ? `${value.slice(0, MAX_DETAIL_CHARS)}…` : value;
|
|
469
|
-
}
|
|
470
|
-
//#endregion
|
|
471
|
-
//#region src/runtime/supervise/coordination-http.ts
|
|
472
|
-
function coordinationHttpLimits(options) {
|
|
473
|
-
const positive = (name, value, fallback) => {
|
|
474
|
-
const resolved = value ?? fallback;
|
|
475
|
-
if (!Number.isSafeInteger(resolved) || resolved <= 0) throw new ConfigError(`coordination ${name} must be a positive safe integer`);
|
|
476
|
-
return resolved;
|
|
477
|
-
};
|
|
478
|
-
const origins = new Set(options.allowedOrigins ?? []);
|
|
479
|
-
for (const origin of origins) {
|
|
480
|
-
const parsed = new URL(origin);
|
|
481
|
-
if (parsed.origin !== origin || !["http:", "https:"].includes(parsed.protocol)) throw new ConfigError("coordination allowedOrigins must contain canonical HTTP origins");
|
|
482
|
-
}
|
|
483
|
-
const requestTimeoutMs = positive("requestTimeoutMs", options.requestTimeoutMs, 3e4);
|
|
484
|
-
if (requestTimeoutMs > 2147483647) throw new ConfigError("coordination requestTimeoutMs exceeds the timer limit");
|
|
485
|
-
return {
|
|
486
|
-
maxRequestBytes: positive("maxRequestBytes", options.maxRequestBytes, 1024 * 1024),
|
|
487
|
-
requestTimeoutMs,
|
|
488
|
-
maxConcurrentRequests: positive("maxConcurrentRequests", options.maxConcurrentRequests, 32),
|
|
489
|
-
requestsPerMinute: positive("requestsPerMinute", options.requestsPerMinute, 600),
|
|
490
|
-
origins
|
|
491
|
-
};
|
|
492
|
-
}
|
|
493
|
-
/** One bounded HTTP adapter around the existing JSON-RPC handler; it owns no run commands. */
|
|
494
|
-
function coordinationHttpHandler(input) {
|
|
495
|
-
const limits = coordinationHttpLimits(input.options);
|
|
496
|
-
let active = 0;
|
|
497
|
-
let controlActive = 0;
|
|
498
|
-
let controlRequests = 0;
|
|
499
|
-
let windowStart = Date.now();
|
|
500
|
-
let requests = 0;
|
|
501
|
-
const audit = async (outcome, status, action) => {
|
|
502
|
-
await input.options.onAudit?.({
|
|
503
|
-
...input.identity,
|
|
504
|
-
outcome,
|
|
505
|
-
status,
|
|
506
|
-
...action ? { action } : {}
|
|
507
|
-
});
|
|
179
|
+
/** One bounded HTTP adapter around the existing JSON-RPC handler; it owns no run commands. */
|
|
180
|
+
function coordinationHttpHandler(input) {
|
|
181
|
+
const limits = coordinationHttpLimits(input.options);
|
|
182
|
+
let active = 0;
|
|
183
|
+
let controlActive = 0;
|
|
184
|
+
let controlRequests = 0;
|
|
185
|
+
let windowStart = Date.now();
|
|
186
|
+
let requests = 0;
|
|
187
|
+
const audit = async (outcome, status, action) => {
|
|
188
|
+
await input.options.onAudit?.({
|
|
189
|
+
...input.identity,
|
|
190
|
+
outcome,
|
|
191
|
+
status,
|
|
192
|
+
...action ? { action } : {}
|
|
193
|
+
});
|
|
508
194
|
};
|
|
509
195
|
return (req, res) => {
|
|
510
196
|
let finished = false;
|
|
@@ -904,6 +590,7 @@ async function serveCoordinationMcp(opts) {
|
|
|
904
590
|
...opts.priorAnalystDefinitions?.length ? { priorAnalystDefinitions: opts.priorAnalystDefinitions } : {},
|
|
905
591
|
...opts.preflightSpawn ? { preflightSpawn: opts.preflightSpawn } : {},
|
|
906
592
|
...opts.resolveSpawnProfile ? { resolveSpawnProfile: opts.resolveSpawnProfile } : {},
|
|
593
|
+
...opts.spawnResourceRoot ? { spawnResourceRoot: opts.spawnResourceRoot } : {},
|
|
907
594
|
...opts.peerMail ? { peerMail: typeof opts.peerMail === "object" && opts.peerMail.limits ? { limits: opts.peerMail.limits } : {} } : {}
|
|
908
595
|
});
|
|
909
596
|
await coord.ready();
|
|
@@ -989,397 +676,407 @@ async function serveCoordinationMcp(opts) {
|
|
|
989
676
|
get credentialExpiresAt() {
|
|
990
677
|
return credentialExpiresAt;
|
|
991
678
|
},
|
|
992
|
-
rotateCredential,
|
|
993
|
-
settled: () => coord.settled(),
|
|
994
|
-
submittedResult: () => coord.submittedResult(),
|
|
995
|
-
drainResolved: () => coord.drainResolved(),
|
|
996
|
-
isStopped: () => coord.isStopped(),
|
|
997
|
-
history: () => coord.history(),
|
|
998
|
-
stats: () => coord.stats(),
|
|
999
|
-
raiseFinding: (finding) => coord.raiseFinding(finding),
|
|
1000
|
-
mailHistory: () => mailbox?.history() ?? [],
|
|
1001
|
-
stopMailThread: (threadId) => mailbox?.stopThread(threadId) ?? false,
|
|
1002
|
-
close: async () => {
|
|
1003
|
-
closed = true;
|
|
1004
|
-
await new Promise((resolve) => {
|
|
1005
|
-
server.close(() => resolve());
|
|
1006
|
-
});
|
|
1007
|
-
await mailListener?.close();
|
|
679
|
+
rotateCredential,
|
|
680
|
+
settled: () => coord.settled(),
|
|
681
|
+
submittedResult: () => coord.submittedResult(),
|
|
682
|
+
drainResolved: () => coord.drainResolved(),
|
|
683
|
+
isStopped: () => coord.isStopped(),
|
|
684
|
+
history: () => coord.history(),
|
|
685
|
+
stats: () => coord.stats(),
|
|
686
|
+
raiseFinding: (finding) => coord.raiseFinding(finding),
|
|
687
|
+
mailHistory: () => mailbox?.history() ?? [],
|
|
688
|
+
stopMailThread: (threadId) => mailbox?.stopThread(threadId) ?? false,
|
|
689
|
+
close: async () => {
|
|
690
|
+
closed = true;
|
|
691
|
+
await new Promise((resolve) => {
|
|
692
|
+
server.close(() => resolve());
|
|
693
|
+
});
|
|
694
|
+
await mailListener?.close();
|
|
695
|
+
}
|
|
696
|
+
};
|
|
697
|
+
}
|
|
698
|
+
/**
|
|
699
|
+
* Stand up the peer-mail capability listener: one HTTP server, one secret path per worker, and on
|
|
700
|
+
* each path a tool server carrying ONLY `send_mail` / `read_mail` with that worker's identity
|
|
701
|
+
* closed over. An unknown path is a flat 404 — the path is the credential, so a request that does
|
|
702
|
+
* not present a minted one is not a client to reason with.
|
|
703
|
+
*
|
|
704
|
+
* The host is the coordination host, which the caller has already had to justify: the loopback gate
|
|
705
|
+
* above governs both listeners, and there is deliberately no way to bind mail somewhere else.
|
|
706
|
+
*/
|
|
707
|
+
async function servePeerMail(mailbox, host) {
|
|
708
|
+
const servers = /* @__PURE__ */ new Map();
|
|
709
|
+
const forCapability = (capabilityId) => {
|
|
710
|
+
const existing = servers.get(capabilityId);
|
|
711
|
+
if (existing) return existing;
|
|
712
|
+
const created = createStdioToolServer({
|
|
713
|
+
serverName: "peer-mail",
|
|
714
|
+
serverVersion: "1",
|
|
715
|
+
tools: mailbox.tools(capabilityId)
|
|
716
|
+
});
|
|
717
|
+
servers.set(capabilityId, created);
|
|
718
|
+
return created;
|
|
719
|
+
};
|
|
720
|
+
const listener = createServer((req, res) => {
|
|
721
|
+
const capabilityId = /^\/mail\/([0-9a-f]{32})$/.exec(req.url ?? "")?.[1];
|
|
722
|
+
if (req.method !== "POST" || capabilityId === void 0 || !mailbox.hasCapability(capabilityId)) {
|
|
723
|
+
res.writeHead(404).end();
|
|
724
|
+
return;
|
|
725
|
+
}
|
|
726
|
+
let body = "";
|
|
727
|
+
req.on("data", (chunk) => {
|
|
728
|
+
body += chunk;
|
|
729
|
+
});
|
|
730
|
+
req.on("end", () => {
|
|
731
|
+
(async () => {
|
|
732
|
+
try {
|
|
733
|
+
const message = JSON.parse(body);
|
|
734
|
+
const response = await forCapability(capabilityId).handle(message);
|
|
735
|
+
if (response === null) {
|
|
736
|
+
res.writeHead(202).end();
|
|
737
|
+
return;
|
|
738
|
+
}
|
|
739
|
+
res.writeHead(200, { "content-type": "application/json" });
|
|
740
|
+
res.end(JSON.stringify(response));
|
|
741
|
+
} catch (e) {
|
|
742
|
+
res.writeHead(200, { "content-type": "application/json" });
|
|
743
|
+
res.end(JSON.stringify({
|
|
744
|
+
jsonrpc: "2.0",
|
|
745
|
+
id: null,
|
|
746
|
+
error: {
|
|
747
|
+
code: -32700,
|
|
748
|
+
message: e instanceof Error ? e.message : "parse error"
|
|
749
|
+
}
|
|
750
|
+
}));
|
|
751
|
+
}
|
|
752
|
+
})();
|
|
753
|
+
});
|
|
754
|
+
});
|
|
755
|
+
const port = await new Promise((resolve, reject) => {
|
|
756
|
+
listener.once("error", reject);
|
|
757
|
+
listener.listen(0, host, () => {
|
|
758
|
+
const addr = listener.address();
|
|
759
|
+
resolve(typeof addr === "object" && addr ? addr.port : 0);
|
|
760
|
+
});
|
|
761
|
+
});
|
|
762
|
+
mailbox.setEndpoint(`http://${host}:${port}/mail`);
|
|
763
|
+
return { close: () => new Promise((resolve) => {
|
|
764
|
+
listener.close(() => resolve());
|
|
765
|
+
}) };
|
|
766
|
+
}
|
|
767
|
+
//#endregion
|
|
768
|
+
//#region src/runtime/supervise/otel-spans.ts
|
|
769
|
+
/**
|
|
770
|
+
* Supervisor tree → OTLP spans. OPT-IN, off by default.
|
|
771
|
+
*
|
|
772
|
+
* WHY. A supervised tree is legible today only by parsing this package's own spawn journal, so
|
|
773
|
+
* every other multi-agent shape on the machine (a coding-CLI's subagents, a pi fanout, ad-hoc tool
|
|
774
|
+
* parallelism) needs its own bespoke reader. A span carrying `parent_span_id` IS a tree, and any
|
|
775
|
+
* system can emit one — so emitting spans makes the supervisor readable by the same viewer as
|
|
776
|
+
* everything else, with no per-system reader.
|
|
777
|
+
*
|
|
778
|
+
* WHAT IT IS NOT. This is telemetry, never the record of truth. The spawn journal remains the sole
|
|
779
|
+
* durable ledger for replay/resume and for cost; nothing here is read back, and a run whose export
|
|
780
|
+
* fails is unaffected in every observable way. The two data models are deliberately separate.
|
|
781
|
+
*
|
|
782
|
+
* HOW IT ATTACHES. This is a pure `RuntimeHooks` observer over the lifecycle events `Scope` ALREADY
|
|
783
|
+
* emits — `agent.spawn` (a node opened), `agent.child` (a node settled), `agent.turn` (a driver
|
|
784
|
+
* inference turn was metered). It adds no event, mutates no journal, and changes no result. Because
|
|
785
|
+
* `Scope` re-seeds the same hooks into every nested scope (`makeNestedScopeSeam`), one observer sees
|
|
786
|
+
* the WHOLE recursion at arbitrary depth.
|
|
787
|
+
*
|
|
788
|
+
* SPAN SHAPE. One span per supervised node, opened at spawn and closed at settle, parented to its
|
|
789
|
+
* parent node's span; the run root is the trace. Driver inference rides as an `LLM` child span under
|
|
790
|
+
* the node that metered it. Attributes reuse the vocabulary the rest of the stack already reads
|
|
791
|
+
* (`openinference.span.kind`, `agent.name`, `llm.token_count.*`, `llm.cost_usd`, `tangle.cost.usd`)
|
|
792
|
+
* — see `@tangle-network/agent-eval`'s `src/trace/attribute-vocabulary.ts`, which is the consumer.
|
|
793
|
+
*
|
|
794
|
+
* UNKNOWN IS NEVER ZERO. `Spend.tokensKnown === false` / `usdKnown === false` mark work that
|
|
795
|
+
* HAPPENED with an unreported count. Those spans OMIT the token/cost attribute entirely and set
|
|
796
|
+
* `tangle.supervise.tokens_known` / `tangle.supervise.cost_known` to `false`, so a reader can never
|
|
797
|
+
* mistake an unmeasured turn for a free one.
|
|
798
|
+
*/
|
|
799
|
+
/** OTEL status codes (`UNSET` / `OK` / `ERROR`) — the numeric wire values `OtelSpan.status` carries. */
|
|
800
|
+
const STATUS_UNSET = 0;
|
|
801
|
+
const STATUS_OK = 1;
|
|
802
|
+
const STATUS_ERROR = 2;
|
|
803
|
+
/** Longest string attribute value written from free-form detail, so an oversized turn payload
|
|
804
|
+
* cannot inflate a span. Identity/label attributes we control are never truncated. */
|
|
805
|
+
const MAX_DETAIL_CHARS = 256;
|
|
806
|
+
/**
|
|
807
|
+
* Build the span recorder for one supervised run, or `undefined` when no exporter resolves — the
|
|
808
|
+
* off-by-default path. A run that passes no `exporter` and no `exportConfig` never reaches this
|
|
809
|
+
* function at all; one that passes an `exportConfig` with no endpoint (and no env endpoint) gets
|
|
810
|
+
* `undefined` here, so "configured but unreachable" also costs nothing.
|
|
811
|
+
*/
|
|
812
|
+
function createSupervisorSpanRecorder(opts) {
|
|
813
|
+
const exporter = opts.exporter ?? createOtelExporter(opts.exportConfig);
|
|
814
|
+
if (!exporter) return void 0;
|
|
815
|
+
const ownsExporter = opts.exporter === void 0;
|
|
816
|
+
const now = opts.now ?? Date.now;
|
|
817
|
+
const traceId = normalizeTraceId(opts.traceId, opts.runId);
|
|
818
|
+
const rootSpanId = generateSpanId();
|
|
819
|
+
const rootStartMs = now();
|
|
820
|
+
const base = {
|
|
821
|
+
"tangle.run.id": opts.runId,
|
|
822
|
+
"tangle.sessionId": opts.runId,
|
|
823
|
+
...opts.attributes ?? {}
|
|
824
|
+
};
|
|
825
|
+
/** Node id → its open span. Seeded with the run id ⇒ the root span, because a depth-0 spawn's
|
|
826
|
+
* `parentId` is the run id itself and every deeper spawn's is a real node id. */
|
|
827
|
+
const open = /* @__PURE__ */ new Map();
|
|
828
|
+
const spanIdOf = /* @__PURE__ */ new Map([[opts.runId, rootSpanId]]);
|
|
829
|
+
let finished = false;
|
|
830
|
+
/** Every export is best-effort: a throwing exporter must never reach the run. */
|
|
831
|
+
const emit = (span) => {
|
|
832
|
+
try {
|
|
833
|
+
exporter.exportSpan(span);
|
|
834
|
+
} catch {}
|
|
835
|
+
};
|
|
836
|
+
const span = (spanId, parentSpanId, name, startMs, endMs, attrs, status, message) => ({
|
|
837
|
+
traceId,
|
|
838
|
+
spanId,
|
|
839
|
+
...parentSpanId ? { parentSpanId } : {},
|
|
840
|
+
name,
|
|
841
|
+
kind: 1,
|
|
842
|
+
startTimeUnixNano: msToNano(startMs),
|
|
843
|
+
endTimeUnixNano: msToNano(Math.max(startMs, endMs)),
|
|
844
|
+
attributes: toOtelAttributes(attrs),
|
|
845
|
+
status: {
|
|
846
|
+
code: status,
|
|
847
|
+
...message ? { message } : {}
|
|
848
|
+
}
|
|
849
|
+
});
|
|
850
|
+
function onSpawn(event) {
|
|
851
|
+
const p = record(event.payload);
|
|
852
|
+
const childId = str(p.childId);
|
|
853
|
+
if (!childId) return;
|
|
854
|
+
const label = str(p.label) ?? "node";
|
|
855
|
+
const runtime = str(p.runtime);
|
|
856
|
+
const isWait = runtime === "wait";
|
|
857
|
+
const attrs = {
|
|
858
|
+
...base,
|
|
859
|
+
"openinference.span.kind": isWait ? "CHAIN" : "AGENT",
|
|
860
|
+
"agent.name": label,
|
|
861
|
+
"tangle.supervise.node.id": childId,
|
|
862
|
+
"tangle.supervise.node.label": label,
|
|
863
|
+
"tangle.supervise.node.kind": isWait ? "wait" : "agent",
|
|
864
|
+
"tangle.supervise.tree.root": event.runId
|
|
865
|
+
};
|
|
866
|
+
if (event.parentId) attrs["tangle.supervise.node.parent_id"] = event.parentId;
|
|
867
|
+
if (runtime) attrs["tangle.supervise.node.runtime"] = runtime;
|
|
868
|
+
if (typeof p.depth === "number") attrs["tangle.supervise.node.depth"] = p.depth;
|
|
869
|
+
if (typeof event.stepIndex === "number") attrs["tangle.supervise.node.ordinal"] = event.stepIndex;
|
|
870
|
+
if (p.resumed === true) attrs["tangle.supervise.node.resumed"] = true;
|
|
871
|
+
assignBudget(attrs, p.budget);
|
|
872
|
+
const spanId = generateSpanId();
|
|
873
|
+
spanIdOf.set(childId, spanId);
|
|
874
|
+
open.set(childId, {
|
|
875
|
+
spanId,
|
|
876
|
+
parentSpanId: event.parentId && spanIdOf.get(event.parentId) || rootSpanId,
|
|
877
|
+
name: label,
|
|
878
|
+
startMs: event.timestamp,
|
|
879
|
+
attrs
|
|
880
|
+
});
|
|
881
|
+
}
|
|
882
|
+
function onSettled(event) {
|
|
883
|
+
const p = record(event.payload);
|
|
884
|
+
const childId = str(p.childId);
|
|
885
|
+
if (!childId) return;
|
|
886
|
+
const node = open.get(childId);
|
|
887
|
+
if (!node) return;
|
|
888
|
+
open.delete(childId);
|
|
889
|
+
const status = str(p.status);
|
|
890
|
+
const down = status === "down";
|
|
891
|
+
const attrs = {
|
|
892
|
+
...node.attrs,
|
|
893
|
+
"tangle.supervise.node.status": status ?? "done"
|
|
894
|
+
};
|
|
895
|
+
if (typeof event.stepIndex === "number") attrs["tangle.supervise.node.seq"] = event.stepIndex;
|
|
896
|
+
if (typeof p.outRef === "string") attrs["tangle.supervise.node.out_ref"] = p.outRef;
|
|
897
|
+
if (typeof p.valid === "boolean") attrs["tangle.supervise.verdict.valid"] = p.valid;
|
|
898
|
+
if (typeof p.score === "number") attrs["tangle.supervise.verdict.score"] = p.score;
|
|
899
|
+
if (down) {
|
|
900
|
+
attrs["error.type"] = p.infra === true ? "infra" : "child-down";
|
|
901
|
+
const reason = str(p.reason);
|
|
902
|
+
if (reason) attrs["error.message"] = truncate(reason);
|
|
903
|
+
if (typeof p.infra === "boolean") attrs["tangle.supervise.node.infra"] = p.infra;
|
|
904
|
+
}
|
|
905
|
+
const wokeBy = str(record(p.wait).settled);
|
|
906
|
+
if (wokeBy) attrs["tangle.supervise.wait.settled"] = wokeBy;
|
|
907
|
+
assignSpend(attrs, p.spent);
|
|
908
|
+
emit(span(node.spanId, node.parentSpanId, node.name, node.startMs, event.timestamp, attrs, down ? STATUS_ERROR : STATUS_OK, down ? str(p.reason) ?? "child down" : void 0));
|
|
909
|
+
}
|
|
910
|
+
function onTurn(event) {
|
|
911
|
+
const p = record(event.payload);
|
|
912
|
+
const parentId = event.parentId;
|
|
913
|
+
const parentSpanId = parentId && spanIdOf.get(parentId) || rootSpanId;
|
|
914
|
+
const attrs = {
|
|
915
|
+
...base,
|
|
916
|
+
"openinference.span.kind": "LLM",
|
|
917
|
+
"inference.observation_kind": "LLM",
|
|
918
|
+
"tangle.supervise.node.kind": "inference"
|
|
919
|
+
};
|
|
920
|
+
if (parentId) attrs["tangle.supervise.node.id"] = parentId;
|
|
921
|
+
for (const [key, value] of Object.entries(p)) {
|
|
922
|
+
if (key === "spend") continue;
|
|
923
|
+
if (key === "driver" && typeof value === "string") {
|
|
924
|
+
attrs["agent.name"] = value;
|
|
925
|
+
attrs["inference.agent_name"] = value;
|
|
926
|
+
continue;
|
|
927
|
+
}
|
|
928
|
+
if (key === "model" && typeof value === "string") {
|
|
929
|
+
attrs["llm.model_name"] = value;
|
|
930
|
+
continue;
|
|
931
|
+
}
|
|
932
|
+
if (Array.isArray(value)) {
|
|
933
|
+
const scalars = value.filter((v) => typeof v === "string" || typeof v === "number");
|
|
934
|
+
if (scalars.length > 0) attrs[`tangle.supervise.turn.${key}`] = truncate(scalars.join(","));
|
|
935
|
+
continue;
|
|
936
|
+
}
|
|
937
|
+
if (typeof value === "string") attrs[`tangle.supervise.turn.${key}`] = truncate(value);
|
|
938
|
+
else if (typeof value === "number" || typeof value === "boolean") attrs[`tangle.supervise.turn.${key}`] = value;
|
|
939
|
+
}
|
|
940
|
+
const spend = assignSpend(attrs, p.spend);
|
|
941
|
+
const endMs = event.timestamp + (spend?.ms ?? 0);
|
|
942
|
+
emit(span(generateSpanId(), parentSpanId, "gen_ai.client.inference", event.timestamp, endMs, attrs, STATUS_OK));
|
|
943
|
+
}
|
|
944
|
+
return {
|
|
945
|
+
hooks: { onEvent(event) {
|
|
946
|
+
if (finished) return;
|
|
947
|
+
try {
|
|
948
|
+
if (event.phase !== "after") return;
|
|
949
|
+
if (event.target === "agent.spawn") onSpawn(event);
|
|
950
|
+
else if (event.target === "agent.child") onSettled(event);
|
|
951
|
+
else if (event.target === "agent.turn") onTurn(event);
|
|
952
|
+
} catch {}
|
|
953
|
+
} },
|
|
954
|
+
traceId,
|
|
955
|
+
rootSpanId,
|
|
956
|
+
workerTrace(spawningNodeId) {
|
|
957
|
+
return {
|
|
958
|
+
traceId,
|
|
959
|
+
parentSpanId: spanIdOf.get(spawningNodeId) ?? rootSpanId
|
|
960
|
+
};
|
|
961
|
+
},
|
|
962
|
+
async finish(outcome) {
|
|
963
|
+
if (finished) return;
|
|
964
|
+
finished = true;
|
|
965
|
+
const endMs = now();
|
|
966
|
+
try {
|
|
967
|
+
for (const [nodeId, node] of open) emit(span(node.spanId, node.parentSpanId, node.name, node.startMs, endMs, {
|
|
968
|
+
...node.attrs,
|
|
969
|
+
"tangle.supervise.node.status": "unsettled",
|
|
970
|
+
"tangle.supervise.node.settled": false,
|
|
971
|
+
"tangle.supervise.node.id": nodeId
|
|
972
|
+
}, STATUS_UNSET));
|
|
973
|
+
open.clear();
|
|
974
|
+
emit(span(rootSpanId, opts.parentSpanId, "supervisor.run", rootStartMs, endMs, rootAttrs(base, opts.agentName ?? "supervisor", outcome), rootStatus(outcome), rootMessage(outcome)));
|
|
975
|
+
await exporter.flush();
|
|
976
|
+
if (ownsExporter) await exporter.shutdown();
|
|
977
|
+
} catch {}
|
|
1008
978
|
}
|
|
1009
979
|
};
|
|
1010
980
|
}
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
*/
|
|
1020
|
-
async function servePeerMail(mailbox, host) {
|
|
1021
|
-
const servers = /* @__PURE__ */ new Map();
|
|
1022
|
-
const forCapability = (capabilityId) => {
|
|
1023
|
-
const existing = servers.get(capabilityId);
|
|
1024
|
-
if (existing) return existing;
|
|
1025
|
-
const created = createStdioToolServer({
|
|
1026
|
-
serverName: "peer-mail",
|
|
1027
|
-
serverVersion: "1",
|
|
1028
|
-
tools: mailbox.tools(capabilityId)
|
|
1029
|
-
});
|
|
1030
|
-
servers.set(capabilityId, created);
|
|
1031
|
-
return created;
|
|
981
|
+
function rootAttrs(base, agentName, outcome) {
|
|
982
|
+
const attrs = {
|
|
983
|
+
...base,
|
|
984
|
+
"openinference.span.kind": "AGENT",
|
|
985
|
+
"inference.observation_kind": "AGENT",
|
|
986
|
+
"agent.name": agentName,
|
|
987
|
+
"inference.agent_name": agentName,
|
|
988
|
+
"tangle.supervise.node.kind": "root"
|
|
1032
989
|
};
|
|
1033
|
-
const
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
990
|
+
const result = outcome?.result;
|
|
991
|
+
if (result) {
|
|
992
|
+
attrs["tangle.supervise.result"] = result.kind;
|
|
993
|
+
if (result.kind === "no-winner") {
|
|
994
|
+
attrs["tangle.supervise.reason"] = result.reason;
|
|
995
|
+
attrs["tangle.supervise.down_count"] = result.downCount;
|
|
996
|
+
if (result.reason === "driver-failed") {
|
|
997
|
+
attrs["error.type"] = result.error.name;
|
|
998
|
+
attrs["error.message"] = truncate(result.error.message);
|
|
999
|
+
}
|
|
1038
1000
|
}
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
const message = JSON.parse(body);
|
|
1047
|
-
const response = await forCapability(capabilityId).handle(message);
|
|
1048
|
-
if (response === null) {
|
|
1049
|
-
res.writeHead(202).end();
|
|
1050
|
-
return;
|
|
1051
|
-
}
|
|
1052
|
-
res.writeHead(200, { "content-type": "application/json" });
|
|
1053
|
-
res.end(JSON.stringify(response));
|
|
1054
|
-
} catch (e) {
|
|
1055
|
-
res.writeHead(200, { "content-type": "application/json" });
|
|
1056
|
-
res.end(JSON.stringify({
|
|
1057
|
-
jsonrpc: "2.0",
|
|
1058
|
-
id: null,
|
|
1059
|
-
error: {
|
|
1060
|
-
code: -32700,
|
|
1061
|
-
message: e instanceof Error ? e.message : "parse error"
|
|
1062
|
-
}
|
|
1063
|
-
}));
|
|
1064
|
-
}
|
|
1065
|
-
})();
|
|
1066
|
-
});
|
|
1067
|
-
});
|
|
1068
|
-
const port = await new Promise((resolve, reject) => {
|
|
1069
|
-
listener.once("error", reject);
|
|
1070
|
-
listener.listen(0, host, () => {
|
|
1071
|
-
const addr = listener.address();
|
|
1072
|
-
resolve(typeof addr === "object" && addr ? addr.port : 0);
|
|
1073
|
-
});
|
|
1074
|
-
});
|
|
1075
|
-
mailbox.setEndpoint(`http://${host}:${port}/mail`);
|
|
1076
|
-
return { close: () => new Promise((resolve) => {
|
|
1077
|
-
listener.close(() => resolve());
|
|
1078
|
-
}) };
|
|
1079
|
-
}
|
|
1080
|
-
//#endregion
|
|
1081
|
-
//#region src/runtime/supervise/driver-retry.ts
|
|
1082
|
-
/**
|
|
1083
|
-
* Root-driver retry — the root gets the same second chance a worker's transport already has.
|
|
1084
|
-
*
|
|
1085
|
-
* A spawned child that dies is typed into a `down` settlement and the driver may re-spawn it. The
|
|
1086
|
-
* ROOT had no such path: one dropped connection, one SIGKILLed harness process, one upstream 5xx
|
|
1087
|
-
* ended a run of arbitrary length with `reason: 'driver-failed'`, and every live child was torn
|
|
1088
|
-
* down with it (#741). The driver's budget and deadline were usually almost untouched.
|
|
1089
|
-
*
|
|
1090
|
-
* This module supplies the missing arm: run the driver again, on the SAME scope, the SAME
|
|
1091
|
-
* coordination server and the SAME live children, until the budget or the deadline says stop.
|
|
1092
|
-
* Nothing here restarts children or replays work — it re-enters the driver, and the bridge backend
|
|
1093
|
-
* reattaches the harness session because the execution id is bound durably per node.
|
|
1094
|
-
*
|
|
1095
|
-
* Two classifications decide everything, and both are conservative:
|
|
1096
|
-
*
|
|
1097
|
-
* - TERMINAL failures are Runtime's own refusals: a `ValidationError`/`ConfigError` guard, an
|
|
1098
|
-
* exhausted budget, an abort, a client-side transport status (401/404/422). Runtime meant them,
|
|
1099
|
-
* so retrying re-runs a decision rather than recovering from an accident. They fail immediately.
|
|
1100
|
-
* - TRANSIENT failures are everything foreign: a harness process that exited without a reason, a
|
|
1101
|
-
* stream that cut mid-turn, a 5xx, a socket reset. Those are accidents, and they are exactly
|
|
1102
|
-
* what a retry exists for.
|
|
1103
|
-
*
|
|
1104
|
-
* The one loop the budget alone cannot bound is a driver that dies INSTANTLY and repeatedly — a
|
|
1105
|
-
* dead-on-arrival credential, a harness that refuses to start. Spending nothing, it would retry
|
|
1106
|
-
* until the deadline hours later. So progress is measured between attempts, and a run of attempts
|
|
1107
|
-
* that changes nothing stops at `maxConsecutiveFailures`. A failure that made progress resets that
|
|
1108
|
-
* counter: a long run may be rescued many times, a hopeless one gives up in seconds.
|
|
1109
|
-
*
|
|
1110
|
-
* WHAT COUNTS AS PROGRESS is the part this module got wrong first, and the correction is measured.
|
|
1111
|
-
* The original mark read metered spend, settled children, and an accepted submission — the
|
|
1112
|
-
* filesystem and the meter, never the goal. Across 1,422 settled discovery-lab runs (2026-09-01)
|
|
1113
|
-
* that reading retried the runs that had produced NOTHING 629 of 827 times (76.1%) while retrying
|
|
1114
|
-
* the runs that HAD left an artifact 21 of 399 times (5.3%): the loop spent its second chances on
|
|
1115
|
-
* the hopeless runs and measured persistence by burn rate. So when the caller declares a completion
|
|
1116
|
-
* check, spend and settlements alone are NOT progress while that check is unmet; only a delivery —
|
|
1117
|
-
* an accepted submission, a child that passed the check, or the contract turning met — resets the
|
|
1118
|
-
* barren counter. A caller that declares no check reports `contract: 'none'` and keeps the exact
|
|
1119
|
-
* historical reading.
|
|
1120
|
-
*
|
|
1121
|
-
* THE SECOND HALF of the same defect: `budgetStop` used to be consulted only after a failure, and a
|
|
1122
|
-
* driver that RETURNED with the contract unmet ended the run silently — 376 of 376 winning lab runs
|
|
1123
|
-
* ended on this loop's own `stop: 'completed'`, with the completion gate left to label the result
|
|
1124
|
-
* rather than to change it. A completed drive whose contract is unmet is now a first-class moment:
|
|
1125
|
-
* `reprompt.maxReprompts` re-enters the SAME live session with the unmet items, and every re-entry
|
|
1126
|
-
* crosses the same budget, deadline, abort, and attempt bounds a retry crosses.
|
|
1127
|
-
*/
|
|
1128
|
-
/**
|
|
1129
|
-
* The instruction a completed-but-undelivered drive is re-entered with when the caller supplies no
|
|
1130
|
-
* `onUnmetContract`. It states the verdict, names what is owed, reports the ledger, and gives the
|
|
1131
|
-
* three steps — the same shape `depthStrategy` re-prompts a resumed session with, said in the
|
|
1132
|
-
* driver's own terms.
|
|
1133
|
-
*/
|
|
1134
|
-
function defaultUnmetContractSteer(context) {
|
|
1135
|
-
const owed = context.describe?.trim();
|
|
1136
|
-
return [
|
|
1137
|
-
"The completion check has not passed. This run has delivered nothing yet.",
|
|
1138
|
-
owed === void 0 || owed.length === 0 ? "The deliverable this run owes is still missing." : `The deliverable this run owes: ${owed}`,
|
|
1139
|
-
`Workers settled: ${context.progress.settledCount}. Workers that passed the check: ${context.progress.deliveredCount ?? 0}.`,
|
|
1140
|
-
"Do the unfinished work with the tools.",
|
|
1141
|
-
"Verify that the check passes.",
|
|
1142
|
-
"Then submit the result.",
|
|
1143
|
-
"Do not restate work you already did."
|
|
1144
|
-
].join("\n");
|
|
1145
|
-
}
|
|
1146
|
-
const DEFAULT_MAX_CONSECUTIVE_FAILURES = 3;
|
|
1147
|
-
const DEFAULT_MAX_ATTEMPTS = 8;
|
|
1148
|
-
const DEFAULT_INITIAL_BACKOFF_MS$1 = 2e3;
|
|
1149
|
-
const DEFAULT_MAX_BACKOFF_MS$1 = 3e4;
|
|
1150
|
-
/**
|
|
1151
|
-
* Bridge error classes the bridge itself never retries: a request that fails identically on
|
|
1152
|
-
* every attempt, mapped below 5xx on its HTTP path (`parse_error` 400, the other two 501). On the
|
|
1153
|
-
* stream path the same failure arrives with no status at all — a profile that cannot materialize
|
|
1154
|
-
* is a `parse_error` — and the status split alone read it as a bad moment and re-drove it to the
|
|
1155
|
-
* attempt ceiling.
|
|
1156
|
-
*/
|
|
1157
|
-
const DETERMINISTIC_BRIDGE_CODES = /* @__PURE__ */ new Set([
|
|
1158
|
-
"parse_error",
|
|
1159
|
-
"not_configured",
|
|
1160
|
-
"capability_denied"
|
|
1161
|
-
]);
|
|
1162
|
-
/**
|
|
1163
|
-
* Classify one driver failure. Runtime's own typed refusals are decisions and stay terminal;
|
|
1164
|
-
* anything foreign is an accident and is retryable. A `BackendTransportError` is split by status
|
|
1165
|
-
* because the taxonomy already promises consumers may branch on it: a 5xx/429/408 is the upstream
|
|
1166
|
-
* having a bad moment, while a 401/404/422 is a request that will fail identically forever. The
|
|
1167
|
-
* bridge's own never-retry classes are terminal whether or not a status rides with them.
|
|
1168
|
-
*/
|
|
1169
|
-
function classifyDriverFailure(error, signal) {
|
|
1170
|
-
if (signal?.aborted) return "terminal";
|
|
1171
|
-
if (error instanceof Error && error.name === "AbortError") return "terminal";
|
|
1172
|
-
if (error instanceof BackendTransportError) {
|
|
1173
|
-
if (error.upstreamCode !== void 0 && DETERMINISTIC_BRIDGE_CODES.has(error.upstreamCode)) return "terminal";
|
|
1174
|
-
const status = error.status;
|
|
1175
|
-
if (status === void 0) return "transient";
|
|
1176
|
-
if (status === 408 || status === 429 || status >= 500) return "transient";
|
|
1177
|
-
return "terminal";
|
|
1001
|
+
assignSpend(attrs, result.spentTotal);
|
|
1002
|
+
}
|
|
1003
|
+
if (outcome?.error !== void 0) {
|
|
1004
|
+
attrs["tangle.supervise.result"] = "error";
|
|
1005
|
+
const err = outcome.error;
|
|
1006
|
+
attrs["error.type"] = err instanceof Error ? err.name : typeof err;
|
|
1007
|
+
attrs["error.message"] = truncate(err instanceof Error ? err.message : String(err));
|
|
1178
1008
|
}
|
|
1179
|
-
|
|
1180
|
-
if (error instanceof AgentEvalError) return "terminal";
|
|
1181
|
-
return "transient";
|
|
1009
|
+
return attrs;
|
|
1182
1010
|
}
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
if (
|
|
1186
|
-
|
|
1187
|
-
if (budget.usdCapped && budget.usdLeft <= 0) return "budget-exhausted";
|
|
1188
|
-
if (budget.usdCapped && budget.usdKnown === false) return "budget-exhausted";
|
|
1189
|
-
if (Object.values(budget.resources ?? {}).some((resource) => !resource.known || resource.remaining <= 0)) return "budget-exhausted";
|
|
1011
|
+
function rootStatus(outcome) {
|
|
1012
|
+
if (outcome?.error !== void 0) return STATUS_ERROR;
|
|
1013
|
+
if (!outcome?.result) return STATUS_UNSET;
|
|
1014
|
+
return outcome.result.kind === "winner" ? STATUS_OK : STATUS_ERROR;
|
|
1190
1015
|
}
|
|
1191
|
-
function
|
|
1192
|
-
return
|
|
1016
|
+
function rootMessage(outcome) {
|
|
1017
|
+
if (outcome?.error !== void 0) return truncate(outcome.error instanceof Error ? outcome.error.message : String(outcome.error));
|
|
1018
|
+
const result = outcome?.result;
|
|
1019
|
+
return result && result.kind === "no-winner" ? result.reason : void 0;
|
|
1193
1020
|
}
|
|
1194
1021
|
/**
|
|
1195
|
-
*
|
|
1196
|
-
*
|
|
1197
|
-
*
|
|
1198
|
-
* contract turning met. Spend and settlements count only while no declared check is outstanding —
|
|
1199
|
-
* with a check unmet they are the burn-rate reading this module's header measures and rejects.
|
|
1022
|
+
* Write a `Spend` onto a span, honouring the "a missing measurement is never zero" invariant: a
|
|
1023
|
+
* channel marked NOT known contributes no number at all and instead flags itself, so nothing
|
|
1024
|
+
* downstream can sum an unmeasured turn as free. Returns the spend it read (for the duration).
|
|
1200
1025
|
*/
|
|
1201
|
-
function
|
|
1202
|
-
if (
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
if (
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
* `driver-failed` carries a diagnosable message instead of one backend's last words. */
|
|
1210
|
-
var DriverAttemptsExhaustedError = class extends RuntimeRunStateError {
|
|
1211
|
-
attempts;
|
|
1212
|
-
stop;
|
|
1213
|
-
constructor(cause, attempts, stop) {
|
|
1214
|
-
const last = attempts[attempts.length - 1];
|
|
1215
|
-
const causeText = cause instanceof Error ? `${cause.name}: ${cause.message}` : describeUnknown(cause);
|
|
1216
|
-
super(`supervisor driver failed after ${attempts.length} attempt(s) — stopped by ${stop}; last cause: ${causeText}` + (last?.classification ? ` (classified ${last.classification})` : ""), { cause });
|
|
1217
|
-
this.attempts = Object.freeze([...attempts]);
|
|
1218
|
-
this.stop = stop;
|
|
1026
|
+
function assignSpend(attrs, value) {
|
|
1027
|
+
if (!isRecord(value)) return void 0;
|
|
1028
|
+
const spend = value;
|
|
1029
|
+
const tokens = isRecord(spend.tokens) ? spend.tokens : void 0;
|
|
1030
|
+
if (spend.tokensKnown === false) attrs["tangle.supervise.tokens_known"] = false;
|
|
1031
|
+
else if (tokens) {
|
|
1032
|
+
if (typeof tokens.input === "number") attrs["llm.token_count.prompt"] = tokens.input;
|
|
1033
|
+
if (typeof tokens.output === "number") attrs["llm.token_count.completion"] = tokens.output;
|
|
1219
1034
|
}
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
return JSON.stringify(value) ?? String(value);
|
|
1225
|
-
} catch {
|
|
1226
|
-
return String(value);
|
|
1035
|
+
if (spend.usdKnown === false) attrs["tangle.supervise.cost_known"] = false;
|
|
1036
|
+
else if (typeof spend.usd === "number" && Number.isFinite(spend.usd)) {
|
|
1037
|
+
attrs["llm.cost_usd"] = spend.usd;
|
|
1038
|
+
attrs["tangle.cost.usd"] = spend.usd;
|
|
1227
1039
|
}
|
|
1040
|
+
if (typeof spend.iterations === "number") attrs["tangle.supervise.iterations"] = spend.iterations;
|
|
1041
|
+
if (typeof spend.ms === "number" && spend.ms > 0) attrs["tangle.supervise.duration_ms"] = spend.ms;
|
|
1042
|
+
if (typeof spend.boxMinutes === "number" && Number.isFinite(spend.boxMinutes)) attrs["tangle.platform.box_minutes"] = spend.boxMinutes;
|
|
1043
|
+
if (spend.boxMinutesProvenance !== void 0) {
|
|
1044
|
+
attrs["tangle.platform.box_minutes_provenance"] = spend.boxMinutesProvenance;
|
|
1045
|
+
attrs["tangle.platform.box_minutes_known"] = spend.boxMinutesKnown === true;
|
|
1046
|
+
}
|
|
1047
|
+
if (spend.tokensProvenance !== void 0) attrs["tangle.supervise.tokens_provenance"] = spend.tokensProvenance;
|
|
1048
|
+
return spend;
|
|
1228
1049
|
}
|
|
1229
|
-
|
|
1230
|
-
if (
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
resolve();
|
|
1236
|
-
};
|
|
1237
|
-
const timer = setTimeout(done, ms);
|
|
1238
|
-
if (typeof timer === "object" && timer !== null && "unref" in timer) timer.unref();
|
|
1239
|
-
signal.addEventListener("abort", done, { once: true });
|
|
1240
|
-
});
|
|
1050
|
+
function assignBudget(attrs, value) {
|
|
1051
|
+
if (!isRecord(value)) return;
|
|
1052
|
+
const budget = value;
|
|
1053
|
+
if (typeof budget.maxTokens === "number") attrs["tangle.supervise.budget.max_tokens"] = budget.maxTokens;
|
|
1054
|
+
if (typeof budget.maxIterations === "number") attrs["tangle.supervise.budget.max_iterations"] = budget.maxIterations;
|
|
1055
|
+
if (typeof budget.maxUsd === "number") attrs["tangle.supervise.budget.max_usd"] = budget.maxUsd;
|
|
1241
1056
|
}
|
|
1242
1057
|
/**
|
|
1243
|
-
*
|
|
1244
|
-
*
|
|
1245
|
-
*
|
|
1246
|
-
* items, up to `reprompt.maxReprompts`. Throws `DriverAttemptsExhaustedError` (cause = the last
|
|
1247
|
-
* real failure) when a FAILURE ends the loop; a completed drive returns, met contract or not,
|
|
1248
|
-
* because deciding what an undelivered run is worth belongs to the finalizer, not to this loop.
|
|
1058
|
+
* A trace id must be 32 hex characters. A caller-supplied one is used verbatim when it already is;
|
|
1059
|
+
* anything else (and the default) is DERIVED from the run id by content address — deterministic, so
|
|
1060
|
+
* a resumed run rejoins the trace its first process opened rather than forking a new one.
|
|
1249
1061
|
*/
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
};
|
|
1268
|
-
/**
|
|
1269
|
-
* Decide what a completed-but-undelivered drive does next. Every bound the FAILURE path applies
|
|
1270
|
-
* is applied here too — that is the fix for `budgetStop` having lived only in the catch arm — and
|
|
1271
|
-
* the caller's hook is consulted last, so no hook can talk the loop past a deadline.
|
|
1272
|
-
*/
|
|
1273
|
-
const decideReprompt = async (attempt, after) => {
|
|
1274
|
-
if (reprompts >= maxReprompts) return { refusedBy: "reprompts-exhausted" };
|
|
1275
|
-
if (run.signal.aborted) return { refusedBy: "aborted" };
|
|
1276
|
-
const byBudget = budgetStop(run.budget(), now());
|
|
1277
|
-
if (byBudget === "deadline" || byBudget === "budget-exhausted") return { refusedBy: byBudget };
|
|
1278
|
-
if (attempt >= maxAttempts) return { refusedBy: "max-attempts" };
|
|
1279
|
-
const context = {
|
|
1280
|
-
attempt,
|
|
1281
|
-
reprompts,
|
|
1282
|
-
maxReprompts,
|
|
1283
|
-
progress: after,
|
|
1284
|
-
budget: run.budget(),
|
|
1285
|
-
...run.reprompt?.describe === void 0 ? {} : { describe: run.reprompt.describe }
|
|
1286
|
-
};
|
|
1287
|
-
const decision = await run.reprompt?.onUnmetContract?.(context) ?? { steer: defaultUnmetContractSteer(context) };
|
|
1288
|
-
if (decision === "stop") return { refusedBy: "caller-stop" };
|
|
1289
|
-
const steer = typeof decision.steer === "string" ? decision.steer.trim() : "";
|
|
1290
|
-
if (steer.length === 0) throw new ValidationError("runDriverWithRetry: onUnmetContract returned an empty steer — return a non-empty instruction or 'stop'");
|
|
1291
|
-
return { steer };
|
|
1292
|
-
};
|
|
1293
|
-
for (let attempt = 1;; attempt += 1) {
|
|
1294
|
-
const before = run.progress();
|
|
1295
|
-
const startedAt = now();
|
|
1296
|
-
try {
|
|
1297
|
-
await run.drive(attempt, reentry);
|
|
1298
|
-
} catch (error) {
|
|
1299
|
-
const durationMs = now() - startedAt;
|
|
1300
|
-
const classification = classifyDriverFailure(error, run.signal);
|
|
1301
|
-
const progressed = madeProgress(before, run.progress());
|
|
1302
|
-
const stop = (() => {
|
|
1303
|
-
if (classification === "terminal") return "terminal-error";
|
|
1304
|
-
if (!retryEnabled) return "retry-disabled";
|
|
1305
|
-
if (run.signal.aborted) return "aborted";
|
|
1306
|
-
const byBudget = budgetStop(run.budget(), now());
|
|
1307
|
-
if (byBudget) return byBudget;
|
|
1308
|
-
if (attempt >= maxAttempts) return "max-attempts";
|
|
1309
|
-
if (progressed) return void 0;
|
|
1310
|
-
return consecutiveBarren + 1 >= maxConsecutive ? "no-progress" : void 0;
|
|
1311
|
-
})();
|
|
1312
|
-
if (stop !== void 0) {
|
|
1313
|
-
await emit({
|
|
1314
|
-
attempt,
|
|
1315
|
-
durationMs,
|
|
1316
|
-
error: error instanceof Error ? error.message : describeUnknown(error),
|
|
1317
|
-
classification,
|
|
1318
|
-
madeProgress: progressed,
|
|
1319
|
-
stop
|
|
1320
|
-
});
|
|
1321
|
-
throw new DriverAttemptsExhaustedError(error, attempts, stop);
|
|
1322
|
-
}
|
|
1323
|
-
consecutiveBarren = progressed ? 0 : consecutiveBarren + 1;
|
|
1324
|
-
reentry = void 0;
|
|
1325
|
-
const backoff = Math.min(maxBackoff, initialBackoff * 2 ** Math.max(0, consecutiveBarren - 1));
|
|
1326
|
-
await emit({
|
|
1327
|
-
attempt,
|
|
1328
|
-
durationMs,
|
|
1329
|
-
error: error instanceof Error ? error.message : describeUnknown(error),
|
|
1330
|
-
classification,
|
|
1331
|
-
madeProgress: progressed,
|
|
1332
|
-
retryInMs: backoff
|
|
1333
|
-
});
|
|
1334
|
-
await sleep(backoff, run.signal);
|
|
1335
|
-
if (run.signal.aborted) throw new DriverAttemptsExhaustedError(error, attempts, "aborted");
|
|
1336
|
-
const afterWait = budgetStop(run.budget(), now());
|
|
1337
|
-
if (afterWait) throw new DriverAttemptsExhaustedError(error, attempts, afterWait);
|
|
1338
|
-
continue;
|
|
1339
|
-
}
|
|
1340
|
-
const durationMs = now() - startedAt;
|
|
1341
|
-
const after = run.progress();
|
|
1342
|
-
const progressed = madeProgress(before, after);
|
|
1343
|
-
const contract = contractOf(after);
|
|
1344
|
-
const contractField = contract === "none" ? {} : { contract };
|
|
1345
|
-
if (contract === "unmet" && maxReprompts > 0) {
|
|
1346
|
-
const decision = await decideReprompt(attempt, after);
|
|
1347
|
-
if ("steer" in decision) {
|
|
1348
|
-
reprompts += 1;
|
|
1349
|
-
reentry = {
|
|
1350
|
-
reason: "unmet-contract",
|
|
1351
|
-
steer: decision.steer,
|
|
1352
|
-
reprompt: reprompts
|
|
1353
|
-
};
|
|
1354
|
-
await emit({
|
|
1355
|
-
attempt,
|
|
1356
|
-
durationMs,
|
|
1357
|
-
madeProgress: progressed,
|
|
1358
|
-
...contractField,
|
|
1359
|
-
reprompted: true,
|
|
1360
|
-
retryInMs: 0
|
|
1361
|
-
});
|
|
1362
|
-
continue;
|
|
1363
|
-
}
|
|
1364
|
-
await emit({
|
|
1365
|
-
attempt,
|
|
1366
|
-
durationMs,
|
|
1367
|
-
madeProgress: progressed,
|
|
1368
|
-
...contractField,
|
|
1369
|
-
repromptRefusedBy: decision.refusedBy,
|
|
1370
|
-
stop: "completed"
|
|
1371
|
-
});
|
|
1372
|
-
return;
|
|
1373
|
-
}
|
|
1374
|
-
await emit({
|
|
1375
|
-
attempt,
|
|
1376
|
-
durationMs,
|
|
1377
|
-
madeProgress: progressed,
|
|
1378
|
-
...contractField,
|
|
1379
|
-
stop: "completed"
|
|
1380
|
-
});
|
|
1381
|
-
return;
|
|
1382
|
-
}
|
|
1062
|
+
function normalizeTraceId(traceId, runId) {
|
|
1063
|
+
if (traceId && /^[0-9a-f]{32}$/i.test(traceId)) return traceId.toLowerCase();
|
|
1064
|
+
return contentAddress(traceId ?? runId).slice(7, 39);
|
|
1065
|
+
}
|
|
1066
|
+
function msToNano(ms) {
|
|
1067
|
+
return (BigInt(Number.isFinite(ms) ? Math.floor(ms) : 0) * 1000000n).toString();
|
|
1068
|
+
}
|
|
1069
|
+
function isRecord(value) {
|
|
1070
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
1071
|
+
}
|
|
1072
|
+
function record(value) {
|
|
1073
|
+
return isRecord(value) ? value : {};
|
|
1074
|
+
}
|
|
1075
|
+
function str(value) {
|
|
1076
|
+
return typeof value === "string" && value.length > 0 ? value : void 0;
|
|
1077
|
+
}
|
|
1078
|
+
function truncate(value) {
|
|
1079
|
+
return value.length > MAX_DETAIL_CHARS ? `${value.slice(0, MAX_DETAIL_CHARS)}…` : value;
|
|
1383
1080
|
}
|
|
1384
1081
|
//#endregion
|
|
1385
1082
|
//#region src/runtime/supervise/supervisor-agent.ts
|
|
@@ -1621,6 +1318,7 @@ function buildSupervisorAgent(profile, deps, testBrain) {
|
|
|
1621
1318
|
...deps.continuityByProfile ? { continuityByProfile: deps.continuityByProfile } : {},
|
|
1622
1319
|
...deps.preflightSpawn ? { preflightSpawn: deps.preflightSpawn } : {},
|
|
1623
1320
|
...deps.resolveSpawnProfile ? { resolveSpawnProfile: deps.resolveSpawnProfile } : {},
|
|
1321
|
+
...deps.spawnResourceRoot ? { spawnResourceRoot: deps.spawnResourceRoot } : {},
|
|
1624
1322
|
...deps.stopRule ? { stopRule: deps.stopRule } : {},
|
|
1625
1323
|
...deps.onProgressStop ? { onProgressStop: deps.onProgressStop } : {},
|
|
1626
1324
|
...deps.maxTurns !== void 0 ? { maxTurns: deps.maxTurns } : {},
|
|
@@ -1708,6 +1406,7 @@ function buildSupervisorAgent(profile, deps, testBrain) {
|
|
|
1708
1406
|
...deps.replaySettlements ? { replaySettlements: true } : {},
|
|
1709
1407
|
...deps.preflightSpawn ? { preflightSpawn: deps.preflightSpawn } : {},
|
|
1710
1408
|
...deps.resolveSpawnProfile ? { resolveSpawnProfile: deps.resolveSpawnProfile } : {},
|
|
1409
|
+
...deps.spawnResourceRoot ? { spawnResourceRoot: deps.spawnResourceRoot } : {},
|
|
1711
1410
|
...deps.peerMail ? { peerMail: deps.peerMail } : {},
|
|
1712
1411
|
...priorCoordination?.questions.length ? { priorQuestions: priorCoordination.questions } : {},
|
|
1713
1412
|
...priorCoordination?.escalations?.length ? { priorEscalations: priorCoordination.escalations } : {},
|
|
@@ -2984,6 +2683,19 @@ function assertNoUncapturedExecutableOption(decisionData) {
|
|
|
2984
2683
|
* detached and frozen; executable ports are copied as the exact references selected at intake.
|
|
2985
2684
|
* Service internals intentionally remain live, while replacing a callback/service on the caller's
|
|
2986
2685
|
* mutable options object can no longer change an in-flight run. */
|
|
2686
|
+
/** The directory a root manager's spawn may resolve an inline resource `path` under. Only a
|
|
2687
|
+
* loopback bridge driver has one this process can read: its `cwd` is a directory on this host.
|
|
2688
|
+
* A remote bridge, a provider sandbox, or an in-process driver with no cwd yields `undefined`. */
|
|
2689
|
+
function rootSpawnResourceRoot(backend) {
|
|
2690
|
+
if (backend === void 0 || backend.backend !== "bridge" || backend.cwd === void 0) return void 0;
|
|
2691
|
+
let host;
|
|
2692
|
+
try {
|
|
2693
|
+
host = new URL(backend.bridgeUrl).hostname;
|
|
2694
|
+
} catch {
|
|
2695
|
+
return;
|
|
2696
|
+
}
|
|
2697
|
+
return isLoopbackHost(host) ? resolve(backend.cwd) : void 0;
|
|
2698
|
+
}
|
|
2987
2699
|
function captureSuperviseOptions(opts) {
|
|
2988
2700
|
assertSuperviseOptionKeys(opts, "supervise");
|
|
2989
2701
|
const { backend, coordination, driverBackend, deliverable, resolveDeliverable, router, compaction, watchWorkers, analysts, makeWorkerAgent, makeLeafAgent, resolveSpawnProfile, blobs, journal, probes, registry, hooks, otel, authorizeSpawn, authorizeMessage, driveHarness, resolveDriveHarness, resolveSupervisorTools, escalateQuestion, onCoordinationEvent, executeExtraTool, stopRule, onProgressStop, onDriverAttempt, onUnmetContract, workerRetry, onWorkerRetry, finalizer, now, signal, rootHandle, ...decisionData } = opts;
|
|
@@ -3301,6 +3013,7 @@ function superviseInternal(profile, task, opts, testBrain) {
|
|
|
3301
3013
|
const managerBackend = options.driverBackend ?? (options.rootDriverFromBackend === false ? void 0 : options.backend);
|
|
3302
3014
|
if (options.driveHarness && options.resolveDriveHarness) throw new ValidationError("supervise: provide driveHarness or resolveDriveHarness, not both");
|
|
3303
3015
|
const hasCustomDriveHarness = Boolean(options.driveHarness || options.resolveDriveHarness);
|
|
3016
|
+
const spawnResourceRoot = hasCustomDriveHarness ? void 0 : rootSpawnResourceRoot(managerBackend);
|
|
3304
3017
|
const spawnPreflight = composeSpawnPreflights(profileToolSpawnPreflight(options.makeWorkerAgent === void 0, options.resolveSupervisorTools !== void 0), coordinationChannelSpawnPreflight(options.makeWorkerAgent === void 0, hasCustomDriveHarness, managerBackend, options.coordination), options.backend?.backend === "bridge" ? bridgeSpawnPreflight(options.backend) : void 0);
|
|
3305
3018
|
const driverMaterialization = hasCustomDriveHarness ? options.driveHarnessMaterialization ?? fullProfileMaterialization : managerBackend && automaticDriverBackendSupported(managerBackend, options.coordination) ? backendProfileMaterialization(managerBackend) : void 0;
|
|
3306
3019
|
if (isExternalSupervisor(canonicalProfile) && !options.driveHarness && !options.resolveDriveHarness && (!managerBackend || !automaticDriverBackendSupported(managerBackend, options.coordination))) throw new ValidationError(`supervise: external supervisor profile.harness=${JSON.stringify(canonicalProfile.harness)} requires a local bridge, a provider with authenticated coordination.publicUrl and runtime MCP attachments, or an explicit driveHarness with reachable coordination transport`);
|
|
@@ -3529,6 +3242,7 @@ function superviseInternal(profile, task, opts, testBrain) {
|
|
|
3529
3242
|
...options.coordination && isExternalSupervisor(canonicalProfile) ? { coordination: options.coordination } : {},
|
|
3530
3243
|
...spawnPreflight ? { preflightSpawn: spawnPreflight } : {},
|
|
3531
3244
|
...options.resolveSpawnProfile ? { resolveSpawnProfile: options.resolveSpawnProfile } : {},
|
|
3245
|
+
...spawnResourceRoot === void 0 ? {} : { spawnResourceRoot },
|
|
3532
3246
|
...options.peerMail ? { peerMail: options.peerMail } : {},
|
|
3533
3247
|
...options.maxLiveWorkers !== void 0 ? { maxLiveWorkers: options.maxLiveWorkers } : {},
|
|
3534
3248
|
...options.router ? { router: options.router } : {},
|
|
@@ -3657,6 +3371,6 @@ function rootProviderModelEvidenceFromExecution(evidence) {
|
|
|
3657
3371
|
return evidence ?? rootProviderModelEvidence([]);
|
|
3658
3372
|
}
|
|
3659
3373
|
//#endregion
|
|
3660
|
-
export {
|
|
3374
|
+
export { mapExecutorResult as _, isPreSpawnExecutorFailure as a, withWorkerSpawnRetry as c, resolveSupervisorProfile as d, supervisorAgent as f, gateOnDeliverable as g, serveCoordinationMcp as h, workerFromBackend as i, assertCoordinationBinding as l, createSupervisorSpanRecorder as m, supervise as n, resolveWorkerSpawnRetry as o, supervisorAgentWithTestBrain as p, superviseWithTestBrain as r, retryPreSpawnRefusals as s, DEFAULT_AUTHORED_PROFILE_SECURITY_POLICY as t, coordinationProfileToolPrefix as u };
|
|
3661
3375
|
|
|
3662
|
-
//# sourceMappingURL=supervise-
|
|
3376
|
+
//# sourceMappingURL=supervise-CSYu0BeH.js.map
|