@dogpile/sdk 0.3.1 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +201 -0
- package/README.md +1 -0
- package/dist/browser/index.js +2328 -237
- package/dist/browser/index.js.map +1 -1
- package/dist/index.d.ts +3 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/providers/openai-compatible.d.ts +11 -0
- package/dist/providers/openai-compatible.d.ts.map +1 -1
- package/dist/providers/openai-compatible.js +88 -2
- package/dist/providers/openai-compatible.js.map +1 -1
- package/dist/runtime/audit.d.ts +42 -0
- package/dist/runtime/audit.d.ts.map +1 -0
- package/dist/runtime/audit.js +73 -0
- package/dist/runtime/audit.js.map +1 -0
- package/dist/runtime/broadcast.d.ts.map +1 -1
- package/dist/runtime/broadcast.js +39 -36
- package/dist/runtime/broadcast.js.map +1 -1
- package/dist/runtime/cancellation.d.ts +26 -0
- package/dist/runtime/cancellation.d.ts.map +1 -1
- package/dist/runtime/cancellation.js +38 -1
- package/dist/runtime/cancellation.js.map +1 -1
- package/dist/runtime/coordinator.d.ts +79 -1
- package/dist/runtime/coordinator.d.ts.map +1 -1
- package/dist/runtime/coordinator.js +979 -61
- package/dist/runtime/coordinator.js.map +1 -1
- package/dist/runtime/decisions.d.ts +25 -3
- package/dist/runtime/decisions.d.ts.map +1 -1
- package/dist/runtime/decisions.js +241 -3
- package/dist/runtime/decisions.js.map +1 -1
- package/dist/runtime/defaults.d.ts +37 -1
- package/dist/runtime/defaults.d.ts.map +1 -1
- package/dist/runtime/defaults.js +359 -4
- package/dist/runtime/defaults.js.map +1 -1
- package/dist/runtime/engine.d.ts +17 -4
- package/dist/runtime/engine.d.ts.map +1 -1
- package/dist/runtime/engine.js +770 -35
- package/dist/runtime/engine.js.map +1 -1
- package/dist/runtime/health.d.ts +51 -0
- package/dist/runtime/health.d.ts.map +1 -0
- package/dist/runtime/health.js +85 -0
- package/dist/runtime/health.js.map +1 -0
- package/dist/runtime/introspection.d.ts +96 -0
- package/dist/runtime/introspection.d.ts.map +1 -0
- package/dist/runtime/introspection.js +31 -0
- package/dist/runtime/introspection.js.map +1 -0
- package/dist/runtime/metrics.d.ts +44 -0
- package/dist/runtime/metrics.d.ts.map +1 -0
- package/dist/runtime/metrics.js +12 -0
- package/dist/runtime/metrics.js.map +1 -0
- package/dist/runtime/model.d.ts.map +1 -1
- package/dist/runtime/model.js +34 -7
- package/dist/runtime/model.js.map +1 -1
- package/dist/runtime/provenance.d.ts +25 -0
- package/dist/runtime/provenance.d.ts.map +1 -0
- package/dist/runtime/provenance.js +13 -0
- package/dist/runtime/provenance.js.map +1 -0
- package/dist/runtime/sequential.d.ts.map +1 -1
- package/dist/runtime/sequential.js +47 -37
- package/dist/runtime/sequential.js.map +1 -1
- package/dist/runtime/shared.d.ts.map +1 -1
- package/dist/runtime/shared.js +39 -36
- package/dist/runtime/shared.js.map +1 -1
- package/dist/runtime/tracing.d.ts +31 -0
- package/dist/runtime/tracing.d.ts.map +1 -0
- package/dist/runtime/tracing.js +18 -0
- package/dist/runtime/tracing.js.map +1 -0
- package/dist/runtime/validation.d.ts +10 -0
- package/dist/runtime/validation.d.ts.map +1 -1
- package/dist/runtime/validation.js +73 -0
- package/dist/runtime/validation.js.map +1 -1
- package/dist/types/events.d.ts +339 -12
- package/dist/types/events.d.ts.map +1 -1
- package/dist/types/replay.d.ts +7 -1
- package/dist/types/replay.d.ts.map +1 -1
- package/dist/types.d.ts +255 -6
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js.map +1 -1
- package/package.json +39 -1
- package/src/index.ts +15 -0
- package/src/providers/openai-compatible.ts +83 -3
- package/src/runtime/audit.ts +121 -0
- package/src/runtime/broadcast.ts +40 -37
- package/src/runtime/cancellation.ts +59 -1
- package/src/runtime/coordinator.ts +1221 -61
- package/src/runtime/decisions.ts +307 -4
- package/src/runtime/defaults.ts +389 -4
- package/src/runtime/engine.ts +1004 -35
- package/src/runtime/health.ts +136 -0
- package/src/runtime/introspection.ts +122 -0
- package/src/runtime/metrics.ts +45 -0
- package/src/runtime/model.ts +38 -6
- package/src/runtime/provenance.ts +43 -0
- package/src/runtime/sequential.ts +49 -38
- package/src/runtime/shared.ts +40 -37
- package/src/runtime/tracing.ts +35 -0
- package/src/runtime/validation.ts +81 -0
- package/src/types/events.ts +369 -12
- package/src/types/replay.ts +14 -1
- package/src/types.ts +279 -4
|
@@ -1,24 +1,89 @@
|
|
|
1
|
+
import { DogpileError } from "../types.js";
|
|
1
2
|
import { createRunId, elapsedMs, nowMs, providerCallIdFor } from "./ids.js";
|
|
2
|
-
import { addCost, createReplayTraceBudget, createReplayTraceBudgetStateChanges, createReplayTraceFinalOutput, createReplayTraceProtocolDecision, createReplayTraceRunInputs, createReplayTraceSeed, createRunAccounting, createRunEventLog, createRunMetadata, createRunUsage, createTranscriptLink, emptyCost, nextProviderCallId } from "./defaults.js";
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
3
|
+
import { addCost, createReplayTraceBudget, createReplayTraceBudgetStateChanges, createReplayTraceFinalOutput, createReplayTraceProtocolDecision, createReplayTraceRunInputs, createReplayTraceSeed, createRunAccounting, createRunEventLog, createRunMetadata, createRunUsage, createTranscriptLink, emptyCost, lastCostBearingEventCost, nextProviderCallId } from "./defaults.js";
|
|
4
|
+
import { computeHealth, DEFAULT_HEALTH_THRESHOLDS } from "./health.js";
|
|
5
|
+
import { classifyAbortReason, classifyChildTimeoutSource, createAbortErrorFromSignal, createEngineDeadlineTimeoutError, throwIfAborted } from "./cancellation.js";
|
|
6
|
+
import { assertDepthWithinLimit, parseAgentDecision } from "./decisions.js";
|
|
5
7
|
import { generateModelTurn } from "./model.js";
|
|
6
8
|
import { evaluateTerminationStop, warnOnProtocolTerminationMisconfiguration } from "./termination.js";
|
|
7
9
|
import { createRuntimeToolExecutor, executeModelResponseToolRequests, runtimeToolAvailability } from "./tools.js";
|
|
8
10
|
import { createWrapUpHintController } from "./wrap-up.js";
|
|
11
|
+
/**
|
|
12
|
+
* Hard-coded loop guard for the delegate dispatch in the coordinator plan
|
|
13
|
+
* turn. After this many consecutive delegate decisions the coordinator throws
|
|
14
|
+
* `invalid-configuration` (T-03-01). Not a public option.
|
|
15
|
+
*/
|
|
16
|
+
const MAX_DISPATCH_PER_TURN = 8;
|
|
17
|
+
const DEFAULT_MAX_CONCURRENT_CHILDREN = 4;
|
|
18
|
+
function createSemaphore(maxConcurrent) {
|
|
19
|
+
let inFlight = 0;
|
|
20
|
+
const waiters = [];
|
|
21
|
+
return {
|
|
22
|
+
acquire() {
|
|
23
|
+
if (inFlight < maxConcurrent) {
|
|
24
|
+
inFlight += 1;
|
|
25
|
+
return Promise.resolve();
|
|
26
|
+
}
|
|
27
|
+
return new Promise((resolve) => {
|
|
28
|
+
waiters.push(() => {
|
|
29
|
+
inFlight += 1;
|
|
30
|
+
resolve();
|
|
31
|
+
});
|
|
32
|
+
});
|
|
33
|
+
},
|
|
34
|
+
release() {
|
|
35
|
+
inFlight -= 1;
|
|
36
|
+
const next = waiters.shift();
|
|
37
|
+
if (next !== undefined) {
|
|
38
|
+
next();
|
|
39
|
+
}
|
|
40
|
+
},
|
|
41
|
+
get inFlight() {
|
|
42
|
+
return inFlight;
|
|
43
|
+
},
|
|
44
|
+
get queued() {
|
|
45
|
+
return waiters.length;
|
|
46
|
+
}
|
|
47
|
+
};
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Walk the coordinator's active provider set and return the FIRST provider
|
|
51
|
+
* whose metadata.locality === "local", or undefined if none found.
|
|
52
|
+
*
|
|
53
|
+
* Walk order (forward-compat): options.model first, then options.agents in
|
|
54
|
+
* declaration order. AgentSpec has no `model` field today (Phase 3 D-11
|
|
55
|
+
* forward-compat scaffolding); the agent walk uses optional chaining and
|
|
56
|
+
* effectively no-ops until a future phase adds AgentSpec.model.
|
|
57
|
+
*/
|
|
58
|
+
function findFirstLocalProvider(options) {
|
|
59
|
+
if (options.model.metadata?.locality === "local") {
|
|
60
|
+
return options.model;
|
|
61
|
+
}
|
|
62
|
+
// Forward-compat: AgentSpec.model not yet declared (Phase 3 D-11). Walk no-ops today; ready for caller-defined trees in a future milestone.
|
|
63
|
+
for (const agent of options.agents) {
|
|
64
|
+
const agentModel = agent.model;
|
|
65
|
+
if (agentModel?.metadata?.locality === "local") {
|
|
66
|
+
return agentModel;
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
return undefined;
|
|
70
|
+
}
|
|
9
71
|
export async function runCoordinator(options) {
|
|
10
72
|
const runId = createRunId();
|
|
11
73
|
const events = [];
|
|
12
74
|
const transcript = [];
|
|
13
75
|
const protocolDecisions = [];
|
|
14
76
|
const providerCalls = [];
|
|
77
|
+
const dispatchedChildren = new Map();
|
|
15
78
|
let totalCost = emptyCost();
|
|
79
|
+
let concurrencyClampEmitted = false; // D-12: emit once per run, never per-engine.
|
|
16
80
|
const maxTurns = options.protocol.maxTurns ?? options.agents.length;
|
|
17
81
|
const activeAgents = options.agents.slice(0, maxTurns);
|
|
18
82
|
const coordinator = activeAgents[0];
|
|
19
83
|
const startedAtMs = nowMs();
|
|
20
84
|
let stopped = false;
|
|
21
85
|
let termination;
|
|
86
|
+
let triggeringFailureForAbortMode;
|
|
22
87
|
const wrapUpHint = createWrapUpHintController({
|
|
23
88
|
protocol: options.protocol,
|
|
24
89
|
tier: options.tier,
|
|
@@ -34,6 +99,61 @@ export async function runCoordinator(options) {
|
|
|
34
99
|
const recordProtocolDecision = (event, decisionOptions) => {
|
|
35
100
|
protocolDecisions.push(createReplayTraceProtocolDecision("coordinator", event, events.length - 1, decisionOptions));
|
|
36
101
|
};
|
|
102
|
+
const drainOnParentAbort = (reasonSource) => {
|
|
103
|
+
const reason = classifyAbortReason(reasonSource);
|
|
104
|
+
for (const child of dispatchedChildren.values()) {
|
|
105
|
+
if (child.closed) {
|
|
106
|
+
continue;
|
|
107
|
+
}
|
|
108
|
+
const partialCost = child.started
|
|
109
|
+
? lastCostBearingEventCost(child.childEvents) ?? emptyCost()
|
|
110
|
+
: emptyCost();
|
|
111
|
+
const partialTrace = buildPartialTrace({
|
|
112
|
+
childRunId: child.childRunId,
|
|
113
|
+
events: [...child.childEvents],
|
|
114
|
+
startedAtMs: child.startedAtMs,
|
|
115
|
+
protocol: child.decision.protocol,
|
|
116
|
+
tier: options.tier,
|
|
117
|
+
modelProviderId: options.model.id,
|
|
118
|
+
agents: options.agents,
|
|
119
|
+
intent: child.decision.intent,
|
|
120
|
+
temperature: options.temperature,
|
|
121
|
+
...(child.childTimeoutMs !== undefined ? { childTimeoutMs: child.childTimeoutMs } : {}),
|
|
122
|
+
...(options.seed !== undefined ? { seed: options.seed } : {})
|
|
123
|
+
});
|
|
124
|
+
const failedEvent = {
|
|
125
|
+
type: "sub-run-failed",
|
|
126
|
+
runId,
|
|
127
|
+
at: new Date().toISOString(),
|
|
128
|
+
childRunId: child.childRunId,
|
|
129
|
+
parentRunId: runId,
|
|
130
|
+
parentDecisionId: child.parentDecisionId,
|
|
131
|
+
parentDecisionArrayIndex: child.parentDecisionArrayIndex,
|
|
132
|
+
error: child.started
|
|
133
|
+
? {
|
|
134
|
+
code: "aborted",
|
|
135
|
+
message: "Parent run aborted.",
|
|
136
|
+
detail: {
|
|
137
|
+
reason
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
: {
|
|
141
|
+
code: "aborted",
|
|
142
|
+
message: "Sibling delegate failed; queued delegate never started.",
|
|
143
|
+
detail: {
|
|
144
|
+
reason: "sibling-failed"
|
|
145
|
+
}
|
|
146
|
+
},
|
|
147
|
+
partialTrace,
|
|
148
|
+
partialCost
|
|
149
|
+
};
|
|
150
|
+
child.closed = true;
|
|
151
|
+
totalCost = addCost(totalCost, partialCost);
|
|
152
|
+
emit(failedEvent);
|
|
153
|
+
recordProtocolDecision(failedEvent);
|
|
154
|
+
}
|
|
155
|
+
};
|
|
156
|
+
options.registerAbortDrain?.(drainOnParentAbort);
|
|
37
157
|
const toolExecutor = createRuntimeToolExecutor({
|
|
38
158
|
runId,
|
|
39
159
|
protocol: "coordinator",
|
|
@@ -61,24 +181,265 @@ export async function runCoordinator(options) {
|
|
|
61
181
|
}
|
|
62
182
|
if (coordinator) {
|
|
63
183
|
if (!stopIfNeeded()) {
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
184
|
+
// Delegate dispatch loop (D-11/D-16/D-17/D-18). Phase 1 limits delegation
|
|
185
|
+
// to the coordinator's plan turn; workers cannot delegate. The loop
|
|
186
|
+
// re-issues the coordinator plan turn after each successful sub-run with
|
|
187
|
+
// the projected D-17 result tagged into the next prompt and a synthetic
|
|
188
|
+
// D-18 transcript entry already appended. `partialTrace` for failed
|
|
189
|
+
// sub-runs is captured via a tee'd emit buffer locally — `runProtocol`'s
|
|
190
|
+
// error contract is unchanged.
|
|
191
|
+
let dispatchInput = buildCoordinatorPlanInput(options.intent, coordinator);
|
|
192
|
+
let dispatchCount = 0;
|
|
193
|
+
while (true) {
|
|
194
|
+
const turnOutcome = await runCoordinatorTurn({
|
|
195
|
+
agent: coordinator,
|
|
196
|
+
coordinator,
|
|
197
|
+
input: dispatchInput,
|
|
198
|
+
phase: "plan",
|
|
199
|
+
options,
|
|
200
|
+
runId,
|
|
201
|
+
transcript,
|
|
202
|
+
totalCost,
|
|
203
|
+
providerCalls,
|
|
204
|
+
toolExecutor,
|
|
205
|
+
toolAvailability,
|
|
206
|
+
events,
|
|
207
|
+
startedAtMs,
|
|
208
|
+
wrapUpHint,
|
|
209
|
+
emit,
|
|
210
|
+
recordProtocolDecision
|
|
211
|
+
});
|
|
212
|
+
totalCost = turnOutcome.totalCost;
|
|
213
|
+
if (turnOutcome.decision === undefined) {
|
|
214
|
+
break;
|
|
215
|
+
}
|
|
216
|
+
const delegates = Array.isArray(turnOutcome.decision)
|
|
217
|
+
? turnOutcome.decision
|
|
218
|
+
: turnOutcome.decision.type === "delegate"
|
|
219
|
+
? [turnOutcome.decision]
|
|
220
|
+
: [];
|
|
221
|
+
if (delegates.length === 0) {
|
|
222
|
+
break;
|
|
223
|
+
}
|
|
224
|
+
if (dispatchCount + delegates.length > MAX_DISPATCH_PER_TURN) {
|
|
225
|
+
throw new DogpileError({
|
|
226
|
+
code: "invalid-configuration",
|
|
227
|
+
message: `Coordinator plan turn delegated ${delegates.length} more children after ${dispatchCount}; max is ${MAX_DISPATCH_PER_TURN}.`,
|
|
228
|
+
retryable: false,
|
|
229
|
+
detail: {
|
|
230
|
+
kind: "delegate-validation",
|
|
231
|
+
path: "decision",
|
|
232
|
+
reason: "loop-guard-exceeded",
|
|
233
|
+
maxDispatchPerTurn: MAX_DISPATCH_PER_TURN
|
|
234
|
+
}
|
|
235
|
+
});
|
|
236
|
+
}
|
|
237
|
+
const parentDecisionId = String(events.length - 1);
|
|
238
|
+
const parentDepth = options.currentDepth ?? 0;
|
|
239
|
+
const decisionMax = delegates.reduce((max, delegate) => Math.min(max, delegate.maxConcurrentChildren ?? Number.POSITIVE_INFINITY), Number.POSITIVE_INFINITY);
|
|
240
|
+
let effectiveForTurn = Math.min(options.effectiveMaxConcurrentChildren ?? DEFAULT_MAX_CONCURRENT_CHILDREN, decisionMax);
|
|
241
|
+
const requestedMax = effectiveForTurn;
|
|
242
|
+
const localProvider = findFirstLocalProvider(options);
|
|
243
|
+
if (localProvider !== undefined) {
|
|
244
|
+
effectiveForTurn = 1;
|
|
245
|
+
if (!concurrencyClampEmitted) {
|
|
246
|
+
const clampEvent = {
|
|
247
|
+
type: "sub-run-concurrency-clamped",
|
|
248
|
+
runId,
|
|
249
|
+
at: new Date().toISOString(),
|
|
250
|
+
requestedMax,
|
|
251
|
+
effectiveMax: 1,
|
|
252
|
+
reason: "local-provider-detected",
|
|
253
|
+
providerId: localProvider.id
|
|
254
|
+
};
|
|
255
|
+
emit(clampEvent);
|
|
256
|
+
recordProtocolDecision(clampEvent);
|
|
257
|
+
concurrencyClampEmitted = true;
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
const semaphore = createSemaphore(effectiveForTurn);
|
|
261
|
+
const childRunIds = delegates.map(() => createRunId());
|
|
262
|
+
const dispatchedForTurn = delegates.map((delegate, index) => {
|
|
263
|
+
const childRunId = childRunIds[index];
|
|
264
|
+
if (childRunId === undefined) {
|
|
265
|
+
throw new Error("missing child run id");
|
|
266
|
+
}
|
|
267
|
+
const dispatchedChild = {
|
|
268
|
+
childRunId,
|
|
269
|
+
decision: delegate,
|
|
270
|
+
parentDecisionId,
|
|
271
|
+
parentDecisionArrayIndex: index,
|
|
272
|
+
parentDepth,
|
|
273
|
+
controller: new AbortController(),
|
|
274
|
+
removeParentListener: undefined,
|
|
275
|
+
childEvents: [],
|
|
276
|
+
started: false,
|
|
277
|
+
closed: false,
|
|
278
|
+
startedAtMs: Date.now(),
|
|
279
|
+
childTimeoutMs: undefined,
|
|
280
|
+
failure: undefined
|
|
281
|
+
};
|
|
282
|
+
dispatchedChildren.set(childRunId, dispatchedChild);
|
|
283
|
+
return dispatchedChild;
|
|
284
|
+
});
|
|
285
|
+
const dispatchResults = [];
|
|
286
|
+
let firstFailureIndex;
|
|
287
|
+
const tasks = delegates.map(async (delegate, index) => {
|
|
288
|
+
const childRunId = childRunIds[index];
|
|
289
|
+
if (childRunId === undefined) {
|
|
290
|
+
throw new Error("missing child run id");
|
|
291
|
+
}
|
|
292
|
+
if (semaphore.inFlight >= effectiveForTurn) {
|
|
293
|
+
const queuedEvent = {
|
|
294
|
+
type: "sub-run-queued",
|
|
295
|
+
runId,
|
|
296
|
+
at: new Date().toISOString(),
|
|
297
|
+
childRunId,
|
|
298
|
+
parentRunId: runId,
|
|
299
|
+
parentDecisionId,
|
|
300
|
+
parentDecisionArrayIndex: index,
|
|
301
|
+
protocol: delegate.protocol,
|
|
302
|
+
intent: delegate.intent,
|
|
303
|
+
depth: parentDepth + 1,
|
|
304
|
+
queuePosition: semaphore.queued
|
|
305
|
+
};
|
|
306
|
+
emit(queuedEvent);
|
|
307
|
+
recordProtocolDecision(queuedEvent);
|
|
308
|
+
}
|
|
309
|
+
await semaphore.acquire();
|
|
310
|
+
try {
|
|
311
|
+
const dispatchedChild = dispatchedForTurn[index];
|
|
312
|
+
if (!dispatchedChild) {
|
|
313
|
+
throw new Error("missing dispatched child");
|
|
314
|
+
}
|
|
315
|
+
if (firstFailureIndex !== undefined) {
|
|
316
|
+
if (dispatchedChild.closed) {
|
|
317
|
+
dispatchResults.push({
|
|
318
|
+
index,
|
|
319
|
+
result: {
|
|
320
|
+
nextInput: "",
|
|
321
|
+
taggedText: `[sub-run ${childRunId}]: skipped because the parent run aborted`,
|
|
322
|
+
completedAtMs: Date.now()
|
|
323
|
+
}
|
|
324
|
+
});
|
|
325
|
+
return;
|
|
326
|
+
}
|
|
327
|
+
const partialCost = emptyCost();
|
|
328
|
+
const partialTrace = buildPartialTrace({
|
|
329
|
+
childRunId,
|
|
330
|
+
events: [],
|
|
331
|
+
startedAtMs: Date.now(),
|
|
332
|
+
protocol: delegate.protocol,
|
|
333
|
+
tier: options.tier,
|
|
334
|
+
modelProviderId: options.model.id,
|
|
335
|
+
agents: options.agents,
|
|
336
|
+
intent: delegate.intent,
|
|
337
|
+
temperature: options.temperature,
|
|
338
|
+
...(options.seed !== undefined ? { seed: options.seed } : {})
|
|
339
|
+
});
|
|
340
|
+
const failedEvent = {
|
|
341
|
+
type: "sub-run-failed",
|
|
342
|
+
runId,
|
|
343
|
+
at: new Date().toISOString(),
|
|
344
|
+
childRunId,
|
|
345
|
+
parentRunId: runId,
|
|
346
|
+
parentDecisionId,
|
|
347
|
+
parentDecisionArrayIndex: index,
|
|
348
|
+
error: {
|
|
349
|
+
code: "aborted",
|
|
350
|
+
message: "Sibling delegate failed; queued delegate never started.",
|
|
351
|
+
detail: {
|
|
352
|
+
reason: "sibling-failed"
|
|
353
|
+
}
|
|
354
|
+
},
|
|
355
|
+
partialTrace,
|
|
356
|
+
partialCost
|
|
357
|
+
};
|
|
358
|
+
emit(failedEvent);
|
|
359
|
+
recordProtocolDecision(failedEvent);
|
|
360
|
+
dispatchedChild.closed = true;
|
|
361
|
+
dispatchResults.push({
|
|
362
|
+
index,
|
|
363
|
+
result: {
|
|
364
|
+
nextInput: "",
|
|
365
|
+
taggedText: `[sub-run ${childRunId}]: skipped because a sibling delegate failed`,
|
|
366
|
+
completedAtMs: Date.now()
|
|
367
|
+
}
|
|
368
|
+
});
|
|
369
|
+
return;
|
|
370
|
+
}
|
|
371
|
+
const result = await dispatchDelegate({
|
|
372
|
+
decision: delegate,
|
|
373
|
+
childRunId,
|
|
374
|
+
parentDecisionId,
|
|
375
|
+
parentDecisionArrayIndex: index,
|
|
376
|
+
parentDepth,
|
|
377
|
+
parentRunId: runId,
|
|
378
|
+
options,
|
|
379
|
+
transcript,
|
|
380
|
+
emit,
|
|
381
|
+
recordProtocolDecision,
|
|
382
|
+
recordSubRunCost: (cost) => {
|
|
383
|
+
totalCost = addCost(totalCost, cost);
|
|
384
|
+
},
|
|
385
|
+
dispatchedChild
|
|
386
|
+
});
|
|
387
|
+
dispatchResults.push({ index, result });
|
|
388
|
+
}
|
|
389
|
+
catch (error) {
|
|
390
|
+
firstFailureIndex ??= index;
|
|
391
|
+
const dispatchedChild = dispatchedForTurn[index];
|
|
392
|
+
const failure = dispatchedChild?.failure;
|
|
393
|
+
if (delegates.length === 1 &&
|
|
394
|
+
(options.onChildFailure === "abort" || failure === undefined || isDelegateValidationError(error))) {
|
|
395
|
+
throw error;
|
|
396
|
+
}
|
|
397
|
+
const failureMessage = error instanceof Error ? error.message : String(error);
|
|
398
|
+
let taggedText = `[sub-run ${childRunId} failed]: ${failureMessage}`;
|
|
399
|
+
if (failure) {
|
|
400
|
+
const error = failure.error;
|
|
401
|
+
taggedText = `[sub-run ${childRunId} failed | code=${error.code} | spent=$${failure.partialCost.usd.toFixed(3)}]: ${error.message}`;
|
|
402
|
+
}
|
|
403
|
+
dispatchResults.push({
|
|
404
|
+
index,
|
|
405
|
+
result: {
|
|
406
|
+
nextInput: "",
|
|
407
|
+
taggedText,
|
|
408
|
+
completedAtMs: Date.now()
|
|
409
|
+
}
|
|
410
|
+
});
|
|
411
|
+
}
|
|
412
|
+
finally {
|
|
413
|
+
semaphore.release();
|
|
414
|
+
}
|
|
415
|
+
});
|
|
416
|
+
const settled = await Promise.allSettled(tasks);
|
|
417
|
+
const firstRejected = settled.find((result) => result.status === "rejected");
|
|
418
|
+
if (firstRejected?.status === "rejected" &&
|
|
419
|
+
delegates.length === 1 &&
|
|
420
|
+
(options.onChildFailure === "abort" || dispatchResults.length === 0)) {
|
|
421
|
+
throw firstRejected.reason;
|
|
422
|
+
}
|
|
423
|
+
dispatchResults.sort((a, b) => a.result.completedAtMs - b.result.completedAtMs);
|
|
424
|
+
const taggedResults = dispatchResults.map((entry) => entry.result.taggedText).join("\n\n");
|
|
425
|
+
const currentWaveFailures = dispatchedForTurn
|
|
426
|
+
.map((child) => child.failure)
|
|
427
|
+
.filter((failure) => failure !== undefined);
|
|
428
|
+
if (options.onChildFailure === "abort" && currentWaveFailures.length > 0) {
|
|
429
|
+
triggeringFailureForAbortMode ??= currentWaveFailures[0];
|
|
430
|
+
break;
|
|
431
|
+
}
|
|
432
|
+
const failuresSection = buildFailuresSection(currentWaveFailures);
|
|
433
|
+
const coordinatorAgent = options.agents[0] ?? { id: "coordinator", role: "coordinator" };
|
|
434
|
+
const baseInput = buildCoordinatorPlanInput(options.intent, coordinatorAgent);
|
|
435
|
+
dispatchInput = [
|
|
436
|
+
baseInput,
|
|
437
|
+
taggedResults,
|
|
438
|
+
failuresSection,
|
|
439
|
+
"Using the sub-run results above, decide the next step (participate or delegate)."
|
|
440
|
+
].filter((section) => Boolean(section)).join("\n\n");
|
|
441
|
+
dispatchCount += delegates.length;
|
|
442
|
+
}
|
|
82
443
|
stopIfNeeded();
|
|
83
444
|
}
|
|
84
445
|
if (!stopIfNeeded()) {
|
|
@@ -136,7 +497,7 @@ export async function runCoordinator(options) {
|
|
|
136
497
|
stopIfNeeded();
|
|
137
498
|
}
|
|
138
499
|
if (!stopIfNeeded()) {
|
|
139
|
-
|
|
500
|
+
const synthesisOutcome = await runCoordinatorTurn({
|
|
140
501
|
agent: coordinator,
|
|
141
502
|
coordinator,
|
|
142
503
|
input: buildFinalSynthesisInput(options.intent, transcript, coordinator),
|
|
@@ -154,6 +515,20 @@ export async function runCoordinator(options) {
|
|
|
154
515
|
emit,
|
|
155
516
|
recordProtocolDecision
|
|
156
517
|
});
|
|
518
|
+
totalCost = synthesisOutcome.totalCost;
|
|
519
|
+
// Phase 1: final-synthesis turn cannot delegate.
|
|
520
|
+
if (Array.isArray(synthesisOutcome.decision) || synthesisOutcome.decision?.type === "delegate") {
|
|
521
|
+
throw new DogpileError({
|
|
522
|
+
code: "invalid-configuration",
|
|
523
|
+
message: "Coordinator final-synthesis turn cannot emit a delegate decision in Phase 1",
|
|
524
|
+
retryable: false,
|
|
525
|
+
detail: {
|
|
526
|
+
kind: "delegate-validation",
|
|
527
|
+
path: "decision",
|
|
528
|
+
phase: "final-synthesis"
|
|
529
|
+
}
|
|
530
|
+
});
|
|
531
|
+
}
|
|
157
532
|
stopIfNeeded();
|
|
158
533
|
}
|
|
159
534
|
}
|
|
@@ -173,44 +548,46 @@ export async function runCoordinator(options) {
|
|
|
173
548
|
transcriptEntryCount: transcript.length
|
|
174
549
|
});
|
|
175
550
|
const finalEvent = events.at(-1);
|
|
551
|
+
const trace = {
|
|
552
|
+
schemaVersion: "1.0",
|
|
553
|
+
runId,
|
|
554
|
+
protocol: "coordinator",
|
|
555
|
+
tier: options.tier,
|
|
556
|
+
modelProviderId: options.model.id,
|
|
557
|
+
agentsUsed: activeAgents,
|
|
558
|
+
inputs: createReplayTraceRunInputs({
|
|
559
|
+
intent: options.intent,
|
|
560
|
+
protocol: options.protocol,
|
|
561
|
+
tier: options.tier,
|
|
562
|
+
modelProviderId: options.model.id,
|
|
563
|
+
agents: activeAgents,
|
|
564
|
+
temperature: options.temperature
|
|
565
|
+
}),
|
|
566
|
+
budget: createReplayTraceBudget({
|
|
567
|
+
tier: options.tier,
|
|
568
|
+
...(options.budget ? { caps: options.budget } : {}),
|
|
569
|
+
...(options.terminate ? { termination: options.terminate } : {})
|
|
570
|
+
}),
|
|
571
|
+
budgetStateChanges: createReplayTraceBudgetStateChanges(events),
|
|
572
|
+
seed: createReplayTraceSeed(options.seed),
|
|
573
|
+
protocolDecisions,
|
|
574
|
+
providerCalls,
|
|
575
|
+
finalOutput: createReplayTraceFinalOutput(output, finalEvent ?? {
|
|
576
|
+
type: "final",
|
|
577
|
+
runId,
|
|
578
|
+
at: "",
|
|
579
|
+
output,
|
|
580
|
+
cost: totalCost,
|
|
581
|
+
transcript: createTranscriptLink(transcript)
|
|
582
|
+
}),
|
|
583
|
+
...(triggeringFailureForAbortMode !== undefined ? { triggeringFailureForAbortMode } : {}),
|
|
584
|
+
events,
|
|
585
|
+
transcript
|
|
586
|
+
};
|
|
176
587
|
return {
|
|
177
588
|
output,
|
|
178
589
|
eventLog: createRunEventLog(runId, "coordinator", events),
|
|
179
|
-
trace
|
|
180
|
-
schemaVersion: "1.0",
|
|
181
|
-
runId,
|
|
182
|
-
protocol: "coordinator",
|
|
183
|
-
tier: options.tier,
|
|
184
|
-
modelProviderId: options.model.id,
|
|
185
|
-
agentsUsed: activeAgents,
|
|
186
|
-
inputs: createReplayTraceRunInputs({
|
|
187
|
-
intent: options.intent,
|
|
188
|
-
protocol: options.protocol,
|
|
189
|
-
tier: options.tier,
|
|
190
|
-
modelProviderId: options.model.id,
|
|
191
|
-
agents: activeAgents,
|
|
192
|
-
temperature: options.temperature
|
|
193
|
-
}),
|
|
194
|
-
budget: createReplayTraceBudget({
|
|
195
|
-
tier: options.tier,
|
|
196
|
-
...(options.budget ? { caps: options.budget } : {}),
|
|
197
|
-
...(options.terminate ? { termination: options.terminate } : {})
|
|
198
|
-
}),
|
|
199
|
-
budgetStateChanges: createReplayTraceBudgetStateChanges(events),
|
|
200
|
-
seed: createReplayTraceSeed(options.seed),
|
|
201
|
-
protocolDecisions,
|
|
202
|
-
providerCalls,
|
|
203
|
-
finalOutput: createReplayTraceFinalOutput(output, finalEvent ?? {
|
|
204
|
-
type: "final",
|
|
205
|
-
runId,
|
|
206
|
-
at: "",
|
|
207
|
-
output,
|
|
208
|
-
cost: totalCost,
|
|
209
|
-
transcript: createTranscriptLink(transcript)
|
|
210
|
-
}),
|
|
211
|
-
events,
|
|
212
|
-
transcript
|
|
213
|
-
},
|
|
590
|
+
trace,
|
|
214
591
|
transcript,
|
|
215
592
|
usage: createRunUsage(totalCost),
|
|
216
593
|
metadata: createRunMetadata({
|
|
@@ -228,7 +605,8 @@ export async function runCoordinator(options) {
|
|
|
228
605
|
cost: totalCost,
|
|
229
606
|
events
|
|
230
607
|
}),
|
|
231
|
-
cost: totalCost
|
|
608
|
+
cost: totalCost,
|
|
609
|
+
health: computeHealth(trace, DEFAULT_HEALTH_THRESHOLDS)
|
|
232
610
|
};
|
|
233
611
|
function stopIfNeeded() {
|
|
234
612
|
throwIfAborted(options.signal, options.model.id);
|
|
@@ -273,6 +651,10 @@ export async function runCoordinator(options) {
|
|
|
273
651
|
});
|
|
274
652
|
}
|
|
275
653
|
}
|
|
654
|
+
function isDelegateValidationError(error) {
|
|
655
|
+
return DogpileError.isInstance(error) && error.code === "invalid-configuration" &&
|
|
656
|
+
error.detail?.["kind"] === "delegate-validation";
|
|
657
|
+
}
|
|
276
658
|
async function runCoordinatorTurn(turn) {
|
|
277
659
|
throwIfAborted(turn.options.signal, turn.options.model.id);
|
|
278
660
|
const request = {
|
|
@@ -319,7 +701,11 @@ async function runCoordinatorTurn(turn) {
|
|
|
319
701
|
turn.providerCalls.push(call);
|
|
320
702
|
}
|
|
321
703
|
});
|
|
322
|
-
const decision = parseAgentDecision(response.text
|
|
704
|
+
const decision = parseAgentDecision(response.text, {
|
|
705
|
+
parentProviderId: turn.options.model.id,
|
|
706
|
+
currentDepth: turn.options.currentDepth ?? 0,
|
|
707
|
+
maxDepth: turn.options.effectiveMaxDepth ?? Number.POSITIVE_INFINITY
|
|
708
|
+
});
|
|
323
709
|
const totalCost = addCost(turn.totalCost, responseCost(response));
|
|
324
710
|
const toolCalls = await executeModelResponseToolRequests({
|
|
325
711
|
response,
|
|
@@ -357,7 +743,7 @@ async function runCoordinatorTurn(turn) {
|
|
|
357
743
|
phase: turn.phase,
|
|
358
744
|
transcriptEntryCount: turn.transcript.length
|
|
359
745
|
});
|
|
360
|
-
return totalCost;
|
|
746
|
+
return { totalCost, decision };
|
|
361
747
|
}
|
|
362
748
|
async function runCoordinatorWorkerTurn(turn) {
|
|
363
749
|
throwIfAborted(turn.options.signal, turn.options.model.id);
|
|
@@ -405,7 +791,23 @@ async function runCoordinatorWorkerTurn(turn) {
|
|
|
405
791
|
turn.providerCallSlots[turn.providerCallIndex] = call;
|
|
406
792
|
}
|
|
407
793
|
});
|
|
408
|
-
const decision = parseAgentDecision(response.text
|
|
794
|
+
const decision = parseAgentDecision(response.text, {
|
|
795
|
+
parentProviderId: turn.options.model.id,
|
|
796
|
+
currentDepth: turn.options.currentDepth ?? 0,
|
|
797
|
+
maxDepth: turn.options.effectiveMaxDepth ?? Number.POSITIVE_INFINITY
|
|
798
|
+
});
|
|
799
|
+
if (Array.isArray(decision) || decision?.type === "delegate") {
|
|
800
|
+
throw new DogpileError({
|
|
801
|
+
code: "invalid-configuration",
|
|
802
|
+
message: "Workers cannot emit delegate decisions in Phase 1",
|
|
803
|
+
retryable: false,
|
|
804
|
+
detail: {
|
|
805
|
+
kind: "delegate-validation",
|
|
806
|
+
path: "decision",
|
|
807
|
+
phase: "worker"
|
|
808
|
+
}
|
|
809
|
+
});
|
|
810
|
+
}
|
|
409
811
|
const toolCalls = await executeModelResponseToolRequests({
|
|
410
812
|
response,
|
|
411
813
|
executor: turn.toolExecutor,
|
|
@@ -436,6 +838,34 @@ function buildSystemPrompt(agent, coordinator) {
|
|
|
436
838
|
function buildCoordinatorPlanInput(intent, coordinator) {
|
|
437
839
|
return `Mission: ${intent}\nCoordinator ${coordinator.id}: assign the work, name the plan, and provide the first contribution.`;
|
|
438
840
|
}
|
|
841
|
+
function buildFailuresSection(failures) {
|
|
842
|
+
if (failures.length === 0) {
|
|
843
|
+
return null;
|
|
844
|
+
}
|
|
845
|
+
return [
|
|
846
|
+
"## Sub-run failures since last decision",
|
|
847
|
+
"",
|
|
848
|
+
"```json",
|
|
849
|
+
JSON.stringify(failures, null, 2),
|
|
850
|
+
"```"
|
|
851
|
+
].join("\n");
|
|
852
|
+
}
|
|
853
|
+
function dispatchWaveFailureFromEvent(intent, event) {
|
|
854
|
+
const reason = typeof event.error.detail?.["reason"] === "string" ? event.error.detail["reason"] : undefined;
|
|
855
|
+
if (reason === "sibling-failed" || reason === "parent-aborted") {
|
|
856
|
+
return undefined;
|
|
857
|
+
}
|
|
858
|
+
return {
|
|
859
|
+
childRunId: event.childRunId,
|
|
860
|
+
intent,
|
|
861
|
+
error: {
|
|
862
|
+
code: event.error.code,
|
|
863
|
+
message: event.error.message,
|
|
864
|
+
...(reason !== undefined ? { detail: { reason } } : {})
|
|
865
|
+
},
|
|
866
|
+
partialCost: { usd: event.partialCost.usd }
|
|
867
|
+
};
|
|
868
|
+
}
|
|
439
869
|
function buildWorkerInput(intent, transcript, coordinator) {
|
|
440
870
|
const prior = transcript
|
|
441
871
|
.map((entry) => `${entry.role} (${entry.agentId}): ${entry.output}`)
|
|
@@ -456,4 +886,492 @@ function responseCost(response) {
|
|
|
456
886
|
totalTokens: response.usage?.totalTokens ?? 0
|
|
457
887
|
};
|
|
458
888
|
}
|
|
889
|
+
/**
|
|
890
|
+
* Dispatch a single delegate decision as a recursive sub-run.
|
|
891
|
+
*
|
|
892
|
+
* D-11: child reuses the parent provider object verbatim.
|
|
893
|
+
* D-16: `recursive: true` flag set when both parent and child protocol are
|
|
894
|
+
* `coordinator`.
|
|
895
|
+
* D-17: tagged result text appended to the next coordinator prompt.
|
|
896
|
+
* D-18: synthetic transcript entry pushed for replay/provenance.
|
|
897
|
+
*
|
|
898
|
+
* On thrown error from the child engine, builds `partialTrace` from a locally
|
|
899
|
+
* tee'd `childEvents` buffer — `runProtocol`'s error contract is unchanged.
|
|
900
|
+
*/
|
|
901
|
+
async function dispatchDelegate(input) {
|
|
902
|
+
const { decision, options } = input;
|
|
903
|
+
// Dispatcher-time depth gate (D-14). Same error shape as the parser; this
|
|
904
|
+
// is the TOCTOU defense for any state mutation between parse and dispatch.
|
|
905
|
+
// Fires BEFORE sub-run-started is emitted so failed dispatches do not show
|
|
906
|
+
// up in the trace as half-started sub-runs.
|
|
907
|
+
if (options.effectiveMaxDepth !== undefined) {
|
|
908
|
+
assertDepthWithinLimit(input.parentDepth, options.effectiveMaxDepth);
|
|
909
|
+
}
|
|
910
|
+
const childRunId = input.childRunId ?? createRunId();
|
|
911
|
+
const recursive = decision.protocol === "coordinator";
|
|
912
|
+
const decisionTimeoutMs = decision.budget?.timeoutMs;
|
|
913
|
+
const parentDeadlineMs = options.parentDeadlineMs;
|
|
914
|
+
// BUDGET-02 / D-12: deadline-based remaining-time math. Children inherit
|
|
915
|
+
// `parentDeadlineMs - now()`, not a static `parent.budget.timeoutMs`. If the
|
|
916
|
+
// parent's deadline has already elapsed, throw `code: "aborted"` with
|
|
917
|
+
// `detail.reason: "timeout"` BEFORE `sub-run-started` is emitted.
|
|
918
|
+
const remainingMs = parentDeadlineMs !== undefined ? Math.max(0, parentDeadlineMs - Date.now()) : undefined;
|
|
919
|
+
if (parentDeadlineMs !== undefined && remainingMs === 0) {
|
|
920
|
+
throw new DogpileError({
|
|
921
|
+
code: "aborted",
|
|
922
|
+
message: "Parent deadline elapsed before sub-run dispatch.",
|
|
923
|
+
retryable: false,
|
|
924
|
+
providerId: options.model.id,
|
|
925
|
+
detail: { reason: "timeout" }
|
|
926
|
+
});
|
|
927
|
+
}
|
|
928
|
+
// Resolve child timeout with precedence (D-12 / D-14):
|
|
929
|
+
// decision.budget.timeoutMs > parent's remaining > defaultSubRunTimeoutMs > undefined.
|
|
930
|
+
// When the decision-level timeout exceeds the parent's remaining, CLAMP
|
|
931
|
+
// (no longer throw) and emit a `sub-run-budget-clamped` event below.
|
|
932
|
+
let childTimeoutMs;
|
|
933
|
+
let clampedFrom;
|
|
934
|
+
if (remainingMs !== undefined) {
|
|
935
|
+
if (decisionTimeoutMs !== undefined) {
|
|
936
|
+
if (decisionTimeoutMs > remainingMs) {
|
|
937
|
+
clampedFrom = decisionTimeoutMs;
|
|
938
|
+
childTimeoutMs = remainingMs;
|
|
939
|
+
}
|
|
940
|
+
else {
|
|
941
|
+
childTimeoutMs = decisionTimeoutMs;
|
|
942
|
+
}
|
|
943
|
+
}
|
|
944
|
+
else {
|
|
945
|
+
childTimeoutMs = remainingMs;
|
|
946
|
+
}
|
|
947
|
+
}
|
|
948
|
+
else if (decisionTimeoutMs !== undefined) {
|
|
949
|
+
childTimeoutMs = decisionTimeoutMs;
|
|
950
|
+
}
|
|
951
|
+
else if (options.defaultSubRunTimeoutMs !== undefined) {
|
|
952
|
+
childTimeoutMs = options.defaultSubRunTimeoutMs;
|
|
953
|
+
}
|
|
954
|
+
if (!options.runProtocol) {
|
|
955
|
+
throw new DogpileError({
|
|
956
|
+
code: "invalid-configuration",
|
|
957
|
+
message: "Coordinator delegate dispatch requires the engine `runProtocol` callback. " +
|
|
958
|
+
"Use `Dogpile.run` / `createEngine` rather than calling `runCoordinator` directly when delegate is in play.",
|
|
959
|
+
retryable: false,
|
|
960
|
+
detail: {
|
|
961
|
+
kind: "delegate-validation",
|
|
962
|
+
path: "runProtocol"
|
|
963
|
+
}
|
|
964
|
+
});
|
|
965
|
+
}
|
|
966
|
+
// Buffered tee for partialTrace capture — see Plan 03 step 8.
|
|
967
|
+
const childEvents = input.dispatchedChild.childEvents;
|
|
968
|
+
const parentEmit = input.emit;
|
|
969
|
+
const teedEmit = (event) => {
|
|
970
|
+
childEvents.push(event);
|
|
971
|
+
if (input.dispatchedChild.closed) {
|
|
972
|
+
return;
|
|
973
|
+
}
|
|
974
|
+
if (options.streamEvents && options.emit) {
|
|
975
|
+
const inbound = event.parentRunIds;
|
|
976
|
+
options.emit({
|
|
977
|
+
...event,
|
|
978
|
+
parentRunIds: [input.parentRunId, ...(inbound ?? [])]
|
|
979
|
+
});
|
|
980
|
+
}
|
|
981
|
+
};
|
|
982
|
+
const childStartedAt = Date.now();
|
|
983
|
+
input.dispatchedChild.startedAtMs = childStartedAt;
|
|
984
|
+
// BUDGET-02 / D-12: emit clamp event BEFORE sub-run-started so the trace
|
|
985
|
+
// records "this child's requested timeout was reduced to fit parent's
|
|
986
|
+
// remaining deadline." Skipped on the happy path (no clamp, no event).
|
|
987
|
+
if (clampedFrom !== undefined && childTimeoutMs !== undefined) {
|
|
988
|
+
const clampEvent = {
|
|
989
|
+
type: "sub-run-budget-clamped",
|
|
990
|
+
runId: input.parentRunId,
|
|
991
|
+
at: new Date().toISOString(),
|
|
992
|
+
childRunId,
|
|
993
|
+
parentRunId: input.parentRunId,
|
|
994
|
+
parentDecisionId: input.parentDecisionId,
|
|
995
|
+
requestedTimeoutMs: clampedFrom,
|
|
996
|
+
clampedTimeoutMs: childTimeoutMs,
|
|
997
|
+
reason: "exceeded-parent-remaining"
|
|
998
|
+
};
|
|
999
|
+
input.emit(clampEvent);
|
|
1000
|
+
input.recordProtocolDecision(clampEvent);
|
|
1001
|
+
}
|
|
1002
|
+
const startEvent = {
|
|
1003
|
+
type: "sub-run-started",
|
|
1004
|
+
runId: input.parentRunId,
|
|
1005
|
+
at: new Date().toISOString(),
|
|
1006
|
+
childRunId,
|
|
1007
|
+
parentRunId: input.parentRunId,
|
|
1008
|
+
parentDecisionId: input.parentDecisionId,
|
|
1009
|
+
parentDecisionArrayIndex: input.parentDecisionArrayIndex,
|
|
1010
|
+
protocol: decision.protocol,
|
|
1011
|
+
intent: decision.intent,
|
|
1012
|
+
depth: input.parentDepth + 1,
|
|
1013
|
+
...(recursive ? { recursive: true } : {})
|
|
1014
|
+
};
|
|
1015
|
+
parentEmit(startEvent);
|
|
1016
|
+
input.recordProtocolDecision(startEvent);
|
|
1017
|
+
// BUDGET-01 / D-07: derive a per-child AbortController so child engines see
|
|
1018
|
+
// their own signal. Listener forwards parent.signal.reason verbatim, so
|
|
1019
|
+
// detail.reason classification (parent-aborted vs timeout) is preserved.
|
|
1020
|
+
// Phase 4 STREAM-03 hook: per-child cancel handle attaches here.
|
|
1021
|
+
const parentSignal = options.signal;
|
|
1022
|
+
let removeParentAbortListener;
|
|
1023
|
+
if (parentSignal !== undefined) {
|
|
1024
|
+
if (parentSignal.aborted) {
|
|
1025
|
+
input.dispatchedChild.controller.abort(parentSignal.reason);
|
|
1026
|
+
}
|
|
1027
|
+
else {
|
|
1028
|
+
const handler = () => {
|
|
1029
|
+
input.dispatchedChild.controller.abort(parentSignal.reason);
|
|
1030
|
+
};
|
|
1031
|
+
parentSignal.addEventListener("abort", handler, { once: true });
|
|
1032
|
+
removeParentAbortListener = () => {
|
|
1033
|
+
parentSignal.removeEventListener("abort", handler);
|
|
1034
|
+
};
|
|
1035
|
+
}
|
|
1036
|
+
}
|
|
1037
|
+
input.dispatchedChild.removeParentListener = removeParentAbortListener;
|
|
1038
|
+
input.dispatchedChild.started = true;
|
|
1039
|
+
input.dispatchedChild.childTimeoutMs = childTimeoutMs;
|
|
1040
|
+
const childDeadlineReason = childTimeoutMs !== undefined && parentDeadlineMs === undefined
|
|
1041
|
+
? createEngineDeadlineTimeoutError(options.model.id, childTimeoutMs)
|
|
1042
|
+
: undefined;
|
|
1043
|
+
const childDeadlineTimer = childDeadlineReason !== undefined
|
|
1044
|
+
? setTimeout(() => {
|
|
1045
|
+
input.dispatchedChild.controller.abort(childDeadlineReason);
|
|
1046
|
+
}, childTimeoutMs)
|
|
1047
|
+
: undefined;
|
|
1048
|
+
const childOptions = {
|
|
1049
|
+
runId: childRunId,
|
|
1050
|
+
intent: decision.intent,
|
|
1051
|
+
protocol: decision.protocol,
|
|
1052
|
+
tier: options.tier,
|
|
1053
|
+
model: options.model, // D-11: same provider instance verbatim
|
|
1054
|
+
agents: options.agents,
|
|
1055
|
+
tools: options.tools,
|
|
1056
|
+
temperature: options.temperature,
|
|
1057
|
+
...(childTimeoutMs !== undefined ? { budget: { timeoutMs: childTimeoutMs } } : {}),
|
|
1058
|
+
signal: input.dispatchedChild.controller.signal,
|
|
1059
|
+
emit: teedEmit,
|
|
1060
|
+
...(options.streamEvents !== undefined ? { streamEvents: options.streamEvents } : {}),
|
|
1061
|
+
currentDepth: input.parentDepth + 1,
|
|
1062
|
+
...(options.effectiveMaxDepth !== undefined ? { effectiveMaxDepth: options.effectiveMaxDepth } : {}),
|
|
1063
|
+
...(options.effectiveMaxConcurrentChildren !== undefined
|
|
1064
|
+
? { effectiveMaxConcurrentChildren: options.effectiveMaxConcurrentChildren }
|
|
1065
|
+
: {}),
|
|
1066
|
+
...(options.onChildFailure !== undefined ? { onChildFailure: options.onChildFailure } : {}),
|
|
1067
|
+
// BUDGET-02 / D-12: forward the ROOT deadline so depth-N grandchildren
|
|
1068
|
+
// see the same `parentDeadlineMs` rather than a fresh per-level snapshot.
|
|
1069
|
+
...(parentDeadlineMs !== undefined ? { parentDeadlineMs } : {}),
|
|
1070
|
+
...(options.defaultSubRunTimeoutMs !== undefined
|
|
1071
|
+
? { defaultSubRunTimeoutMs: options.defaultSubRunTimeoutMs }
|
|
1072
|
+
: {})
|
|
1073
|
+
};
|
|
1074
|
+
let subResult;
|
|
1075
|
+
try {
|
|
1076
|
+
subResult = await options.runProtocol(childOptions);
|
|
1077
|
+
}
|
|
1078
|
+
catch (error) {
|
|
1079
|
+
if (childDeadlineTimer !== undefined) {
|
|
1080
|
+
clearTimeout(childDeadlineTimer);
|
|
1081
|
+
}
|
|
1082
|
+
removeParentAbortListener?.();
|
|
1083
|
+
if (input.dispatchedChild.closed) {
|
|
1084
|
+
const enrichedError = enrichAbortErrorWithParentReason(error, parentSignal);
|
|
1085
|
+
if (DogpileError.isInstance(enrichedError)) {
|
|
1086
|
+
throw enrichedError;
|
|
1087
|
+
}
|
|
1088
|
+
throw error;
|
|
1089
|
+
}
|
|
1090
|
+
const failedDecision = {
|
|
1091
|
+
type: "delegate",
|
|
1092
|
+
protocol: decision.protocol,
|
|
1093
|
+
intent: decision.intent,
|
|
1094
|
+
...(decision.model !== undefined ? { model: decision.model } : {}),
|
|
1095
|
+
...(decision.budget !== undefined ? { budget: decision.budget } : {})
|
|
1096
|
+
};
|
|
1097
|
+
const partialTrace = buildPartialTrace({
|
|
1098
|
+
childRunId,
|
|
1099
|
+
events: childEvents,
|
|
1100
|
+
startedAtMs: childStartedAt,
|
|
1101
|
+
protocol: decision.protocol,
|
|
1102
|
+
tier: options.tier,
|
|
1103
|
+
modelProviderId: options.model.id,
|
|
1104
|
+
agents: options.agents,
|
|
1105
|
+
intent: decision.intent,
|
|
1106
|
+
temperature: options.temperature,
|
|
1107
|
+
...(childTimeoutMs !== undefined ? { childTimeoutMs } : {}),
|
|
1108
|
+
...(options.seed !== undefined ? { seed: options.seed } : {})
|
|
1109
|
+
});
|
|
1110
|
+
// BUDGET-01 / D-08: when the child aborted because the parent.signal
|
|
1111
|
+
// aborted, lock detail.reason on the surfaced error. Upstream engine
|
|
1112
|
+
// wrapping (e.g., createStreamCancellationError) attaches its own
|
|
1113
|
+
// detail.status; we add detail.reason so consumers can discriminate
|
|
1114
|
+
// parent-aborted vs timeout regardless of which engine path produced the
|
|
1115
|
+
// abort error.
|
|
1116
|
+
const enrichedError = enrichProviderTimeoutSource(enrichAbortErrorWithParentReason(error, parentSignal), {
|
|
1117
|
+
...(decisionTimeoutMs !== undefined ? { decisionTimeoutMs } : {}),
|
|
1118
|
+
...(options.defaultSubRunTimeoutMs !== undefined
|
|
1119
|
+
? { engineDefaultTimeoutMs: options.defaultSubRunTimeoutMs }
|
|
1120
|
+
: {})
|
|
1121
|
+
});
|
|
1122
|
+
if (DogpileError.isInstance(enrichedError)) {
|
|
1123
|
+
options.failureInstancesByChildRunId?.set(childRunId, enrichedError);
|
|
1124
|
+
}
|
|
1125
|
+
const errorPayload = errorPayloadFromUnknown(enrichedError, failedDecision);
|
|
1126
|
+
// BUDGET-03 / D-02: capture real provider spend before the throw and
|
|
1127
|
+
// roll it into the parent's totalCost BEFORE emitting sub-run-failed.
|
|
1128
|
+
const partialCost = lastCostBearingEventCost(childEvents) ?? emptyCost();
|
|
1129
|
+
input.recordSubRunCost(partialCost);
|
|
1130
|
+
const failEvent = {
|
|
1131
|
+
type: "sub-run-failed",
|
|
1132
|
+
runId: input.parentRunId,
|
|
1133
|
+
at: new Date().toISOString(),
|
|
1134
|
+
childRunId,
|
|
1135
|
+
parentRunId: input.parentRunId,
|
|
1136
|
+
parentDecisionId: input.parentDecisionId,
|
|
1137
|
+
parentDecisionArrayIndex: input.parentDecisionArrayIndex,
|
|
1138
|
+
error: errorPayload,
|
|
1139
|
+
partialTrace,
|
|
1140
|
+
partialCost
|
|
1141
|
+
};
|
|
1142
|
+
parentEmit(failEvent);
|
|
1143
|
+
input.recordProtocolDecision(failEvent);
|
|
1144
|
+
input.dispatchedChild.closed = true;
|
|
1145
|
+
input.dispatchedChild.failure = dispatchWaveFailureFromEvent(decision.intent, failEvent);
|
|
1146
|
+
// Re-throw a DogpileError so the parent run terminates with a typed error.
|
|
1147
|
+
if (DogpileError.isInstance(enrichedError)) {
|
|
1148
|
+
throw enrichedError;
|
|
1149
|
+
}
|
|
1150
|
+
throw new DogpileError({
|
|
1151
|
+
code: "invalid-configuration",
|
|
1152
|
+
message: error instanceof Error ? error.message : String(error),
|
|
1153
|
+
retryable: false,
|
|
1154
|
+
detail: {
|
|
1155
|
+
kind: "delegate-validation",
|
|
1156
|
+
path: "decision",
|
|
1157
|
+
reason: "child-run-failed"
|
|
1158
|
+
}
|
|
1159
|
+
});
|
|
1160
|
+
}
|
|
1161
|
+
if (childDeadlineTimer !== undefined) {
|
|
1162
|
+
clearTimeout(childDeadlineTimer);
|
|
1163
|
+
}
|
|
1164
|
+
removeParentAbortListener?.();
|
|
1165
|
+
// BUDGET-03 / D-01: roll child's full cost into the parent's totalCost
|
|
1166
|
+
// BEFORE emitting sub-run-completed. The next agent-turn / final event will
|
|
1167
|
+
// read totalCost from the closure scope, preserving the existing
|
|
1168
|
+
// "last cost-bearing event === final.cost" invariant.
|
|
1169
|
+
input.recordSubRunCost(subResult.cost);
|
|
1170
|
+
const completedEvent = {
|
|
1171
|
+
type: "sub-run-completed",
|
|
1172
|
+
runId: input.parentRunId,
|
|
1173
|
+
at: new Date().toISOString(),
|
|
1174
|
+
childRunId,
|
|
1175
|
+
parentRunId: input.parentRunId,
|
|
1176
|
+
parentDecisionId: input.parentDecisionId,
|
|
1177
|
+
parentDecisionArrayIndex: input.parentDecisionArrayIndex,
|
|
1178
|
+
subResult
|
|
1179
|
+
};
|
|
1180
|
+
parentEmit(completedEvent);
|
|
1181
|
+
input.recordProtocolDecision(completedEvent);
|
|
1182
|
+
input.dispatchedChild.closed = true;
|
|
1183
|
+
// BUDGET-01 / D-10: parent.signal aborted AFTER the child completed but
|
|
1184
|
+
// before we advance to the next coordinator turn. Emit a marker event so
|
|
1185
|
+
// streaming subscribers see "parent gave up after sub-run" provenance,
|
|
1186
|
+
// then re-throw the parent's abort reason. Non-streaming run() rejects with
|
|
1187
|
+
// the thrown error and does NOT preserve the marker — engine.ts does not
|
|
1188
|
+
// attach the parent events array to the rejected error (verified at
|
|
1189
|
+
// engine.ts:230-239). Streaming-subscriber observability is the contract.
|
|
1190
|
+
if (parentSignal?.aborted) {
|
|
1191
|
+
const abortMarker = {
|
|
1192
|
+
type: "sub-run-parent-aborted",
|
|
1193
|
+
runId: input.parentRunId,
|
|
1194
|
+
at: new Date().toISOString(),
|
|
1195
|
+
childRunId,
|
|
1196
|
+
parentRunId: input.parentRunId,
|
|
1197
|
+
reason: "parent-aborted"
|
|
1198
|
+
};
|
|
1199
|
+
parentEmit(abortMarker);
|
|
1200
|
+
input.recordProtocolDecision(abortMarker);
|
|
1201
|
+
throw enrichAbortErrorWithParentReason(createAbortErrorFromSignal(parentSignal, options.model.id), parentSignal);
|
|
1202
|
+
}
|
|
1203
|
+
// D-18 synthetic transcript entry.
|
|
1204
|
+
const decisionAsJson = {
|
|
1205
|
+
type: "delegate",
|
|
1206
|
+
protocol: decision.protocol,
|
|
1207
|
+
intent: decision.intent,
|
|
1208
|
+
...(decision.model !== undefined ? { model: decision.model } : {}),
|
|
1209
|
+
...(decision.budget !== undefined ? { budget: decision.budget } : {})
|
|
1210
|
+
};
|
|
1211
|
+
const taggedText = renderSubRunResult(childRunId, subResult);
|
|
1212
|
+
input.transcript.push({
|
|
1213
|
+
agentId: `sub-run:${childRunId}`,
|
|
1214
|
+
role: "delegate-result",
|
|
1215
|
+
input: JSON.stringify(decisionAsJson),
|
|
1216
|
+
output: taggedText
|
|
1217
|
+
});
|
|
1218
|
+
// Build the next coordinator prompt by appending the D-17 tagged block.
|
|
1219
|
+
const coordinatorAgent = options.agents[0];
|
|
1220
|
+
const baseInput = buildCoordinatorPlanInput(input.options.intent, coordinatorAgent ?? {
|
|
1221
|
+
id: "coordinator",
|
|
1222
|
+
role: "coordinator"
|
|
1223
|
+
});
|
|
1224
|
+
return {
|
|
1225
|
+
nextInput: `${baseInput}\n\n${taggedText}\n\nUsing the sub-run result above, decide the next step (participate or delegate).`,
|
|
1226
|
+
taggedText,
|
|
1227
|
+
completedAtMs: Date.now()
|
|
1228
|
+
};
|
|
1229
|
+
}
|
|
1230
|
+
/**
|
|
1231
|
+
* D-17 prompt-injection helper. Renders a child `RunResult` as the canonical
|
|
1232
|
+
* tagged-result block injected into the parent coordinator's next prompt.
|
|
1233
|
+
*
|
|
1234
|
+
* Format:
|
|
1235
|
+
* `[sub-run <childRunId>]: <output>`
|
|
1236
|
+
* `[sub-run <childRunId> stats]: turns=<N> costUsd=<X> durationMs=<Y>`
|
|
1237
|
+
*
|
|
1238
|
+
* The stats line is a soft contract — field names stable, ordering stable.
|
|
1239
|
+
*/
|
|
1240
|
+
function renderSubRunResult(childRunId, subResult) {
|
|
1241
|
+
const turns = subResult.transcript.length;
|
|
1242
|
+
const costUsd = subResult.cost.usd ?? 0;
|
|
1243
|
+
const startedAt = eventTimestamp(subResult.trace.events[0]);
|
|
1244
|
+
const endedAt = eventTimestamp(subResult.trace.events.at(-1));
|
|
1245
|
+
const durationMs = startedAt && endedAt
|
|
1246
|
+
? Math.max(0, Date.parse(endedAt) - Date.parse(startedAt))
|
|
1247
|
+
: 0;
|
|
1248
|
+
return [
|
|
1249
|
+
`[sub-run ${childRunId}]: ${subResult.output}`,
|
|
1250
|
+
`[sub-run ${childRunId} stats]: turns=${turns} costUsd=${costUsd} durationMs=${durationMs}`
|
|
1251
|
+
].join("\n");
|
|
1252
|
+
}
|
|
1253
|
+
function eventTimestamp(event) {
|
|
1254
|
+
if (event === undefined)
|
|
1255
|
+
return undefined;
|
|
1256
|
+
if ("at" in event)
|
|
1257
|
+
return event.at;
|
|
1258
|
+
return event.type === "model-response" ? event.completedAt : event.startedAt;
|
|
1259
|
+
}
|
|
1260
|
+
/**
|
|
1261
|
+
* Build a JSON-serializable {@link Trace} for `sub-run-failed.partialTrace`
|
|
1262
|
+
* from a buffered tee of child emits. Keeps `runProtocol`'s error contract
|
|
1263
|
+
* unchanged — Plan 03 step 8.
|
|
1264
|
+
*/
|
|
1265
|
+
function buildPartialTrace(input) {
|
|
1266
|
+
const protocolName = typeof input.protocol === "string" ? input.protocol : input.protocol.kind;
|
|
1267
|
+
const protocolConfig = typeof input.protocol === "string"
|
|
1268
|
+
? { kind: input.protocol }
|
|
1269
|
+
: input.protocol;
|
|
1270
|
+
return {
|
|
1271
|
+
schemaVersion: "1.0",
|
|
1272
|
+
runId: input.childRunId,
|
|
1273
|
+
protocol: protocolName,
|
|
1274
|
+
tier: input.tier,
|
|
1275
|
+
modelProviderId: input.modelProviderId,
|
|
1276
|
+
agentsUsed: input.agents,
|
|
1277
|
+
inputs: createReplayTraceRunInputs({
|
|
1278
|
+
intent: input.intent,
|
|
1279
|
+
protocol: protocolConfig,
|
|
1280
|
+
tier: input.tier,
|
|
1281
|
+
modelProviderId: input.modelProviderId,
|
|
1282
|
+
agents: input.agents,
|
|
1283
|
+
temperature: input.temperature
|
|
1284
|
+
}),
|
|
1285
|
+
budget: createReplayTraceBudget({
|
|
1286
|
+
tier: input.tier,
|
|
1287
|
+
...(input.childTimeoutMs !== undefined ? { caps: { timeoutMs: input.childTimeoutMs } } : {})
|
|
1288
|
+
}),
|
|
1289
|
+
budgetStateChanges: createReplayTraceBudgetStateChanges(input.events),
|
|
1290
|
+
seed: createReplayTraceSeed(input.seed),
|
|
1291
|
+
protocolDecisions: [],
|
|
1292
|
+
providerCalls: [],
|
|
1293
|
+
finalOutput: {
|
|
1294
|
+
kind: "replay-trace-final-output",
|
|
1295
|
+
output: "",
|
|
1296
|
+
cost: emptyCost(),
|
|
1297
|
+
completedAt: new Date().toISOString(),
|
|
1298
|
+
transcript: createTranscriptLink([])
|
|
1299
|
+
},
|
|
1300
|
+
events: input.events,
|
|
1301
|
+
transcript: []
|
|
1302
|
+
};
|
|
1303
|
+
}
|
|
1304
|
+
/**
|
|
1305
|
+
* BUDGET-01 / D-08: when a child sub-run threw because the parent's signal
|
|
1306
|
+
* aborted, lock the `detail.reason` discriminator on the resulting
|
|
1307
|
+
* `code: "aborted"` error. Preserves any pre-existing detail keys (e.g.,
|
|
1308
|
+
* `detail.status: "cancelled"` attached by `createStreamCancellationError`).
|
|
1309
|
+
*
|
|
1310
|
+
* No-op when:
|
|
1311
|
+
* - parent.signal is undefined or not aborted (child failure was unrelated)
|
|
1312
|
+
* - error is not a DogpileError with `code: "aborted"`
|
|
1313
|
+
* - error already has a `detail.reason` set (preserve upstream classification)
|
|
1314
|
+
*/
|
|
1315
|
+
function enrichAbortErrorWithParentReason(error, parentSignal) {
|
|
1316
|
+
if (parentSignal === undefined || !parentSignal.aborted) {
|
|
1317
|
+
return error;
|
|
1318
|
+
}
|
|
1319
|
+
if (!DogpileError.isInstance(error) || error.code !== "aborted") {
|
|
1320
|
+
return error;
|
|
1321
|
+
}
|
|
1322
|
+
const existingDetail = error.detail ?? {};
|
|
1323
|
+
if (existingDetail["reason"] !== undefined) {
|
|
1324
|
+
return error;
|
|
1325
|
+
}
|
|
1326
|
+
const reason = classifyAbortReason(parentSignal.reason);
|
|
1327
|
+
return new DogpileError({
|
|
1328
|
+
code: "aborted",
|
|
1329
|
+
message: error.message,
|
|
1330
|
+
retryable: error.retryable ?? false,
|
|
1331
|
+
...(error.providerId !== undefined ? { providerId: error.providerId } : {}),
|
|
1332
|
+
detail: { ...existingDetail, reason },
|
|
1333
|
+
...(error.cause !== undefined ? { cause: error.cause } : {})
|
|
1334
|
+
});
|
|
1335
|
+
}
|
|
1336
|
+
function enrichProviderTimeoutSource(error, context) {
|
|
1337
|
+
if (!DogpileError.isInstance(error) || error.code !== "provider-timeout") {
|
|
1338
|
+
return error;
|
|
1339
|
+
}
|
|
1340
|
+
const existingDetail = error.detail ?? {};
|
|
1341
|
+
if (existingDetail["source"] !== undefined) {
|
|
1342
|
+
return error;
|
|
1343
|
+
}
|
|
1344
|
+
const source = classifyChildTimeoutSource(error, {
|
|
1345
|
+
...context,
|
|
1346
|
+
isProviderError: true
|
|
1347
|
+
});
|
|
1348
|
+
return new DogpileError({
|
|
1349
|
+
code: "provider-timeout",
|
|
1350
|
+
message: error.message,
|
|
1351
|
+
retryable: error.retryable ?? true,
|
|
1352
|
+
...(error.providerId !== undefined ? { providerId: error.providerId } : {}),
|
|
1353
|
+
detail: { ...existingDetail, source },
|
|
1354
|
+
...(error.cause !== undefined ? { cause: error.cause } : {})
|
|
1355
|
+
});
|
|
1356
|
+
}
|
|
1357
|
+
function errorPayloadFromUnknown(error, failedDecision) {
|
|
1358
|
+
if (DogpileError.isInstance(error)) {
|
|
1359
|
+
const detail = {
|
|
1360
|
+
...(error.detail ?? {}),
|
|
1361
|
+
failedDecision
|
|
1362
|
+
};
|
|
1363
|
+
return {
|
|
1364
|
+
code: error.code,
|
|
1365
|
+
message: error.message,
|
|
1366
|
+
...(error.providerId !== undefined ? { providerId: error.providerId } : {}),
|
|
1367
|
+
detail
|
|
1368
|
+
};
|
|
1369
|
+
}
|
|
1370
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1371
|
+
return {
|
|
1372
|
+
code: "invalid-configuration",
|
|
1373
|
+
message,
|
|
1374
|
+
detail: { failedDecision }
|
|
1375
|
+
};
|
|
1376
|
+
}
|
|
459
1377
|
//# sourceMappingURL=coordinator.js.map
|