@dogpile/sdk 0.4.0 → 0.6.0

package/src/runtime/coordinator.ts

@@ -46,6 +46,7 @@ import {
   lastCostBearingEventCost,
   nextProviderCallId
 } from "./defaults.js";
+import { computeHealth, DEFAULT_HEALTH_THRESHOLDS } from "./health.js";
 import {
   classifyAbortReason,
   classifyChildTimeoutSource,
@@ -64,6 +65,11 @@ import { createWrapUpHintController } from "./wrap-up.js";
  * in by `engine.ts` so coordinator avoids a circular import.
  */
 export type RunProtocolFn = (input: {
+  /**
+   * Planned child run id emitted on sub-run lifecycle events before dispatch.
+   * The engine callback uses this to look up the matching sub-run span.
+   */
+  readonly runId: string;
   readonly intent: string;
   readonly protocol: ProtocolSelection;
   readonly tier: Tier;
@@ -81,6 +87,7 @@ export type RunProtocolFn = (input: {
   readonly currentDepth?: number;
   readonly effectiveMaxDepth?: number;
   readonly effectiveMaxConcurrentChildren?: number;
+  readonly effectiveMaxConcurrentAgentTurns?: number;
   readonly onChildFailure?: DogpileOptions["onChildFailure"];
   /**
    * Root-run deadline (epoch ms). Children inherit `parentDeadlineMs - now()`
@@ -120,11 +127,12 @@ interface CoordinatorRunOptions {
    */
   readonly currentDepth?: number;
   /**
-   * Effective max recursion depth resolved at run start. Plan 04 enforces;
-   * Plan 03 only plumbs the value.
+   * Effective max recursion depth resolved at run start and enforced before
+   * delegate dispatch.
    */
   readonly effectiveMaxDepth?: number;
   readonly effectiveMaxConcurrentChildren?: number;
+  readonly effectiveMaxConcurrentAgentTurns?: number;
   readonly onChildFailure?: DogpileOptions["onChildFailure"];
   /**
    * Engine `runProtocol` callback used by the delegate dispatch loop to
@@ -156,6 +164,7 @@ interface CoordinatorRunOptions {
  */
 const MAX_DISPATCH_PER_TURN = 8;
 const DEFAULT_MAX_CONCURRENT_CHILDREN = 4;
+const DEFAULT_MAX_CONCURRENT_AGENT_TURNS = 4;
 
 type DispatchWaveFailure = {
   readonly childRunId: string;
@@ -177,7 +186,8 @@ interface Semaphore {
 
 function createSemaphore(maxConcurrent: number): Semaphore {
   let inFlight = 0;
-  const waiters: Array<() => void> = [];
+  const waiters: Array<(() => void) | undefined> = [];
+  let waiterHead = 0;
   return {
     acquire(): Promise<void> {
       if (inFlight < maxConcurrent) {
@@ -193,8 +203,14 @@ function createSemaphore(maxConcurrent: number): Semaphore {
     },
     release(): void {
       inFlight -= 1;
-      const next = waiters.shift();
+      const next = waiters[waiterHead];
       if (next !== undefined) {
+        waiters[waiterHead] = undefined;
+        waiterHead += 1;
+        if (waiterHead > 32 && waiterHead * 2 > waiters.length) {
+          waiters.splice(0, waiterHead);
+          waiterHead = 0;
+        }
         next();
       }
     },
@@ -202,7 +218,7 @@ function createSemaphore(maxConcurrent: number): Semaphore {
       return inFlight;
     },
     get queued() {
-      return waiters.length;
+      return waiters.length - waiterHead;
     }
   };
 }
@@ -212,15 +228,16 @@ function createSemaphore(maxConcurrent: number): Semaphore {
  * whose metadata.locality === "local", or undefined if none found.
  *
  * Walk order (forward-compat): options.model first, then options.agents in
- * declaration order. AgentSpec has no `model` field today (Phase 3 D-11
- * forward-compat scaffolding); the agent walk uses optional chaining and
- * effectively no-ops until a future phase adds AgentSpec.model.
+ * declaration order. AgentSpec has no `model` field today; the agent walk
+ * uses optional chaining and effectively no-ops until a future caller-defined
+ * tree surface adds AgentSpec.model.
  */
 function findFirstLocalProvider(options: CoordinatorRunOptions): ConfiguredModelProvider | undefined {
   if (options.model.metadata?.locality === "local") {
     return options.model;
   }
-  // Forward-compat: AgentSpec.model not yet declared (Phase 3 D-11). Walk no-ops today; ready for caller-defined trees in a future milestone.
+  // Forward-compat: AgentSpec.model is not yet declared; this no-ops today and
+  // is ready for a future caller-defined tree surface.
   for (const agent of options.agents) {
     const agentModel = (agent as { readonly model?: ConfiguredModelProvider }).model;
     if (agentModel?.metadata?.locality === "local") {
@@ -357,13 +374,12 @@ export async function runCoordinator(options: CoordinatorRunOptions): Promise<Ru
 
   if (coordinator) {
     if (!stopIfNeeded()) {
-      // Delegate dispatch loop (D-11/D-16/D-17/D-18). Phase 1 limits delegation
-      // to the coordinator's plan turn; workers cannot delegate. The loop
-      // re-issues the coordinator plan turn after each successful sub-run with
-      // the projected D-17 result tagged into the next prompt and a synthetic
-      // D-18 transcript entry already appended. `partialTrace` for failed
-      // sub-runs is captured via a tee'd emit buffer locally — `runProtocol`'s
-      // error contract is unchanged.
+      // Delegate dispatch is restricted to the coordinator's plan turn.
+      // Workers and final synthesis cannot delegate. The loop re-issues the
+      // coordinator plan turn after each successful sub-run with the tagged
+      // child result in the next prompt and a synthetic transcript entry
+      // already appended. Failed children use the local tee buffer for
+      // partialTrace capture; runProtocol's error contract stays unchanged.
       let dispatchInput = buildCoordinatorPlanInput(options.intent, coordinator);
       let dispatchCount = 0;
       while (true) {
@@ -638,29 +654,39 @@ export async function runCoordinator(options: CoordinatorRunOptions): Promise<Ru
       const workers = activeAgents.slice(1);
       const providerCallSlots: ReplayTraceProviderCall[] = [];
       const planTranscript = [...transcript];
-      const workerResults = await Promise.all(
-        workers.map((agent, index) =>
-          runCoordinatorWorkerTurn({
-            agent,
-            coordinator,
-            input: buildWorkerInput(options.intent, planTranscript, coordinator),
-            options,
-            runId,
-            turn: transcript.length + index + 1,
-            providerCallId: providerCallIdFor(runId, providerCalls.length + index + 1),
-            providerCallIndex: index,
-            providerCallSlots,
-            toolExecutor,
-            toolAvailability,
-            totalCost,
-            events,
-            transcript: planTranscript,
-            startedAtMs,
-            wrapUpHint,
-            emit
-          })
-        )
-      );
+      const fanout = createFanoutAbortController(options.signal);
+      const workerResults = await (async () => {
+        try {
+          return await mapWithConcurrency(
+            workers,
+            options.effectiveMaxConcurrentAgentTurns ?? DEFAULT_MAX_CONCURRENT_AGENT_TURNS,
+            fanout,
+            (agent, index) =>
+              runCoordinatorWorkerTurn({
+                agent,
+                coordinator,
+                input: buildWorkerInput(options.intent, planTranscript, coordinator),
+                options,
+                runId,
+                turn: transcript.length + index + 1,
+                providerCallId: providerCallIdFor(runId, providerCalls.length + index + 1),
+                providerCallIndex: index,
+                providerCallSlots,
+                toolExecutor,
+                toolAvailability,
+                totalCost,
+                events,
+                transcript: planTranscript,
+                startedAtMs,
+                wrapUpHint,
+                emit,
+                fanoutSignal: fanout.signal
+              })
+          );
+        } finally {
+          fanout.cleanup();
+        }
+      })();
       providerCalls.push(...providerCallSlots.filter((call): call is ReplayTraceProviderCall => call !== undefined));
 
       for (const result of workerResults) {
@@ -715,11 +741,11 @@ export async function runCoordinator(options: CoordinatorRunOptions): Promise<Ru
         recordProtocolDecision
       });
       totalCost = synthesisOutcome.totalCost;
-      // Phase 1: final-synthesis turn cannot delegate.
+      // Final synthesis is terminal and cannot dispatch children.
       if (Array.isArray(synthesisOutcome.decision) || synthesisOutcome.decision?.type === "delegate") {
         throw new DogpileError({
           code: "invalid-configuration",
-          message: "Coordinator final-synthesis turn cannot emit a delegate decision in Phase 1",
+          message: "Coordinator final-synthesis turn cannot emit a delegate decision.",
           retryable: false,
           detail: {
             kind: "delegate-validation",
@@ -748,46 +774,47 @@ export async function runCoordinator(options: CoordinatorRunOptions): Promise<Ru
     transcriptEntryCount: transcript.length
   });
   const finalEvent = events.at(-1);
+  const trace: Trace = {
+    schemaVersion: "1.0",
+    runId,
+    protocol: "coordinator",
+    tier: options.tier,
+    modelProviderId: options.model.id,
+    agentsUsed: activeAgents,
+    inputs: createReplayTraceRunInputs({
+      intent: options.intent,
+      protocol: options.protocol,
+      tier: options.tier,
+      modelProviderId: options.model.id,
+      agents: activeAgents,
+      temperature: options.temperature
+    }),
+    budget: createReplayTraceBudget({
+      tier: options.tier,
+      ...(options.budget ? { caps: options.budget } : {}),
+      ...(options.terminate ? { termination: options.terminate } : {})
+    }),
+    budgetStateChanges: createReplayTraceBudgetStateChanges(events),
+    seed: createReplayTraceSeed(options.seed),
+    protocolDecisions,
+    providerCalls,
+    finalOutput: createReplayTraceFinalOutput(output, finalEvent ?? {
+      type: "final",
+      runId,
+      at: "",
+      output,
+      cost: totalCost,
+      transcript: createTranscriptLink(transcript)
+    }),
+    ...(triggeringFailureForAbortMode !== undefined ? { triggeringFailureForAbortMode } : {}),
+    events,
+    transcript
+  };
 
   return {
     output,
     eventLog: createRunEventLog(runId, "coordinator", events),
-    trace: {
-      schemaVersion: "1.0",
-      runId,
-      protocol: "coordinator",
-      tier: options.tier,
-      modelProviderId: options.model.id,
-      agentsUsed: activeAgents,
-      inputs: createReplayTraceRunInputs({
-        intent: options.intent,
-        protocol: options.protocol,
-        tier: options.tier,
-        modelProviderId: options.model.id,
-        agents: activeAgents,
-        temperature: options.temperature
-      }),
-      budget: createReplayTraceBudget({
-        tier: options.tier,
-        ...(options.budget ? { caps: options.budget } : {}),
-        ...(options.terminate ? { termination: options.terminate } : {})
-      }),
-      budgetStateChanges: createReplayTraceBudgetStateChanges(events),
-      seed: createReplayTraceSeed(options.seed),
-      protocolDecisions,
-      providerCalls,
-      finalOutput: createReplayTraceFinalOutput(output, finalEvent ?? {
-        type: "final",
-        runId,
-        at: "",
-        output,
-        cost: totalCost,
-        transcript: createTranscriptLink(transcript)
-      }),
-      ...(triggeringFailureForAbortMode !== undefined ? { triggeringFailureForAbortMode } : {}),
-      events,
-      transcript
-    },
+    trace,
     transcript,
     usage: createRunUsage(totalCost),
     metadata: createRunMetadata({
@@ -805,7 +832,8 @@ export async function runCoordinator(options: CoordinatorRunOptions): Promise<Ru
       cost: totalCost,
       events
     }),
-    cost: totalCost
+    cost: totalCost,
+    health: computeHealth(trace, DEFAULT_HEALTH_THRESHOLDS)
   };
 
   function stopIfNeeded(): boolean {
@@ -1008,6 +1036,7 @@ interface CoordinatorWorkerTurnOptions {
   readonly startedAtMs: number;
   readonly wrapUpHint: ReturnType<typeof createWrapUpHintController>;
   readonly emit: (event: RunEvent) => void;
+  readonly fanoutSignal?: AbortSignal;
 }
 
 interface CoordinatorWorkerTurnResult {
@@ -1021,10 +1050,11 @@ interface CoordinatorWorkerTurnResult {
 
 async function runCoordinatorWorkerTurn(turn: CoordinatorWorkerTurnOptions): Promise<CoordinatorWorkerTurnResult> {
   throwIfAborted(turn.options.signal, turn.options.model.id);
+  throwIfAborted(turn.fanoutSignal, turn.options.model.id);
 
   const request: ModelRequest = {
     temperature: turn.options.temperature,
-    ...(turn.options.signal !== undefined ? { signal: turn.options.signal } : {}),
+    ...(turn.fanoutSignal !== undefined ? { signal: turn.fanoutSignal } : turn.options.signal !== undefined ? { signal: turn.options.signal } : {}),
     metadata: {
       runId: turn.runId,
       protocol: "coordinator",
@@ -1077,7 +1107,7 @@ async function runCoordinatorWorkerTurn(turn: CoordinatorWorkerTurnOptions): Pro
   if (Array.isArray(decision) || decision?.type === "delegate") {
     throw new DogpileError({
       code: "invalid-configuration",
-      message: "Workers cannot emit delegate decisions in Phase 1",
+      message: "Workers cannot emit delegate decisions.",
       retryable: false,
       detail: {
         kind: "delegate-validation",
@@ -1097,6 +1127,7 @@ async function runCoordinatorWorkerTurn(turn: CoordinatorWorkerTurnOptions): Pro
     }
   });
   throwIfAborted(turn.options.signal, turn.options.model.id);
+  throwIfAborted(turn.fanoutSignal, turn.options.model.id);
 
   return {
     agent: turn.agent,
@@ -1185,6 +1216,76 @@ function responseCost(response: ModelResponse): CostSummary {
   };
 }
 
+interface FanoutAbortController {
+  readonly signal: AbortSignal;
+  abort(reason: unknown): void;
+  cleanup(): void;
+}
+
+function createFanoutAbortController(parentSignal: AbortSignal | undefined): FanoutAbortController {
+  const controller = new AbortController();
+  let removeParentListener = (): void => {};
+
+  if (parentSignal?.aborted) {
+    controller.abort(parentSignal.reason);
+  } else if (parentSignal !== undefined) {
+    const abortFromParent = (): void => {
+      controller.abort(parentSignal.reason);
+    };
+    parentSignal.addEventListener("abort", abortFromParent, { once: true });
+    removeParentListener = (): void => {
+      parentSignal.removeEventListener("abort", abortFromParent);
+    };
+  }
+
+  return {
+    signal: controller.signal,
+    abort(reason: unknown): void {
+      if (!controller.signal.aborted) {
+        controller.abort(reason);
+      }
+    },
+    cleanup(): void {
+      removeParentListener();
+    }
+  };
+}
+
+async function mapWithConcurrency<T, R>(
+  items: readonly T[],
+  maxConcurrent: number,
+  fanout: FanoutAbortController,
+  mapper: (item: T, index: number) => Promise<R>
+): Promise<R[]> {
+  if (items.length === 0) {
+    return [];
+  }
+
+  const results: R[] = new Array(items.length);
+  let nextIndex = 0;
+  let firstError: unknown;
+  const workerCount = Math.min(maxConcurrent, items.length);
+
+  await Promise.all(Array.from({ length: workerCount }, async () => {
+    while (nextIndex < items.length && firstError === undefined) {
+      const index = nextIndex;
+      nextIndex += 1;
+      try {
+        results[index] = await mapper(items[index]!, index);
+      } catch (error) {
+        firstError ??= error;
+        fanout.abort(error);
+      }
+    }
+  }));
+
+  if (firstError !== undefined) {
+    throw firstError;
+  }
+
+  return results;
+}
+
 interface DispatchDelegateOptions {
   readonly decision: DelegateAgentDecision;
   readonly childRunId?: string;
@@ -1230,7 +1331,7 @@ interface DispatchedChild {
   startedAtMs: number;
   childTimeoutMs: number | undefined;
   failure: DispatchWaveFailure | undefined;
-  /** STREAM-03 hook (Phase 4). Reserved; do not use. */
+  /** Reserved child stream handle slot; do not use. */
   readonly streamHandle?: never;
 }
 
@@ -1316,7 +1417,7 @@ async function dispatchDelegate(input: DispatchDelegateOptions): Promise<Dispatc
     });
   }
 
-  // Buffered tee for partialTrace capture — see Plan 03 step 8.
+  // Buffered tee captures the child event prefix used for partialTrace.
   const childEvents = input.dispatchedChild.childEvents;
   const parentEmit = input.emit;
   const teedEmit = (event: RunEvent): void => {
@@ -1373,7 +1474,7 @@ async function dispatchDelegate(input: DispatchDelegateOptions): Promise<Dispatc
   // BUDGET-01 / D-07: derive a per-child AbortController so child engines see
   // their own signal. Listener forwards parent.signal.reason verbatim, so
   // detail.reason classification (parent-aborted vs timeout) is preserved.
-  // Phase 4 STREAM-03 hook: per-child cancel handle attaches here.
+  // Reserved per-child stream cancel hook attaches here if that surface ships.
   const parentSignal = options.signal;
   let removeParentAbortListener: (() => void) | undefined;
   if (parentSignal !== undefined) {
@@ -1404,6 +1505,7 @@ async function dispatchDelegate(input: DispatchDelegateOptions): Promise<Dispatc
       : undefined;
 
   const childOptions = {
+    runId: childRunId,
     intent: decision.intent,
     protocol: decision.protocol,
     tier: options.tier,
@@ -1420,6 +1522,9 @@ async function dispatchDelegate(input: DispatchDelegateOptions): Promise<Dispatc
     ...(options.effectiveMaxConcurrentChildren !== undefined
       ? { effectiveMaxConcurrentChildren: options.effectiveMaxConcurrentChildren }
       : {}),
+    ...(options.effectiveMaxConcurrentAgentTurns !== undefined
+      ? { effectiveMaxConcurrentAgentTurns: options.effectiveMaxConcurrentAgentTurns }
+      : {}),
     ...(options.onChildFailure !== undefined ? { onChildFailure: options.onChildFailure } : {}),
     // BUDGET-02 / D-12: forward the ROOT deadline so depth-N grandchildren
     // see the same `parentDeadlineMs` rather than a fresh per-level snapshot.
@@ -1614,8 +1719,8 @@ async function dispatchDelegate(input: DispatchDelegateOptions): Promise<Dispatc
 function renderSubRunResult(childRunId: string, subResult: RunResult): string {
   const turns = subResult.transcript.length;
   const costUsd = subResult.cost.usd ?? 0;
-  const startedAt = subResult.trace.events[0]?.at;
-  const endedAt = subResult.trace.events.at(-1)?.at;
+  const startedAt = eventTimestamp(subResult.trace.events[0]);
+  const endedAt = eventTimestamp(subResult.trace.events.at(-1));
   const durationMs =
     startedAt && endedAt
       ? Math.max(0, Date.parse(endedAt) - Date.parse(startedAt))
@@ -1626,10 +1731,16 @@ function renderSubRunResult(childRunId: string, subResult: RunResult): string {
   ].join("\n");
 }
 
+function eventTimestamp(event: RunEvent | undefined): string | undefined {
+  if (event === undefined) return undefined;
+  if ("at" in event) return event.at;
+  return event.type === "model-response" ? event.completedAt : event.startedAt;
+}
+
 /**
  * Build a JSON-serializable {@link Trace} for `sub-run-failed.partialTrace`
  * from a buffered tee of child emits. Keeps `runProtocol`'s error contract
- * unchanged — Plan 03 step 8.
+ * unchanged.
  */
 function buildPartialTrace(input: {
   readonly childRunId: string;

package/src/runtime/defaults.ts

@@ -236,8 +236,8 @@ export function createRunMetadata(options: {
     tier: options.tier,
     modelProviderId: options.modelProviderId,
     agentsUsed: options.agentsUsed,
-    startedAt: firstEvent?.at ?? "",
-    completedAt: lastEvent?.at ?? ""
+    startedAt: eventTimestamp(firstEvent) ?? "",
+    completedAt: eventTimestamp(lastEvent) ?? ""
   };
 }
 
@@ -365,7 +365,7 @@ export function createReplayTraceProtocolDecision(
     eventType: event.type,
     protocol,
     decision: options.decision ?? defaultProtocolDecision(event),
-    at: event.at,
+    at: eventTimestamp(event),
     ...(options.turn !== undefined ? { turn: options.turn } : {}),
     ...(options.phase !== undefined ? { phase: options.phase } : {}),
     ...(options.round !== undefined ? { round: options.round } : {}),
@@ -404,7 +404,8 @@ export function createReplayTraceProtocolDecision(
         agentId: event.agentId,
         role: event.role,
         input: event.input,
-        output: event.output
+        output: event.text,
+        outputLength: event.outputLength
       };
     case "tool-call":
       return {
@@ -550,7 +551,7 @@ export function createReplayTraceFinalOutput(output: string, event: RunEvent): R
     kind: "replay-trace-final-output",
     output,
     cost: emptyCost(),
-    completedAt: event.at,
+    completedAt: eventTimestamp(event),
     transcript: {
       kind: "trace-transcript",
       entryCount: 0,
@@ -559,6 +560,14 @@ export function createReplayTraceFinalOutput(output: string, event: RunEvent): R
   };
 }
 
+function eventTimestamp(event: RunEvent): string;
+function eventTimestamp(event: RunEvent | undefined): string | undefined;
+function eventTimestamp(event: RunEvent | undefined): string | undefined {
+  if (event === undefined) return undefined;
+  if ("at" in event) return event.at;
+  return event.type === "model-response" ? event.completedAt : event.startedAt;
+}
+
 export function nextProviderCallId(
   runId: string,
   providerCalls: readonly ReplayTraceProviderCall[]
@@ -589,6 +598,7 @@ export function canonicalizeRunResult(result: RunResult): RunResult {
     cost: canonicalizeSerializable(result.cost),
     ...(result.evaluation !== undefined ? { evaluation: canonicalizeSerializable(result.evaluation) } : {}),
     eventLog,
+    health: canonicalizeSerializable(result.health),
     metadata: canonicalizeSerializable(result.metadata),
     output: result.output,
     ...(result.quality !== undefined ? { quality: canonicalizeSerializable(result.quality) } : {}),