@dogpile/sdk 0.4.0 → 0.6.0

package/src/runtime/health.ts

@@ -0,0 +1,136 @@
+/**
+ * Health diagnostics computation for completed run traces.
+ *
+ * @module
+ */
+import type { HealthAnomaly, RunHealthSummary, Trace } from "../types.js";
+import type { TurnEvent } from "../types/events.js";
+
+// Re-export types so callers who import from this subpath get them directly.
+export type { HealthAnomaly, RunHealthSummary } from "../types.js";
+
+/**
+ * Thresholds for health anomaly detection.
+ *
+ * Both fields are optional. When absent, the corresponding threshold-gated
+ * anomaly is suppressed entirely. Threshold-free anomalies (`empty-contribution`)
+ * always fire when qualifying events are present regardless of this config.
+ *
+ * Note: `provider-error-recovered` is in the AnomalyCode union but is never
+ * emitted by computeHealth in Phase 7 - no trace signal exists without an
+ * event-shape change. See STATE.md: "Phase 6 is the only event-shape change."
+ */
+export interface HealthThresholds {
+  /**
+   * Per-agent turn count threshold. If an agent produces more than this many
+   * agent-turn events, a "runaway-turns" anomaly is emitted with severity "error".
+   * The threshold value in the anomaly record equals this number.
+   */
+  readonly runawayTurns?: number;
+  /**
+   * Budget utilization percentage threshold (0-100). If budget utilization
+   * (finalCost / maxUsd * 100) >= this value, a "budget-near-miss" anomaly is
+   * emitted with severity "warning". Suppressed when no USD cap is configured.
+   */
+  readonly budgetNearMissPct?: number;
+}
+
+/**
+ * Default health thresholds used for `result.health` auto-computation.
+ *
+ * Both threshold-gated anomalies (runaway-turns, budget-near-miss) are suppressed
+ * by default. Only threshold-free anomalies (empty-contribution) can fire on the
+ * auto-compute path.
+ */
+export const DEFAULT_HEALTH_THRESHOLDS: HealthThresholds = Object.freeze({});
+
+/**
+ * Compute a health summary from a completed run trace.
+ *
+ * Pure function - no side effects, no I/O, no storage access. Deterministic:
+ * given the same trace and thresholds, always produces the same result.
+ *
+ * @param trace - Completed run trace (from RunResult.trace or a stored trace).
+ * @param thresholds - Optional threshold overrides. Defaults to DEFAULT_HEALTH_THRESHOLDS.
+ */
+export function computeHealth(
+  trace: Trace,
+  thresholds: HealthThresholds = DEFAULT_HEALTH_THRESHOLDS
+): RunHealthSummary {
+  assertFiniteNonNegativeThreshold(thresholds.runawayTurns, "runawayTurns");
+  assertBudgetNearMissThreshold(thresholds.budgetNearMissPct);
+
+  const turnEvents = trace.events.filter((event): event is TurnEvent => event.type === "agent-turn");
+  const agentIds = new Set(turnEvents.map((event) => event.agentId));
+  const totalTurns = turnEvents.length;
+  const agentCount = agentIds.size;
+
+  const maxUsd = trace.budget.caps?.maxUsd;
+  const finalCost = trace.finalOutput.cost.usd;
+  const budgetUtilizationPct: number | null =
+    maxUsd !== undefined ? (maxUsd === 0 ? (finalCost === 0 ? 0 : 100) : (finalCost / maxUsd) * 100) : null;
+
+  const anomalies: HealthAnomaly[] = [];
+
+  if (thresholds.runawayTurns !== undefined) {
+    for (const agentId of agentIds) {
+      const count = turnEvents.filter((event) => event.agentId === agentId).length;
+      if (count > thresholds.runawayTurns) {
+        anomalies.push({
+          code: "runaway-turns",
+          severity: "error",
+          value: count,
+          threshold: thresholds.runawayTurns,
+          agentId
+        });
+      }
+    }
+  }
+
+  if (thresholds.budgetNearMissPct !== undefined && budgetUtilizationPct !== null) {
+    if (budgetUtilizationPct >= thresholds.budgetNearMissPct) {
+      anomalies.push({
+        code: "budget-near-miss",
+        severity: "warning",
+        value: budgetUtilizationPct,
+        threshold: thresholds.budgetNearMissPct
+      });
+    }
+  }
+
+  for (const event of turnEvents) {
+    if (event.output.trim() === "") {
+      anomalies.push({
+        code: "empty-contribution",
+        severity: "error",
+        value: 0,
+        threshold: 0,
+        agentId: event.agentId
+      });
+    }
+  }
+
+  // provider-error-recovered is deferred: no trace signal exists in Phase 7.
+  return {
+    anomalies,
+    stats: {
+      totalTurns,
+      agentCount,
+      budgetUtilizationPct
+    }
+  };
+}
+
+function assertFiniteNonNegativeThreshold(value: number | undefined, name: string): void {
+  if (value !== undefined && (!Number.isFinite(value) || value < 0)) {
+    throw new RangeError(`${name} must be a finite non-negative number`);
+  }
+}
+
+function assertBudgetNearMissThreshold(value: number | undefined): void {
+  assertFiniteNonNegativeThreshold(value, "budgetNearMissPct");
+
+  if (value !== undefined && value > 100) {
+    throw new RangeError("budgetNearMissPct must be between 0 and 100");
+  }
+}

package/src/runtime/introspection.ts

@@ -0,0 +1,122 @@
+/**
+ * Typed event query function for filtering completed trace events.
+ *
+ * @module
+ */
+import type {
+  BroadcastEvent,
+  BudgetStopEvent,
+  FinalEvent,
+  ModelOutputChunkEvent,
+  ModelRequestEvent,
+  ModelResponseEvent,
+  RoleAssignmentEvent,
+  RunEvent,
+  SubRunBudgetClampedEvent,
+  SubRunCompletedEvent,
+  SubRunConcurrencyClampedEvent,
+  SubRunFailedEvent,
+  SubRunParentAbortedEvent,
+  SubRunQueuedEvent,
+  SubRunStartedEvent,
+  ToolCallEvent,
+  ToolResultEvent,
+  TurnEvent
+} from "../types.js";
+
+/**
+ * Filter criteria for querying a completed trace event log.
+ *
+ * All fields are optional. AND semantics: all present fields must match.
+ * An empty filter object returns all events. An unmatched filter returns [].
+ *
+ * `costRange` matches only events with a `cost.usd` field: TurnEvent and
+ * BroadcastEvent. Events without a cost field are excluded from results when
+ * `costRange` is set (not returned as unmatched - silently excluded).
+ *
+ * `turnRange` uses the global 1-based position of agent-turn events across
+ * all agents. Position 1 is the first TurnEvent in the event array regardless
+ * of which agent produced it. BroadcastEvent.round is a separate concept and
+ * is not matched by turnRange.
+ */
+export interface EventQueryFilter {
+  /** Filter to events with this exact type discriminant. */
+  readonly type?: RunEvent["type"];
+  /** Filter to events where agentId === this value. Events without agentId are excluded. */
+  readonly agentId?: string;
+  /**
+   * Filter to agent-turn events at the specified global 1-based position range.
+   * Only TurnEvents are included in results when this filter is set.
+   */
+  readonly turnRange?: {
+    readonly min?: number;
+    readonly max?: number;
+  };
+  /**
+   * Filter to events where cost.usd is within [min, max].
+   * Only TurnEvent and BroadcastEvent have cost.usd - all other events are excluded.
+   */
+  readonly costRange?: {
+    readonly min?: number;
+    readonly max?: number;
+  };
+}
+
+// One overload per RunEvent discriminant (D-03: hand-written overloads, IDE-reliable)
+export function queryEvents(events: readonly RunEvent[], filter: EventQueryFilter & { type: "role-assignment" }): RoleAssignmentEvent[];
+export function queryEvents(events: readonly RunEvent[], filter: EventQueryFilter & { type: "model-request" }): ModelRequestEvent[];
+export function queryEvents(events: readonly RunEvent[], filter: EventQueryFilter & { type: "model-response" }): ModelResponseEvent[];
+export function queryEvents(events: readonly RunEvent[], filter: EventQueryFilter & { type: "model-output-chunk" }): ModelOutputChunkEvent[];
+export function queryEvents(events: readonly RunEvent[], filter: EventQueryFilter & { type: "tool-call" }): ToolCallEvent[];
+export function queryEvents(events: readonly RunEvent[], filter: EventQueryFilter & { type: "tool-result" }): ToolResultEvent[];
+export function queryEvents(events: readonly RunEvent[], filter: EventQueryFilter & { type: "agent-turn" }): TurnEvent[];
+export function queryEvents(events: readonly RunEvent[], filter: EventQueryFilter & { type: "broadcast" }): BroadcastEvent[];
+export function queryEvents(events: readonly RunEvent[], filter: EventQueryFilter & { type: "sub-run-started" }): SubRunStartedEvent[];
+export function queryEvents(events: readonly RunEvent[], filter: EventQueryFilter & { type: "sub-run-completed" }): SubRunCompletedEvent[];
+export function queryEvents(events: readonly RunEvent[], filter: EventQueryFilter & { type: "sub-run-failed" }): SubRunFailedEvent[];
+export function queryEvents(events: readonly RunEvent[], filter: EventQueryFilter & { type: "sub-run-parent-aborted" }): SubRunParentAbortedEvent[];
+export function queryEvents(events: readonly RunEvent[], filter: EventQueryFilter & { type: "sub-run-budget-clamped" }): SubRunBudgetClampedEvent[];
+export function queryEvents(events: readonly RunEvent[], filter: EventQueryFilter & { type: "sub-run-queued" }): SubRunQueuedEvent[];
+export function queryEvents(events: readonly RunEvent[], filter: EventQueryFilter & { type: "sub-run-concurrency-clamped" }): SubRunConcurrencyClampedEvent[];
+export function queryEvents(events: readonly RunEvent[], filter: EventQueryFilter & { type: "budget-stop" }): BudgetStopEvent[];
+export function queryEvents(events: readonly RunEvent[], filter: EventQueryFilter & { type: "final" }): FinalEvent[];
+// Fallback overload: no type constraint -> returns full RunEvent[]
+export function queryEvents(events: readonly RunEvent[], filter: EventQueryFilter): RunEvent[];
+// Implementation signature (not visible to callers):
+export function queryEvents(events: readonly RunEvent[], filter: EventQueryFilter): RunEvent[] {
+  let result: RunEvent[] = filter.type !== undefined
+    ? events.filter((event) => event.type === filter.type)
+    : [...events];
+
+  if (filter.agentId !== undefined) {
+    const { agentId } = filter;
+    result = result.filter((event) => "agentId" in event && (event as { agentId?: string }).agentId === agentId);
+  }
+
+  if (filter.turnRange !== undefined) {
+    const { min, max } = filter.turnRange;
+    const agentTurnEvents = events.filter((event): event is TurnEvent => event.type === "agent-turn");
+    const inRangeSet = new Set<RunEvent>(
+      agentTurnEvents.filter((_, index) => {
+        const position = index + 1;
+        return (min === undefined || position >= min) && (max === undefined || position <= max);
+      })
+    );
+
+    result = result.filter((event) => event.type === "agent-turn" && inRangeSet.has(event));
+  }
+
+  if (filter.costRange !== undefined) {
+    const { min, max } = filter.costRange;
+    result = result.filter((event) => {
+      if (event.type !== "agent-turn" && event.type !== "broadcast") {
+        return false;
+      }
+
+      const usd = event.cost.usd;
+      return (min === undefined || usd >= min) && (max === undefined || usd <= max);
+    });
+  }
+
+  return result;
+}

package/src/runtime/metrics.ts

@@ -0,0 +1,45 @@
+/**
+ * Metrics hook interface for run-completion counters (Phase 10 / METR-01..METR-02).
+ *
+ * The SDK does not import any metrics backend. Callers provide an object
+ * satisfying `MetricsHook` to receive named counters at run and sub-run
+ * completion. When absent, zero overhead — no allocations, no branch cost.
+ *
+ * `replay()` and `replayStream()` ignore `metricsHook` on engine options —
+ * counters for historical replays would be misleading.
+ */
+
+export interface RunMetricsSnapshot {
+  readonly outcome: "completed" | "budget-stopped" | "aborted";
+  /** Direct tokens for this run, excluding nested sub-runs. */
+  readonly inputTokens: number;
+  /** Direct tokens for this run, excluding nested sub-runs. */
+  readonly outputTokens: number;
+  /** Direct cost for this run, excluding nested sub-runs. */
+  readonly costUsd: number;
+  /** Total tokens including the full sub-run subtree (already rolled up). */
+  readonly totalInputTokens: number;
+  /** Total tokens including the full sub-run subtree. */
+  readonly totalOutputTokens: number;
+  /** Total cost including the full sub-run subtree. */
+  readonly totalCostUsd: number;
+  /** Count of agent-turn events directly in this run (own-only, not nested sub-runs). */
+  readonly turns: number;
+  /** Wall-clock duration in milliseconds from run start to terminal state. */
+  readonly durationMs: number;
+}
+
+export interface MetricsHook {
+  /**
+   * Called once at every terminal state of the top-level run (completed,
+   * budget-stopped, aborted). When the hook is async, the SDK attaches
+   * `.catch` and does NOT await — hook latency never delays run completion.
+   */
+  readonly onRunComplete?: (snapshot: RunMetricsSnapshot) => void | Promise<void>;
+  /**
+   * Called once for each coordinator-dispatched child run that completes.
+   * Fires from the parent run's emit closure on the `sub-run-completed` event.
+   * Does NOT fire for failed sub-runs (`sub-run-failed`).
+   */
+  readonly onSubRunComplete?: (snapshot: RunMetricsSnapshot) => void | Promise<void>;
+}

package/src/runtime/model.ts

@@ -24,18 +24,33 @@ type ModelUsage = NonNullable<ModelResponse["usage"]>;
 
 export async function generateModelTurn(options: GenerateModelTurnOptions): Promise<ModelResponse> {
   const startedAt = new Date().toISOString();
+  const modelId = options.model.modelId ?? options.model.id;
+  const traceRequest = requestForTrace(options.request);
   let response: ModelResponse;
 
   throwIfAborted(options.request.signal, options.model.id);
 
+  options.emit({
+    type: "model-request",
+    runId: options.runId,
+    callId: options.callId,
+    providerId: options.model.id,
+    modelId,
+    startedAt,
+    agentId: options.agent.id,
+    role: options.agent.role,
+    request: traceRequest
+  });
+
   if (!options.model.stream) {
     response = await options.model.generate(options.request);
     throwIfAborted(options.request.signal, options.model.id);
-    recordProviderCall(response, startedAt, options);
+    recordProviderCall(response, startedAt, modelId, traceRequest, options);
     return response;
   }
 
-  let text = "";
+  const chunks: string[] = [];
+  let outputLength = 0;
   let chunkIndex = 0;
   let usage: ModelUsage | undefined;
   let costUsd: number | undefined;
@@ -45,7 +60,8 @@ export async function generateModelTurn(options: GenerateModelTurnOptions): Prom
 
   for await (const chunk of options.model.stream(options.request)) {
     throwIfAborted(options.request.signal, options.model.id);
-    text += chunk.text;
+    chunks.push(chunk.text);
+    outputLength += chunk.text.length;
 
     options.emit({
       type: "model-output-chunk",
@@ -56,7 +72,7 @@ export async function generateModelTurn(options: GenerateModelTurnOptions): Prom
       input: options.input,
       chunkIndex,
       text: chunk.text,
-      output: text
+      outputLength
     });
     chunkIndex += 1;
 
@@ -77,6 +93,7 @@ export async function generateModelTurn(options: GenerateModelTurnOptions): Prom
     }
   }
 
+  const text = chunks.join("");
   response = {
     text,
     ...(finishReason !== undefined ? { finishReason } : {}),
@@ -86,32 +103,50 @@ export async function generateModelTurn(options: GenerateModelTurnOptions): Prom
     ...(metadata !== undefined ? { metadata } : {})
   };
   throwIfAborted(options.request.signal, options.model.id);
-  recordProviderCall(response, startedAt, options);
+  recordProviderCall(response, startedAt, modelId, traceRequest, options);
   return response;
 }
 
 function recordProviderCall(
   response: ModelResponse,
   startedAt: string,
+  modelId: string,
+  request: ModelRequest,
   options: GenerateModelTurnOptions
 ): void {
+  const completedAt = new Date().toISOString();
+
+  options.emit({
+    type: "model-response",
+    runId: options.runId,
+    callId: options.callId,
+    providerId: options.model.id,
+    modelId,
+    startedAt,
+    completedAt,
+    agentId: options.agent.id,
+    role: options.agent.role,
+    response
+  });
+
   options.onProviderCall?.({
     kind: "replay-trace-provider-call",
     callId: options.callId,
     providerId: options.model.id,
+    modelId,
     startedAt,
-    completedAt: new Date().toISOString(),
+    completedAt,
     agentId: options.agent.id,
     role: options.agent.role,
-    request: requestForTrace(options.request),
+    request,
     response
   });
 }
 
 function requestForTrace(request: ModelRequest): ModelRequest {
   return {
-    messages: request.messages,
+    messages: request.messages.map((message) => ({ ...message })),
     temperature: request.temperature,
-    metadata: request.metadata
+    metadata: JSON.parse(JSON.stringify(request.metadata)) as ModelRequest["metadata"]
   };
 }

package/src/runtime/provenance.ts

@@ -0,0 +1,43 @@
+import type { ModelRequestEvent, ModelResponseEvent } from "../types.js";
+
+/**
+ * Normalized provenance fields from a completed model-response event.
+ * All five fields are present and JSON-serializable.
+ */
+export interface ProvenanceRecord {
+  readonly modelId: string;
+  readonly providerId: string;
+  readonly callId: string;
+  readonly startedAt: string;
+  readonly completedAt: string;
+}
+
+/**
+ * Normalized provenance fields from a model-request event.
+ * completedAt is absent because the call has not completed at this point.
+ */
+export interface PartialProvenanceRecord {
+  readonly modelId: string;
+  readonly providerId: string;
+  readonly callId: string;
+  readonly startedAt: string;
+}
+
+export function getProvenance(event: ModelResponseEvent): ProvenanceRecord;
+export function getProvenance(event: ModelRequestEvent): PartialProvenanceRecord;
+export function getProvenance(
+  event: ModelRequestEvent | ModelResponseEvent
+): ProvenanceRecord | PartialProvenanceRecord {
+  const base: PartialProvenanceRecord = {
+    modelId: event.modelId,
+    providerId: event.providerId,
+    callId: event.callId,
+    startedAt: event.startedAt
+  };
+
+  if (event.type === "model-response") {
+    return { ...base, completedAt: event.completedAt };
+  }
+
+  return base;
+}