@ai-sdk/harness 0.0.0 → 1.0.0-beta.15

package/src/agent/internal/turn-telemetry.ts

@@ -0,0 +1,359 @@
+import { generateId, type ModelMessage } from '@ai-sdk/provider-utils';
+import { createTelemetryDispatcher } from 'ai/internal';
+import type { TelemetryOptions } from 'ai';
+
+/*
+ * Drives AI SDK's pluggable `Telemetry` lifecycle from a harness turn.
+ *
+ * A harness turn is not a `streamText` call — it has no language model, prompt
+ * standardization, or sampling settings — but the AI SDK telemetry contract is
+ * shaped around `generateText`/`streamText` events, and `@ai-sdk/otel` (the
+ * main integration) only produces spans when the full lifecycle fires. So we
+ * map the turn onto that contract: turn = operation, each `finish-step` = a
+ * step boundary, tool-calls = tool executions, `finish` = operation end. The
+ * model-call-only event fields the harness has no value for (sampling params,
+ * standardized prompt) are left `undefined` / cast; the fields the integrations
+ * actually read (`callId`, `operationId`, `provider`, `modelId`, `messages`,
+ * `toolCall`, `usage`, `finishReason`) carry real values.
+ *
+ * Telemetry is opt-in: the framework only drives it when `settings.telemetry`
+ * is set (the dispatcher then also honours globally-registered integrations).
+ */
+
+type Dispatcher = ReturnType<typeof createTelemetryDispatcher>;
+type EventArg<K extends keyof Dispatcher> = Dispatcher[K] extends
+  | ((event: infer E) => unknown)
+  | undefined
+  ? E
+  : never;
+
+/**
+ * An output content part accumulated over a step — the model's assistant turn.
+ * Shaped for the gen_ai output-message conventions `@ai-sdk/otel` reads.
+ */
+export type TurnContentPart =
+  | { type: 'text'; text: string }
+  | { type: 'reasoning'; text: string }
+  | { type: 'tool-call'; toolCallId: string; toolName: string; input: unknown };
+
+export interface TurnTelemetry {
+  /**
+   * Begin the operation span. Called on `stream-start`, optionally with the
+   * model the runtime resolved to (overriding the session's configured id).
+   * Idempotent — the first call wins.
+   */
+  start(modelId?: string): void;
+  /** Open a step span lazily, before the first content of a step. */
+  ensureStepOpen(): void;
+  /** Close the current step (on a harness `finish-step`). */
+  stepFinish(info: {
+    finishReason: unknown;
+    usage: unknown;
+    providerMetadata?: unknown;
+    /** The model's output content for this step (text/reasoning/tool-calls). */
+    content?: TurnContentPart[];
+  }): void;
+  /** A tool execution began (on a `tool-call`). */
+  toolStart(call: {
+    toolCallId: string;
+    toolName: string;
+    input: unknown;
+  }): void;
+  /**
+   * A tool execution completed (on its `tool-result` or after host execution).
+   * Idempotent per `toolCallId` — the first caller wins, so provider-executed
+   * and host-executed paths can both call it without double-counting.
+   */
+  toolEnd(
+    toolCallId: string,
+    output: { ok: true; output: unknown } | { ok: false; error: unknown },
+  ): void;
+  /** The turn ended (on a harness `finish`). */
+  end(info: { finishReason: unknown; usage: unknown }): void;
+  /** The turn failed. */
+  error(err: unknown): void;
+}
+
+const NOOP: TurnTelemetry = {
+  start() {},
+  ensureStepOpen() {},
+  stepFinish() {},
+  toolStart() {},
+  toolEnd() {},
+  end() {},
+  error() {},
+};
+
+export function createTurnTelemetry(opts: {
+  telemetry: TelemetryOptions | undefined;
+  harnessId: string;
+  modelId: string | undefined;
+  instructions: string | undefined;
+  promptText: string;
+  runtimeContext: unknown;
+}): TurnTelemetry {
+  // Opt-in: with no telemetry settings we do no work and construct no events.
+  if (opts.telemetry == null) return NOOP;
+
+  const dispatcher = createTelemetryDispatcher({ telemetry: opts.telemetry });
+
+  const callId = generateId();
+  const provider = opts.harnessId;
+  // The configured session model; `start(modelId)` may override it with the
+  // model the runtime actually resolved to.
+  let modelId = opts.modelId ?? '';
+  const runtimeContext = opts.runtimeContext;
+  const inputMessages: ModelMessage[] = [
+    { role: 'user', content: opts.promptText },
+  ];
+
+  let started = false;
+  let stepOpen = false;
+  let stepNumber = 0;
+  let ended = false;
+  /** Tool calls started in the current turn and not yet ended. */
+  const openTools = new Map<
+    string,
+    { toolCallId: string; toolName: string; input: unknown }
+  >();
+
+  const cast = <K extends keyof Dispatcher>(event: unknown): EventArg<K> =>
+    event as EventArg<K>;
+
+  // onStart — open the operation (root) span. Deferred until `start()` so the
+  // runtime-resolved model can be attached to the operation span + trace label.
+  const fireStart = (): void => {
+    if (started) return;
+    started = true;
+    dispatcher.onStart?.(
+      cast<'onStart'>({
+        callId,
+        operationId: 'ai.harness',
+        provider,
+        modelId,
+        tools: undefined,
+        toolChoice: undefined,
+        activeTools: undefined,
+        maxRetries: 0,
+        timeout: undefined,
+        headers: undefined,
+        providerOptions: undefined,
+        output: undefined,
+        toolsContext: undefined,
+        runtimeContext,
+        instructions: opts.instructions,
+        messages: inputMessages,
+      }),
+    );
+  };
+
+  const start = (overrideModelId?: string): void => {
+    if (started) return;
+    if (overrideModelId) modelId = overrideModelId;
+    fireStart();
+  };
+
+  const ensureStepOpen = (): void => {
+    if (!started) fireStart();
+    if (stepOpen || ended) return;
+    stepOpen = true;
+    dispatcher.onStepStart?.(
+      cast<'onStepStart'>({
+        callId,
+        provider,
+        modelId,
+        stepNumber,
+        tools: undefined,
+        toolChoice: undefined,
+        activeTools: undefined,
+        steps: new Array(stepNumber),
+        providerOptions: undefined,
+        output: undefined,
+        runtimeContext,
+        messages: inputMessages,
+      }),
+    );
+    // Open the inference (language-model call) span — the gen_ai home for the
+    // step's input and (on end) output messages.
+    dispatcher.onLanguageModelCallStart?.(
+      cast<'onLanguageModelCallStart'>({
+        callId,
+        provider,
+        modelId,
+        messages: inputMessages,
+        tools: undefined,
+      }),
+    );
+  };
+
+  /** Close the inference span with the step's output content. */
+  const inferenceEnd = (info: {
+    finishReason: unknown;
+    usage: unknown;
+    content: TurnContentPart[];
+  }): void => {
+    dispatcher.onLanguageModelCallEnd?.(
+      cast<'onLanguageModelCallEnd'>({
+        callId,
+        finishReason: info.finishReason,
+        responseId: callId,
+        usage: info.usage,
+        content: info.content,
+      }),
+    );
+  };
+
+  const closeOpenTools = (): void => {
+    for (const call of openTools.values()) {
+      dispatcher.onToolExecutionEnd?.(
+        cast<'onToolExecutionEnd'>({
+          callId,
+          toolExecutionMs: 0,
+          messages: [],
+          toolCall: {
+            type: 'tool-call',
+            toolCallId: call.toolCallId,
+            toolName: call.toolName,
+            input: call.input,
+            dynamic: true,
+          },
+          toolContext: undefined,
+          toolOutput: { type: 'error', error: new Error('tool span unclosed') },
+        }),
+      );
+    }
+    openTools.clear();
+  };
+
+  return {
+    start,
+    ensureStepOpen,
+
+    stepFinish(info) {
+      if (!stepOpen) return;
+      const content = info.content ?? [];
+      closeOpenTools();
+      inferenceEnd({
+        finishReason: info.finishReason,
+        usage: info.usage,
+        content,
+      });
+      dispatcher.onStepEnd?.(
+        cast<'onStepEnd'>({
+          callId,
+          finishReason: info.finishReason,
+          usage: info.usage,
+          providerMetadata: info.providerMetadata,
+          content,
+          response: {
+            id: callId,
+            modelId,
+            timestamp: new Date(0),
+            messages: [],
+          },
+        }),
+      );
+      stepOpen = false;
+      stepNumber += 1;
+    },
+
+    toolStart(call) {
+      ensureStepOpen();
+      openTools.set(call.toolCallId, call);
+      dispatcher.onToolExecutionStart?.(
+        cast<'onToolExecutionStart'>({
+          callId,
+          messages: [],
+          toolCall: {
+            type: 'tool-call',
+            toolCallId: call.toolCallId,
+            toolName: call.toolName,
+            input: call.input,
+            dynamic: true,
+          },
+          toolContext: undefined,
+        }),
+      );
+    },
+
+    toolEnd(toolCallId, output) {
+      const call = openTools.get(toolCallId);
+      if (call == null) return;
+      openTools.delete(toolCallId);
+      dispatcher.onToolExecutionEnd?.(
+        cast<'onToolExecutionEnd'>({
+          callId,
+          toolExecutionMs: 0,
+          messages: [],
+          toolCall: {
+            type: 'tool-call',
+            toolCallId: call.toolCallId,
+            toolName: call.toolName,
+            input: call.input,
+            dynamic: true,
+          },
+          toolContext: undefined,
+          toolOutput: output.ok
+            ? { type: 'tool-result', output: output.output }
+            : { type: 'error', error: output.error },
+        }),
+      );
+    },
+
+    end(info) {
+      if (ended) return;
+      if (!started) fireStart();
+      if (stepOpen) {
+        closeOpenTools();
+        inferenceEnd({
+          finishReason: info.finishReason,
+          usage: info.usage,
+          content: [],
+        });
+        dispatcher.onStepEnd?.(
+          cast<'onStepEnd'>({
+            callId,
+            finishReason: info.finishReason,
+            usage: info.usage,
+            providerMetadata: undefined,
+            content: [],
+            response: {
+              id: callId,
+              modelId,
+              timestamp: new Date(0),
+              messages: [],
+            },
+          }),
+        );
+        stepOpen = false;
+      }
+      ended = true;
+      dispatcher.onEnd?.(
+        cast<'onEnd'>({
+          callId,
+          operationId: 'ai.harness',
+          finishReason: info.finishReason,
+          usage: info.usage,
+          totalUsage: info.usage,
+          content: [],
+          steps: new Array(stepNumber),
+          response: {
+            id: callId,
+            modelId,
+            timestamp: new Date(0),
+            messages: [],
+          },
+          runtimeContext,
+        }),
+      );
+    },
+
+    error(err) {
+      if (ended) return;
+      if (!started) fireStart();
+      closeOpenTools();
+      ended = true;
+      dispatcher.onError?.(err);
+    },
+  };
+}

package/src/agent/observability/file-reporter.ts

@@ -0,0 +1,206 @@
+import { appendFileSync, mkdirSync } from 'node:fs';
+import type { Telemetry } from 'ai';
+import type { HarnessDiagnostic, HarnessDiagnosticConsumer } from './types';
+
+/**
+ * A harness observability reporter that writes a unified, non-lossy
+ * `events.jsonl` containing **both** the telemetry span lifecycle (turn / step
+ * / tool) **and** the forwarded bridge diagnostics (console lines + structured
+ * events). It is a single object registered in `telemetry.integrations`: the
+ * framework drives its `Telemetry` methods for spans and calls
+ * `ingestDiagnostic` for diagnostics.
+ *
+ * No external collector or OTel setup required — this is the AI-SDK-idiomatic
+ * replacement for the original SDK's host-side artifact files.
+ */
+export interface FileReporterOptions {
+  /** Directory for `events.jsonl` (created if absent). */
+  dir: string;
+  /**
+   * Buffer a turn's records in memory and write them only if the turn produced
+   * an error (an `error`-level diagnostic, a failed tool, or an error finish).
+   * Default false (write everything).
+   */
+  failOnly?: boolean;
+  /** File name within `dir`. Default `events.jsonl`. */
+  fileName?: string;
+}
+
+type Record_ = { ts: number } & Record<string, unknown>;
+
+export type FileReporter = Telemetry & HarnessDiagnosticConsumer;
+
+export function createFileReporter(options: FileReporterOptions): FileReporter {
+  const fileName = options.fileName ?? 'events.jsonl';
+  const path = `${options.dir}/${fileName}`;
+  const failOnly = options.failOnly ?? false;
+
+  // Per-turn buffers, keyed by the telemetry callId. Diagnostics (which carry a
+  // sessionId, not a callId) attach to the most recently started, open turn.
+  const turns = new Map<string, { lines: Record_[]; errored: boolean }>();
+  let lastOpenCallId: string | undefined;
+  let dirReady = false;
+
+  const bucketFor = (callId: string) => {
+    let bucket = turns.get(callId);
+    if (!bucket) {
+      bucket = { lines: [], errored: false };
+      turns.set(callId, bucket);
+    }
+    return bucket;
+  };
+
+  const record = (callId: string | undefined, rec: Record_): void => {
+    const id = callId ?? lastOpenCallId;
+    if (id == null) {
+      // No active turn — write standalone (best-effort).
+      flushLines([rec]);
+      return;
+    }
+    bucketFor(id).lines.push(rec);
+  };
+
+  const flushLines = (lines: Record_[]): void => {
+    if (lines.length === 0) return;
+    try {
+      if (!dirReady) {
+        mkdirSync(options.dir, { recursive: true });
+        dirReady = true;
+      }
+      appendFileSync(path, lines.map(l => JSON.stringify(l)).join('\n') + '\n');
+    } catch {
+      // Best-effort: never let observability break a turn.
+    }
+  };
+
+  const finishTurn = (callId: string): void => {
+    const bucket = turns.get(callId);
+    if (!bucket) return;
+    turns.delete(callId);
+    if (lastOpenCallId === callId) lastOpenCallId = undefined;
+    if (failOnly && !bucket.errored) return;
+    flushLines(bucket.lines);
+  };
+
+  return {
+    onStart(event) {
+      const e = event as {
+        callId: string;
+        operationId?: string;
+        modelId?: string;
+        provider?: string;
+        messages?: unknown;
+        instructions?: unknown;
+        recordInputs?: boolean;
+      };
+      lastOpenCallId = e.callId;
+      record(e.callId, {
+        ts: Date.now(),
+        kind: 'turn-start',
+        callId: e.callId,
+        operationId: e.operationId,
+        provider: e.provider,
+        modelId: e.modelId,
+        // Input prompt, unless the consumer opted out via `recordInputs: false`.
+        ...(e.recordInputs === false
+          ? {}
+          : { input: { messages: e.messages, instructions: e.instructions } }),
+      });
+    },
+    onStepStart(event) {
+      const e = event as { callId: string; stepNumber?: number };
+      record(e.callId, {
+        ts: Date.now(),
+        kind: 'step-start',
+        callId: e.callId,
+        step: e.stepNumber,
+      });
+    },
+    onToolExecutionStart(event) {
+      const e = event as {
+        callId: string;
+        toolCall: { toolName: string; toolCallId: string; input: unknown };
+      };
+      record(e.callId, {
+        ts: Date.now(),
+        kind: 'tool-start',
+        callId: e.callId,
+        toolName: e.toolCall.toolName,
+        toolCallId: e.toolCall.toolCallId,
+        input: e.toolCall.input,
+      });
+    },
+    onToolExecutionEnd(event) {
+      const e = event as {
+        callId: string;
+        toolCall: { toolCallId: string };
+        toolOutput: { type: string; output?: unknown; error?: unknown };
+      };
+      const isError = e.toolOutput.type === 'error';
+      if (isError) bucketFor(e.callId).errored = true;
+      record(e.callId, {
+        ts: Date.now(),
+        kind: 'tool-end',
+        callId: e.callId,
+        toolCallId: e.toolCall.toolCallId,
+        isError,
+        output: isError ? e.toolOutput.error : e.toolOutput.output,
+      });
+    },
+    onStepFinish(event) {
+      const e = event as {
+        callId: string;
+        usage?: unknown;
+        content?: unknown[];
+        recordOutputs?: boolean;
+      };
+      record(e.callId, {
+        ts: Date.now(),
+        kind: 'step-finish',
+        callId: e.callId,
+        usage: e.usage,
+        // The model's output content, unless `recordOutputs: false`.
+        ...(e.recordOutputs === false || e.content == null
+          ? {}
+          : { output: e.content }),
+      });
+    },
+    onEnd(event) {
+      const e = event as {
+        callId: string;
+        finishReason?: unknown;
+        usage?: unknown;
+        totalUsage?: unknown;
+      };
+      record(e.callId, {
+        ts: Date.now(),
+        kind: 'turn-finish',
+        callId: e.callId,
+        finishReason: e.finishReason,
+        usage: e.totalUsage ?? e.usage,
+      });
+      finishTurn(e.callId);
+    },
+    onError(error) {
+      if (lastOpenCallId != null) bucketFor(lastOpenCallId).errored = true;
+      record(lastOpenCallId, {
+        ts: Date.now(),
+        kind: 'error',
+        error:
+          error instanceof Error
+            ? { name: error.name, message: error.message }
+            : error,
+      });
+    },
+    ingestDiagnostic(diagnostic: HarnessDiagnostic) {
+      if (diagnostic.level === 'error' && lastOpenCallId != null) {
+        bucketFor(lastOpenCallId).errored = true;
+      }
+      record(lastOpenCallId, {
+        ts: diagnostic.timestamp,
+        kind: 'diagnostic',
+        diagnostic,
+      });
+    },
+  };
+}

package/src/agent/observability/index.ts

@@ -0,0 +1,15 @@
+export {
+  createFileReporter,
+  type FileReporter,
+  type FileReporterOptions,
+} from './file-reporter';
+export {
+  createTraceTreeReporter,
+  type TraceTreeReporterOptions,
+} from './trace-tree-reporter';
+export type {
+  HarnessDebugConfig,
+  HarnessDebugLevel,
+  HarnessDiagnostic,
+  HarnessDiagnosticConsumer,
+} from './types';

package/src/agent/observability/trace-tree-reporter.ts

@@ -0,0 +1,122 @@
+import type { Telemetry } from 'ai';
+
+/**
+ * A harness observability reporter that renders an ASCII trace tree of a
+ * turn's span lifecycle (turn → steps → tools) to a stream at turn end. It is a
+ * `Telemetry` integration — register it in `telemetry.integrations`. Useful for
+ * zero-setup local debugging when no OTel collector is wired up; a real OTel
+ * backend (via `@ai-sdk/otel`) is a strict superset.
+ */
+export interface TraceTreeReporterOptions {
+  /** Where to write the rendered tree. Default `process.stderr.write`. */
+  write?: (chunk: string) => void;
+}
+
+type Node = {
+  label: string;
+  startMs: number;
+  endMs?: number;
+  children: Node[];
+};
+
+export function createTraceTreeReporter(
+  options: TraceTreeReporterOptions = {},
+): Telemetry {
+  const write =
+    options.write ?? ((chunk: string) => void process.stderr.write(chunk));
+
+  type TurnState = {
+    root: Node;
+    step?: Node;
+    tools: Map<string, Node>;
+  };
+  const turns = new Map<string, TurnState>();
+
+  const render = (node: Node, depth: number): string => {
+    const indent = '  '.repeat(depth);
+    const dur =
+      node.endMs != null
+        ? `${Math.max(0, Math.round(node.endMs - node.startMs))}ms`
+        : '(open)';
+    let out = `${indent}- ${node.label} ${dur}\n`;
+    for (const child of node.children) out += render(child, depth + 1);
+    return out;
+  };
+
+  return {
+    onStart(event) {
+      const e = event as {
+        callId: string;
+        operationId?: string;
+        modelId?: string;
+      };
+      turns.set(e.callId, {
+        root: {
+          label: `${e.operationId ?? 'turn'}${e.modelId ? ` ${e.modelId}` : ''}`,
+          startMs: Date.now(),
+          children: [],
+        },
+        tools: new Map(),
+      });
+    },
+    onStepStart(event) {
+      const e = event as { callId: string; stepNumber?: number };
+      const turn = turns.get(e.callId);
+      if (!turn) return;
+      const step: Node = {
+        label: `step ${(e.stepNumber ?? turn.root.children.length) + 1}`,
+        startMs: Date.now(),
+        children: [],
+      };
+      turn.step = step;
+      turn.root.children.push(step);
+    },
+    onToolExecutionStart(event) {
+      const e = event as {
+        callId: string;
+        toolCall: { toolName: string; toolCallId: string };
+      };
+      const turn = turns.get(e.callId);
+      const parent = turn?.step ?? turn?.root;
+      if (!turn || !parent) return;
+      const node: Node = {
+        label: `tool ${e.toolCall.toolName}`,
+        startMs: Date.now(),
+        children: [],
+      };
+      turn.tools.set(e.toolCall.toolCallId, node);
+      parent.children.push(node);
+    },
+    onToolExecutionEnd(event) {
+      const e = event as {
+        callId: string;
+        toolCall: { toolCallId: string };
+        toolOutput: { type: string };
+      };
+      const node = turns.get(e.callId)?.tools.get(e.toolCall.toolCallId);
+      if (!node) return;
+      node.endMs = Date.now();
+      if (e.toolOutput.type === 'error') node.label += ' [error]';
+    },
+    onStepFinish(event) {
+      const e = event as { callId: string };
+      const turn = turns.get(e.callId);
+      if (turn?.step) {
+        turn.step.endMs = Date.now();
+        turn.step = undefined;
+      }
+    },
+    onEnd(event) {
+      const e = event as { callId: string };
+      const turn = turns.get(e.callId);
+      if (!turn) return;
+      turn.root.endMs = Date.now();
+      turns.delete(e.callId);
+      try {
+        write(`\n${render(turn.root, 0)}`);
+      } catch {
+        // Never let rendering break the turn.
+      }
+    },
+  };
+}