@ai-sdk/harness 0.0.0 → 1.0.0-canary.3

package/src/errors/harness-capability-unsupported-error.ts

@@ -0,0 +1,41 @@
+import { AISDKError } from '@ai-sdk/provider';
+import { HarnessError } from './harness-error';
+
+const name = 'AI_HarnessCapabilityUnsupportedError';
+const marker = `vercel.ai.error.${name}`;
+const symbol = Symbol.for(marker);
+
+/**
+ * Thrown when a caller asks the harness to do something the adapter (or the
+ * supplied sandbox) does not support, e.g. requesting manual compaction from
+ * an adapter that only auto-compacts, or invoking `getPortUrl` on a sandbox
+ * that does not expose one.
+ *
+ * The caller supplies the full human-readable message. Optional `harnessId`
+ * is recorded as structured context for tooling.
+ */
+export class HarnessCapabilityUnsupportedError extends HarnessError {
+  private readonly [symbol] = true;
+
+  readonly harnessId?: string;
+
+  constructor({
+    message,
+    harnessId,
+    cause,
+  }: {
+    message: string;
+    harnessId?: string;
+    cause?: unknown;
+  }) {
+    super({ message, cause });
+    Object.defineProperty(this, 'name', { value: name });
+    this.harnessId = harnessId;
+  }
+
+  static isInstance(
+    error: unknown,
+  ): error is HarnessCapabilityUnsupportedError {
+    return AISDKError.hasMarker(error, marker);
+  }
+}

package/src/errors/harness-error.ts

@@ -0,0 +1,22 @@
+import { AISDKError } from '@ai-sdk/provider';
+
+const name = 'AI_HarnessError';
+const marker = `vercel.ai.error.${name}`;
+const symbol = Symbol.for(marker);
+
+/**
+ * Base error type for failures originating in or signalled by a harness
+ * adapter. Specific failure modes (e.g. unsupported capability) extend this
+ * class.
+ */
+export class HarnessError extends AISDKError {
+  private readonly [symbol] = true;
+
+  constructor({ message, cause }: { message: string; cause?: unknown }) {
+    super({ name, message, cause });
+  }
+
+  static isInstance(error: unknown): error is HarnessError {
+    return AISDKError.hasMarker(error, marker);
+  }
+}

package/src/index.ts

@@ -0,0 +1,3 @@
+export * from './v1';
+export * from './errors/harness-error';
+export * from './errors/harness-capability-unsupported-error';

package/src/observability/file-reporter.ts

@@ -0,0 +1,209 @@
+import { appendFileSync, mkdirSync } from 'node:fs';
+import type { Telemetry } from 'ai';
+import type {
+  HarnessDiagnostic,
+  HarnessDiagnosticConsumer,
+} from '../agent/harness-diagnostics';
+
+/**
+ * A harness observability reporter that writes a unified, non-lossy
+ * `events.jsonl` containing **both** the telemetry span lifecycle (turn / step
+ * / tool) **and** the forwarded bridge diagnostics (console lines + structured
+ * events). It is a single object registered in `telemetry.integrations`: the
+ * framework drives its `Telemetry` methods for spans and calls
+ * `ingestDiagnostic` for diagnostics.
+ *
+ * No external collector or OTel setup required — this is the AI-SDK-idiomatic
+ * replacement for the original SDK's host-side artifact files.
+ */
+export interface FileReporterOptions {
+  /** Directory for `events.jsonl` (created if absent). */
+  dir: string;
+  /**
+   * Buffer a turn's records in memory and write them only if the turn produced
+   * an error (an `error`-level diagnostic, a failed tool, or an error finish).
+   * Default false (write everything).
+   */
+  failOnly?: boolean;
+  /** File name within `dir`. Default `events.jsonl`. */
+  fileName?: string;
+}
+
+type Record_ = { ts: number } & Record<string, unknown>;
+
+export type FileReporter = Telemetry & HarnessDiagnosticConsumer;
+
+export function createFileReporter(options: FileReporterOptions): FileReporter {
+  const fileName = options.fileName ?? 'events.jsonl';
+  const path = `${options.dir}/${fileName}`;
+  const failOnly = options.failOnly ?? false;
+
+  // Per-turn buffers, keyed by the telemetry callId. Diagnostics (which carry a
+  // sessionId, not a callId) attach to the most recently started, open turn.
+  const turns = new Map<string, { lines: Record_[]; errored: boolean }>();
+  let lastOpenCallId: string | undefined;
+  let dirReady = false;
+
+  const bucketFor = (callId: string) => {
+    let bucket = turns.get(callId);
+    if (!bucket) {
+      bucket = { lines: [], errored: false };
+      turns.set(callId, bucket);
+    }
+    return bucket;
+  };
+
+  const record = (callId: string | undefined, rec: Record_): void => {
+    const id = callId ?? lastOpenCallId;
+    if (id == null) {
+      // No active turn — write standalone (best-effort).
+      flushLines([rec]);
+      return;
+    }
+    bucketFor(id).lines.push(rec);
+  };
+
+  const flushLines = (lines: Record_[]): void => {
+    if (lines.length === 0) return;
+    try {
+      if (!dirReady) {
+        mkdirSync(options.dir, { recursive: true });
+        dirReady = true;
+      }
+      appendFileSync(path, lines.map(l => JSON.stringify(l)).join('\n') + '\n');
+    } catch {
+      // Best-effort: never let observability break a turn.
+    }
+  };
+
+  const finishTurn = (callId: string): void => {
+    const bucket = turns.get(callId);
+    if (!bucket) return;
+    turns.delete(callId);
+    if (lastOpenCallId === callId) lastOpenCallId = undefined;
+    if (failOnly && !bucket.errored) return;
+    flushLines(bucket.lines);
+  };
+
+  return {
+    onStart(event) {
+      const e = event as {
+        callId: string;
+        operationId?: string;
+        modelId?: string;
+        provider?: string;
+        messages?: unknown;
+        instructions?: unknown;
+        recordInputs?: boolean;
+      };
+      lastOpenCallId = e.callId;
+      record(e.callId, {
+        ts: Date.now(),
+        kind: 'turn-start',
+        callId: e.callId,
+        operationId: e.operationId,
+        provider: e.provider,
+        modelId: e.modelId,
+        // Input prompt, unless the consumer opted out via `recordInputs: false`.
+        ...(e.recordInputs === false
+          ? {}
+          : { input: { messages: e.messages, instructions: e.instructions } }),
+      });
+    },
+    onStepStart(event) {
+      const e = event as { callId: string; stepNumber?: number };
+      record(e.callId, {
+        ts: Date.now(),
+        kind: 'step-start',
+        callId: e.callId,
+        step: e.stepNumber,
+      });
+    },
+    onToolExecutionStart(event) {
+      const e = event as {
+        callId: string;
+        toolCall: { toolName: string; toolCallId: string; input: unknown };
+      };
+      record(e.callId, {
+        ts: Date.now(),
+        kind: 'tool-start',
+        callId: e.callId,
+        toolName: e.toolCall.toolName,
+        toolCallId: e.toolCall.toolCallId,
+        input: e.toolCall.input,
+      });
+    },
+    onToolExecutionEnd(event) {
+      const e = event as {
+        callId: string;
+        toolCall: { toolCallId: string };
+        toolOutput: { type: string; output?: unknown; error?: unknown };
+      };
+      const isError = e.toolOutput.type === 'error';
+      if (isError) bucketFor(e.callId).errored = true;
+      record(e.callId, {
+        ts: Date.now(),
+        kind: 'tool-end',
+        callId: e.callId,
+        toolCallId: e.toolCall.toolCallId,
+        isError,
+        output: isError ? e.toolOutput.error : e.toolOutput.output,
+      });
+    },
+    onStepFinish(event) {
+      const e = event as {
+        callId: string;
+        usage?: unknown;
+        content?: unknown[];
+        recordOutputs?: boolean;
+      };
+      record(e.callId, {
+        ts: Date.now(),
+        kind: 'step-finish',
+        callId: e.callId,
+        usage: e.usage,
+        // The model's output content, unless `recordOutputs: false`.
+        ...(e.recordOutputs === false || e.content == null
+          ? {}
+          : { output: e.content }),
+      });
+    },
+    onEnd(event) {
+      const e = event as {
+        callId: string;
+        finishReason?: unknown;
+        usage?: unknown;
+        totalUsage?: unknown;
+      };
+      record(e.callId, {
+        ts: Date.now(),
+        kind: 'turn-finish',
+        callId: e.callId,
+        finishReason: e.finishReason,
+        usage: e.totalUsage ?? e.usage,
+      });
+      finishTurn(e.callId);
+    },
+    onError(error) {
+      if (lastOpenCallId != null) bucketFor(lastOpenCallId).errored = true;
+      record(lastOpenCallId, {
+        ts: Date.now(),
+        kind: 'error',
+        error:
+          error instanceof Error
+            ? { name: error.name, message: error.message }
+            : error,
+      });
+    },
+    ingestDiagnostic(diagnostic: HarnessDiagnostic) {
+      if (diagnostic.level === 'error' && lastOpenCallId != null) {
+        bucketFor(lastOpenCallId).errored = true;
+      }
+      record(lastOpenCallId, {
+        ts: diagnostic.timestamp,
+        kind: 'diagnostic',
+        diagnostic,
+      });
+    },
+  };
+}

package/src/observability/index.ts

@@ -0,0 +1,13 @@
+export {
+  createFileReporter,
+  type FileReporter,
+  type FileReporterOptions,
+} from './file-reporter';
+export {
+  createTraceTreeReporter,
+  type TraceTreeReporterOptions,
+} from './trace-tree-reporter';
+export type {
+  HarnessDiagnostic,
+  HarnessDiagnosticConsumer,
+} from '../agent/harness-diagnostics';

package/src/observability/trace-tree-reporter.ts

@@ -0,0 +1,122 @@
+import type { Telemetry } from 'ai';
+
+/**
+ * A harness observability reporter that renders an ASCII trace tree of a
+ * turn's span lifecycle (turn → steps → tools) to a stream at turn end. It is a
+ * `Telemetry` integration — register it in `telemetry.integrations`. Useful for
+ * zero-setup local debugging when no OTel collector is wired up; a real OTel
+ * backend (via `@ai-sdk/otel`) is a strict superset.
+ */
+export interface TraceTreeReporterOptions {
+  /** Where to write the rendered tree. Default `process.stderr.write`. */
+  write?: (chunk: string) => void;
+}
+
+type Node = {
+  label: string;
+  startMs: number;
+  endMs?: number;
+  children: Node[];
+};
+
+export function createTraceTreeReporter(
+  options: TraceTreeReporterOptions = {},
+): Telemetry {
+  const write =
+    options.write ?? ((chunk: string) => void process.stderr.write(chunk));
+
+  type TurnState = {
+    root: Node;
+    step?: Node;
+    tools: Map<string, Node>;
+  };
+  const turns = new Map<string, TurnState>();
+
+  const render = (node: Node, depth: number): string => {
+    const indent = '  '.repeat(depth);
+    const dur =
+      node.endMs != null
+        ? `${Math.max(0, Math.round(node.endMs - node.startMs))}ms`
+        : '(open)';
+    let out = `${indent}- ${node.label} ${dur}\n`;
+    for (const child of node.children) out += render(child, depth + 1);
+    return out;
+  };
+
+  return {
+    onStart(event) {
+      const e = event as {
+        callId: string;
+        operationId?: string;
+        modelId?: string;
+      };
+      turns.set(e.callId, {
+        root: {
+          label: `${e.operationId ?? 'turn'}${e.modelId ? ` ${e.modelId}` : ''}`,
+          startMs: Date.now(),
+          children: [],
+        },
+        tools: new Map(),
+      });
+    },
+    onStepStart(event) {
+      const e = event as { callId: string; stepNumber?: number };
+      const turn = turns.get(e.callId);
+      if (!turn) return;
+      const step: Node = {
+        label: `step ${(e.stepNumber ?? turn.root.children.length) + 1}`,
+        startMs: Date.now(),
+        children: [],
+      };
+      turn.step = step;
+      turn.root.children.push(step);
+    },
+    onToolExecutionStart(event) {
+      const e = event as {
+        callId: string;
+        toolCall: { toolName: string; toolCallId: string };
+      };
+      const turn = turns.get(e.callId);
+      const parent = turn?.step ?? turn?.root;
+      if (!turn || !parent) return;
+      const node: Node = {
+        label: `tool ${e.toolCall.toolName}`,
+        startMs: Date.now(),
+        children: [],
+      };
+      turn.tools.set(e.toolCall.toolCallId, node);
+      parent.children.push(node);
+    },
+    onToolExecutionEnd(event) {
+      const e = event as {
+        callId: string;
+        toolCall: { toolCallId: string };
+        toolOutput: { type: string };
+      };
+      const node = turns.get(e.callId)?.tools.get(e.toolCall.toolCallId);
+      if (!node) return;
+      node.endMs = Date.now();
+      if (e.toolOutput.type === 'error') node.label += ' [error]';
+    },
+    onStepFinish(event) {
+      const e = event as { callId: string };
+      const turn = turns.get(e.callId);
+      if (turn?.step) {
+        turn.step.endMs = Date.now();
+        turn.step = undefined;
+      }
+    },
+    onEnd(event) {
+      const e = event as { callId: string };
+      const turn = turns.get(e.callId);
+      if (!turn) return;
+      turn.root.endMs = Date.now();
+      turns.delete(e.callId);
+      try {
+        write(`\n${render(turn.root, 0)}`);
+      } catch {
+        // Never let rendering break the turn.
+      }
+    },
+  };
+}

package/src/utils/classify-disk-log.ts

@@ -0,0 +1,43 @@
+import { safeParseJSON } from '@ai-sdk/provider-utils';
+
+/**
+ * Recovery rung selected from an on-disk bridge event log when attach is not
+ * possible (the bridge process is gone): `'replay'` when the log holds a
+ * complete turn the host can resume from its cursor, `'rerun'` otherwise.
+ */
+export type DiskLogRecoveryMode = 'replay' | 'rerun';
+
+/**
+ * Decide whether a respawned bridge can `replay` a turn from its persisted
+ * `event-log.ndjson`, or must `rerun` it from scratch.
+ *
+ * A turn is replayable only when its log ends in a terminal `finish` event —
+ * a log that is missing, empty, or ends mid-turn means the bridge died before
+ * completing the turn, so there is no coherent tail to deliver and the runtime
+ * must re-run it (continuing its own thread from the sandbox snapshot).
+ *
+ * @param eventLog Raw contents of `event-log.ndjson` (newline-delimited JSON),
+ *   or `null`/`undefined`/empty when the file is absent.
+ */
+export async function classifyDiskLog(
+  eventLog: string | null | undefined,
+): Promise<DiskLogRecoveryMode> {
+  if (!eventLog) return 'rerun';
+  const lines = eventLog
+    .split('\n')
+    .map(line => line.trim())
+    .filter(Boolean);
+  const lastLine = lines.at(-1);
+  if (lastLine == null) return 'rerun';
+
+  const parsed = await safeParseJSON({ text: lastLine });
+  if (
+    parsed.success &&
+    parsed.value != null &&
+    typeof parsed.value === 'object' &&
+    (parsed.value as { type?: unknown }).type === 'finish'
+  ) {
+    return 'replay';
+  }
+  return 'rerun';
+}

package/src/utils/index.ts

@@ -0,0 +1,7 @@
+export {
+  SandboxChannel,
+  type SandboxChannelDebugEvent,
+  type SandboxChannelOptions,
+  type SandboxChannelReconnectOptions,
+} from './sandbox-channel';
+export { classifyDiskLog, type DiskLogRecoveryMode } from './classify-disk-log';