@ai-sdk/harness 0.0.0-6b196531-20260710185421

package/src/agent/observability/file-reporter.ts

@@ -0,0 +1,206 @@
+import { appendFileSync, mkdirSync } from 'node:fs';
+import type { Telemetry } from 'ai';
+import type { HarnessDiagnostic, HarnessDiagnosticConsumer } from './types';
+
+/**
+ * A harness observability reporter that writes a unified, non-lossy
+ * `events.jsonl` containing **both** the telemetry span lifecycle (turn / step
+ * / tool) **and** the forwarded bridge diagnostics (console lines + structured
+ * events). It is a single object registered in `telemetry.integrations`: the
+ * framework drives its `Telemetry` methods for spans and calls
+ * `ingestDiagnostic` for diagnostics.
+ *
+ * No external collector or OTel setup required — this is the AI-SDK-idiomatic
+ * replacement for the original SDK's host-side artifact files.
+ */
+export interface FileReporterOptions {
+  /** Directory for `events.jsonl` (created if absent). */
+  dir: string;
+  /**
+   * Buffer a turn's records in memory and write them only if the turn produced
+   * an error (an `error`-level diagnostic, a failed tool, or an error finish).
+   * Default false (write everything).
+   */
+  failOnly?: boolean;
+  /** File name within `dir`. Default `events.jsonl`. */
+  fileName?: string;
+}
+
+type Record_ = { ts: number } & Record<string, unknown>;
+
+export type FileReporter = Telemetry & HarnessDiagnosticConsumer;
+
+export function createFileReporter(options: FileReporterOptions): FileReporter {
+  const fileName = options.fileName ?? 'events.jsonl';
+  const path = `${options.dir}/${fileName}`;
+  const failOnly = options.failOnly ?? false;
+
+  // Per-turn buffers, keyed by the telemetry callId. Diagnostics (which carry a
+  // sessionId, not a callId) attach to the most recently started, open turn.
+  const turns = new Map<string, { lines: Record_[]; errored: boolean }>();
+  let lastOpenCallId: string | undefined;
+  let dirReady = false;
+
+  const bucketFor = (callId: string) => {
+    let bucket = turns.get(callId);
+    if (!bucket) {
+      bucket = { lines: [], errored: false };
+      turns.set(callId, bucket);
+    }
+    return bucket;
+  };
+
+  const record = (callId: string | undefined, rec: Record_): void => {
+    const id = callId ?? lastOpenCallId;
+    if (id == null) {
+      // No active turn — write standalone (best-effort).
+      flushLines([rec]);
+      return;
+    }
+    bucketFor(id).lines.push(rec);
+  };
+
+  const flushLines = (lines: Record_[]): void => {
+    if (lines.length === 0) return;
+    try {
+      if (!dirReady) {
+        mkdirSync(options.dir, { recursive: true });
+        dirReady = true;
+      }
+      appendFileSync(path, lines.map(l => JSON.stringify(l)).join('\n') + '\n');
+    } catch {
+      // Best-effort: never let observability break a turn.
+    }
+  };
+
+  const finishTurn = (callId: string): void => {
+    const bucket = turns.get(callId);
+    if (!bucket) return;
+    turns.delete(callId);
+    if (lastOpenCallId === callId) lastOpenCallId = undefined;
+    if (failOnly && !bucket.errored) return;
+    flushLines(bucket.lines);
+  };
+
+  return {
+    onStart(event) {
+      const e = event as {
+        callId: string;
+        operationId?: string;
+        modelId?: string;
+        provider?: string;
+        messages?: unknown;
+        instructions?: unknown;
+        recordInputs?: boolean;
+      };
+      lastOpenCallId = e.callId;
+      record(e.callId, {
+        ts: Date.now(),
+        kind: 'turn-start',
+        callId: e.callId,
+        operationId: e.operationId,
+        provider: e.provider,
+        modelId: e.modelId,
+        // Input prompt, unless the consumer opted out via `recordInputs: false`.
+        ...(e.recordInputs === false
+          ? {}
+          : { input: { messages: e.messages, instructions: e.instructions } }),
+      });
+    },
+    onStepStart(event) {
+      const e = event as { callId: string; stepNumber?: number };
+      record(e.callId, {
+        ts: Date.now(),
+        kind: 'step-start',
+        callId: e.callId,
+        step: e.stepNumber,
+      });
+    },
+    onToolExecutionStart(event) {
+      const e = event as {
+        callId: string;
+        toolCall: { toolName: string; toolCallId: string; input: unknown };
+      };
+      record(e.callId, {
+        ts: Date.now(),
+        kind: 'tool-start',
+        callId: e.callId,
+        toolName: e.toolCall.toolName,
+        toolCallId: e.toolCall.toolCallId,
+        input: e.toolCall.input,
+      });
+    },
+    onToolExecutionEnd(event) {
+      const e = event as {
+        callId: string;
+        toolCall: { toolCallId: string };
+        toolOutput: { type: string; output?: unknown; error?: unknown };
+      };
+      const isError = e.toolOutput.type === 'error';
+      if (isError) bucketFor(e.callId).errored = true;
+      record(e.callId, {
+        ts: Date.now(),
+        kind: 'tool-end',
+        callId: e.callId,
+        toolCallId: e.toolCall.toolCallId,
+        isError,
+        output: isError ? e.toolOutput.error : e.toolOutput.output,
+      });
+    },
+    onStepFinish(event) {
+      const e = event as {
+        callId: string;
+        usage?: unknown;
+        content?: unknown[];
+        recordOutputs?: boolean;
+      };
+      record(e.callId, {
+        ts: Date.now(),
+        kind: 'step-finish',
+        callId: e.callId,
+        usage: e.usage,
+        // The model's output content, unless `recordOutputs: false`.
+        ...(e.recordOutputs === false || e.content == null
+          ? {}
+          : { output: e.content }),
+      });
+    },
+    onEnd(event) {
+      const e = event as {
+        callId: string;
+        finishReason?: unknown;
+        usage?: unknown;
+        totalUsage?: unknown;
+      };
+      record(e.callId, {
+        ts: Date.now(),
+        kind: 'turn-finish',
+        callId: e.callId,
+        finishReason: e.finishReason,
+        usage: e.totalUsage ?? e.usage,
+      });
+      finishTurn(e.callId);
+    },
+    onError(error) {
+      if (lastOpenCallId != null) bucketFor(lastOpenCallId).errored = true;
+      record(lastOpenCallId, {
+        ts: Date.now(),
+        kind: 'error',
+        error:
+          error instanceof Error
+            ? { name: error.name, message: error.message }
+            : error,
+      });
+    },
+    ingestDiagnostic(diagnostic: HarnessDiagnostic) {
+      if (diagnostic.level === 'error' && lastOpenCallId != null) {
+        bucketFor(lastOpenCallId).errored = true;
+      }
+      record(lastOpenCallId, {
+        ts: diagnostic.timestamp,
+        kind: 'diagnostic',
+        diagnostic,
+      });
+    },
+  };
+}

package/src/agent/observability/index.ts

@@ -0,0 +1,15 @@
+export {
+  createFileReporter,
+  type FileReporter,
+  type FileReporterOptions,
+} from './file-reporter';
+export {
+  createTraceTreeReporter,
+  type TraceTreeReporterOptions,
+} from './trace-tree-reporter';
+export type {
+  HarnessDebugConfig,
+  HarnessDebugLevel,
+  HarnessDiagnostic,
+  HarnessDiagnosticConsumer,
+} from './types';

package/src/agent/observability/trace-tree-reporter.ts

@@ -0,0 +1,122 @@
+import type { Telemetry } from 'ai';
+
+/**
+ * A harness observability reporter that renders an ASCII trace tree of a
+ * turn's span lifecycle (turn → steps → tools) to a stream at turn end. It is a
+ * `Telemetry` integration — register it in `telemetry.integrations`. Useful for
+ * zero-setup local debugging when no OTel collector is wired up; a real OTel
+ * backend (via `@ai-sdk/otel`) is a strict superset.
+ */
+export interface TraceTreeReporterOptions {
+  /** Where to write the rendered tree. Default `process.stderr.write`. */
+  write?: (chunk: string) => void;
+}
+
+type Node = {
+  label: string;
+  startMs: number;
+  endMs?: number;
+  children: Node[];
+};
+
+export function createTraceTreeReporter(
+  options: TraceTreeReporterOptions = {},
+): Telemetry {
+  const write =
+    options.write ?? ((chunk: string) => void process.stderr.write(chunk));
+
+  type TurnState = {
+    root: Node;
+    step?: Node;
+    tools: Map<string, Node>;
+  };
+  const turns = new Map<string, TurnState>();
+
+  const render = (node: Node, depth: number): string => {
+    const indent = '  '.repeat(depth);
+    const dur =
+      node.endMs != null
+        ? `${Math.max(0, Math.round(node.endMs - node.startMs))}ms`
+        : '(open)';
+    let out = `${indent}- ${node.label} ${dur}\n`;
+    for (const child of node.children) out += render(child, depth + 1);
+    return out;
+  };
+
+  return {
+    onStart(event) {
+      const e = event as {
+        callId: string;
+        operationId?: string;
+        modelId?: string;
+      };
+      turns.set(e.callId, {
+        root: {
+          label: `${e.operationId ?? 'turn'}${e.modelId ? ` ${e.modelId}` : ''}`,
+          startMs: Date.now(),
+          children: [],
+        },
+        tools: new Map(),
+      });
+    },
+    onStepStart(event) {
+      const e = event as { callId: string; stepNumber?: number };
+      const turn = turns.get(e.callId);
+      if (!turn) return;
+      const step: Node = {
+        label: `step ${(e.stepNumber ?? turn.root.children.length) + 1}`,
+        startMs: Date.now(),
+        children: [],
+      };
+      turn.step = step;
+      turn.root.children.push(step);
+    },
+    onToolExecutionStart(event) {
+      const e = event as {
+        callId: string;
+        toolCall: { toolName: string; toolCallId: string };
+      };
+      const turn = turns.get(e.callId);
+      const parent = turn?.step ?? turn?.root;
+      if (!turn || !parent) return;
+      const node: Node = {
+        label: `tool ${e.toolCall.toolName}`,
+        startMs: Date.now(),
+        children: [],
+      };
+      turn.tools.set(e.toolCall.toolCallId, node);
+      parent.children.push(node);
+    },
+    onToolExecutionEnd(event) {
+      const e = event as {
+        callId: string;
+        toolCall: { toolCallId: string };
+        toolOutput: { type: string };
+      };
+      const node = turns.get(e.callId)?.tools.get(e.toolCall.toolCallId);
+      if (!node) return;
+      node.endMs = Date.now();
+      if (e.toolOutput.type === 'error') node.label += ' [error]';
+    },
+    onStepFinish(event) {
+      const e = event as { callId: string };
+      const turn = turns.get(e.callId);
+      if (turn?.step) {
+        turn.step.endMs = Date.now();
+        turn.step = undefined;
+      }
+    },
+    onEnd(event) {
+      const e = event as { callId: string };
+      const turn = turns.get(e.callId);
+      if (!turn) return;
+      turn.root.endMs = Date.now();
+      turns.delete(e.callId);
+      try {
+        write(`\n${render(turn.root, 0)}`);
+      } catch {
+        // Never let rendering break the turn.
+      }
+    },
+  };
+}

package/src/agent/observability/types.ts

@@ -0,0 +1,86 @@
+/*
+ * Host/consumer-facing observability types.
+ *
+ * These are deliberately NOT part of the versioned adapter spec: no harness
+ * adapter implements or consumes them. Adapters only deal with bridge wire
+ * frames and the observability handle the framework hands to `doStart`. The
+ * framework normalizes those frames into the `HarnessDiagnostic` shape below
+ * for consumers (`HarnessAgentSettings.onLog`) and reporters.
+ */
+
+/** Severity of a diagnostic. */
+export type HarnessDebugLevel = 'error' | 'warn' | 'info' | 'debug' | 'trace';
+
+/**
+ * Consumer-facing diagnostics configuration. Set on `HarnessAgentSettings` to
+ * enable bridge log forwarding and the `HARNESS_DEBUG` stderr default in code.
+ * `HARNESS_DEBUG` / `HARNESS_DEBUG_LEVEL` / `HARNESS_DEBUG_SUBSYSTEMS` env vars
+ * fill any unset field — a convenience default, never the only path.
+ */
+export type HarnessDebugConfig = {
+  /** Master switch. Nothing is captured or forwarded when false/unset. */
+  readonly enabled?: boolean;
+  /** Threshold; events at or above this severity are emitted. Default `debug`. */
+  readonly level?: HarnessDebugLevel;
+  /** Dotted-prefix subsystem filter for structured events. */
+  readonly subsystems?: ReadonlyArray<string>;
+};
+
+/**
+ * A forwarded bridge diagnostic, normalized for host consumers.
+ *
+ * The bridge emits two raw frame kinds — captured console lines (`sandbox-log`)
+ * and structured events (`debug-event`). The framework normalizes both into
+ * this single shape before handing them to a consumer's `onLog` callback, the
+ * `HARNESS_DEBUG` stderr default, and observability reporters. Diagnostics are
+ * kept first-class and per-line — they are never folded into telemetry spans.
+ */
+export type HarnessDiagnostic = {
+  /**
+   * Severity. Structured events carry their own level; captured console lines
+   * map `stderr` → `'warn'` and `stdout` → `'info'`.
+   */
+  readonly level: HarnessDebugLevel;
+
+  /** Human-readable line (console capture) or message (structured event). */
+  readonly message: string;
+
+  /**
+   * Dotted subsystem. For captured console output this is
+   * `sandbox.log.<source>`; for structured events it is the adapter-supplied
+   * subsystem (e.g. `bridge.turn`).
+   */
+  readonly subsystem: string;
+
+  /** `'log'` = captured console line; `'event'` = structured `bridgeLog`. */
+  readonly kind: 'log' | 'event';
+
+  /** Originating sandbox source label (console capture). */
+  readonly source?: string;
+
+  /** Which standard stream the line came from (console capture). */
+  readonly stream?: 'stdout' | 'stderr';
+
+  /** Structured attributes (structured events only). */
+  readonly attrs?: Record<string, unknown>;
+
+  /** Error payload (structured events only). */
+  readonly error?: { name?: string; message: string; stack?: string };
+
+  /** The harness session this diagnostic originated from. */
+  readonly sessionId?: string;
+
+  /** Host receipt time (epoch ms). */
+  readonly timestamp: number;
+};
+
+/**
+ * A telemetry integration that also wants the per-line diagnostics stream. The
+ * framework calls `ingestDiagnostic` for every forwarded bridge diagnostic in
+ * addition to driving the standard `Telemetry` span lifecycle, so a single
+ * reporter object (e.g. `createFileReporter`) registered in
+ * `telemetry.integrations` receives both spans and logs.
+ */
+export interface HarnessDiagnosticConsumer {
+  ingestDiagnostic?(diagnostic: HarnessDiagnostic): void;
+}

package/src/agent/prepare-harness-sandbox-template.ts

@@ -0,0 +1,68 @@
+import type { HarnessV1SandboxProvider } from '../v1';
+import type { HarnessAgentAdapter } from './harness-agent-types';
+import type { HarnessAgentSandboxConfig } from './harness-agent-settings';
+import { applyBootstrapRecipe } from './internal/bootstrap-recipe';
+import {
+  createSandboxBootstrapPlan,
+  validateSandboxBootstrapSettings,
+} from './internal/sandbox-bootstrap';
+
+type SandboxBootstrapSettings = Omit<HarnessAgentSandboxConfig, 'onSession'>;
+
+/**
+ * Prepare a harness's sandbox template without running an agent. Idempotent: if
+ * the template already exists (snapshot present, or marker on a non-snapshot
+ * provider), this resolves quickly.
+ *
+ * Use from a CI/deploy script to amortize the first-session cost so production
+ * sessions always resume from snapshot. For adapters without a bootstrap
+ * recipe (no `getBootstrap`) this is a no-op.
+ *
+ * The temporary network sandbox session created during preparation is stopped
+ * before the function resolves; the snapshot/template state persists in the
+ * provider's native storage (for Vercel: as the `currentSnapshotId` of the
+ * named template sandbox).
+ */
+export async function prepareHarnessSandboxTemplate(options: {
+  readonly harness: HarnessAgentAdapter;
+  readonly sandboxProvider: HarnessV1SandboxProvider;
+  readonly sandboxConfig?: SandboxBootstrapSettings;
+  readonly abortSignal?: AbortSignal;
+}): Promise<void> {
+  const sandboxConfig = options.sandboxConfig ?? {};
+  validateSandboxBootstrapSettings(sandboxConfig);
+  const recipe = await options.harness.getBootstrap?.({
+    abortSignal: options.abortSignal,
+  });
+  const bootstrapPlan = await createSandboxBootstrapPlan({
+    recipe,
+    settings: sandboxConfig,
+  });
+  if (bootstrapPlan.identity == null || bootstrapPlan.onFirstCreate == null) {
+    return;
+  }
+
+  const sandboxSession = await options.sandboxProvider.createSession({
+    abortSignal: options.abortSignal,
+    identity: bootstrapPlan.identity,
+    onFirstCreate: bootstrapPlan.onFirstCreate,
+  });
+
+  try {
+    if (bootstrapPlan.recipe != null && bootstrapPlan.recipeIdentity != null) {
+      await applyBootstrapRecipe(
+        sandboxSession.restricted(),
+        bootstrapPlan.recipe,
+        bootstrapPlan.recipeIdentity,
+        {
+          abortSignal: options.abortSignal,
+        },
+      );
+    }
+  } finally {
+    await Promise.resolve(sandboxSession.stop()).catch(() => {});
+  }
+}
+
+/** @deprecated Use `prepareHarnessSandboxTemplate` instead. */
+export const prewarmHarness = prepareHarnessSandboxTemplate;

package/src/agent/prepare-sandbox-for-harness.ts

@@ -0,0 +1,165 @@
+import type { Experimental_SandboxSession as SandboxSession } from '@ai-sdk/provider-utils';
+import type { HarnessAgentSandboxConfig } from './harness-agent-settings';
+import type { HarnessAgentAdapter } from './harness-agent-types';
+import {
+  applyBootstrapRecipe,
+  hashHarnessBootstrap,
+} from './internal/bootstrap-recipe';
+import {
+  normalizeSandboxWorkDir,
+  runSandboxBootstrap,
+  validateSandboxBootstrapSettings,
+} from './internal/sandbox-bootstrap';
+
+const PREPARED_SANDBOX_IDENTITY_VERSION = 1;
+
+export type PrepareSandboxForHarnessResult = {
+  readonly identity?: string;
+  readonly recipeIdentities: Record<string, string>;
+  readonly skippedHarnessIds: ReadonlyArray<string>;
+};
+
+/**
+ * Apply one or more harness bootstrap recipes to an existing sandbox session.
+ *
+ * Use this when a sandbox provider or caller wants to prepare a reusable
+ * sandbox template, image, or snapshot before creating live harness sessions.
+ *
+ * The function writes each adapter's bridge/bootstrap files, runs its install
+ * commands, and then returns a deterministic identity derived from the applied
+ * recipes and optional sandbox bootstrap configuration. Providers can use that
+ * identity as the cache key for the prepared artifact.
+ *
+ * This function only mutates the supplied sandbox; the caller is responsible for
+ * committing, snapshotting, or otherwise persisting that modified filesystem.
+ * When a later `HarnessAgent` session uses a sandbox created from the persisted
+ * artifact, the adapter recomputes the same recipe identity and the existing
+ * bootstrap marker makes the bootstrap logic a no-op.
+ */
+export async function prepareSandboxForHarness(options: {
+  readonly session: SandboxSession;
+  readonly harnesses: ReadonlyArray<HarnessAgentAdapter>;
+  readonly sandboxConfig?: HarnessAgentSandboxConfig;
+  readonly abortSignal?: AbortSignal;
+}): Promise<PrepareSandboxForHarnessResult> {
+  const sandboxConfig = options.sandboxConfig ?? {};
+  validateSandboxBootstrapSettings(sandboxConfig);
+
+  if (options.harnesses.length === 0) {
+    throw new Error(
+      'prepareSandboxForHarness: at least one harness must be provided.',
+    );
+  }
+
+  const harnesses = [...options.harnesses].sort((a, b) =>
+    a.harnessId.localeCompare(b.harnessId),
+  );
+  assertUniqueHarnessIds(harnesses);
+
+  const workDir =
+    sandboxConfig.workDir == null
+      ? undefined
+      : normalizeSandboxWorkDir(sandboxConfig.workDir);
+  const recipeIdentities: Record<string, string> = {};
+  const skippedHarnessIds: string[] = [];
+
+  for (const harness of harnesses) {
+    const recipe = await harness.getBootstrap?.({
+      abortSignal: options.abortSignal,
+    });
+    if (recipe == null) {
+      skippedHarnessIds.push(harness.harnessId);
+      continue;
+    }
+
+    const recipeIdentity = await hashHarnessBootstrap(recipe);
+    recipeIdentities[harness.harnessId] = recipeIdentity;
+    await applyBootstrapRecipe(options.session, recipe, recipeIdentity, {
+      abortSignal: options.abortSignal,
+    });
+  }
+
+  if (sandboxConfig.onBootstrap != null) {
+    await runSandboxBootstrap({
+      session: options.session,
+      workDir,
+      onBootstrap: sandboxConfig.onBootstrap,
+      abortSignal: options.abortSignal,
+    });
+  }
+
+  const identity = await resolvePreparedSandboxIdentity({
+    recipeIdentities,
+    bootstrapHash: sandboxConfig.bootstrapHash,
+    workDir,
+  });
+
+  return {
+    ...(identity != null ? { identity } : {}),
+    recipeIdentities,
+    skippedHarnessIds,
+  };
+}
+
+function assertUniqueHarnessIds(
+  harnesses: ReadonlyArray<HarnessAgentAdapter>,
+): void {
+  const seen = new Set<string>();
+  for (const harness of harnesses) {
+    if (seen.has(harness.harnessId)) {
+      throw new Error(
+        `prepareSandboxForHarness: duplicate harness id "${harness.harnessId}".`,
+      );
+    }
+    seen.add(harness.harnessId);
+  }
+}
+
+async function resolvePreparedSandboxIdentity({
+  recipeIdentities,
+  bootstrapHash,
+  workDir,
+}: {
+  readonly recipeIdentities: Record<string, string>;
+  readonly bootstrapHash?: string;
+  readonly workDir?: string;
+}): Promise<string | undefined> {
+  const entries = Object.entries(recipeIdentities).sort(([a], [b]) =>
+    a.localeCompare(b),
+  );
+  if (entries.length === 0 && bootstrapHash == null) {
+    return undefined;
+  }
+
+  const encoder = new TextEncoder();
+  const chunks: Uint8Array[] = [];
+  const pushString = (value: string) => {
+    chunks.push(encoder.encode(value));
+    chunks.push(encoder.encode('\0'));
+  };
+
+  pushString(String(PREPARED_SANDBOX_IDENTITY_VERSION));
+  pushString(workDir ?? '');
+  pushString(bootstrapHash ?? '');
+
+  for (const [harnessId, identity] of entries) {
+    pushString(harnessId);
+    pushString(identity);
+  }
+
+  const totalLength = chunks.reduce((sum, chunk) => sum + chunk.length, 0);
+  const buffer = new Uint8Array(totalLength);
+  let offset = 0;
+  for (const chunk of chunks) {
+    buffer.set(chunk, offset);
+    offset += chunk.length;
+  }
+
+  const digest = await crypto.subtle.digest('SHA-256', buffer);
+  const bytes = new Uint8Array(digest);
+  let hex = '';
+  for (let i = 0; i < 8; i++) {
+    hex += bytes[i].toString(16).padStart(2, '0');
+  }
+  return hex;
+}