@relayflows/sdk 2.0.42 → 2.0.43

package/src/flow-eval-version.ts

@@ -0,0 +1,316 @@
+// What a flow evaluation judges, by content: the flow version.
+//
+// For a flow file the version is the sha256 of a manifest of every local
+// source the run reads, not of the entry file alone. Editing a helper module,
+// a `use`d flow, or the project's extension lock changes what executes, so it
+// must change the version too — otherwise `expectVersion` would accept code it
+// never judged. Packages under node_modules are not part of the manifest; they
+// are pinned by the project's own lockfile, which is. An evaluation executes a
+// sealed copy of exactly these bytes (`sealFlowEvalSnapshot`).
+
+import { createHash } from 'node:crypto';
+import { existsSync, statSync } from 'node:fs';
+import { chmod, mkdir, mkdtemp, readdir, readFile, realpath, rm, symlink, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { basename, dirname, join, relative, resolve, sep } from 'node:path';
+import { canonicalize, specHash } from './canonical.js';
+import { compileSpec, toKernelSpec } from './compile.js';
+import { FlowEvalError } from './flow-eval-error.js';
+import { parsePluginLock, type PluginLock } from './plugin-lock.js';
+import { pluginStoreDirectory } from './plugin-store.js';
+import type { FlowSpec } from './spec.js';
+
+/** A flow source path (`.flow.ts`, `.flow.yaml`, spec JSON) or an in-memory spec. */
+export type FlowEvalTarget = { path: string } | FlowSpec;
+
+export interface FlowEvalVersion {
+  /** `sha256:<hex>` over the manifest (or the kernel spec hash for an in-memory spec). */
+  version: string;
+  /** Manifest paths, relative to the entry's directory; empty for an in-memory spec. */
+  files: string[];
+}
+
+/** Most local source files one version may span; past it the evaluation is refused, not truncated. */
+export const MAX_FLOW_EVAL_SOURCES = 2_000;
+
+const SOURCE_EXTENSIONS = ['.ts', '.mts', '.cts', '.js', '.mjs', '.cjs', '.json'];
+const PROJECT_FILES = ['package.json', 'flows.json', 'flows.lock.json', 'package-lock.json', 'bun.lock', 'pnpm-lock.yaml', 'yarn.lock'];
+
+export function isPathTarget(flow: FlowEvalTarget): flow is { path: string } {
+  return typeof (flow as { path?: unknown }).path === 'string' && !('steps' in flow);
+}
+
+/** The version string alone. */
+export async function flowEvalVersion(flow: FlowEvalTarget): Promise<string> {
+  return (await flowEvalSources(flow)).version;
+}
+
+/** The version and the files it covers. */
+export async function flowEvalSources(flow: FlowEvalTarget): Promise<FlowEvalVersion> {
+  if (!isPathTarget(flow)) return { version: `sha256:${specHash(toKernelSpec(compileSpec(flow)))}`, files: [] };
+  const { version, files } = await collectSources(flow.path);
+  return { version, files: files.map(file => file.path) };
+}
+
+/** The sealed bytes a version names, from which every case gets its own copy. */
+export interface FlowEvalSnapshot extends FlowEvalVersion {
+  /**
+   * A fresh, read-only copy for one case: its own directory, files and
+   * directories without write permission, written from the buffers the
+   * version hashed. An in-memory spec yields a deep-frozen clone.
+   */
+  materialize(): Promise<{ flow: FlowEvalTarget; dispose(): Promise<void> }>;
+}
+
+/**
+ * Seal the flow before any case runs, and execute only copies of the seal.
+ *
+ * Hashing the working tree and then running it would let the tree change
+ * between the two — an edit made after a check and restored before the next
+ * would run bytes the report never named. The sources are read once, hashed,
+ * and held in memory; each case then runs a fresh read-only copy written from
+ * those buffers, so nothing a case does to its copy reaches the next case. A
+ * local module the manifest could not see (a computed
+ * `import(\`./rules/${kind}.js\`)`, a file read at runtime) is simply absent
+ * from the copy, so such a run fails instead of executing unjudged code.
+ *
+ * Trust boundary: evaluated code runs as the caller, so it can still reach
+ * anything the caller can — including `node_modules`, which is linked rather
+ * than copied and pinned by the lockfile in the manifest. Containing hostile
+ * code needs a sandbox; this seal makes sure the code that runs is the code
+ * the version names.
+ */
+export async function sealFlowEvalSnapshot(flow: FlowEvalTarget): Promise<FlowEvalSnapshot> {
+  if (!isPathTarget(flow)) {
+    // Hash and execute the same detached copy: a caller mutating its object
+    // later cannot change what the remaining cases run.
+    const frozen = deepFreeze(structuredClone(flow));
+    return { ...(await flowEvalSources(frozen)), materialize: async () => ({ flow: frozen, dispose: async () => {} }) };
+  }
+  const sources = await collectSources(flow.path);
+  // Every node_modules Node would consult from the entry, at the same relative
+  // place: a nested package resolves some dependencies from higher levels.
+  const modules = ancestorNodeModules(dirname(sources.entry));
+  const root = commonDirectory([...sources.absolute.map(file => file.path), ...modules.map(dir => join(dir, '.'))]);
+  return {
+    version: sources.version,
+    files: sources.files.map(file => file.path),
+    materialize: async () => {
+      const directory = await mkdtemp(join(tmpdir(), 'flows-eval-snapshot-'));
+      const dispose = async (): Promise<void> => {
+        await makeWritable(directory);
+        await rm(directory, { recursive: true, force: true });
+      };
+      try {
+        for (const file of sources.absolute) {
+          const target = join(directory, relative(root, file.path));
+          await mkdir(dirname(target), { recursive: true });
+          await writeFile(target, file.bytes, { mode: 0o444 });
+        }
+        for (const dir of modules) {
+          const link = join(directory, relative(root, dir));
+          await mkdir(dirname(link), { recursive: true });
+          await symlink(dir, link, 'dir');
+        }
+        await makeReadOnly(directory);
+      } catch (error) {
+        await dispose();
+        throw error;
+      }
+      return { flow: { path: join(directory, relative(root, sources.entry)) }, dispose };
+    },
+  };
+}
+
+async function makeReadOnly(directory: string): Promise<void> {
+  for (const entry of await readdir(directory, { withFileTypes: true })) {
+    if (entry.isDirectory()) await makeReadOnly(join(directory, entry.name));
+  }
+  await chmod(directory, 0o555);
+}
+
+async function makeWritable(directory: string): Promise<void> {
+  try {
+    await chmod(directory, 0o755);
+    for (const entry of await readdir(directory, { withFileTypes: true })) {
+      if (entry.isDirectory()) await makeWritable(join(directory, entry.name));
+    }
+  } catch { /* already gone */ }
+}
+
+export function deepFreeze<T>(value: T): T {
+  if (value !== null && typeof value === 'object' && !Object.isFrozen(value)) {
+    Object.freeze(value);
+    for (const nested of Object.values(value as Record<string, unknown>)) deepFreeze(nested);
+  }
+  return value;
+}
+
+interface CollectedSources {
+  entry: string;
+  version: string;
+  files: Array<{ path: string; sha256: string }>;
+  absolute: Array<{ path: string; bytes: Buffer }>;
+}
+
+async function collectSources(path: string): Promise<CollectedSources> {
+  let entry: string;
+  try {
+    entry = await realpath(path);
+  } catch {
+    throw new FlowEvalError('unreadable_flow', `Flow "${path}" is not readable.`);
+  }
+  const root = dirname(entry);
+  const contents = new Map<string, Buffer>();
+  const missing = new Set<string>();
+  // One cap over everything sealed — sources, manifests, extension payloads —
+  // so no category can grow a version past it.
+  const add = async (file: string): Promise<Buffer> => {
+    const known = contents.get(file);
+    if (known !== undefined) return known;
+    if (contents.size >= MAX_FLOW_EVAL_SOURCES) {
+      throw new FlowEvalError('unreadable_flow', `Flow "${path}" reaches more than ${MAX_FLOW_EVAL_SOURCES} local files.`);
+    }
+    let bytes: Buffer;
+    try {
+      bytes = await readFile(file);
+    } catch {
+      throw new FlowEvalError('unreadable_flow', `Flow source "${file}" is not readable.`);
+    }
+    contents.set(file, bytes);
+    return bytes;
+  };
+  const pending = [entry];
+  while (pending.length > 0) {
+    const file = pending.pop()!;
+    if (contents.has(file)) continue;
+    const bytes = await add(file);
+    // The package scope that decides how this file loads (`"type"`), per file:
+    // a nested package.json can differ from the entry's.
+    const scope = nearestFile(dirname(file), 'package.json');
+    if (scope !== undefined) await add(scope);
+    if (!/\.(?:[mc]?[jt]s)$/u.test(file)) continue;
+    for (const specifier of localSpecifiers(bytes.toString('utf8'))) {
+      const targets = resolveLocal(dirname(file), specifier);
+      if (targets.length === 0) missing.add(`${relative(root, dirname(file)) || '.'}:${specifier}`);
+      for (const target of targets) if (!contents.has(target)) pending.push(target);
+    }
+  }
+  // The nearest of each: package.json decides module semantics, flows.json and
+  // its lock the extensions, and a package lockfile pins node_modules. Each is
+  // found by walking up on its own, so a lockfile beside package.json (not
+  // beside flows.json) still pins the version.
+  for (const name of PROJECT_FILES) {
+    const file = nearestFile(root, name);
+    if (file !== undefined) await add(file);
+  }
+  // Locked flow extensions load from the project's content-addressed store,
+  // so their payloads are part of what runs: hash and seal them too.
+  const lockPath = nearestFile(root, 'flows.lock.json');
+  if (lockPath !== undefined) {
+    let lock: PluginLock;
+    try {
+      lock = parsePluginLock(JSON.parse(contents.get(lockPath)!.toString('utf8')));
+    } catch (error) {
+      throw new FlowEvalError('unreadable_flow', `flows.lock.json is not a valid plugin lock: ${error instanceof Error ? error.message : String(error)}`);
+    }
+    for (const entry of lock.plugins) {
+      const store = pluginStoreDirectory(dirname(lockPath), entry.name, entry.digest);
+      if (!existsSync(store)) {
+        throw new FlowEvalError('unreadable_flow', `Locked extension ${entry.name} is not materialized at ${store}; run \`flows plugin verify\`.`);
+      }
+      for (const file of await listFiles(store)) await add(file);
+    }
+  }
+  const absolute = [...contents.entries()].map(([file, bytes]) => ({ path: file, bytes }));
+  const files = absolute
+    .map(file => ({ path: relative(root, file.path) || basename(file.path), sha256: sha256(file.bytes) }))
+    .sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0));
+  // An unresolved import is part of the identity too: creating that file later changes what runs.
+  const manifest = { entry: basename(entry), files, missing: [...missing].sort() };
+  return { entry, version: `sha256:${sha256(canonicalize(manifest))}`, files, absolute };
+}
+
+async function listFiles(directory: string): Promise<string[]> {
+  const files: string[] = [];
+  for (const entry of await readdir(directory, { withFileTypes: true })) {
+    const path = join(directory, entry.name);
+    if (entry.isDirectory()) files.push(...await listFiles(path));
+    else if (entry.isFile()) files.push(path);
+  }
+  return files;
+}
+
+function commonDirectory(paths: readonly string[]): string {
+  let common = dirname(paths[0]!);
+  for (const path of paths) {
+    while (common !== dirname(common) && path !== common && !path.startsWith(common + sep)) common = dirname(common);
+  }
+  return common;
+}
+
+function nearestFile(start: string, name: string): string | undefined {
+  for (let directory = start; ; directory = dirname(directory)) {
+    const candidate = join(directory, name);
+    if (existsSync(candidate)) return candidate;
+    if (dirname(directory) === directory) return undefined;
+  }
+}
+
+function ancestorNodeModules(start: string): string[] {
+  const found: string[] = [];
+  for (let directory = start; ; directory = dirname(directory)) {
+    const candidate = join(directory, 'node_modules');
+    try {
+      if (statSync(candidate).isDirectory()) found.push(candidate);
+    } catch { /* none here */ }
+    if (dirname(directory) === directory) return found;
+  }
+}
+
+/**
+ * Relative module specifiers and relative `.flow.ts` string literals (the
+ * `use:` header) in a source file. A lexical scan, deliberately: computing a
+ * version must never execute the flow.
+ */
+export function localSpecifiers(source: string): string[] {
+  const found = new Set<string>();
+  const patterns = [
+    /\b(?:import|export)\s+(?:type\s+)?[^'"`;]*?\bfrom\s*['"](\.{1,2}\/[^'"]+)['"]/gu,
+    /\bimport\s*['"](\.{1,2}\/[^'"]+)['"]/gu,
+    /\b(?:import|require)\s*\(\s*['"](\.{1,2}\/[^'"]+)['"]\s*\)/gu,
+    /['"](\.{1,2}\/[^'"\s]+\.flow\.ts)['"]/gu,
+  ];
+  for (const pattern of patterns) for (const match of source.matchAll(pattern)) found.add(match[1]!);
+  return [...found];
+}
+
+/**
+ * The files a relative specifier can load. Where a TypeScript sibling and a
+ * JavaScript file both match (`./x.js` beside `./x.ts`), both are returned:
+ * which one a loader picks is the loader's business, and the version must not
+ * hash one while the run executes the other.
+ */
+function resolveLocal(from: string, specifier: string): string[] {
+  const base = resolve(from, specifier.replace(/[?#].*$/u, ''));
+  const isFile = (candidate: string): boolean => {
+    try {
+      return statSync(candidate).isFile();
+    } catch {
+      return false;
+    }
+  };
+  // TypeScript's NodeNext convention: import './x.js' names ./x.ts.
+  const swapped = base.replace(/\.([mc]?)js$/u, '.$1ts');
+  const direct = [...new Set([swapped, base])].filter(isFile);
+  if (direct.length > 0) return direct;
+  for (const candidate of [
+    ...SOURCE_EXTENSIONS.map(extension => base + extension),
+    ...SOURCE_EXTENSIONS.map(extension => join(base, `index${extension}`)),
+  ]) if (isFile(candidate)) return [candidate];
+  return [];
+}
+
+function sha256(data: Buffer | string): string {
+  return createHash('sha256').update(data).digest('hex');
+}

package/src/flow-eval.ts

@@ -0,0 +1,234 @@
+// Re-run evaluation: execute one exact flow version against a frozen set of
+// input cases and score the result, so a deploy can be gated on it.
+//
+// `flows replay` reads a finished run's journal and never executes anything.
+// This module is the other thing: it RE-EXECUTES the flow, once per case,
+// through an executor (local daemon, Cloud, or one the caller supplies), and
+// returns a report whose `gate.pass` is the deploy decision.
+//
+// It is a library first. `evaluateFlow` is what a step calls — from a
+// deterministic step's script or an authored body — and `flows eval` is a thin
+// CLI over it whose exit code is the same verdict (0 pass, 1 fail, 2 refused).
+//
+// Two things are frozen and recorded in every report, so a verdict names
+// exactly what it judged: the flow version (flow-eval-version.ts) and the suite
+// (flow-eval-suite.ts). Every case executes its own fresh, read-only copy
+// written from the very bytes the version hashes, never the working tree, so
+// neither an edit during the evaluation nor an earlier case can change what
+// runs under the judged version.
+// Aggregation and the verdict itself live in flow-eval-gate.ts.
+
+import { FlowEvalError } from './flow-eval-error.js';
+import { decideFlowEvalGate, parseFlowEvalBaseline, summarizeFlowEval } from './flow-eval-gate.js';
+import {
+  FLOW_EVAL_REPORT_SCHEMA_VERSION,
+  type FlowEvalCaseResult, type FlowEvalExecutor, type FlowEvalReport, type FlowEvalRun, type FlowEvalScorer,
+} from './flow-eval-report.js';
+import { flowEvalSuiteSha256, parseFlowEvalSuite, type FlowEvalCase, type FlowEvalSuite } from './flow-eval-suite.js';
+import { isPathTarget, sealFlowEvalSnapshot, type FlowEvalSnapshot, type FlowEvalTarget } from './flow-eval-version.js';
+
+export { FlowEvalError, type FlowEvalErrorCode } from './flow-eval-error.js';
+export { parseFlowEvalBaseline, type FlowEvalBaseline } from './flow-eval-gate.js';
+export * from './flow-eval-report.js';
+export {
+  MAX_FLOW_EVAL_CASES, flowEvalSuiteSha256, loadFlowEvalSuite, parseFlowEvalSuite,
+  type FlowEvalCase, type FlowEvalExpectation, type FlowEvalSuite, type FlowEvalThresholds,
+} from './flow-eval-suite.js';
+export {
+  MAX_FLOW_EVAL_SOURCES, flowEvalSources, flowEvalVersion, sealFlowEvalSnapshot,
+  type FlowEvalSnapshot, type FlowEvalTarget, type FlowEvalVersion,
+} from './flow-eval-version.js';
+
+export interface EvaluateFlowOptions {
+  flow: FlowEvalTarget;
+  suite: FlowEvalSuite;
+  executor: FlowEvalExecutor;
+  /** Recorded in the report; defaults to `custom`. */
+  executorName?: string;
+  scorers?: FlowEvalScorer[];
+  /** Cases executed at once. Defaults to 1, which keeps runs from contending. */
+  concurrency?: number;
+  /** Refuse unless the flow version is exactly this (`sha256:<hex>`). */
+  expectVersion?: string;
+  /** Refuse unless the suite hashes to exactly this (`sha256:<hex>`). */
+  expectSuiteSha256?: string;
+  /**
+   * A prior report on the same suite (parsed JSON is fine; it is validated
+   * before any case runs). Any case that passed there and does not pass here
+   * is a regression, and a regression fails the gate.
+   */
+  baseline?: unknown;
+  signal?: AbortSignal;
+  /** Clock injection for tests. */
+  now?: () => number;
+}
+
+/**
+ * Re-execute `flow` once per suite case and score the results.
+ *
+ * Never throws for a case that fails or errors: those are verdicts, recorded
+ * in the report. Throws `FlowEvalError` only when the evaluation itself is
+ * invalid (bad suite, baseline or options, or a pinned version/suite that does
+ * not match), and rethrows an abort.
+ */
+export async function evaluateFlow(options: EvaluateFlowOptions): Promise<FlowEvalReport> {
+  const now = options.now ?? Date.now;
+  const suite = parseFlowEvalSuite(options.suite);
+  const concurrency = options.concurrency ?? 1;
+  if (!Number.isSafeInteger(concurrency) || concurrency < 1 || concurrency > 64) {
+    throw new FlowEvalError('invalid_options', 'concurrency must be an integer from 1 to 64.');
+  }
+  const names = new Set<string>();
+  for (const scorer of options.scorers ?? []) {
+    if (typeof scorer.name !== 'string' || scorer.name === '' || names.has(scorer.name)) {
+      throw new FlowEvalError('invalid_options', 'Each scorer needs a unique, nonempty name.');
+    }
+    names.add(scorer.name);
+  }
+  const suiteSha256 = flowEvalSuiteSha256(suite);
+  if (options.expectSuiteSha256 !== undefined && options.expectSuiteSha256 !== suiteSha256) {
+    throw new FlowEvalError('suite_mismatch', `Suite hashes to ${suiteSha256}; expected ${options.expectSuiteSha256}.`);
+  }
+  const baseline = options.baseline === undefined ? undefined : parseFlowEvalBaseline(options.baseline);
+  if (baseline !== undefined && baseline.suiteSha256 !== suiteSha256) {
+    throw new FlowEvalError('suite_mismatch', 'The baseline report was produced from a different suite; regressions would not be comparable.');
+  }
+  if (baseline !== undefined) {
+    // The hash is a claim; the cases are the evidence. A baseline missing a
+    // case would silently skip that case's regression check.
+    const ids = new Set(suite.cases.map(c => c.id));
+    const extra = [...baseline.outcomes.keys()].filter(id => !ids.has(id));
+    const absent = [...ids].filter(id => !baseline.outcomes.has(id));
+    if (extra.length > 0 || absent.length > 0) {
+      throw new FlowEvalError('invalid_baseline', 'The baseline report does not cover exactly this suite\'s cases'
+        + (absent.length > 0 ? `; missing ${absent.join(', ')}` : '') + (extra.length > 0 ? `; unknown ${extra.join(', ')}` : '') + '.');
+    }
+  }
+  const snapshot = await sealFlowEvalSnapshot(options.flow);
+  const { version } = snapshot;
+  if (options.expectVersion !== undefined && options.expectVersion !== version) {
+    throw new FlowEvalError('version_mismatch', `Flow version is ${version}; expected ${options.expectVersion}.`);
+  }
+  const startedAt = now();
+  const results: FlowEvalCaseResult[] = new Array(suite.cases.length);
+  let next = 0;
+  const worker = async (): Promise<void> => {
+    for (;;) {
+      options.signal?.throwIfAborted();
+      const index = next++;
+      if (index >= suite.cases.length) return;
+      results[index] = await runCase(suite.cases[index]!, options, now, snapshot);
+    }
+  };
+  await Promise.all(Array.from({ length: Math.min(concurrency, suite.cases.length) }, worker));
+  options.signal?.throwIfAborted();
+
+  const summary = summarizeFlowEval(results);
+  const gate = decideFlowEvalGate(suite, results, summary, baseline);
+  return {
+    schemaVersion: FLOW_EVAL_REPORT_SCHEMA_VERSION,
+    kind: 'flows.eval.report',
+    flow: { ...(isPathTarget(options.flow) ? { path: options.flow.path } : {}), version, files: snapshot.files },
+    suite: { name: suite.name, sha256: suiteSha256, cases: suite.cases.length },
+    executor: options.executorName ?? 'custom',
+    startedAt: new Date(startedAt).toISOString(),
+    finishedAt: new Date(now()).toISOString(),
+    cases: results,
+    summary,
+    gate,
+  };
+}
+
+async function runCase(
+  testCase: FlowEvalCase, options: EvaluateFlowOptions, now: () => number, snapshot: FlowEvalSnapshot,
+): Promise<FlowEvalCaseResult> {
+  const copy = await snapshot.materialize();
+  try {
+    return await scoreCase(testCase, options, now, copy.flow);
+  } finally {
+    await copy.dispose();
+  }
+}
+
+async function scoreCase(
+  testCase: FlowEvalCase, options: EvaluateFlowOptions, now: () => number, sealed: FlowEvalTarget,
+): Promise<FlowEvalCaseResult> {
+  const started = now();
+  let run: FlowEvalRun;
+  try {
+    run = await options.executor({
+      flow: sealed, caseId: testCase.id,
+      ...(testCase.input === undefined ? {} : { input: testCase.input }),
+      ...(options.signal === undefined ? {} : { signal: options.signal }),
+    });
+  } catch (error) {
+    // An abort is the caller stopping the evaluation, not this case's verdict.
+    if (options.signal?.aborted) throw options.signal.reason ?? error;
+    return {
+      id: testCase.id, outcome: 'error', latencyMs: Math.max(0, now() - started),
+      costUsd: null, tokensIn: null, tokensOut: null, metrics: {},
+      failures: [`executor failed: ${error instanceof Error ? error.message : String(error)}`],
+    };
+  }
+  const latencyMs = Math.max(0, now() - started);
+  const costUsd = finiteOrNull(run.costUsd);
+  const failures: string[] = [];
+  const expect = testCase.expect ?? {};
+  const wantReason = expect.completionReason ?? 'success';
+  if (run.completionReason !== wantReason) {
+    failures.push(`completionReason was ${run.completionReason}; expected ${wantReason}`
+      + (run.completionDetail ? ` (${truncate(run.completionDetail)})` : ''));
+  }
+  const detail = run.completionDetail ?? '';
+  for (const fragment of expect.detailIncludes ?? []) {
+    if (!detail.includes(fragment)) failures.push(`completion detail does not include ${JSON.stringify(fragment)}`);
+  }
+  if (expect.detailMatches !== undefined && !new RegExp(expect.detailMatches, 'u').test(detail)) {
+    failures.push(`completion detail does not match /${expect.detailMatches}/`);
+  }
+  if (expect.maxLatencyMs !== undefined && latencyMs > expect.maxLatencyMs) {
+    failures.push(`latency ${latencyMs}ms exceeds ${expect.maxLatencyMs}ms`);
+  }
+  if (expect.maxCostUsd !== undefined) {
+    if (costUsd === null) failures.push(`cost is unknown; the case caps it at $${expect.maxCostUsd}`);
+    else if (costUsd > expect.maxCostUsd) failures.push(`cost $${costUsd} exceeds $${expect.maxCostUsd}`);
+  }
+  const metrics: Record<string, number> = {};
+  for (const [name, value] of Object.entries(run.metrics ?? {})) {
+    if (Number.isFinite(value)) metrics[name] = value;
+  }
+  let errored = false;
+  for (const scorer of options.scorers ?? []) {
+    try {
+      const score = await scorer.score(run, testCase);
+      if (score.value !== undefined && Number.isFinite(score.value)) metrics[scorer.name] = score.value;
+      if (score.pass === false) failures.push(`scorer ${scorer.name} failed` + (score.detail ? `: ${score.detail}` : ''));
+    } catch (error) {
+      errored = true;
+      failures.push(`scorer ${scorer.name} threw: ${error instanceof Error ? error.message : String(error)}`);
+    }
+  }
+  return {
+    id: testCase.id,
+    outcome: errored ? 'error' : failures.length === 0 ? 'pass' : 'fail',
+    ...(run.runId === undefined ? {} : { runId: run.runId }),
+    completionReason: run.completionReason,
+    latencyMs,
+    costUsd,
+    tokensIn: finiteOrNull(run.tokensIn),
+    tokensOut: finiteOrNull(run.tokensOut),
+    metrics,
+    ...(run.steps === undefined ? {} : { steps: run.steps }),
+    failures,
+  };
+}
+
+/** A negative or non-finite figure is malformed; as unknown it fails every ceiling instead of offsetting other cases. */
+function finiteOrNull(value: number | null | undefined): number | null {
+  return typeof value === 'number' && Number.isFinite(value) && value >= 0 ? value : null;
+}
+
+function truncate(text: string): string {
+  const line = text.replace(/\s+/gu, ' ').trim();
+  return line.length > 200 ? `${line.slice(0, 197)}...` : line;
+}