@relayflows/sdk 2.0.42 → 2.0.43

package/src/flow-eval-local.ts

@@ -0,0 +1,236 @@
+// The local executor for `evaluateFlow`: one fresh `flows run` per case
+// against relayflowd, and the case's spend read back from the journals.
+// Memoization (`--reuse-from`) is never used: an evaluation must judge the
+// candidate's own step results, not a prior version's.
+
+import { spawn } from 'node:child_process';
+import { existsSync } from 'node:fs';
+import { mkdtemp, rm, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { dirname, join, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { canonicalize } from './canonical.js';
+import { isAuthoredFlowPath } from './direct-input.js';
+import { walkJournal } from './journal-reader.js';
+import type { FlowEvalExecutor, FlowEvalRun, FlowEvalStep } from './flow-eval-report.js';
+import { runReportOutcome } from './flow-eval-outcome.js';
+import type { RunExecution } from './cli/run.js';
+
+export interface LocalFlowEvalExecutorOptions {
+  /** Daemon data directory. Defaults to `.relayflowd`. */
+  dataDir?: string;
+  /** Attach a local agent/LLM worker, as `flows run --local-agent`. */
+  localAgent?: boolean;
+  agentCapacity?: number;
+  /** `{ spawn: false }` refuses instead of starting a daemon, as `flows run --no-spawn`. */
+  daemon?: { spawn?: boolean };
+  /** The `flows` CLI entry point. Defaults to this package's own `dist/cli.js`. */
+  cliPath?: string;
+  /**
+   * Working directory for each run. Defaults to the snapshot directory holding
+   * the flow, so a relative read (`readFileSync('./prompt.txt')`) resolves
+   * inside the sealed copy — and fails there if the file is not part of the
+   * version — instead of reading the caller's mutable working tree. Setting it
+   * deliberately reintroduces that dependency.
+   */
+  cwd?: string;
+}
+
+/**
+ * Execute each case as a fresh local run, in its own `flows run` process.
+ *
+ * A child process per case is deliberate. Authored source is loaded with
+ * `import()`, and an ES module is cached by URL for the life of a process: an
+ * in-process run would keep executing the FIRST version of a `.flow.ts` it saw,
+ * while the report named the edited bytes on disk. A gate must run what it
+ * judges, so each case gets a process that has never seen the flow before.
+ *
+ * Authored `.flow.ts` receives the case input (`{}` when the case has none);
+ * declarative YAML/JSON specs take no input, so a case that declares one is
+ * refused rather than silently run without it.
+ */
+export function localFlowEvalExecutor(options: LocalFlowEvalExecutorOptions = {}): FlowEvalExecutor {
+  const dataDir = resolve(options.dataDir ?? '.relayflowd');
+  return async ({ flow, input, signal }) => {
+    if (!('path' in flow) || typeof flow.path !== 'string') {
+      throw new Error('The local executor runs flow files; write the spec to a file or use a custom executor.');
+    }
+    const authored = isAuthoredFlowPath(flow.path);
+    if (!authored && input !== undefined) throw new Error('Declarative flows take no input; remove `input` from the case.');
+    const scratch = authored ? await mkdtemp(join(tmpdir(), 'flows-eval-')) : undefined;
+    try {
+      const args = [options.cliPath ?? defaultCliPath(), 'run', '--json', '--no-observer-link', '--data-dir', dataDir];
+      if (options.daemon?.spawn === false) args.push('--no-spawn');
+      if (options.localAgent) args.push('--local-agent');
+      if (options.agentCapacity !== undefined) args.push('--agent-capacity', String(options.agentCapacity));
+      args.push(resolve(flow.path));
+      if (scratch !== undefined) {
+        // A file, never inline: an inline argument that happens to name a file
+        // in the working directory would be read as that file.
+        const inputPath = join(scratch, 'input.json');
+        await writeFile(inputPath, canonicalize(input ?? {}));
+        args.push('--input', inputPath);
+      }
+      const execution = await runCliProcess(args, options.cwd ?? dirname(resolve(flow.path)), signal);
+      const run = localRunOutcome(execution);
+      if (run.runId !== undefined && run.completionReason !== 'refused') {
+        Object.assign(run, await localRunSpend(run.runId, dataDir));
+      }
+      return run;
+    } finally {
+      if (scratch !== undefined) await rm(scratch, { recursive: true, force: true });
+    }
+  };
+}
+
+function defaultCliPath(): string {
+  // Built: this file sits beside cli.js in dist/. From source (tests), use the build.
+  const sibling = fileURLToPath(new URL('./cli.js', import.meta.url));
+  return existsSync(sibling) ? sibling : fileURLToPath(new URL('../dist/cli.js', import.meta.url));
+}
+
+function runCliProcess(args: string[], cwd: string | undefined, signal: AbortSignal | undefined): Promise<RunExecution> {
+  return new Promise((resolvePromise, reject) => {
+    const child = spawn(process.execPath, args, {
+      ...(cwd === undefined ? {} : { cwd }), stdio: ['ignore', 'pipe', 'pipe'],
+      ...(signal === undefined ? {} : { signal }),
+    });
+    let stdout = '';
+    let stderr = '';
+    child.stdout.setEncoding('utf8').on('data', (chunk: string) => { stdout += chunk; });
+    child.stderr.setEncoding('utf8').on('data', (chunk: string) => { stderr = (stderr + chunk).slice(-4_000); });
+    child.once('error', reject);
+    child.once('close', (code) => {
+      const line = stdout.trim().split('\n').reverse().find(candidate => candidate.startsWith('{'));
+      let report: RunExecution['report'] | undefined;
+      try {
+        report = line === undefined ? undefined : JSON.parse(line) as RunExecution['report'];
+      } catch { /* reported below */ }
+      if (report === undefined || typeof report.ok !== 'boolean' || !Array.isArray(report.diagnostics)) {
+        reject(new Error(`flows run exited ${code} without a JSON report: ${stderr.trim() || stdout.trim() || '(no output)'}`));
+        return;
+      }
+      resolvePromise({ exitCode: (code ?? 1) as RunExecution['exitCode'], report });
+    });
+  });
+}
+
+/** Map a local `RunExecution` onto the eval vocabulary. Exported for tests. */
+export function localRunOutcome(execution: RunExecution): FlowEvalRun {
+  const { report } = execution;
+  // The root, not the failing child: a failed authored run reports the child
+  // that failed as `runId`, and spend must be read from the whole run.
+  const runId = report.rootRunId ?? report.runId;
+  return { ...(runId === undefined ? {} : { runId }), ...runReportOutcome(report, execution.exitCode) };
+}
+
+/**
+ * Spend from the local journals.
+ *
+ * Summed from each step's own `step.completed` budget, never from a run's
+ * `budget_total`: an authored child run starts from the flow's `prior_spend`,
+ * so its total already includes every earlier child, and adding totals would
+ * count the same dollars repeatedly.
+ *
+ * An authored root runs each step as a child run with its own journal. The
+ * children are found in the root's `authored-steps` stream, which is written
+ * as each child opens and so survives a failed body, and in the root step's
+ * success output (`journalSteps`).
+ */
+export async function localRunSpend(runId: string, dataDir: string): Promise<Pick<FlowEvalRun, 'costUsd' | 'tokensIn' | 'tokensOut' | 'steps'>> {
+  try {
+    const spend: Spend = { dollars: 0, tokensIn: 0, tokensOut: 0 };
+    const steps: FlowEvalStep[] = [];
+    const visited = new Set<string>();
+    const queue: Array<{ runId: string; depth: number }> = [{ runId, depth: 0 }];
+    while (queue.length > 0) {
+      const next = queue.shift()!;
+      if (visited.has(next.runId)) continue;
+      visited.add(next.runId);
+      const read = await readJournal(next.runId, dataDir);
+      addSpend(spend, read.spend);
+      steps.push(...read.steps);
+      // Bounded like `f.dispatch` (depth <= 3).
+      if (next.depth < 3) for (const child of read.children) queue.push({ runId: child, depth: next.depth + 1 });
+    }
+    return { costUsd: spend.dollars === null ? null : roundUsd(spend.dollars), tokensIn: spend.tokensIn, tokensOut: spend.tokensOut, steps };
+  } catch {
+    // Spend is evidence, not the verdict: an unreadable journal leaves cost unknown.
+    return { costUsd: null, tokensIn: null, tokensOut: null };
+  }
+}
+
+interface Spend { dollars: number | null; tokensIn: number; tokensOut: number }
+
+const AUTHORED_STEP_RECORD = 'relayflows.authored-step.v1';
+const RUN_ID = /^[A-Za-z0-9][A-Za-z0-9_-]*$/u;
+
+async function readJournal(runId: string, dataDir: string): Promise<{ spend: Spend; steps: FlowEvalStep[]; children: string[] }> {
+  const spend: Spend = { dollars: 0, tokensIn: 0, tokensOut: 0 };
+  const steps = new Map<string, FlowEvalStep & { startedAt?: number }>();
+  const children = new Set<string>();
+  for await (const event of walkJournal(runId, dataDir)) {
+    const payload = (event.payload !== null && typeof event.payload === 'object' ? event.payload : {}) as Record<string, unknown>;
+    if (event.entry_type === 'step.attempt.started' && event.step_id !== null) {
+      const step = steps.get(event.step_id) ?? { id: event.step_id, status: 'running' };
+      step.startedAt ??= event.at_ms;
+      steps.set(event.step_id, step);
+    } else if (event.entry_type === 'step.completed' && event.step_id !== null) {
+      const budget = readBudget(payload['budget']);
+      const step = steps.get(event.step_id) ?? { id: event.step_id, status: 'done' };
+      step.status = typeof payload['completionReason'] === 'string' ? payload['completionReason'] : 'done';
+      if (step.startedAt !== undefined) step.durationMs = event.at_ms - step.startedAt;
+      if (budget !== undefined) {
+        step.costUsd = budget.dollars;
+        addSpend(spend, budget);
+      }
+      steps.set(event.step_id, step);
+      for (const child of journalStepRunIds(payload['output'])) children.add(child);
+    } else if (event.entry_type === 'stream.appended') {
+      for (const child of authoredStepRunIds(payload)) children.add(child);
+    }
+  }
+  children.delete(runId);
+  return { spend, steps: [...steps.values()].map(({ startedAt: _startedAt, ...step }) => step), children: [...children] };
+}
+
+function journalStepRunIds(output: unknown): string[] {
+  if (output === null || typeof output !== 'object') return [];
+  const journalSteps = (output as Record<string, unknown>)['journalSteps'];
+  if (!Array.isArray(journalSteps)) return [];
+  return journalSteps.flatMap(step => {
+    const id = step !== null && typeof step === 'object' ? (step as Record<string, unknown>)['runId'] : undefined;
+    return typeof id === 'string' && RUN_ID.test(id) ? [id] : [];
+  });
+}
+
+/** Child run ids from an `authored-steps` index record, wherever the stream envelope nests it. */
+function authoredStepRunIds(value: unknown, depth = 0): string[] {
+  if (value === null || typeof value !== 'object' || depth > 4) return [];
+  const record = value as Record<string, unknown>;
+  if (record['index'] === AUTHORED_STEP_RECORD) {
+    return typeof record['runId'] === 'string' && RUN_ID.test(record['runId']) ? [record['runId']] : [];
+  }
+  return Object.values(record).flatMap(nested => authoredStepRunIds(nested, depth + 1));
+}
+
+function addSpend(into: Spend, add: Spend): void {
+  into.tokensIn += add.tokensIn;
+  into.tokensOut += add.tokensOut;
+  into.dollars = into.dollars === null || add.dollars === null ? null : into.dollars + add.dollars;
+}
+
+function roundUsd(value: number): number {
+  return Math.round(value * 1e6) / 1e6;
+}
+
+function readBudget(value: unknown): { dollars: number | null; tokensIn: number; tokensOut: number } | undefined {
+  if (value === null || typeof value !== 'object') return undefined;
+  const budget = value as Record<string, unknown>;
+  const dollars = typeof budget['dollars'] === 'string' ? Number(budget['dollars']) : NaN;
+  return {
+    dollars: budget['dollars_unmetered'] === true || !Number.isFinite(dollars) || dollars < 0 ? null : dollars,
+    tokensIn: typeof budget['tokens_in'] === 'number' ? budget['tokens_in'] : 0,
+    tokensOut: typeof budget['tokens_out'] === 'number' ? budget['tokens_out'] : 0,
+  };
+}

package/src/flow-eval-outcome.ts

@@ -0,0 +1,43 @@
+// One vocabulary for a finished run's verdict, whoever ran it: a local
+// `flows run` report and a hosted run's stored result are the same RunReport
+// shape, and must map to the same completion reason, or a suite would pass
+// locally and fail on Cloud for the same flow behavior.
+
+import type { FlowEvalRun } from './flow-eval-report.js';
+
+/** The fields of a `RunReport` the verdict reads. */
+export interface RunReportVerdict {
+  ok?: unknown;
+  status?: unknown;
+  completionReason?: unknown;
+  completionDetail?: unknown;
+  parkCause?: unknown;
+  diagnostics?: unknown;
+}
+
+/**
+ * Map a run report onto the eval vocabulary.
+ *
+ * - `declined`: the report says `success` with a `run_declined` diagnostic —
+ *   the body chose not to act. Scoring it as `success` would let a flow that
+ *   did nothing pass a suite expecting it to act.
+ * - `worker_unavailable` vs `needs_human`: both park (exit 3); only the second
+ *   reached a human wait.
+ * - `refused`: preflight refused the run (exit 2); nothing executed.
+ */
+export function runReportOutcome(report: RunReportVerdict, exitCode?: number): Pick<FlowEvalRun, 'completionReason' | 'completionDetail'> {
+  const diagnostics = Array.isArray(report.diagnostics)
+    ? report.diagnostics.filter((d): d is Record<string, unknown> => d !== null && typeof d === 'object') : [];
+  const message = diagnostics.find(d => ['failure', 'refusal', 'parked'].includes(d['severity'] as string))?.['message'];
+  const detail = typeof report.completionDetail === 'string' ? report.completionDetail
+    : typeof message === 'string' ? message : undefined;
+  let completionReason: string;
+  if (exitCode === 2) completionReason = 'refused';
+  else if (report.status === 'parked' || exitCode === 3) {
+    completionReason = report.parkCause === 'worker_unavailable' ? 'worker_unavailable' : 'needs_human';
+  } else if (report.ok === true && diagnostics.some(d => d['kind'] === 'run_declined')) completionReason = 'declined';
+  else if (report.ok === true) completionReason = 'success';
+  else if (typeof report.completionReason === 'string') completionReason = report.completionReason;
+  else completionReason = report.status === 'suspended' ? 'suspended' : 'failed';
+  return { completionReason, ...(detail === undefined ? {} : { completionDetail: detail }) };
+}

package/src/flow-eval-report.ts

@@ -0,0 +1,102 @@
+// The data a flow evaluation exchanges: what an executor returns for one run,
+// and the report the evaluation produces. Types only.
+
+import type { FlowEvalCase } from './flow-eval-suite.js';
+import type { JsonValue } from './json-value.js';
+import type { FlowEvalTarget } from './flow-eval-version.js';
+
+export const FLOW_EVAL_REPORT_SCHEMA_VERSION = 1;
+
+/** One step of an executed case, as far as the executor can say. */
+export interface FlowEvalStep {
+  id: string;
+  status: string;
+  durationMs?: number | null;
+  costUsd?: number | null;
+}
+
+/** What an executor reports for one execution. */
+export interface FlowEvalRun {
+  runId?: string;
+  /**
+   * `success`, `declined`, `step_failed`, `budget_exceeded`, `canceled`,
+   * `needs_human`, `worker_unavailable` (parked with nothing attached to run
+   * it), or `refused` (preflight refused the run).
+   */
+  completionReason: string;
+  completionDetail?: string;
+  /** `null` when the executor cannot attest a dollar cost. */
+  costUsd?: number | null;
+  tokensIn?: number | null;
+  tokensOut?: number | null;
+  steps?: FlowEvalStep[];
+  /** Executor-supplied numeric metrics, merged into the case's metrics. */
+  metrics?: Record<string, number>;
+}
+
+export interface FlowEvalExecutionRequest {
+  /** The sealed snapshot to execute (a path inside it, or the in-memory spec), never the working tree. */
+  flow: FlowEvalTarget;
+  caseId: string;
+  /** Absent when the case declares no input. */
+  input?: JsonValue;
+  signal?: AbortSignal;
+}
+
+/** Executes the flow once. Throwing marks the case `error`, never `pass`. */
+export type FlowEvalExecutor = (request: FlowEvalExecutionRequest) => Promise<FlowEvalRun>;
+
+export interface FlowEvalScore {
+  /** Numeric value recorded under the scorer's name in the case's metrics. */
+  value?: number;
+  /** False fails the case. Absent means the scorer only measures. */
+  pass?: boolean;
+  detail?: string;
+}
+
+/** A caller-defined metric over one executed case. */
+export interface FlowEvalScorer {
+  name: string;
+  score(run: FlowEvalRun, testCase: FlowEvalCase): FlowEvalScore | Promise<FlowEvalScore>;
+}
+
+export type FlowEvalOutcome = 'pass' | 'fail' | 'error';
+
+export interface FlowEvalCaseResult {
+  id: string;
+  outcome: FlowEvalOutcome;
+  runId?: string;
+  completionReason?: string;
+  latencyMs: number;
+  costUsd: number | null;
+  tokensIn: number | null;
+  tokensOut: number | null;
+  metrics: Record<string, number>;
+  steps?: FlowEvalStep[];
+  /** Why the case did not pass; empty when it did. */
+  failures: string[];
+}
+
+export interface FlowEvalReport {
+  schemaVersion: typeof FLOW_EVAL_REPORT_SCHEMA_VERSION;
+  kind: 'flows.eval.report';
+  /** `files`: the local sources the version covers, relative to the entry's directory. */
+  flow: { path?: string; version: string; files: string[] };
+  suite: { name: string; sha256: string; cases: number };
+  executor: string;
+  startedAt: string;
+  finishedAt: string;
+  cases: FlowEvalCaseResult[];
+  summary: {
+    total: number;
+    passed: number;
+    failed: number;
+    errored: number;
+    passRate: number;
+    /** `null` when any case's cost is unknown. */
+    totalCostUsd: number | null;
+    latencyMs: { p50: number; p95: number; max: number };
+  };
+  /** The deploy decision: pass only when every threshold and regression check holds. */
+  gate: { pass: boolean; reasons: string[]; regressions: string[] };
+}

package/src/flow-eval-suite.ts

@@ -0,0 +1,169 @@
+// The frozen input set of a flow evaluation: its shape, strict parsing of
+// untrusted suite JSON, and its identity hash.
+
+import { createHash } from 'node:crypto';
+import { readFile } from 'node:fs/promises';
+import { canonicalize } from './canonical.js';
+import { FlowEvalError } from './flow-eval-error.js';
+import { snapshotJsonValue, type JsonSnapshotLimits, type JsonValue } from './json-value.js';
+import { MAX_DIRECT_INPUT_BYTES } from './direct-input.js';
+import { deepFreeze } from './flow-eval-version.js';
+
+export const MAX_FLOW_EVAL_CASES = 1_000;
+
+/** One case's input: no larger than `flows run --input` accepts. */
+const INPUT_LIMITS: JsonSnapshotLimits = Object.freeze({ maxDepth: 64, maxNodes: 262_144, maxBytes: MAX_DIRECT_INPUT_BYTES });
+
+/** What a case must observe to pass. Every field present must hold. */
+export interface FlowEvalExpectation {
+  /**
+   * The run's outcome. Defaults to `success`. `declined` is a successful run
+   * whose body chose not to act; `needs_human` is a parked run.
+   */
+  completionReason?: string;
+  /** Substrings the run's completion detail must contain. */
+  detailIncludes?: string[];
+  /** A regular expression (JavaScript syntax) the completion detail must match. */
+  detailMatches?: string;
+  /** Upper bound on this case's wall-clock latency. */
+  maxLatencyMs?: number;
+  /** Upper bound on this case's reported cost; an unknown cost fails it. */
+  maxCostUsd?: number;
+}
+
+export interface FlowEvalCase {
+  /** Stable identity across suite revisions; unique within the suite. */
+  id: string;
+  /** The run's input. Required by authored `.flow.ts`; refused by declarative specs on Cloud. */
+  input?: JsonValue;
+  expect?: FlowEvalExpectation;
+}
+
+export interface FlowEvalThresholds {
+  /** Fraction of cases that must pass, 0..1. Defaults to 1: every case. */
+  minPassRate?: number;
+  /** Ceiling on the summed cost of every case; an unknown cost fails it. */
+  maxTotalCostUsd?: number;
+  /** Ceiling on the 95th-percentile case latency. */
+  maxP95LatencyMs?: number;
+}
+
+export interface FlowEvalSuite {
+  name: string;
+  cases: FlowEvalCase[];
+  thresholds?: FlowEvalThresholds;
+}
+
+/** Validate untrusted suite data (parsed JSON) into a frozen suite. */
+export function parseFlowEvalSuite(value: unknown): FlowEvalSuite {
+  const fail = (message: string): never => { throw new FlowEvalError('invalid_suite', message); };
+  if (!isRecord(value)) fail('Suite must be a JSON object.');
+  const raw = value as Record<string, unknown>;
+  allowOnly(raw, ['name', 'cases', 'thresholds'], 'suite', fail);
+  if (typeof raw.name !== 'string' || raw.name.trim() === '') fail('Suite needs a nonempty `name`.');
+  if (!Array.isArray(raw.cases) || raw.cases.length === 0) fail('Suite needs a nonempty `cases` array.');
+  const rawCases = raw.cases as unknown[];
+  if (rawCases.length > MAX_FLOW_EVAL_CASES) fail(`Suite has ${rawCases.length} cases; the limit is ${MAX_FLOW_EVAL_CASES}.`);
+  const seen = new Set<string>();
+  const cases = rawCases.map((entry, index): FlowEvalCase => {
+    const at = `cases[${index}]`;
+    if (!isRecord(entry)) fail(`${at} must be an object.`);
+    const item = entry as Record<string, unknown>;
+    allowOnly(item, ['id', 'input', 'expect'], at, fail);
+    if (typeof item.id !== 'string' || !/^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/u.test(item.id)) {
+      fail(`${at}.id must be 1-128 characters of letters, digits, '.', '_', ':' or '-'.`);
+    }
+    const id = item.id as string;
+    if (seen.has(id)) fail(`${at}.id "${id}" is not unique.`);
+    seen.add(id);
+    const testCase: FlowEvalCase = { id };
+    if (Object.prototype.hasOwnProperty.call(item, 'input')) {
+      try {
+        testCase.input = snapshotJsonValue(item.input, `${at}.input`, INPUT_LIMITS);
+      } catch (error) {
+        fail(`${at}.input is not JSON data: ${error instanceof Error ? error.message : String(error)}`);
+      }
+    }
+    if (item.expect !== undefined) testCase.expect = parseExpectation(item.expect, `${at}.expect`, fail);
+    return testCase;
+  });
+  const suite: FlowEvalSuite = { name: raw.name as string, cases };
+  if (raw.thresholds !== undefined) {
+    if (!isRecord(raw.thresholds)) fail('`thresholds` must be an object.');
+    const t = raw.thresholds as Record<string, unknown>;
+    allowOnly(t, ['minPassRate', 'maxTotalCostUsd', 'maxP95LatencyMs'], 'thresholds', fail);
+    const thresholds: FlowEvalThresholds = {};
+    if (t.minPassRate !== undefined) {
+      if (typeof t.minPassRate !== 'number' || !(t.minPassRate >= 0 && t.minPassRate <= 1)) fail('`thresholds.minPassRate` must be a number from 0 to 1.');
+      thresholds.minPassRate = t.minPassRate as number;
+    }
+    if (t.maxTotalCostUsd !== undefined) thresholds.maxTotalCostUsd = nonNegative(t.maxTotalCostUsd, 'thresholds.maxTotalCostUsd', fail);
+    if (t.maxP95LatencyMs !== undefined) thresholds.maxP95LatencyMs = nonNegative(t.maxP95LatencyMs, 'thresholds.maxP95LatencyMs', fail);
+    suite.thresholds = thresholds;
+  }
+  // Frozen in fact, not just by name: a scorer receives these cases, and must
+  // not be able to rename one or loosen an expectation for the cases after it.
+  return deepFreeze(suite);
+}
+
+/** Read and validate a suite file (JSON). */
+export async function loadFlowEvalSuite(path: string): Promise<FlowEvalSuite> {
+  let text: string;
+  try {
+    text = await readFile(path, 'utf8');
+  } catch {
+    throw new FlowEvalError('invalid_suite', `Suite file "${path}" is not readable.`);
+  }
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(text);
+  } catch {
+    throw new FlowEvalError('invalid_suite', `Suite file "${path}" is not valid JSON.`);
+  }
+  return parseFlowEvalSuite(parsed);
+}
+
+/** The suite's identity: sha256 of its canonical JSON. */
+export function flowEvalSuiteSha256(suite: FlowEvalSuite): string {
+  return `sha256:${createHash('sha256').update(canonicalize(suite as unknown as JsonValue)).digest('hex')}`;
+}
+
+function parseExpectation(value: unknown, at: string, fail: (message: string) => never): FlowEvalExpectation {
+  if (!isRecord(value)) fail(`${at} must be an object.`);
+  const raw = value as Record<string, unknown>;
+  allowOnly(raw, ['completionReason', 'detailIncludes', 'detailMatches', 'maxLatencyMs', 'maxCostUsd'], at, fail);
+  const expect: FlowEvalExpectation = {};
+  if (raw.completionReason !== undefined) {
+    if (typeof raw.completionReason !== 'string' || raw.completionReason === '') fail(`${at}.completionReason must be a nonempty string.`);
+    expect.completionReason = raw.completionReason as string;
+  }
+  if (raw.detailIncludes !== undefined) {
+    if (!Array.isArray(raw.detailIncludes) || raw.detailIncludes.some(item => typeof item !== 'string')) fail(`${at}.detailIncludes must be an array of strings.`);
+    expect.detailIncludes = [...(raw.detailIncludes as string[])];
+  }
+  if (raw.detailMatches !== undefined) {
+    if (typeof raw.detailMatches !== 'string') fail(`${at}.detailMatches must be a string.`);
+    try {
+      new RegExp(raw.detailMatches as string, 'u');
+    } catch {
+      fail(`${at}.detailMatches is not a valid regular expression.`);
+    }
+    expect.detailMatches = raw.detailMatches as string;
+  }
+  if (raw.maxLatencyMs !== undefined) expect.maxLatencyMs = nonNegative(raw.maxLatencyMs, `${at}.maxLatencyMs`, fail);
+  if (raw.maxCostUsd !== undefined) expect.maxCostUsd = nonNegative(raw.maxCostUsd, `${at}.maxCostUsd`, fail);
+  return expect;
+}
+
+function allowOnly(raw: Record<string, unknown>, keys: readonly string[], at: string, fail: (message: string) => never): void {
+  for (const key of Object.keys(raw)) if (!keys.includes(key)) fail(`${at} has unknown field "${key}".`);
+}
+
+function nonNegative(value: unknown, at: string, fail: (message: string) => never): number {
+  if (typeof value !== 'number' || !Number.isFinite(value) || value < 0) fail(`\`${at}\` must be a non-negative number.`);
+  return value as number;
+}
+
+export function isRecord(value: unknown): value is Record<string, unknown> {
+  return value !== null && typeof value === 'object' && !Array.isArray(value);
+}