@bjoernrochel/sevro 0.1.0-rc.1

package/src/report.ts

@@ -0,0 +1,351 @@
+import { readFile } from "node:fs/promises";
+import { isAbsolute } from "node:path";
+import { parseArgs } from "node:util";
+import { canonicalJson } from "./identity";
+import { assertCliResult, assertReport, assertRunEvidence } from "./schema";
+
+type State = { status: string };
+type Verdict = { verdict: string };
+type DomainOutcome = { id: string; status: string };
+type Trial = {
+  trial: number;
+  execution: State;
+  grading: State;
+  task: Verdict;
+  domainOutcomes?: DomainOutcome[];
+};
+type Case = {
+  caseId: string;
+  execution: State;
+  grading: State;
+  task: Verdict;
+  trials: Trial[];
+};
+type CliResult = {
+  format: string;
+  runId: string | null;
+  execution: State;
+  grading: State;
+  task: Verdict;
+  exitCode: number;
+  evidencePath: string | null;
+  cases: Case[];
+};
+type Usage = {
+  inputTokens: number | null;
+  outputTokens: number | null;
+  costUsd: number | null;
+  complete: boolean;
+};
+type TrialEvidence = {
+  caseId: string;
+  trial: number;
+  executionMode: string;
+  candidateDurationMs?: number | null;
+  usage: Usage;
+};
+type Evidence = {
+  runId: string;
+  result: CliResult;
+  runner: { buildDigest: string };
+  evaluationIdentity: { digest: string };
+  condition: { requested: string; actual: string };
+  routes: Array<{ role: string; host: string; model: string; effort: string }>;
+  trials: TrialEvidence[];
+};
+
+export type ReportRow = {
+  resultFile: string;
+  evidencePath: string | null;
+  runId: string | null;
+  caseId: string;
+  execution: string;
+  grading: string;
+  task: string;
+  exitCode: number;
+  trialCount: number;
+  taskPassRate: number | null;
+  candidateDurationMs: number | null;
+  inputTokens: number | null;
+  outputTokens: number | null;
+  costUsd: number | null;
+  runnerBuildDigest: string | null;
+  evaluationDigest: string | null;
+  requestedCondition: string | null;
+  actualCondition: string | null;
+  candidateRoute: { host: string; model: string; effort: string } | null;
+  routes: Array<{ role: string; host: string; model: string; effort: string }>;
+  domainOutcomes: Array<{ trial: number; id: string; status: string }>;
+};
+
+function sumKnown(values: Array<number | null | undefined>): number | null {
+  return values.length && values.every((value) => value != null)
+    ? values.reduce<number>((total, value) => total + value!, 0)
+    : null;
+}
+
+function passRate(trials: Trial[]): number | null {
+  if (
+    !trials.length ||
+    trials.some(
+      (trial) =>
+        trial.execution.status !== "completed" ||
+        trial.grading.status !== "completed" ||
+        trial.task.verdict === "not_assessed",
+    )
+  )
+    return null;
+  return (
+    trials.filter((trial) => trial.task.verdict === "passed").length /
+    trials.length
+  );
+}
+
+async function loadResult(path: string) {
+  if (!isAbsolute(path))
+    throw new Error("report result paths must be absolute");
+  const result = JSON.parse(await readFile(path, "utf8")) as unknown;
+  assertCliResult(result);
+  return result as CliResult;
+}
+
+async function loadEvidence(result: CliResult): Promise<Evidence | null> {
+  if (!result.evidencePath) {
+    if (result.cases.length || result.exitCode === 0)
+      throw new Error("completed result has no retained evidence");
+    return null;
+  }
+  if (!isAbsolute(result.evidencePath))
+    throw new Error("evidence path must be absolute");
+  const value = JSON.parse(
+    await readFile(result.evidencePath, "utf8"),
+  ) as unknown;
+  assertRunEvidence(value);
+  const evidence = value as Evidence;
+  if (
+    evidence.runId !== result.runId ||
+    canonicalJson(evidence.result) !== canonicalJson(result)
+  )
+    throw new Error("retained evidence differs from report result");
+  return evidence;
+}
+
+function trialEvidence(
+  caseId: string,
+  trials: Trial[],
+  evidence: Evidence | null,
+) {
+  if (!evidence) return [];
+  const matching = evidence.trials.filter((item) => item.caseId === caseId);
+  if (
+    matching.length !== trials.length ||
+    trials.some(
+      (trial) =>
+        matching.filter((item) => item.trial === trial.trial).length !== 1,
+    )
+  )
+    throw new Error("retained trial evidence differs from report result");
+  return matching;
+}
+
+function reportRow(
+  path: string,
+  result: CliResult,
+  selected: Case,
+  evidence: Evidence | null,
+): ReportRow {
+  const retained = trialEvidence(selected.caseId, selected.trials, evidence);
+  const candidateRoutes =
+    evidence?.routes.filter((route) => route.role === "candidate") ?? [];
+  if (evidence && candidateRoutes.length !== 1)
+    throw new Error("retained candidate route is ambiguous");
+  const completeUsage =
+    retained.length > 0 && retained.every((trial) => trial.usage.complete);
+  const route = candidateRoutes[0];
+  return {
+    resultFile: path,
+    evidencePath: result.evidencePath,
+    runId: result.runId,
+    caseId: selected.caseId,
+    execution: selected.execution.status,
+    grading: selected.grading.status,
+    task: selected.task.verdict,
+    exitCode: result.exitCode,
+    trialCount: selected.trials.length,
+    taskPassRate: passRate(selected.trials),
+    candidateDurationMs: sumKnown(
+      retained.map((trial) => trial.candidateDurationMs),
+    ),
+    inputTokens: completeUsage
+      ? sumKnown(retained.map((trial) => trial.usage.inputTokens))
+      : null,
+    outputTokens: completeUsage
+      ? sumKnown(retained.map((trial) => trial.usage.outputTokens))
+      : null,
+    costUsd: completeUsage
+      ? sumKnown(retained.map((trial) => trial.usage.costUsd))
+      : null,
+    runnerBuildDigest: evidence?.runner.buildDigest ?? null,
+    evaluationDigest: evidence?.evaluationIdentity.digest ?? null,
+    requestedCondition: evidence?.condition.requested ?? null,
+    actualCondition: evidence?.condition.actual ?? null,
+    candidateRoute: route
+      ? { host: route.host, model: route.model, effort: route.effort }
+      : null,
+    routes: evidence?.routes ?? [],
+    domainOutcomes: selected.trials.flatMap((trial) =>
+      (trial.domainOutcomes ?? []).map((outcome) => ({
+        trial: trial.trial,
+        id: outcome.id,
+        status: outcome.status,
+      })),
+    ),
+  };
+}
+
+export async function createReport(paths: string[]) {
+  if (!paths.length || new Set(paths).size !== paths.length)
+    throw new Error("report needs distinct result files");
+  const rows: ReportRow[] = [];
+  const inputs: Array<{
+    resultFile: string;
+    execution: string;
+    grading: string;
+    task: string;
+    exitCode: number;
+    caseCount: number;
+  }> = [];
+  for (const path of paths) {
+    const result = await loadResult(path);
+    const evidence = await loadEvidence(result);
+    inputs.push({
+      resultFile: path,
+      execution: result.execution.status,
+      grading: result.grading.status,
+      task: result.task.verdict,
+      exitCode: result.exitCode,
+      caseCount: result.cases.length,
+    });
+    for (const selected of result.cases)
+      rows.push(reportRow(path, result, selected, evidence));
+  }
+  const report = {
+    format: "sevro.report.v1",
+    inputs,
+    rows,
+    summary: {
+      cases: rows.length,
+      passed: rows.filter((row) => row.task === "passed").length,
+      failed: rows.filter((row) => row.task === "failed").length,
+      notAssessed: rows.filter((row) => row.task === "not_assessed").length,
+    },
+  };
+  assertReport(report);
+  return report;
+}
+
+function cell(value: string): string {
+  return value.replaceAll("|", "\\|").replaceAll(/[\r\n]/g, " ");
+}
+
+function measured(value: number | null, digits = 0): string {
+  return value === null ? "unknown" : value.toFixed(digits);
+}
+
+function seconds(value: number | null): string {
+  return value === null ? "unknown" : `${(value / 1000).toFixed(2)}s`;
+}
+
+export function renderReport(
+  report: Awaited<ReturnType<typeof createReport>>,
+): string {
+  const lines = [
+    "# Sevro evaluation report",
+    "",
+    "Task verdict, execution, and grading are separate. Unknown measurements are not zero. Rows do not claim matched comparisons.",
+    "",
+    `Cases: ${report.summary.cases}; passed: ${report.summary.passed}; failed: ${report.summary.failed}; not assessed: ${report.summary.notAssessed}.`,
+    "",
+    "## Input runs",
+    "",
+    "| Result file | Exit | Execution | Grading | Task | Cases |",
+    "| --- | ---: | --- | --- | --- | ---: |",
+    ...report.inputs.map(
+      (input) =>
+        `| ${cell(input.resultFile)} | ${input.exitCode} | ${input.execution} | ${input.grading} | ${input.task} | ${input.caseCount} |`,
+    ),
+    "",
+    "## Case outcomes",
+    "",
+    "| Case | Task | Execution | Grading | Pass rate | Candidate time | Candidate tokens | Candidate cost | Condition | Candidate route |",
+    "| --- | --- | --- | --- | ---: | ---: | ---: | ---: | --- | --- |",
+    ...report.rows.map(
+      (row) =>
+        `| ${cell(row.caseId)} | ${row.task} | ${row.execution} | ${row.grading} | ${row.taskPassRate === null ? "unknown" : `${(row.taskPassRate * 100).toFixed(0)}%`} | ${seconds(row.candidateDurationMs)} | ${measured(row.inputTokens)} / ${measured(row.outputTokens)} | ${row.costUsd === null ? "unknown" : `$${row.costUsd.toFixed(4)}`} | ${row.requestedCondition ?? "unknown"} / ${row.actualCondition ?? "unknown"} | ${row.candidateRoute ? cell(`${row.candidateRoute.host}/${row.candidateRoute.model}@${row.candidateRoute.effort}`) : "unknown"} |`,
+    ),
+    "",
+    "## Retained evidence",
+    "",
+    ...report.rows.map(
+      (row) => `- ${cell(row.caseId)}: ${row.evidencePath ?? "unavailable"}`,
+    ),
+    "",
+  ];
+  const outcomes = report.rows.flatMap((row) =>
+    row.domainOutcomes.map((outcome) => ({ caseId: row.caseId, ...outcome })),
+  );
+  if (outcomes.length)
+    lines.push(
+      "## Domain outcomes",
+      "",
+      "Domain outcomes are independent of the task verdict.",
+      "",
+      "| Case | Trial | Outcome | Status |",
+      "| --- | ---: | --- | --- |",
+      ...outcomes.map(
+        (outcome) =>
+          `| ${cell(outcome.caseId)} | ${outcome.trial} | ${cell(outcome.id)} | ${outcome.status} |`,
+      ),
+      "",
+    );
+  lines.push(
+    "## Routes",
+    "",
+    "| Case | Role | Host | Model | Effort |",
+    "| --- | --- | --- | --- | --- |",
+    ...report.rows.flatMap((row) =>
+      row.routes.map(
+        (route) =>
+          `| ${cell(row.caseId)} | ${route.role} | ${cell(route.host)} | ${cell(route.model)} | ${cell(route.effort)} |`,
+      ),
+    ),
+    "",
+  );
+  return lines.join("\n");
+}
+
+export async function reportCommand(argv: string[]): Promise<number> {
+  try {
+    const { values, positionals } = parseArgs({
+      args: argv,
+      allowPositionals: true,
+      strict: true,
+      options: {
+        json: { type: "boolean" },
+        "result-file": { type: "string", multiple: true },
+      },
+    });
+    if (positionals.length !== 1 || positionals[0] !== "report")
+      throw new Error("expected the report command");
+    const report = await createReport(values["result-file"] ?? []);
+    process.stdout.write(
+      values.json ? `${JSON.stringify(report)}\n` : renderReport(report),
+    );
+    return 0;
+  } catch (error) {
+    process.stderr.write(
+      `${error instanceof Error ? error.message : "invalid report input"}\n`,
+    );
+    return 64;
+  }
+}

package/src/repository-fixture.ts

@@ -0,0 +1,286 @@
+import { lstat, mkdir, realpath, stat, writeFile } from "node:fs/promises";
+import { isAbsolute, join, relative, sep } from "node:path";
+import { fileURLToPath } from "node:url";
+import { canonicalJson } from "./identity";
+import { prepareFixtureBin } from "./fixture-bin";
+import { prepareGitHooks } from "./git-hooks";
+import type { PreparationSources } from "./preparation";
+import { fixtureParts } from "./preparation";
+
+const MAX_GIT_OUTPUT_BYTES = 64 * 1024;
+const MAX_OVERLAY_FILES = 1024;
+const MAX_OVERLAY_BYTES = 32 * 1024 * 1024;
+
+export interface RepositoryFixture {
+  kind: "repository";
+  sourceRef: string;
+  files?: Record<string, string>;
+  staged?: string[];
+  commitFiles?: boolean;
+  hooks?: Record<string, string>;
+  bin?: Record<string, string>;
+}
+
+/** Validate repository source identity and bounded working-tree changes. */
+export function prepareRepositoryFixture(value: unknown): RepositoryFixture {
+  if (!value || typeof value !== "object" || Array.isArray(value))
+    throw new Error("invalid repository fixture");
+  const fixture = value as Record<string, unknown>;
+  if (
+    fixture.kind !== "repository" ||
+    Object.keys(fixture).some(
+      (key) =>
+        ![
+          "kind",
+          "sourceRef",
+          "files",
+          "staged",
+          "commitFiles",
+          "hooks",
+          "bin",
+        ].includes(key),
+    ) ||
+    typeof fixture.sourceRef !== "string" ||
+    !fixture.sourceRef ||
+    (fixture.files !== undefined &&
+      (!fixture.files ||
+        typeof fixture.files !== "object" ||
+        Array.isArray(fixture.files) ||
+        Object.values(fixture.files).some(
+          (item) => typeof item !== "string",
+        ))) ||
+    (fixture.staged !== undefined &&
+      (!Array.isArray(fixture.staged) ||
+        fixture.staged.some((item) => typeof item !== "string"))) ||
+    (fixture.commitFiles !== undefined &&
+      typeof fixture.commitFiles !== "boolean")
+  )
+    throw new Error("invalid repository fixture");
+  prepareGitHooks(fixture.hooks);
+  prepareFixtureBin(fixture.bin);
+  const files = (fixture.files ?? {}) as Record<string, string>;
+  const staged = (fixture.staged ?? []) as string[];
+  const paths = Object.keys(files);
+  if (paths.length > MAX_OVERLAY_FILES)
+    throw new Error("repository fixture exceeds the file limit");
+  let bytes = 0;
+  for (const [path, content] of Object.entries(files)) {
+    const parts = fixtureParts(path);
+    if (parts.some((part) => part.toLowerCase() === ".git"))
+      throw new Error("repository fixture cannot write repository metadata");
+    bytes +=
+      Buffer.byteLength(path, "utf8") + Buffer.byteLength(content, "utf8");
+  }
+  bytes += staged.reduce(
+    (size, path) => size + Buffer.byteLength(path, "utf8"),
+    0,
+  );
+  if (bytes > MAX_OVERLAY_BYTES)
+    throw new Error("repository fixture exceeds the size limit");
+  const normalized = paths.map((path) => path.toLowerCase());
+  for (let index = 0; index < normalized.length; index++) {
+    for (let other = 0; other < index; other++) {
+      if (
+        normalized[index] === normalized[other] ||
+        normalized[index]!.startsWith(`${normalized[other]}/`) ||
+        normalized[other]!.startsWith(`${normalized[index]}/`)
+      )
+        throw new Error("repository fixture overlay paths collide");
+    }
+  }
+  if (
+    new Set(staged.map((path) => path.toLowerCase())).size !== staged.length ||
+    staged.some((path) => !Object.hasOwn(files, path)) ||
+    (fixture.commitFiles === true && paths.length === 0)
+  )
+    throw new Error("invalid repository fixture staging");
+  return JSON.parse(canonicalJson(fixture)) as RepositoryFixture;
+}
+
+export interface RepositorySource {
+  path: string;
+  revision: string;
+}
+
+async function boundedText(
+  stream: ReadableStream<Uint8Array>,
+): Promise<string> {
+  const chunks: Uint8Array[] = [];
+  let size = 0;
+  for await (const chunk of stream) {
+    size += chunk.byteLength;
+    if (size > MAX_GIT_OUTPUT_BYTES)
+      throw new Error("repository fixture Git output is too large");
+    chunks.push(chunk);
+  }
+  return new TextDecoder("utf-8", { fatal: true }).decode(
+    Buffer.concat(chunks),
+  );
+}
+
+async function git(args: string[], cwd: string): Promise<string> {
+  const proc = Bun.spawn(["git", ...args], {
+    cwd,
+    stdout: "pipe",
+    stderr: "pipe",
+    env: {
+      ...process.env,
+      GIT_CONFIG_NOSYSTEM: "1",
+      GIT_CONFIG_GLOBAL: process.platform === "win32" ? "NUL" : "/dev/null",
+      GIT_TERMINAL_PROMPT: "0",
+      GIT_AUTHOR_NAME: "Sevro Fixture",
+      GIT_AUTHOR_EMAIL: "fixture@sevro.invalid",
+      GIT_COMMITTER_NAME: "Sevro Fixture",
+      GIT_COMMITTER_EMAIL: "fixture@sevro.invalid",
+      GIT_AUTHOR_DATE: "2000-01-01T00:00:00+00:00",
+      GIT_COMMITTER_DATE: "2000-01-01T00:00:00+00:00",
+    },
+  });
+  try {
+    const [stdout, _stderr, code] = await Promise.all([
+      boundedText(proc.stdout),
+      boundedText(proc.stderr),
+      proc.exited,
+    ]);
+    if (code !== 0) throw new Error("repository fixture Git operation failed");
+    return stdout.trim();
+  } catch (error) {
+    proc.kill("SIGKILL");
+    await proc.exited;
+    throw error;
+  }
+}
+
+/** Resolve a declared local repository without accepting a dirty source tree. */
+export async function resolveRepositorySource(
+  sourceRef: string,
+  sources?: PreparationSources,
+): Promise<RepositorySource> {
+  if (!sources || !isAbsolute(sources.root))
+    throw new Error("repository source root is required and must be absolute");
+  if (!Object.hasOwn(sources.refs, sourceRef))
+    throw new Error("repository source reference is not declared");
+  const url = sources.refs[sourceRef];
+  if (typeof url !== "string" || !url.startsWith("file:///"))
+    throw new Error("repository source reference must be a file URL");
+  let root: string;
+  let path: string;
+  try {
+    [root, path] = await Promise.all([
+      realpath(sources.root),
+      realpath(fileURLToPath(url)),
+    ]);
+    if (!(await stat(path)).isDirectory()) throw new Error();
+  } catch {
+    throw new Error("repository source is unreadable");
+  }
+  const child = relative(root, path);
+  if (child === ".." || child.startsWith(`..${sep}`) || isAbsolute(child))
+    throw new Error("repository source escapes its declared root");
+  const top = await git(["rev-parse", "--show-toplevel"], path);
+  if ((await realpath(top)) !== path)
+    throw new Error("repository source must name the repository root");
+  const revision = await git(["rev-parse", "--verify", "HEAD^{commit}"], path);
+  if (!/^[a-f0-9]{40,64}$/.test(revision))
+    throw new Error("repository source has no valid commit");
+  if (await git(["status", "--porcelain=v1", "--untracked-files=all"], path))
+    throw new Error("repository source has uncommitted changes");
+  if (await git(["submodule", "status"], path))
+    throw new Error("repository submodules are not supported");
+  return { path, revision };
+}
+
+/** Clone a fixed commit without hardlinks or a remote back to the source. */
+export async function cloneRepositorySource(
+  sourceRef: string,
+  sources: PreparationSources,
+  expected: RepositorySource,
+  workspace: string,
+): Promise<void> {
+  const current = await resolveRepositorySource(sourceRef, sources);
+  if (current.path !== expected.path || current.revision !== expected.revision)
+    throw new Error("repository source changed during run");
+  await git(
+    [
+      "clone",
+      "--local",
+      "--no-hardlinks",
+      "--no-checkout",
+      "--quiet",
+      current.path,
+      workspace,
+    ],
+    workspace,
+  );
+  await git(["checkout", "--detach", "--quiet", expected.revision], workspace);
+  if ((await git(["rev-parse", "HEAD"], workspace)) !== expected.revision)
+    throw new Error("repository fixture revision changed during clone");
+  for (const remote of (await git(["remote"], workspace))
+    .split("\n")
+    .filter(Boolean))
+    await git(["remote", "remove", remote], workspace);
+  const after = await resolveRepositorySource(sourceRef, sources);
+  if (after.path !== expected.path || after.revision !== expected.revision)
+    throw new Error("repository source changed during clone");
+}
+
+async function safeOverlayTarget(
+  workspace: string,
+  path: string,
+): Promise<string> {
+  const parts = fixtureParts(path);
+  let parent = workspace;
+  for (const part of parts.slice(0, -1)) {
+    parent = join(parent, part);
+    try {
+      const entry = await lstat(parent);
+      if (!entry.isDirectory() || entry.isSymbolicLink())
+        throw new Error("repository fixture overlay traverses a non-directory");
+    } catch (error) {
+      if (!(
+        error instanceof Error &&
+        "code" in error &&
+        error.code === "ENOENT"
+      ))
+        throw error;
+      await mkdir(parent, { mode: 0o700 });
+    }
+  }
+  const target = join(workspace, ...parts);
+  try {
+    const entry = await lstat(target);
+    if (!entry.isFile() || entry.isSymbolicLink())
+      throw new Error("repository fixture overlay targets a non-file");
+  } catch (error) {
+    if (!(error instanceof Error && "code" in error && error.code === "ENOENT"))
+      throw error;
+  }
+  return target;
+}
+
+/** Apply declared files after cloning, optionally committing or staging them. */
+export async function applyRepositoryOverlay(
+  fixture: RepositoryFixture,
+  workspace: string,
+): Promise<void> {
+  const files = fixture.files ?? {};
+  for (const [path, content] of Object.entries(files))
+    await writeFile(await safeOverlayTarget(workspace, path), content, {
+      flag: "w",
+      mode: 0o600,
+    });
+  const paths = Object.keys(files);
+  const noHooks = [
+    "-c",
+    `core.hooksPath=${process.platform === "win32" ? "NUL" : "/dev/null"}`,
+  ];
+  if (fixture.commitFiles) {
+    await git([...noHooks, "add", "--", ...paths], workspace);
+    await git(
+      [...noHooks, "commit", "--quiet", "-m", "Add evaluation scaffolding"],
+      workspace,
+    );
+  }
+  if (fixture.staged?.length)
+    await git([...noHooks, "add", "--", ...fixture.staged], workspace);
+}