@bjoernrochel/sevro 0.1.0-rc.2 → 0.1.0-rc.21

package/src/report.ts

@@ -1,86 +1,26 @@
+import type {
+  CliResultData,
+  ReportData,
+  RunEvidenceData,
+} from "./schema-types";
 import { readFile } from "node:fs/promises";
-import { isAbsolute } from "node:path";
+import { isAbsolute, join, resolve } from "node:path";
 import { parseArgs } from "node:util";
 import { canonicalJson } from "./identity";
 import { assertCliResult, assertReport, assertRunEvidence } from "./schema";
+import { canonicalRuntimeRoot } from "./runtime/paths";
+import { nativeTranscriptRetention } from "./transcript-retention";
 
-type State = { status: string };
-type Verdict = { verdict: string };
-type DomainOutcome = { id: string; status: string };
-type Trial = {
-  trial: number;
-  execution: State;
-  grading: State;
-  task: Verdict;
-  domainOutcomes?: DomainOutcome[];
-};
-type Case = {
-  caseId: string;
-  execution: State;
-  grading: State;
-  task: Verdict;
-  trials: Trial[];
-};
-type CliResult = {
-  format: string;
-  runId: string | null;
-  execution: State;
-  grading: State;
-  task: Verdict;
-  exitCode: number;
-  evidencePath: string | null;
-  cases: Case[];
-};
-type Usage = {
-  inputTokens: number | null;
-  outputTokens: number | null;
-  costUsd: number | null;
-  complete: boolean;
-};
-type TrialEvidence = {
-  caseId: string;
-  trial: number;
-  executionMode: string;
-  candidateDurationMs?: number | null;
-  usage: Usage;
-};
-type Evidence = {
-  runId: string;
-  result: CliResult;
-  runner: { buildDigest: string };
-  evaluationIdentity: { digest: string };
-  condition: { requested: string; actual: string };
-  routes: Array<{ role: string; host: string; model: string; effort: string }>;
-  trials: TrialEvidence[];
-};
+type CliResult = CliResultData;
+type Evidence = RunEvidenceData;
+type Case = CliResult["cases"][number];
+type Trial = Case["trials"][number];
 
-export type ReportRow = {
-  resultFile: string;
-  evidencePath: string | null;
-  runId: string | null;
-  caseId: string;
-  execution: string;
-  grading: string;
-  task: string;
-  exitCode: number;
-  trialCount: number;
-  taskPassRate: number | null;
-  candidateDurationMs: number | null;
-  inputTokens: number | null;
-  outputTokens: number | null;
-  costUsd: number | null;
-  runnerBuildDigest: string | null;
-  evaluationDigest: string | null;
-  requestedCondition: string | null;
-  actualCondition: string | null;
-  candidateRoute: { host: string; model: string; effort: string } | null;
-  routes: Array<{ role: string; host: string; model: string; effort: string }>;
-  domainOutcomes: Array<{ trial: number; id: string; status: string }>;
-};
+export type ReportRow = ReportData["rows"][number];
 
 function sumKnown(values: Array<number | null | undefined>): number | null {
   return values.length && values.every((value) => value != null)
-    ? values.reduce<number>((total, value) => total + value!, 0)
+    ? values.reduce<number>((total, value) => total + value, 0)
     : null;
 }
 
@@ -104,24 +44,17 @@ function passRate(trials: Trial[]): number | null {
 async function loadResult(path: string) {
   if (!isAbsolute(path))
     throw new Error("report result paths must be absolute");
-  const result = JSON.parse(await readFile(path, "utf8")) as unknown;
+  const result: unknown = JSON.parse(await readFile(path, "utf8"));
   assertCliResult(result);
-  return result as CliResult;
+  return result;
 }
 
 async function loadEvidence(result: CliResult): Promise<Evidence | null> {
-  if (!result.evidencePath) {
-    if (result.cases.length || result.exitCode === 0)
-      throw new Error("completed result has no retained evidence");
-    return null;
-  }
-  if (!isAbsolute(result.evidencePath))
-    throw new Error("evidence path must be absolute");
-  const value = JSON.parse(
-    await readFile(result.evidencePath, "utf8"),
-  ) as unknown;
+  const path = retainedEvidencePath(result);
+  if (path === null) return null;
+  const value: unknown = JSON.parse(await readFile(path, "utf8"));
   assertRunEvidence(value);
-  const evidence = value as Evidence;
+  const evidence = value;
   if (
     evidence.runId !== result.runId ||
     canonicalJson(evidence.result) !== canonicalJson(result)
@@ -130,6 +63,17 @@ async function loadEvidence(result: CliResult): Promise<Evidence | null> {
   return evidence;
 }
 
+function retainedEvidencePath(result: CliResult): string | null {
+  if (!result.evidencePath) {
+    if (result.cases.length || result.exitCode === 0)
+      throw new Error("completed result has no retained evidence");
+    return null;
+  }
+  if (!isAbsolute(result.evidencePath))
+    throw new Error("evidence path must be absolute");
+  return result.evidencePath;
+}
+
 function trialEvidence(
   caseId: string,
   trials: Trial[],
@@ -153,15 +97,11 @@ function reportRow(
   result: CliResult,
   selected: Case,
   evidence: Evidence | null,
+  transcripts: Awaited<ReturnType<typeof nativeTranscriptRetention>>,
 ): ReportRow {
   const retained = trialEvidence(selected.caseId, selected.trials, evidence);
-  const candidateRoutes =
-    evidence?.routes.filter((route) => route.role === "candidate") ?? [];
-  if (evidence && candidateRoutes.length !== 1)
-    throw new Error("retained candidate route is ambiguous");
-  const completeUsage =
-    retained.length > 0 && retained.every((trial) => trial.usage.complete);
-  const route = candidateRoutes[0];
+  const candidateRoute = retainedCandidateRoute(evidence);
+  const usage = retainedUsage(retained);
   return {
     resultFile: path,
     evidencePath: result.evidencePath,
@@ -176,23 +116,17 @@ function reportRow(
     candidateDurationMs: sumKnown(
       retained.map((trial) => trial.candidateDurationMs),
     ),
-    inputTokens: completeUsage
-      ? sumKnown(retained.map((trial) => trial.usage.inputTokens))
-      : null,
-    outputTokens: completeUsage
-      ? sumKnown(retained.map((trial) => trial.usage.outputTokens))
-      : null,
-    costUsd: completeUsage
-      ? sumKnown(retained.map((trial) => trial.usage.costUsd))
-      : null,
-    runnerBuildDigest: evidence?.runner.buildDigest ?? null,
-    evaluationDigest: evidence?.evaluationIdentity.digest ?? null,
-    requestedCondition: evidence?.condition.requested ?? null,
-    actualCondition: evidence?.condition.actual ?? null,
-    candidateRoute: route
-      ? { host: route.host, model: route.model, effort: route.effort }
-      : null,
-    routes: evidence?.routes ?? [],
+    ...usage,
+    ...retainedIdentity(evidence),
+    candidateRoute,
+    nativeTranscripts: {
+      retained: transcripts.filter(
+        (item) => item.caseId === selected.caseId && !item.pruned,
+      ).length,
+      pruned: transcripts.filter(
+        (item) => item.caseId === selected.caseId && item.pruned,
+      ).length,
+    },
     domainOutcomes: selected.trials.flatMap((trial) =>
       (trial.domainOutcomes ?? []).map((outcome) => ({
         trial: trial.trial,
@@ -203,21 +137,58 @@ function reportRow(
   };
 }
 
+function retainedCandidateRoute(
+  evidence: Evidence | null,
+): ReportRow["candidateRoute"] {
+  if (!evidence) return null;
+  const routes = evidence.routes.filter((route) => route.role === "candidate"),
+    [route] = routes;
+  if (routes.length !== 1 || route === undefined)
+    throw new Error("retained candidate route is ambiguous");
+  return { host: route.host, model: route.model, effort: route.effort };
+}
+function retainedUsage(retained: Evidence["trials"]) {
+  const complete =
+    retained.length > 0 && retained.every((trial) => trial.usage.complete);
+  return {
+    inputTokens: complete
+      ? sumKnown(retained.map((trial) => trial.usage.inputTokens))
+      : null,
+    outputTokens: complete
+      ? sumKnown(retained.map((trial) => trial.usage.outputTokens))
+      : null,
+    costUsd: complete
+      ? sumKnown(retained.map((trial) => trial.usage.costUsd))
+      : null,
+  };
+}
+function retainedIdentity(evidence: Evidence | null) {
+  if (!evidence)
+    return {
+      runnerBuildDigest: null,
+      evaluationDigest: null,
+      requestedCondition: null,
+      actualCondition: null,
+      routes: [],
+    };
+  return {
+    runnerBuildDigest: evidence.runner.buildDigest,
+    evaluationDigest: evidence.evaluationIdentity.digest,
+    requestedCondition: evidence.condition.requested,
+    actualCondition: evidence.condition.actual,
+    routes: evidence.routes,
+  };
+}
+
 export async function createReport(paths: string[]) {
   if (!paths.length || new Set(paths).size !== paths.length)
     throw new Error("report needs distinct result files");
   const rows: ReportRow[] = [];
-  const inputs: Array<{
-    resultFile: string;
-    execution: string;
-    grading: string;
-    task: string;
-    exitCode: number;
-    caseCount: number;
-  }> = [];
+  const inputs: ReportData["inputs"] = [];
   for (const path of paths) {
     const result = await loadResult(path);
     const evidence = await loadEvidence(result);
+    const transcripts = await nativeTranscriptRetention(evidence);
     inputs.push({
       resultFile: path,
       execution: result.execution.status,
@@ -227,7 +198,7 @@ export async function createReport(paths: string[]) {
       caseCount: result.cases.length,
     });
     for (const selected of result.cases)
-      rows.push(reportRow(path, result, selected, evidence));
+      rows.push(reportRow(path, result, selected, evidence, transcripts));
   }
   const report = {
     format: "sevro.report.v1",
@@ -279,15 +250,13 @@ export function renderReport(
     "",
     "| Case | Task | Execution | Grading | Pass rate | Candidate time | Candidate tokens | Candidate cost | Condition | Candidate route |",
     "| --- | --- | --- | --- | ---: | ---: | ---: | ---: | --- | --- |",
-    ...report.rows.map(
-      (row) =>
-        `| ${cell(row.caseId)} | ${row.task} | ${row.execution} | ${row.grading} | ${row.taskPassRate === null ? "unknown" : `${(row.taskPassRate * 100).toFixed(0)}%`} | ${seconds(row.candidateDurationMs)} | ${measured(row.inputTokens)} / ${measured(row.outputTokens)} | ${row.costUsd === null ? "unknown" : `$${row.costUsd.toFixed(4)}`} | ${row.requestedCondition ?? "unknown"} / ${row.actualCondition ?? "unknown"} | ${row.candidateRoute ? cell(`${row.candidateRoute.host}/${row.candidateRoute.model}@${row.candidateRoute.effort}`) : "unknown"} |`,
-    ),
+    ...report.rows.map(caseOutcomeLine),
     "",
     "## Retained evidence",
     "",
     ...report.rows.map(
-      (row) => `- ${cell(row.caseId)}: ${row.evidencePath ?? "unavailable"}`,
+      (row) =>
+        `- ${cell(row.caseId)}: ${row.evidencePath ?? "unavailable"}${transcriptSummary(row)}`,
     ),
     "",
   ];
@@ -324,28 +293,91 @@ export function renderReport(
   return lines.join("\n");
 }
 
+function transcriptSummary(row: ReportRow): string {
+  if (!row.nativeTranscripts) return "";
+  return `; native transcripts: ${row.nativeTranscripts.retained} retained, ${row.nativeTranscripts.pruned} pruned`;
+}
+
+function passRateCell(rate: number | null): string {
+  return rate === null ? "unknown" : `${(rate * 100).toFixed(0)}%`;
+}
+function costCell(cost: number | null): string {
+  return cost === null ? "unknown" : `$${cost.toFixed(4)}`;
+}
+function routeCell(route: ReportRow["candidateRoute"]): string {
+  return route
+    ? cell(`${route.host}/${route.model}@${route.effort}`)
+    : "unknown";
+}
+function conditionCell(row: ReportRow): string {
+  return `${row.requestedCondition ?? "unknown"} / ${row.actualCondition ?? "unknown"}`;
+}
+function caseOutcomeLine(row: ReportRow): string {
+  return `| ${cell(row.caseId)} | ${row.task} | ${row.execution} | ${row.grading} | ${passRateCell(row.taskPassRate)} | ${seconds(row.candidateDurationMs)} | ${measured(row.inputTokens)} / ${measured(row.outputTokens)} | ${costCell(row.costUsd)} | ${conditionCell(row)} | ${routeCell(row.candidateRoute)} |`;
+}
+
 export async function reportCommand(argv: string[]): Promise<number> {
   try {
-    const { values, positionals } = parseArgs({
-      args: argv,
-      allowPositionals: true,
-      strict: true,
-      options: {
-        json: { type: "boolean" },
-        "result-file": { type: "string", multiple: true },
-      },
-    });
-    if (positionals.length !== 1 || positionals[0] !== "report")
-      throw new Error("expected the report command");
-    const report = await createReport(values["result-file"] ?? []);
+    const values = reportInvocation(argv);
+    const report = await createReport(await reportPaths(values));
     process.stdout.write(
       values.json ? `${JSON.stringify(report)}\n` : renderReport(report),
     );
     return 0;
   } catch (error) {
-    process.stderr.write(
-      `${error instanceof Error ? error.message : "invalid report input"}\n`,
-    );
+    process.stderr.write(`${reportErrorMessage(error)}\n`);
     return 64;
   }
 }
+
+function reportInvocation(argv: string[]) {
+  const { values, positionals } = parseArgs({
+    args: argv,
+    allowPositionals: true,
+    strict: true,
+    options: {
+      json: { type: "boolean" },
+      "result-file": { type: "string", multiple: true },
+      latest: { type: "boolean" },
+      "results-root": { type: "string" },
+    },
+  });
+  if (positionals.length !== 1 || positionals[0] !== "report")
+    throw new Error("expected the report command");
+  return values;
+}
+
+async function reportPaths(
+  values: ReturnType<typeof reportInvocation>,
+): Promise<string[]> {
+  validateReportSource(values);
+  const paths = selectedReportPaths(values);
+  if (paths.some((path) => !path.trim()))
+    throw new Error("--result-file must name a readable retained result file");
+  return Promise.all(paths.map((path) => canonicalRuntimeRoot(resolve(path))));
+}
+
+function validateReportSource(
+  values: ReturnType<typeof reportInvocation>,
+): void {
+  if (values.latest && values["result-file"] !== undefined)
+    throw new Error("--latest and --result-file are exclusive");
+  if (values["results-root"] !== undefined && !values.latest)
+    throw new Error("--results-root requires --latest");
+}
+
+function selectedReportPaths(
+  values: ReturnType<typeof reportInvocation>,
+): string[] {
+  return values.latest
+    ? [
+        join(
+          values["results-root"] ?? join(process.cwd(), ".sevro", "results"),
+          "latest.json",
+        ),
+      ]
+    : (values["result-file"] ?? []);
+}
+function reportErrorMessage(error: unknown): string {
+  return error instanceof Error ? error.message : "invalid report input";
+}

package/src/resolved-case.ts

@@ -0,0 +1,33 @@
+import type { ExtensionCase } from "./extension-session";
+import type { RepositoryFixture } from "./fixtures/repository";
+
+function repositoryOverlay(fixture: RepositoryFixture) {
+  return {
+    ...(fixture.files ? { files: fixture.files } : {}),
+    ...(fixture.staged ? { staged: fixture.staged } : {}),
+    ...(fixture.commitFiles ? { commitFiles: true } : {}),
+  };
+}
+
+function repositoryTools(fixture: RepositoryFixture) {
+  return {
+    ...(fixture.hooks ? { hooks: fixture.hooks } : {}),
+    ...(fixture.bin ? { bin: fixture.bin } : {}),
+  };
+}
+
+/** Translate negotiated fixture declarations into the standalone case representation. */
+export function resolvedFixture(fixture: ExtensionCase["fixture"]) {
+  switch (fixture.kind) {
+    case "inline":
+      return { files: fixture.files };
+    case "repository":
+      return {
+        sourceRef: fixture.sourceRef,
+        ...repositoryOverlay(fixture),
+        ...repositoryTools(fixture),
+      };
+    case "generated":
+      return fixture;
+  }
+}

package/src/results-lock.ts

@@ -0,0 +1,29 @@
+import { Database } from "bun:sqlite";
+import { join } from "node:path";
+import { lstat } from "node:fs/promises";
+
+export async function retainedFile(path: string) {
+  const info = await lstat(path).catch((cause: unknown) => {
+    if ((cause as NodeJS.ErrnoException).code === "ENOENT") return null;
+    throw cause;
+  });
+  if (info && !info.isFile())
+    throw new Error(`retained storage must be a regular file: ${path}`);
+  return info;
+}
+
+/** Serialize result publication and pruning across CLI processes. Closing rolls back the lock transaction. */
+export async function withResultsLock<T>(
+  root: string,
+  action: () => Promise<T>,
+): Promise<T> {
+  await retainedFile(join(root, ".retention-lock.sqlite"));
+  const database = new Database(join(root, ".retention-lock.sqlite"));
+  try {
+    database.run("PRAGMA busy_timeout = 5000");
+    database.run("BEGIN IMMEDIATE");
+    return await action();
+  } finally {
+    database.close();
+  }
+}