@bjoernrochel/sevro 0.1.0-rc.8 → 1.1.0

package/src/prune.ts

@@ -0,0 +1,300 @@
+import { lstat, readFile, readdir, realpath, rm, stat } from "node:fs/promises";
+import { basename, dirname, join, resolve } from "node:path";
+import { parseArgs } from "node:util";
+import { assertCliResult, assertRunEvidence } from "./schema";
+import type { CliResultData, RunEvidenceData } from "./schema-types";
+import { createReport } from "./report";
+import { atomicWriteJson } from "./storage";
+import { isRecord } from "./value-guards";
+import {
+  acquireResultsLock,
+  retainedFile,
+  withResultsLock,
+} from "./results-lock";
+import { pruneNativeTranscripts } from "./transcript-retention";
+
+interface RetainedResult {
+  directory: string;
+  result: CliResultData;
+  evidence: RunEvidenceData;
+  time: number;
+}
+
+interface RetainedStorage {
+  directories: string[];
+  roots: string[];
+  releases: (() => void)[];
+}
+
+type RetentionPolicy =
+  { keep: number; cutoff: null } | { keep: null; cutoff: number };
+
+const durationUnits: Record<string, number> = {
+  ms: 1,
+  s: 1000,
+  m: 60_000,
+  h: 3_600_000,
+  d: 86_400_000,
+  w: 604_800_000,
+};
+
+function duration(value: string): number {
+  const match = /^([1-9]\d*)(ms|s|m|h|d|w)$/.exec(value);
+  if (!match)
+    throw new Error(
+      "--older-than requires a positive integer and ms, s, m, h, d or w",
+    );
+  const amount =
+    Number.parseInt(value, 10) *
+    Number(durationUnits[value.replace(/^\d+/, "")]);
+  if (!Number.isSafeInteger(amount))
+    throw new Error("--older-than exceeds the safe duration range");
+  return amount;
+}
+
+function count(value: string): number {
+  const amount = Number(value);
+  if (!/^[1-9]\d*$/.test(value) || !Number.isSafeInteger(amount))
+    throw new Error("--keep must be a positive safe integer");
+  return amount;
+}
+
+function parseInvocation(args: string[]) {
+  const { values } = parseArgs({
+    args,
+    strict: true,
+    options: {
+      "results-root": { type: "string" },
+      keep: { type: "string" },
+      latest: { type: "boolean" },
+      "older-than": { type: "string" },
+      "native-transcripts": { type: "boolean" },
+    },
+  });
+  return values;
+}
+
+function policy(values: ReturnType<typeof parseInvocation>): RetentionPolicy {
+  if (values.latest) return { keep: 1, cutoff: null };
+  if (values["older-than"] !== undefined)
+    return { keep: null, cutoff: Date.now() - duration(values["older-than"]) };
+  if (values.keep !== undefined)
+    return { keep: count(values.keep), cutoff: null };
+  return { keep: values["native-transcripts"] ? 0 : 10, cutoff: null };
+}
+
+function invocation(args: string[]) {
+  const values = parseInvocation(args);
+  const policies = [values.latest, values.keep, values["older-than"]].filter(
+    (value) => value !== undefined,
+  );
+  if (policies.length > 1) throw new Error("retention policies are exclusive");
+  if (values["results-root"] === "")
+    throw new Error("--results-root must not be empty");
+  return {
+    root: resolve(values["results-root"] ?? ".sevro/results"),
+    policy: policy(values),
+    nativeTranscripts: values["native-transcripts"] === true,
+  };
+}
+
+async function retainedResults(
+  root: string,
+  directories: string[],
+): Promise<RetainedResult[]> {
+  const results: RetainedResult[] = [];
+  for (const directory of directories) {
+    const path = join(directory, "result.json");
+    await retainedFile(path);
+    const result: unknown = JSON.parse(await readFile(path, "utf8"));
+    assertCliResult(result);
+    if (!(await finalizedAncestors(root, directory, result.runId))) continue;
+    if (result.evidencePath !== join(directory, "run.json"))
+      throw new Error(
+        `retained result references evidence outside its directory: ${path}`,
+      );
+    await retainedFile(result.evidencePath);
+    const evidence: unknown = JSON.parse(
+      await readFile(result.evidencePath, "utf8"),
+    );
+    assertRunEvidence(evidence);
+    await createReport([path]);
+    results.push({
+      directory,
+      result,
+      evidence,
+      time: (await stat(path)).mtimeMs,
+    });
+  }
+  return results.sort(
+    (left, right) =>
+      right.time - left.time || left.directory.localeCompare(right.directory),
+  );
+}
+
+async function resultDirectories(
+  parent: string,
+  storage: RetainedStorage,
+): Promise<void> {
+  let children = await resultChildren(parent);
+  if (
+    !storage.roots.includes(parent) &&
+    ((await resultsRoot(parent)) || children.results.length > 0)
+  ) {
+    storage.releases.push(await acquireResultsLock(parent));
+    storage.roots.push(parent);
+    children = await resultChildren(parent);
+  }
+  storage.directories.push(...children.results);
+  for (const directory of children.containers)
+    await resultDirectories(directory, storage);
+}
+
+async function resultChildren(parent: string) {
+  const results: string[] = [];
+  const containers: string[] = [];
+  const entries = await readdir(parent, { withFileTypes: true });
+  for (const entry of entries.sort((left, right) =>
+    left.name.localeCompare(right.name),
+  )) {
+    if (!entry.isDirectory()) continue;
+    const directory = join(parent, entry.name);
+    if (await retainedFile(join(directory, "result.json")))
+      results.push(directory);
+    else containers.push(directory);
+  }
+  return { results, containers };
+}
+
+async function resultsRoot(directory: string): Promise<boolean> {
+  const active = await lstat(join(directory, "active")).catch(
+    (cause: unknown) => {
+      if ((cause as NodeJS.ErrnoException).code === "ENOENT") return null;
+      throw cause;
+    },
+  );
+  if (active && !active.isDirectory())
+    throw new Error(`retained ownership must be a directory: ${directory}`);
+  return Boolean(
+    active || (await retainedFile(join(directory, ".retention-lock.sqlite"))),
+  );
+}
+
+async function finalized(root: string, runId: string | null): Promise<boolean> {
+  if (!runId || basename(runId) !== runId) return false;
+  const path = join(root, "active", `${runId}.json`);
+  if (!(await retainedFile(path))) return true;
+  const value: unknown = JSON.parse(await readFile(path, "utf8"));
+  if (!isRecord(value)) return false;
+  return finalizedRecord(value, runId);
+}
+
+async function finalizedAncestors(
+  root: string,
+  directory: string,
+  runId: string | null,
+): Promise<boolean> {
+  let ancestor = dirname(directory);
+  while (ancestor !== dirname(root)) {
+    if (!(await finalized(ancestor, runId))) return false;
+    ancestor = dirname(ancestor);
+  }
+  return true;
+}
+
+function finalizedRecord(
+  value: Record<string, unknown>,
+  runId: string,
+): boolean {
+  return (
+    value.format === "sevro.active-run.v1" &&
+    value.runId === runId &&
+    ["complete", "interrupted", "diagnostic"].includes(String(value.status))
+  );
+}
+
+function retainedByPolicy(results: RetainedResult[], policy: RetentionPolicy) {
+  const cutoff = policy.cutoff;
+  return cutoff === null
+    ? results.slice(0, policy.keep)
+    : results.filter((result) => result.time >= cutoff);
+}
+
+async function pruneResults(
+  root: string,
+  policy: RetentionPolicy,
+  storage: RetainedStorage,
+) {
+  const results = await retainedResults(root, storage.directories);
+  const remaining = retainedByPolicy(results, policy);
+  for (const resultsRoot of storage.roots) {
+    const latest = remaining.find((item) =>
+      item.directory.startsWith(`${resultsRoot}/`),
+    );
+    if (latest)
+      await atomicWriteJson(join(resultsRoot, "latest.json"), latest.result);
+    else await rm(join(resultsRoot, "latest.json"), { force: true });
+  }
+  const removed = results.filter((result) => !remaining.includes(result));
+  for (const result of removed) await rm(result.directory, { recursive: true });
+  return { removed: removed.length, kept: remaining.length };
+}
+
+async function pruneTranscripts(
+  root: string,
+  policy: RetentionPolicy,
+  directories: string[],
+) {
+  const results = await retainedResults(root, directories);
+  const preserved = retainedByPolicy(results, policy);
+  let removed = 0;
+  for (const { evidence } of results.filter(
+    (item) => !preserved.includes(item),
+  )) {
+    removed += await pruneNativeTranscripts(evidence);
+  }
+  return { removed, kept: results.length };
+}
+
+async function prune(options: ReturnType<typeof invocation>) {
+  const root = await realpath(options.root).catch((cause: unknown) => {
+    if ((cause as NodeJS.ErrnoException).code === "ENOENT") return null;
+    throw cause;
+  });
+  if (root === null) return { removed: 0, kept: 0 };
+  return withResultsLock(root, async () => {
+    const storage: RetainedStorage = {
+      directories: [],
+      roots: [root],
+      releases: [],
+    };
+    try {
+      await resultDirectories(root, storage);
+      return options.nativeTranscripts
+        ? await pruneTranscripts(root, options.policy, storage.directories)
+        : await pruneResults(root, options.policy, storage);
+    } finally {
+      for (const release of storage.releases.reverse()) release();
+    }
+  });
+}
+
+/** Explicitly remove old retained runs while keeping report's latest index usable. */
+export async function pruneCommand(args: string[]): Promise<number> {
+  try {
+    const options = invocation(args);
+    const result = await prune(options);
+    const unit = options.nativeTranscripts
+      ? "native transcript bundle(s)"
+      : "result(s)";
+    process.stdout.write(
+      `Removed ${result.removed} ${unit}; kept ${result.kept}${options.nativeTranscripts ? " result(s)" : ""}.\n`,
+    );
+    return 0;
+  } catch (error) {
+    process.stderr.write(
+      `sevro prune: ${error instanceof Error ? error.message : "pruning failed"}\n`,
+    );
+    return 64;
+  }
+}

package/src/report.ts

@@ -1,38 +1,22 @@
-import type { CliResultData, RunEvidenceData } from "./schema-types";
+import type {
+  CliResultData,
+  ReportData,
+  RunEvidenceData,
+} from "./schema-types";
 import { readFile } from "node:fs/promises";
-import { isAbsolute } from "node:path";
+import { isAbsolute, join, resolve } from "node:path";
 import { parseArgs } from "node:util";
 import { canonicalJson } from "./identity";
 import { assertCliResult, assertReport, assertRunEvidence } from "./schema";
+import { canonicalRuntimeRoot } from "./runtime/paths";
+import { nativeTranscriptRetention } from "./transcript-retention";
 
 type CliResult = CliResultData;
 type Evidence = RunEvidenceData;
 type Case = CliResult["cases"][number];
 type Trial = Case["trials"][number];
 
-export type ReportRow = {
-  resultFile: string;
-  evidencePath: string | null;
-  runId: string | null;
-  caseId: string;
-  execution: string;
-  grading: string;
-  task: string;
-  exitCode: number;
-  trialCount: number;
-  taskPassRate: number | null;
-  candidateDurationMs: number | null;
-  inputTokens: number | null;
-  outputTokens: number | null;
-  costUsd: number | null;
-  runnerBuildDigest: string | null;
-  evaluationDigest: string | null;
-  requestedCondition: string | null;
-  actualCondition: string | null;
-  candidateRoute: { host: string; model: string; effort: string } | null;
-  routes: Array<{ role: string; host: string; model: string; effort: string }>;
-  domainOutcomes: Array<{ trial: number; id: string; status: string }>;
-};
+export type ReportRow = ReportData["rows"][number];
 
 function sumKnown(values: Array<number | null | undefined>): number | null {
   return values.length && values.every((value) => value != null)
@@ -113,6 +97,7 @@ function reportRow(
   result: CliResult,
   selected: Case,
   evidence: Evidence | null,
+  transcripts: Awaited<ReturnType<typeof nativeTranscriptRetention>>,
 ): ReportRow {
   const retained = trialEvidence(selected.caseId, selected.trials, evidence);
   const candidateRoute = retainedCandidateRoute(evidence);
@@ -134,6 +119,14 @@ function reportRow(
     ...usage,
     ...retainedIdentity(evidence),
     candidateRoute,
+    nativeTranscripts: {
+      retained: transcripts.filter(
+        (item) => item.caseId === selected.caseId && !item.pruned,
+      ).length,
+      pruned: transcripts.filter(
+        (item) => item.caseId === selected.caseId && item.pruned,
+      ).length,
+    },
     domainOutcomes: selected.trials.flatMap((trial) =>
       (trial.domainOutcomes ?? []).map((outcome) => ({
         trial: trial.trial,
@@ -191,17 +184,11 @@ export async function createReport(paths: string[]) {
   if (!paths.length || new Set(paths).size !== paths.length)
     throw new Error("report needs distinct result files");
   const rows: ReportRow[] = [];
-  const inputs: Array<{
-    resultFile: string;
-    execution: string;
-    grading: string;
-    task: string;
-    exitCode: number;
-    caseCount: number;
-  }> = [];
+  const inputs: ReportData["inputs"] = [];
   for (const path of paths) {
     const result = await loadResult(path);
     const evidence = await loadEvidence(result);
+    const transcripts = await nativeTranscriptRetention(evidence);
     inputs.push({
       resultFile: path,
       execution: result.execution.status,
@@ -211,7 +198,7 @@ export async function createReport(paths: string[]) {
       caseCount: result.cases.length,
     });
     for (const selected of result.cases)
-      rows.push(reportRow(path, result, selected, evidence));
+      rows.push(reportRow(path, result, selected, evidence, transcripts));
   }
   const report = {
     format: "sevro.report.v1",
@@ -268,7 +255,8 @@ export function renderReport(
     "## Retained evidence",
     "",
     ...report.rows.map(
-      (row) => `- ${cell(row.caseId)}: ${row.evidencePath ?? "unavailable"}`,
+      (row) =>
+        `- ${cell(row.caseId)}: ${row.evidencePath ?? "unavailable"}${transcriptSummary(row)}`,
     ),
     "",
   ];
@@ -305,6 +293,11 @@ export function renderReport(
   return lines.join("\n");
 }
 
+function transcriptSummary(row: ReportRow): string {
+  if (!row.nativeTranscripts) return "";
+  return `; native transcripts: ${row.nativeTranscripts.retained} retained, ${row.nativeTranscripts.pruned} pruned`;
+}
+
 function passRateCell(rate: number | null): string {
   return rate === null ? "unknown" : `${(rate * 100).toFixed(0)}%`;
 }
@@ -326,7 +319,7 @@ function caseOutcomeLine(row: ReportRow): string {
 export async function reportCommand(argv: string[]): Promise<number> {
   try {
     const values = reportInvocation(argv);
-    const report = await createReport(values["result-file"] ?? []);
+    const report = await createReport(await reportPaths(values));
     process.stdout.write(
       values.json ? `${JSON.stringify(report)}\n` : renderReport(report),
     );
@@ -345,12 +338,46 @@ function reportInvocation(argv: string[]) {
     options: {
       json: { type: "boolean" },
       "result-file": { type: "string", multiple: true },
+      latest: { type: "boolean" },
+      "results-root": { type: "string" },
     },
   });
   if (positionals.length !== 1 || positionals[0] !== "report")
     throw new Error("expected the report command");
   return values;
 }
+
+async function reportPaths(
+  values: ReturnType<typeof reportInvocation>,
+): Promise<string[]> {
+  validateReportSource(values);
+  const paths = selectedReportPaths(values);
+  if (paths.some((path) => !path.trim()))
+    throw new Error("--result-file must name a readable retained result file");
+  return Promise.all(paths.map((path) => canonicalRuntimeRoot(resolve(path))));
+}
+
+function validateReportSource(
+  values: ReturnType<typeof reportInvocation>,
+): void {
+  if (values.latest && values["result-file"] !== undefined)
+    throw new Error("--latest and --result-file are exclusive");
+  if (values["results-root"] !== undefined && !values.latest)
+    throw new Error("--results-root requires --latest");
+}
+
+function selectedReportPaths(
+  values: ReturnType<typeof reportInvocation>,
+): string[] {
+  return values.latest
+    ? [
+        join(
+          values["results-root"] ?? join(process.cwd(), ".sevro", "results"),
+          "latest.json",
+        ),
+      ]
+    : (values["result-file"] ?? []);
+}
 function reportErrorMessage(error: unknown): string {
   return error instanceof Error ? error.message : "invalid report input";
 }

package/src/resolved-case.ts

@@ -1,5 +1,5 @@
 import type { ExtensionCase } from "./extension-session";
-import type { RepositoryFixture } from "./repository-fixture";
+import type { RepositoryFixture } from "./fixtures/repository";
 
 function repositoryOverlay(fixture: RepositoryFixture) {
   return {

package/src/results-lock.ts

@@ -0,0 +1,42 @@
+import { openLockDatabase } from "./sqlite-lock";
+import { join } from "node:path";
+import { lstat } from "node:fs/promises";
+
+export async function retainedFile(path: string) {
+  const info = await lstat(path).catch((cause: unknown) => {
+    if ((cause as NodeJS.ErrnoException).code === "ENOENT") return null;
+    throw cause;
+  });
+  if (info && !info.isFile())
+    throw new Error(`retained storage must be a regular file: ${path}`);
+  return info;
+}
+
+/** Serialize result publication and pruning across CLI processes. Closing rolls back the lock transaction. */
+export async function withResultsLock<T>(
+  root: string,
+  action: () => Promise<T>,
+): Promise<T> {
+  const release = await acquireResultsLock(root);
+  try {
+    return await action();
+  } finally {
+    release();
+  }
+}
+
+/** Hold a root lock while discovering and validating additional nested roots. */
+export async function acquireResultsLock(root: string): Promise<() => void> {
+  await retainedFile(join(root, ".retention-lock.sqlite"));
+  const database = openLockDatabase(join(root, ".retention-lock.sqlite"));
+  try {
+    database.exec("PRAGMA busy_timeout = 5000");
+    database.exec("BEGIN IMMEDIATE");
+    return () => {
+      database.close();
+    };
+  } catch (cause) {
+    database.close();
+    throw cause;
+  }
+}

package/src/results.ts

@@ -1,21 +1,17 @@
-export type ExecutionStatus = "completed" | "failed" | "cancelled" | "not_run";
-export type GradingStatus =
-  "completed" | "error" | "unavailable" | "not_requested";
-export type TaskVerdict = "passed" | "failed" | "not_assessed";
-
-export interface CheckOutcome {
-  id: string;
-  grader: string;
-  status: "passed" | "failed" | "unavailable";
-  detail?: string;
-  evidenceRefs?: string[];
-}
-
-export interface Assessment {
-  execution: { status: ExecutionStatus };
-  grading: { status: GradingStatus };
-  task: { verdict: TaskVerdict };
-}
+import type {
+  Assessment,
+  CheckOutcome,
+  ExecutionStatus,
+  GradingStatus,
+  TaskVerdict,
+} from "./contract-types";
+export type {
+  Assessment,
+  CheckOutcome,
+  ExecutionStatus,
+  GradingStatus,
+  TaskVerdict,
+} from "./contract-types";
 
 export interface TrialInput {
   execution: ExecutionStatus;

package/src/run-owner.ts

@@ -1,5 +1,6 @@
+import { spawnSync } from "./runtime-process";
 import { isRecord } from "./value-guards";
-import { Database } from "bun:sqlite";
+import { openLockDatabase } from "./sqlite-lock";
 import { randomUUID } from "node:crypto";
 import {
   mkdirSync,
@@ -115,11 +116,16 @@ function atomicJson(path: string, value: unknown): void {
 function withClaimLock<T>(claimPath: string, action: () => T): T {
   const locks = join(dirname(dirname(claimPath)), "locks");
   mkdirSync(locks, { recursive: true, mode: 0o700 });
-  const database = new Database(join(locks, `${basename(claimPath)}.sqlite`));
+  const database = openLockDatabase(
+    join(locks, `${basename(claimPath)}.sqlite`),
+  );
   try {
-    database.run("PRAGMA busy_timeout = 5000");
-    database.run("CREATE TABLE IF NOT EXISTS mutex (id INTEGER PRIMARY KEY)");
-    return database.transaction(action).immediate();
+    database.exec("PRAGMA busy_timeout = 5000");
+    database.exec("CREATE TABLE IF NOT EXISTS mutex (id INTEGER PRIMARY KEY)");
+    database.exec("BEGIN IMMEDIATE");
+    const result = action();
+    database.exec("COMMIT");
+    return result;
   } finally {
     database.close();
   }
@@ -159,16 +165,16 @@ function identityEnvironment(windows: boolean) {
     : { PATH: "/usr/bin:/bin", LC_ALL: "C" };
 }
 
-function processStartTime(result: ReturnType<typeof Bun.spawnSync>): string {
-  return result.stdout?.toString().trim() ?? "";
+function processStartTime(result: ReturnType<typeof spawnSync>): string {
+  return result.stdout.toString().trim();
 }
 
 export function processIdentity(pid: number): ProcessIdentity | null {
   const windows = process.platform === "win32";
   const argv = identityCommand(pid, windows);
-  let result: ReturnType<typeof Bun.spawnSync>;
+  let result: ReturnType<typeof spawnSync>;
   try {
-    result = Bun.spawnSync(argv, {
+    result = spawnSync(argv, {
       env: identityEnvironment(windows),
       stdout: "pipe",
       stderr: "ignore",

package/src/runner-paths.ts

@@ -0,0 +1,21 @@
+import { resolve, join } from "node:path";
+
+export const sourceRuntime = import.meta.filename.endsWith(".ts");
+export const runnerRoot = resolve(
+  import.meta.dirname,
+  sourceRuntime ? ".." : "../..",
+);
+
+export function examplePath(...parts: string[]): string {
+  const path = join(
+    runnerRoot,
+    ...(sourceRuntime ? [] : ["dist"]),
+    "examples",
+    ...parts,
+  );
+  return runtimeModule(path);
+}
+
+export function runtimeModule(path: string): string {
+  return sourceRuntime ? path : path.replace(/\.ts$/, ".js");
+}