@bjoernrochel/sevro 0.1.0-rc.2 → 0.1.0-rc.21

package/src/provenance.ts

@@ -1,42 +1,57 @@
-import { createHash } from "node:crypto";
+import type { Stats } from "node:fs";
 import { lstat, readFile, readdir, readlink, realpath } from "node:fs/promises";
+import { tmpdir } from "node:os";
 import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
 import { pathToFileURL } from "node:url";
 import packageJson from "../package.json";
-import { hashJson } from "./identity";
+import { hashJson, sha256 } from "./identity";
+import { insideRuntimeRoot } from "./runtime/paths";
 
 const MAX_DIRTY_BYTES = 32 * 1024 * 1024;
 const MAX_BUILD_BYTES = 64 * 1024 * 1024;
 const MAX_BUILD_FILES = 4096;
 
-function sha256(bytes: Uint8Array): string {
-  return createHash("sha256").update(bytes).digest("hex");
+type BuildFile = { path: string; sha256: string };
+type ProjectFile = BuildFile & { executable: boolean };
+function fileOrder(left: BuildFile, right: BuildFile): number {
+  return left.path < right.path ? -1 : left.path > right.path ? 1 : 0;
 }
 
-/** Digest the packaged runtime and public contract files in path order. */
-export async function packageBuildDigest(
-  root = join(import.meta.dir, ".."),
-): Promise<string> {
-  const canonicalRoot = await realpath(root);
-  const files: { path: string; sha256: string }[] = [];
-  let totalBytes = 0;
-  async function collect(path: string): Promise<void> {
-    const absolute = join(canonicalRoot, path);
-    const info = await lstat(absolute);
+class BuildSnapshot {
+  readonly files: BuildFile[] = [];
+  private bytes = 0;
+  constructor(private root: string) {}
+  async collect(path: string): Promise<void> {
+    const absolute = join(this.root, path),
+      info = await lstat(absolute);
     if (info.isSymbolicLink())
       throw new Error("package build contains a symbolic link");
     if (info.isDirectory()) {
       for (const entry of (await readdir(absolute)).sort())
-        await collect(join(path, entry));
+        await this.collect(join(path, entry));
       return;
     }
+    await this.collectFile(path, absolute, info);
+  }
+  private async collectFile(
+    path: string,
+    absolute: string,
+    info: Stats,
+  ): Promise<void> {
     if (!info.isFile()) throw new Error("package build contains a non-file");
     const bytes = await readFile(absolute);
-    totalBytes += bytes.byteLength;
-    if (files.length >= MAX_BUILD_FILES || totalBytes > MAX_BUILD_BYTES)
+    this.bytes += bytes.byteLength;
+    if (this.files.length >= MAX_BUILD_FILES || this.bytes > MAX_BUILD_BYTES)
       throw new Error("package build exceeds the size limit");
-    files.push({ path: path.split(sep).join("/"), sha256: sha256(bytes) });
+    this.files.push({ path: path.split(sep).join("/"), sha256: sha256(bytes) });
   }
+}
+
+/** Digest the packaged runtime and public contract files in path order. */
+export async function packageBuildDigest(
+  root = join(import.meta.dir, ".."),
+): Promise<string> {
+  const snapshot = new BuildSnapshot(await realpath(root));
   for (const path of [
     "package.json",
     "README.md",
@@ -45,57 +60,80 @@ export async function packageBuildDigest(
     "schemas",
     "src",
   ])
-    await collect(path);
-  files.sort((left, right) =>
-    left.path < right.path ? -1 : left.path > right.path ? 1 : 0,
-  );
-  return hashJson({ format: "sevro.build.v1", files });
+    await snapshot.collect(path);
+  return hashJson({
+    format: "sevro.build.v1",
+    files: snapshot.files.sort(fileOrder),
+  });
 }
 
-export async function projectIdentityDigest(
-  root: string,
-  provenance: { revision: string | null; dirtyPatchDigest: string | null },
-  excludedRoots: string[] = [],
-): Promise<string> {
-  if (provenance.revision)
-    return hashJson({
-      format: "sevro.project.v1",
-      revision: provenance.revision,
-      dirtyPatchDigest: provenance.dirtyPatchDigest,
-    });
-  const projectRoot = resolve(root);
-  const excluded = new Set(
-    excludedRoots.map((path) => relative(projectRoot, resolve(path))),
-  );
-  const files: { path: string; sha256: string; executable: boolean }[] = [];
-  let totalBytes = 0;
-  async function collect(path: string): Promise<void> {
-    if (path === ".git" || excluded.has(path)) return;
-    const absolute = join(projectRoot, path);
-    const info = await lstat(absolute);
+class ProjectSnapshot {
+  readonly files: ProjectFile[] = [];
+  private bytes = 0;
+  constructor(
+    private root: string,
+    private excluded: Set<string>,
+  ) {}
+  private isExcluded(path: string): boolean {
+    return path === ".git" || this.excluded.has(path);
+  }
+  async collect(path: string): Promise<void> {
+    if (this.isExcluded(path)) return;
+    const absolute = join(this.root, path),
+      info = await lstat(absolute);
     if (info.isSymbolicLink())
       throw new Error("project snapshot contains a symbolic link");
     if (info.isDirectory()) {
-      for (const entry of (await readdir(absolute)).sort())
-        await collect(path === "" ? entry : join(path, entry));
+      await this.collectDirectory(path, absolute);
       return;
     }
+    await this.collectFile(path, absolute, info);
+  }
+  private async collectDirectory(
+    path: string,
+    absolute: string,
+  ): Promise<void> {
+    for (const entry of (await readdir(absolute)).sort())
+      await this.collect(path === "" ? entry : join(path, entry));
+  }
+  private async collectFile(
+    path: string,
+    absolute: string,
+    info: Stats,
+  ): Promise<void> {
     if (!info.isFile()) throw new Error("project snapshot contains a non-file");
     const bytes = await readFile(absolute);
-    totalBytes += bytes.byteLength;
-    if (files.length >= MAX_BUILD_FILES || totalBytes > MAX_BUILD_BYTES)
+    this.bytes += bytes.byteLength;
+    if (this.files.length >= MAX_BUILD_FILES || this.bytes > MAX_BUILD_BYTES)
       throw new Error("project snapshot exceeds the size limit");
-    files.push({
+    this.files.push({
       path: path.split(sep).join("/"),
       sha256: sha256(bytes),
       executable: Boolean(info.mode & 0o111),
     });
   }
-  await collect("");
-  files.sort((left, right) =>
-    left.path < right.path ? -1 : left.path > right.path ? 1 : 0,
-  );
-  return hashJson({ format: "sevro.project.v1", files });
+}
+export async function projectIdentityDigest(
+  root: string,
+  provenance: { revision: string | null; dirtyPatchDigest: string | null },
+  excludedRoots: string[] = [],
+): Promise<string> {
+  if (provenance.revision)
+    return hashJson({
+      format: "sevro.project.v1",
+      revision: provenance.revision,
+      dirtyPatchDigest: provenance.dirtyPatchDigest,
+    });
+  const projectRoot = resolve(root),
+    excluded = new Set(
+      excludedRoots.map((path) => relative(projectRoot, resolve(path))),
+    );
+  const snapshot = new ProjectSnapshot(projectRoot, excluded);
+  await snapshot.collect("");
+  return hashJson({
+    format: "sevro.project.v1",
+    files: snapshot.files.sort(fileOrder),
+  });
 }
 
 function git(root: string, ...arguments_: string[]): Buffer {
@@ -110,7 +148,10 @@ function git(root: string, ...arguments_: string[]): Buffer {
   return Buffer.from(result.stdout);
 }
 
-async function dirtyPatchDigest(gitRoot: string): Promise<string | null> {
+async function dirtyPatchDigest(
+  gitRoot: string,
+  excludedRoots: string[],
+): Promise<string | null> {
   const trackedPatch = git(gitRoot, "diff", "--binary", "HEAD", "--");
   const untrackedPaths = git(
     gitRoot,
@@ -122,26 +163,17 @@ async function dirtyPatchDigest(gitRoot: string): Promise<string | null> {
     .toString("utf8")
     .split("\0")
     .filter(Boolean)
+    .filter(
+      (path) =>
+        !excludedRoots.some((root) =>
+          insideRuntimeRoot(root, resolve(gitRoot, path)),
+        ),
+    )
     .sort();
   let dirtyBytes = trackedPatch.byteLength;
   const untracked: { path: string; sha256: string }[] = [];
   for (const path of untrackedPaths) {
-    const absolute = resolve(gitRoot, path);
-    const relativePath = relative(gitRoot, absolute);
-    if (
-      isAbsolute(relativePath) ||
-      relativePath === ".." ||
-      relativePath.startsWith(`..${sep}`)
-    )
-      throw new Error("Git checkout has an unsafe untracked path");
-    const info = await lstat(absolute);
-    const bytes = info.isSymbolicLink()
-      ? Buffer.from(await readlink(absolute), "utf8")
-      : info.isFile()
-        ? await readFile(absolute)
-        : null;
-    if (!bytes)
-      throw new Error("Git checkout has an unsupported untracked entry");
+    const bytes = await untrackedIdentityBytes(gitRoot, path);
     dirtyBytes += bytes.byteLength;
     if (dirtyBytes > MAX_DIRTY_BYTES)
       throw new Error("Git identity exceeds the size limit");
@@ -152,6 +184,24 @@ async function dirtyPatchDigest(gitRoot: string): Promise<string | null> {
     : null;
 }
 
+function requireSafeUntrackedPath(root: string, absolute: string): void {
+  const path = relative(root, absolute);
+  if (isAbsolute(path) || path === ".." || path.startsWith(`..${sep}`))
+    throw new Error("Git checkout has an unsafe untracked path");
+}
+async function untrackedIdentityBytes(
+  root: string,
+  path: string,
+): Promise<Buffer> {
+  const absolute = resolve(root, path);
+  requireSafeUntrackedPath(root, absolute);
+  const info = await lstat(absolute);
+  if (info.isSymbolicLink())
+    return Buffer.from(await readlink(absolute), "utf8");
+  if (info.isFile()) return readFile(absolute);
+  throw new Error("Git checkout has an unsupported untracked entry");
+}
+
 /** Local development identity; packaged use never calls Git. */
 export async function checkoutProvenance(root: string, buildDigest: string) {
   const canonicalRoot = await realpath(root);
@@ -169,27 +219,50 @@ export async function checkoutProvenance(root: string, buildDigest: string) {
     source: "checkout" as const,
     root: pathToFileURL(canonicalRoot).href,
     revision,
-    dirtyPatchDigest: await dirtyPatchDigest(canonicalRoot),
+    dirtyPatchDigest: await dirtyPatchDigest(canonicalRoot, []),
     buildDigest,
   };
 }
 
 async function enclosingGitRoot(root: string): Promise<string | null> {
   let path = root;
-  while (true) {
-    try {
-      await lstat(join(path, ".git"));
-      return path;
-    } catch (error) {
-      if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
-    }
+  for (;;) {
+    if (await gitMetadataAt(path)) return path;
     const parent = dirname(path);
     if (parent === path) return null;
     path = parent;
   }
 }
 
-export async function projectProvenance(root: string) {
+async function gitMetadataAt(path: string): Promise<boolean> {
+  try {
+    const metadataPath = join(path, ".git");
+    const metadata = await lstat(metadataPath);
+    return !(await emptyTemporaryGitPlaceholder(path, metadataPath, metadata));
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === "ENOENT") return false;
+    throw error;
+  }
+}
+
+async function emptyTemporaryGitPlaceholder(
+  directory: string,
+  metadataPath: string,
+  metadata: Stats,
+): Promise<boolean> {
+  // Codex's Linux sandbox can leave an empty mount placeholder at /tmp/.git.
+  return (
+    process.platform === "linux" &&
+    (directory === "/tmp" || directory === tmpdir()) &&
+    metadata.isDirectory() &&
+    (await readdir(metadataPath)).length === 0
+  );
+}
+
+export async function projectProvenance(
+  root: string,
+  excludedRoots?: string[],
+) {
   const canonicalRoot = await realpath(root);
   const gitEntryRoot = await enclosingGitRoot(canonicalRoot);
   if (!gitEntryRoot)
@@ -219,10 +292,17 @@ export async function projectProvenance(root: string) {
   return {
     root: pathToFileURL(canonicalRoot).href,
     revision,
-    dirtyPatchDigest: await dirtyPatchDigest(gitRoot),
+    dirtyPatchDigest: await projectDirtyDigest(gitRoot, excludedRoots),
   };
 }
 
+async function projectDirtyDigest(
+  root: string,
+  excludedRoots: string[] | undefined,
+) {
+  return dirtyPatchDigest(root, excludedRoots ?? []);
+}
+
 export async function runnerProvenance(
   buildDigest: string,
   checkoutRoot?: string,

package/src/prune.ts

@@ -0,0 +1,215 @@
+import { readFile, readdir, realpath, rm, stat } from "node:fs/promises";
+import { basename, join, resolve } from "node:path";
+import { parseArgs } from "node:util";
+import { assertCliResult, assertRunEvidence } from "./schema";
+import type { CliResultData, RunEvidenceData } from "./schema-types";
+import { createReport } from "./report";
+import { atomicWriteJson } from "./storage";
+import { isRecord } from "./value-guards";
+import { retainedFile, withResultsLock } from "./results-lock";
+import { pruneNativeTranscripts } from "./transcript-retention";
+
+interface RetainedResult {
+  directory: string;
+  result: CliResultData;
+  evidence: RunEvidenceData;
+  time: number;
+}
+
+type RetentionPolicy =
+  { keep: number; cutoff: null } | { keep: null; cutoff: number };
+
+const durationUnits: Record<string, number> = {
+  ms: 1,
+  s: 1000,
+  m: 60_000,
+  h: 3_600_000,
+  d: 86_400_000,
+  w: 604_800_000,
+};
+
+function duration(value: string): number {
+  const match = /^([1-9]\d*)(ms|s|m|h|d|w)$/.exec(value);
+  if (!match)
+    throw new Error(
+      "--older-than requires a positive integer and ms, s, m, h, d or w",
+    );
+  const amount =
+    Number.parseInt(value, 10) *
+    Number(durationUnits[value.replace(/^\d+/, "")]);
+  if (!Number.isSafeInteger(amount))
+    throw new Error("--older-than exceeds the safe duration range");
+  return amount;
+}
+
+function count(value: string): number {
+  const amount = Number(value);
+  if (!/^[1-9]\d*$/.test(value) || !Number.isSafeInteger(amount))
+    throw new Error("--keep must be a positive safe integer");
+  return amount;
+}
+
+function parseInvocation(args: string[]) {
+  const { values } = parseArgs({
+    args,
+    strict: true,
+    options: {
+      "results-root": { type: "string" },
+      keep: { type: "string" },
+      latest: { type: "boolean" },
+      "older-than": { type: "string" },
+      "native-transcripts": { type: "boolean" },
+    },
+  });
+  return values;
+}
+
+function policy(values: ReturnType<typeof parseInvocation>): RetentionPolicy {
+  if (values.latest) return { keep: 1, cutoff: null };
+  if (values["older-than"] !== undefined)
+    return { keep: null, cutoff: Date.now() - duration(values["older-than"]) };
+  if (values.keep !== undefined)
+    return { keep: count(values.keep), cutoff: null };
+  return { keep: values["native-transcripts"] ? 0 : 10, cutoff: null };
+}
+
+function invocation(args: string[]) {
+  const values = parseInvocation(args);
+  const policies = [values.latest, values.keep, values["older-than"]].filter(
+    (value) => value !== undefined,
+  );
+  if (policies.length > 1) throw new Error("retention policies are exclusive");
+  if (values["results-root"] === "")
+    throw new Error("--results-root must not be empty");
+  return {
+    root: resolve(values["results-root"] ?? ".sevro/results"),
+    policy: policy(values),
+    nativeTranscripts: values["native-transcripts"] === true,
+  };
+}
+
+async function retainedResults(root: string): Promise<RetainedResult[]> {
+  const results: RetainedResult[] = [];
+  for (const directory of await resultDirectories(root)) {
+    const path = join(directory, "result.json");
+    await retainedFile(path);
+    const result: unknown = JSON.parse(await readFile(path, "utf8"));
+    assertCliResult(result);
+    if (!(await finalized(root, result.runId))) continue;
+    if (result.evidencePath !== join(directory, "run.json"))
+      throw new Error(
+        `retained result references evidence outside its directory: ${path}`,
+      );
+    await retainedFile(result.evidencePath);
+    const evidence: unknown = JSON.parse(
+      await readFile(result.evidencePath, "utf8"),
+    );
+    assertRunEvidence(evidence);
+    await createReport([path]);
+    results.push({
+      directory,
+      result,
+      evidence,
+      time: (await stat(path)).mtimeMs,
+    });
+  }
+  return results.sort(
+    (left, right) =>
+      right.time - left.time || left.directory.localeCompare(right.directory),
+  );
+}
+
+async function resultDirectories(parent: string): Promise<string[]> {
+  const paths: string[] = [];
+  for (const entry of await readdir(parent, { withFileTypes: true })) {
+    if (!entry.isDirectory()) continue;
+    const directory = join(parent, entry.name);
+    if (await retainedFile(join(directory, "result.json")))
+      paths.push(directory);
+    else paths.push(...(await resultDirectories(directory)));
+  }
+  return paths;
+}
+
+async function finalized(root: string, runId: string | null): Promise<boolean> {
+  if (!runId || basename(runId) !== runId) return false;
+  const path = join(root, "active", `${runId}.json`);
+  if (!(await retainedFile(path))) return true;
+  const value: unknown = JSON.parse(await readFile(path, "utf8"));
+  if (!isRecord(value)) return false;
+  return finalizedRecord(value, runId);
+}
+
+function finalizedRecord(
+  value: Record<string, unknown>,
+  runId: string,
+): boolean {
+  return (
+    value.format === "sevro.active-run.v1" &&
+    value.runId === runId &&
+    ["complete", "interrupted", "diagnostic"].includes(String(value.status))
+  );
+}
+
+function retainedByPolicy(results: RetainedResult[], policy: RetentionPolicy) {
+  const cutoff = policy.cutoff;
+  return cutoff === null
+    ? results.slice(0, policy.keep)
+    : results.filter((result) => result.time >= cutoff);
+}
+
+async function pruneResults(root: string, policy: RetentionPolicy) {
+  const results = await retainedResults(root);
+  const remaining = retainedByPolicy(results, policy);
+  const latest = remaining[0];
+  if (latest) await atomicWriteJson(join(root, "latest.json"), latest.result);
+  else await rm(join(root, "latest.json"), { force: true });
+  const removed = results.filter((result) => !remaining.includes(result));
+  for (const result of removed) await rm(result.directory, { recursive: true });
+  return { removed: removed.length, kept: remaining.length };
+}
+
+async function pruneTranscripts(root: string, policy: RetentionPolicy) {
+  const results = await retainedResults(root);
+  const preserved = retainedByPolicy(results, policy);
+  let removed = 0;
+  for (const { evidence } of results.filter(
+    (item) => !preserved.includes(item),
+  )) {
+    removed += await pruneNativeTranscripts(evidence);
+  }
+  return { removed, kept: results.length };
+}
+
+async function prune(options: ReturnType<typeof invocation>) {
+  const root = await realpath(options.root).catch((cause: unknown) => {
+    if ((cause as NodeJS.ErrnoException).code === "ENOENT") return null;
+    throw cause;
+  });
+  if (root === null) return { removed: 0, kept: 0 };
+  return withResultsLock(root, () =>
+    options.nativeTranscripts
+      ? pruneTranscripts(root, options.policy)
+      : pruneResults(root, options.policy),
+  );
+}
+
+/** Explicitly remove old retained runs while keeping report's latest index usable. */
+export async function pruneCommand(args: string[]): Promise<number> {
+  try {
+    const options = invocation(args);
+    const result = await prune(options);
+    const unit = options.nativeTranscripts
+      ? "native transcript bundle(s)"
+      : "result(s)";
+    process.stdout.write(
+      `Removed ${result.removed} ${unit}; kept ${result.kept}${options.nativeTranscripts ? " result(s)" : ""}.\n`,
+    );
+    return 0;
+  } catch (error) {
+    process.stderr.write(
+      `sevro prune: ${error instanceof Error ? error.message : "pruning failed"}\n`,
+    );
+    return 64;
+  }
+}