@bjoernrochel/sevro 0.1.0-rc.8 → 1.0.0

package/src/candidate-artifacts.ts

@@ -1,11 +1,20 @@
 import { lstat, readdir, realpath, rm } from "node:fs/promises";
+import { containsControl } from "./value-guards";
 import { join, relative, sep } from "node:path";
+import {
+  allocateCodexStateLease,
+  candidateCodexArtifactBinding,
+  candidateCodexArtifactRoots,
+  codexStateEnabled,
+  releaseCandidateCodexState,
+  requireUnreservedArtifactName,
+} from "./candidate-codex-state";
 import {
   allocateNativeState,
   nativeStatePath,
 } from "./native-transcript-state";
-import type { RuntimePolicy, RuntimeRole } from "./runtime-config";
-import { clearFixtureContents } from "./fixture-cleanup";
+import type { RuntimePolicy, RuntimeRole } from "./runtime/config";
+import { clearFixtureContents } from "./fixtures/cleanup";
 
 export interface RuntimeArtifactDirectory {
   name: string;
@@ -35,6 +44,7 @@ export function requireArtifactDirectories(
 function requireArtifactDirectory(directory: RuntimeArtifactDirectory): void {
   if (!/^[a-z][a-z0-9_]{0,63}$/.test(directory.name))
     throw new Error("runtime artifact directory name is invalid");
+  requireUnreservedArtifactName(directory.name);
   const path = directory.path;
   if (!validArtifactPath(path))
     throw new Error("runtime artifact directory path is invalid");
@@ -47,24 +57,33 @@ function requireArtifactDirectory(directory: RuntimeArtifactDirectory): void {
 function validArtifactPath(path: string): boolean {
   if (typeof path !== "string" || path.length > 4096 || path.includes("\\"))
     return false;
-  return !Array.from(path).some(
-    (character) =>
-      character.charCodeAt(0) < 0x20 || character.charCodeAt(0) === 0x7f,
-  );
+  return !containsControl(path);
 }
 
 /** Allocate before host execution so exceptions cannot lose cleanup ownership. */
 export async function allocateCandidateRuntime(
   policy: RuntimePolicy | undefined,
+  retainHome = false,
 ): Promise<string | undefined> {
   const directories = policy?.artifactDirectories ?? [];
-  if (!directories.length) return undefined;
+  const codexState = codexStateEnabled(policy);
+  if (!candidateRuntimeRequired(directories, codexState, retainHome))
+    return undefined;
   requireArtifactDirectories(directories);
   const root = await allocateNativeState("candidate-runtime-");
   leases.set(root, directories);
+  allocateCodexStateLease(root, codexState);
   return root;
 }
 
+function candidateRuntimeRequired(
+  directories: RuntimeArtifactDirectory[],
+  codexState: boolean,
+  retainHome: boolean,
+): boolean {
+  return directories.length > 0 || codexState || retainHome;
+}
+
 export async function requireCandidateRuntime(
   root: string | undefined,
   role?: RuntimeRole,
@@ -91,6 +110,7 @@ export async function candidateArtifactRoots(
   const roots = Object.fromEntries(
     directories.map(({ name, path }) => [name, join(root, "home", path)]),
   );
+  Object.assign(roots, candidateCodexArtifactRoots(root));
   await requireCandidateArtifactRoots(roots);
   return roots;
 }
@@ -100,19 +120,29 @@ export async function requireCandidateArtifactRoots(
   roots: CandidateArtifactRoots | undefined,
 ): Promise<void> {
   for (const [name, path] of Object.entries(roots ?? {})) {
-    const { root, directory } = artifactRootBinding(name, path);
-    await requireArtifactAncestors(root, directory.path);
+    const binding = artifactRootBinding(name, path);
+    await requireArtifactAncestors(binding.root, binding.directoryPath);
+    if (binding.file && !(await lstat(path)).isFile())
+      throw new Error(
+        "candidate Codex configuration must remain a regular file",
+      );
     await requireArtifactTree(path);
   }
 }
 
 function artifactRootBinding(name: string, path: string) {
+  const codex = candidateCodexArtifactBinding(name, path);
+  if (codex) return codex;
+  return homeArtifactBinding(name, path);
+}
+
+function homeArtifactBinding(name: string, path: string) {
   const parts = relative(nativeStatePath(), path).split(sep);
   const root = join(nativeStatePath(), parts[0] ?? "");
   const directory = leases.get(root)?.find((item) => item.name === name);
   if (!directory || path !== join(root, "home", directory.path))
     throw new Error("candidate artifact grant is not bound to an active trial");
-  return { root, directory };
+  return { root, directoryPath: join("home", directory.path), file: false };
 }
 
 async function requireArtifactAncestors(
@@ -120,7 +150,7 @@ async function requireArtifactAncestors(
   path: string,
 ): Promise<void> {
   let current = root;
-  for (const part of ["", "home", ...path.split("/")]) {
+  for (const part of ["", ...path.split(sep)]) {
     current = join(current, part);
     const info = await lstat(current);
     if (!info.isDirectory() || (await realpath(current)) !== current)
@@ -169,6 +199,7 @@ export async function releaseCandidateRuntime(
   root: string | undefined,
 ): Promise<void> {
   if (root === undefined) return;
+  await releaseCandidateCodexState(root);
   await clearFixtureContents(root);
   await rm(root, { recursive: true, force: true });
   leases.delete(root);

package/src/candidate-codex-state.ts

@@ -0,0 +1,185 @@
+import { lstat, realpath, rm } from "node:fs/promises";
+import { basename, dirname, join, relative } from "node:path";
+import { userInfo } from "node:os";
+import { nativeStatePath } from "./native-transcript-state";
+import { clearFixtureContents } from "./fixtures/cleanup";
+import type { RuntimePolicy } from "./runtime/config";
+
+interface CodexPaths {
+  codexHome: string;
+  parentHome: string;
+  commandHome: string;
+}
+interface ArtifactBinding {
+  root: string;
+  path: string;
+  directoryPath: string;
+  file?: boolean;
+}
+interface CodexLease {
+  state?: { stateRoot: string; artifacts: Record<string, ArtifactBinding> };
+}
+const leases = new Map<string, CodexLease>();
+const ARTIFACT_NAMES = [
+  "codex_config",
+  "codex_skills",
+  "agent_skills",
+  "command_agent_skills",
+  "command_codex_skills",
+];
+
+export function codexStateEnabled(policy: RuntimePolicy | undefined): boolean {
+  return policy?.codexState === true;
+}
+
+export function requireUnreservedArtifactName(name: string): void {
+  if (ARTIFACT_NAMES.includes(name))
+    throw new Error(
+      "runtime artifact directory name is reserved for Codex state",
+    );
+}
+
+export function allocateCodexStateLease(root: string, enabled: boolean): void {
+  if (enabled) leases.set(root, {});
+}
+
+/** Bind the actual host state to one trial before execution; never substitute a grader home. */
+export async function retainCandidateCodexState(
+  runtimeRoot: string | undefined,
+  stateRoot: string,
+  paths: CodexPaths,
+): Promise<void> {
+  if (runtimeRoot === undefined) return;
+  const lease = leases.get(runtimeRoot);
+  if (!lease) return;
+  if (lease.state)
+    throw new Error(
+      "candidate Codex state is already bound to a native request",
+    );
+  requireCodexLayout(stateRoot, runtimeRoot, paths);
+  await requirePrivateState(stateRoot);
+  await requirePrivateState(paths.codexHome);
+  lease.state = {
+    stateRoot,
+    artifacts: codexArtifacts(stateRoot, runtimeRoot, paths),
+  };
+}
+
+function requireCodexLayout(
+  stateRoot: string,
+  runtimeRoot: string,
+  paths: CodexPaths,
+): void {
+  if (
+    dirname(stateRoot) !== nativeStatePath() ||
+    !basename(stateRoot).startsWith("codex-")
+  )
+    throw new Error("candidate Codex state must be a native request directory");
+  requireCodexHomes(stateRoot, runtimeRoot, paths);
+}
+
+function requireCodexHomes(
+  stateRoot: string,
+  runtimeRoot: string,
+  paths: CodexPaths,
+): void {
+  if (
+    paths.codexHome !== join(stateRoot, "codex-home") ||
+    paths.parentHome !== join(stateRoot, "home") ||
+    paths.commandHome !== join(runtimeRoot, "home")
+  )
+    throw new Error("candidate Codex homes must belong to the current trial");
+}
+
+async function requirePrivateState(root: string): Promise<void> {
+  const info = await lstat(root);
+  if (
+    !info.isDirectory() ||
+    info.uid !== userInfo().uid ||
+    (info.mode & 0o077) !== 0 ||
+    (await realpath(root)) !== root
+  )
+    throw new Error(
+      "candidate Codex state must be owned, private and canonical",
+    );
+}
+
+function binding(root: string, path: string, file = false): ArtifactBinding {
+  return {
+    root,
+    path,
+    directoryPath: relative(root, file ? dirname(path) : path),
+    file,
+  };
+}
+
+function codexArtifacts(
+  stateRoot: string,
+  runtimeRoot: string,
+  paths: CodexPaths,
+): Record<string, ArtifactBinding> {
+  return {
+    codex_config: binding(
+      stateRoot,
+      join(paths.codexHome, "config.toml"),
+      true,
+    ),
+    codex_skills: binding(stateRoot, join(paths.codexHome, "skills")),
+    agent_skills: binding(
+      stateRoot,
+      join(paths.parentHome, ".agents", "skills"),
+    ),
+    command_agent_skills: binding(
+      runtimeRoot,
+      join(paths.commandHome, ".agents", "skills"),
+    ),
+    command_codex_skills: binding(
+      runtimeRoot,
+      join(paths.commandHome, ".codex", "skills"),
+    ),
+  };
+}
+
+export function candidateCodexArtifactRoots(
+  root: string,
+): Record<string, string> {
+  const lease = leases.get(root);
+  if (!lease) return {};
+  if (!lease.state) throw new Error("candidate Codex state is unavailable");
+  return Object.fromEntries(
+    Object.entries(lease.state.artifacts).map(([name, item]) => [
+      name,
+      item.path,
+    ]),
+  );
+}
+
+export function candidateCodexArtifactBinding(
+  name: string,
+  path: string,
+): ArtifactBinding | undefined {
+  for (const lease of leases.values()) {
+    const artifact = lease.state?.artifacts[name];
+    if (artifact?.path === path) return artifact;
+  }
+  return undefined;
+}
+
+export function candidateCodexStateRetained(
+  runtimeRoot: string | undefined,
+  stateRoot: string,
+): boolean {
+  return (
+    runtimeRoot !== undefined &&
+    leases.get(runtimeRoot)?.state?.stateRoot === stateRoot
+  );
+}
+
+export async function releaseCandidateCodexState(root: string): Promise<void> {
+  const state = leases.get(root)?.state;
+  if (state) {
+    await clearFixtureContents(state.stateRoot);
+    await rm(state.stateRoot, { recursive: true, force: true });
+  }
+  leases.delete(root);
+}

package/src/case-validation.ts

@@ -1,27 +1,19 @@
 import { type ResolvedCase } from "./engine";
-import { prepareGeneratedFixture } from "./generated-fixture";
-import { prepareRepositoryFixture } from "./repository-fixture";
-import {
-  isOptionalNonblankString,
-  isRecord,
-  isStringArray,
-} from "./value-guards";
+import { prepareGeneratedFixture } from "./fixtures/generated";
+import { prepareRepositoryFixture } from "./fixtures/repository";
+import { isOptionalNonblankString, isRecord } from "./value-guards";
 
-function validGeneratedFixture(value: Record<string, unknown>): boolean {
+function fixtureDeclarationIssue(
+  value: Record<string, unknown>,
+): string | undefined {
   try {
-    prepareGeneratedFixture(value);
-    return true;
-  } catch {
-    return false;
-  }
-}
-
-function validRepositoryFixture(value: Record<string, unknown>): boolean {
-  try {
-    prepareRepositoryFixture({ kind: "repository", ...value });
-    return true;
-  } catch {
-    return false;
+    if (value.kind === "generated") prepareGeneratedFixture(value);
+    else prepareRepositoryFixture({ kind: "repository", ...value });
+    return undefined;
+  } catch (error) {
+    const detail =
+      error instanceof Error ? error.message : "invalid declaration";
+    return `fixture: ${detail}; correct the fixture declaration`;
   }
 }
 
@@ -34,49 +26,80 @@ export function validInlineFixture(value: Record<string, unknown>): boolean {
   );
 }
 
-function fixtureFields(value: Record<string, unknown>): boolean {
-  if (Object.hasOwn(value, "sourceRef")) return validRepositoryFixture(value);
-  return validInlineFixture(value);
+function inlineFixtureIssue(
+  value: Record<string, unknown>,
+): string | undefined {
+  if (!Object.hasOwn(value, "files") || !isRecord(value.files))
+    return "fixture.files: expected an object mapping file paths to string contents";
+  const entry = Object.entries(value.files).find(
+    ([, content]) => typeof content !== "string",
+  );
+  return entry
+    ? `fixture.files[${JSON.stringify(entry[0])}]: expected string contents`
+    : undefined;
 }
 
-function validFixture(value: unknown): boolean {
-  if (!isRecord(value)) return false;
-  if (value.kind === "generated") return validGeneratedFixture(value);
-  if (value.kind !== undefined) return false;
-  return fixtureFields(value);
+function fixtureIssue(value: unknown): string | undefined {
+  if (!isRecord(value)) return "fixture: expected an object";
+  if (value.kind === "generated") return fixtureDeclarationIssue(value);
+  if (value.kind !== undefined)
+    return "fixture.kind: omit for inline/repository fixtures, or use generated";
+  if (Object.hasOwn(value, "sourceRef")) return fixtureDeclarationIssue(value);
+  return inlineFixtureIssue(value);
 }
 
 function nonemptyString(value: unknown): value is string {
   return typeof value === "string" && value.length > 0;
 }
 
-function validCaseDetails(value: Record<string, unknown>): boolean {
-  return (
-    nonemptyString(value.id) &&
-    nonemptyString(value.prompt) &&
-    isOptionalNonblankString(value.followUpPrompt) &&
-    validFixture(value.fixture)
-  );
+function caseDetailsIssue(value: Record<string, unknown>): string | undefined {
+  for (const field of ["id", "prompt"])
+    if (!nonemptyString(value[field]))
+      return `${field}: expected a nonempty string`;
+  if (!isOptionalNonblankString(value.followUpPrompt))
+    return "followUpPrompt: omit or supply a nonblank string";
+  return fixtureIssue(value.fixture);
 }
 
-function validCheck(value: unknown): boolean {
-  return (
-    isRecord(value) &&
-    typeof value.id === "string" &&
-    typeof value.grader === "string" &&
-    isRecord(value.configuration)
-  );
+function checkIssue(value: unknown, location: string): string | undefined {
+  if (!isRecord(value)) return `${location}: expected an object`;
+  for (const field of ["id", "grader"])
+    if (typeof value[field] !== "string")
+      return `${location}.${field}: expected a string`;
+  if (!isRecord(value.configuration))
+    return `${location}.configuration: expected an object`;
+  return undefined;
 }
 
-function validChecks(value: unknown): boolean {
-  return Array.isArray(value) && value.every(validCheck);
+function checksIssue(value: unknown): string | undefined {
+  if (!Array.isArray(value))
+    return "checks: expected an array of check objects";
+  for (const [index, check] of value.entries()) {
+    const issue = checkIssue(check, `checks[${index}]`);
+    if (issue) return issue;
+  }
+  return undefined;
 }
 
-export function isResolvedCase(value: unknown): value is ResolvedCase {
+function requiredEvidenceIssue(value: unknown): string | undefined {
+  if (!Array.isArray(value))
+    return "requiredEvidence: expected an array of strings";
+  const index = value.findIndex((entry: unknown) => typeof entry !== "string");
+  return index < 0
+    ? undefined
+    : `requiredEvidence[${index}]: expected a string`;
+}
+
+/** Explain the first invalid field using the same checks as the case type guard. */
+export function resolvedCaseIssue(value: unknown): string | undefined {
+  if (!isRecord(value)) return "case: expected an object";
   return (
-    isRecord(value) &&
-    validCaseDetails(value) &&
-    validChecks(value.checks) &&
-    isStringArray(value.requiredEvidence)
+    caseDetailsIssue(value) ??
+    checksIssue(value.checks) ??
+    requiredEvidenceIssue(value.requiredEvidence)
   );
 }
+
+export function isResolvedCase(value: unknown): value is ResolvedCase {
+  return resolvedCaseIssue(value) === undefined;
+}

package/src/cli/cancellation.ts

@@ -0,0 +1,20 @@
+export class CommandCancellation {
+  private readonly controller = new AbortController();
+  private readonly interrupt = () => {
+    this.controller.abort("SIGINT");
+  };
+  private readonly terminate = () => {
+    this.controller.abort("SIGTERM");
+  };
+  get signal(): AbortSignal {
+    return this.controller.signal;
+  }
+  constructor() {
+    process.on("SIGINT", this.interrupt);
+    process.on("SIGTERM", this.terminate);
+  }
+  dispose(): void {
+    process.off("SIGINT", this.interrupt);
+    process.off("SIGTERM", this.terminate);
+  }
+}

package/src/cli/command.ts

@@ -0,0 +1,87 @@
+import { prepareRunInvocation, type PreparedInvocation } from "./preparation";
+import { evaluateInvocation } from "./evaluation";
+import { CommandCancellation } from "./cancellation";
+import { display, displayFailure, runnerFailureCategory } from "./presentation";
+import { displayEffectiveSettings } from "./settings";
+import { jsonOutputRequested } from "./run-options";
+import { retainCliResult } from "./completion";
+
+type CommandRelease = (() => Promise<void>) | undefined;
+
+export async function main(
+  argv: string[],
+  release?: () => Promise<void>,
+): Promise<void> {
+  const json = jsonOutputRequested(argv);
+  let prepared: PreparedInvocation;
+  try {
+    prepared = await prepareRunInvocation(argv);
+  } catch (error) {
+    await releaseBeforeDisplay(release, 64);
+    displayFailure(
+      error,
+      json,
+      64,
+      "invalid invocation",
+      argv[0] === "run" ? "run" : "",
+    );
+    return;
+  }
+  const cancellation = new CommandCancellation();
+  try {
+    displayEffectiveSettings(prepared);
+    const { result } = await evaluateInvocation(prepared, cancellation.signal);
+    const completion = await retainCompletion(
+      result,
+      prepared.invocation.resultsRoot,
+    );
+    await releaseBeforeDisplay(release, result.exitCode);
+    display(result, prepared.invocation.json, completion);
+    process.exitCode = result.exitCode;
+  } catch (error) {
+    await releaseBeforeDisplay(release, 70);
+    displayFailure(
+      error,
+      prepared.invocation.json,
+      runnerFailureCategory(error),
+      "runner failure",
+    );
+  } finally {
+    cancellation.dispose();
+  }
+}
+
+async function retainCompletion(
+  result: Parameters<typeof retainCliResult>[0],
+  resultsRoot: string,
+) {
+  try {
+    return await retainCliResult(result, resultsRoot);
+  } catch (cause) {
+    if (result.exitCode === 0)
+      throw new Error(
+        "CLI result/latest could not be retained; check --results-root permissions",
+        { cause },
+      );
+    process.stderr.write(
+      "sevro: CLI result/latest could not be retained; original run outcome preserved; inspect evidence directly\n",
+    );
+    return undefined;
+  }
+}
+
+async function releaseBeforeDisplay(
+  release: CommandRelease,
+  exitCode: number,
+): Promise<void> {
+  try {
+    await release?.();
+  } catch (error) {
+    if (exitCode === 0) throw error;
+    const detail =
+      error instanceof Error
+        ? error.message
+        : "temporary project could not be removed";
+    process.stderr.write(`sevro: demo cleanup failed: ${detail}\n`);
+  }
+}

package/src/cli/completion.ts

@@ -0,0 +1,15 @@
+import { dirname, join } from "node:path";
+import { atomicWriteJson } from "../storage";
+import type { CliResult } from "../engine";
+import { createReport } from "../report";
+import { withResultsLock } from "../results-lock";
+
+/** Latest means the last successfully retained CLI result, including dry/failing runs. */
+export async function retainCliResult(result: CliResult, resultsRoot: string) {
+  return withResultsLock(resultsRoot, async () => {
+    const resultFile = join(dirname(result.evidencePath), "result.json");
+    await atomicWriteJson(resultFile, result);
+    await atomicWriteJson(join(resultsRoot, "latest.json"), result);
+    return { resultFile, report: await createReport([resultFile]) };
+  });
+}

package/src/cli/demo.ts

@@ -0,0 +1,60 @@
+import { mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { main } from "./command";
+import { InvocationError } from "./invocation";
+import { displayFailure } from "./presentation";
+
+/** Run the packaged deterministic case without inheriting a caller's project policy. */
+export async function demoCommand(argv: string[]): Promise<void> {
+  const json = argv.includes("--json");
+  if (argv.some((arg) => arg !== "--json")) {
+    displayFailure(
+      new InvocationError("demo accepts only --json; run sevro demo --help"),
+      json,
+      64,
+      "invalid demo invocation",
+      "demo",
+    );
+    return;
+  }
+  let projectRoot: string;
+  try {
+    projectRoot = await mkdtemp(join(tmpdir(), "sevro-demo-"));
+  } catch (cause) {
+    displayFailure(
+      new Error(
+        `demo temporary project could not be created under ${tmpdir()}; set TMPDIR to a writable directory`,
+        { cause },
+      ),
+      json,
+      70,
+      "demo preparation failed",
+    );
+    return;
+  }
+  await main(
+    [
+      "run",
+      ...(json ? ["--json"] : []),
+      "--case-file",
+      join(import.meta.dir, "..", "..", "examples", "basic", "graded.json"),
+      "--adapter-module",
+      join(import.meta.dir, "..", "..", "examples", "basic", "host.ts"),
+      "--project-root",
+      projectRoot,
+      "--results-root",
+      join(process.cwd(), ".sevro", "results"),
+    ],
+    demoCleanup(projectRoot),
+  );
+}
+
+function demoCleanup(projectRoot: string): () => Promise<void> {
+  let attempted = false;
+  return async () => {
+    if (attempted) return;
+    attempted = true;
+    await rm(projectRoot, { recursive: true, force: true });
+  };
+}