@bjoernrochel/sevro 0.1.0-rc.2 → 0.1.0-rc.21

package/src/candidate-artifacts.ts

@@ -0,0 +1,208 @@
+import { lstat, readdir, realpath, rm } from "node:fs/promises";
+import { join, relative, sep } from "node:path";
+import {
+  allocateCodexStateLease,
+  candidateCodexArtifactBinding,
+  candidateCodexArtifactRoots,
+  codexStateEnabled,
+  releaseCandidateCodexState,
+  requireUnreservedArtifactName,
+} from "./candidate-codex-state";
+import {
+  allocateNativeState,
+  nativeStatePath,
+} from "./native-transcript-state";
+import type { RuntimePolicy, RuntimeRole } from "./runtime/config";
+import { clearFixtureContents } from "./fixtures/cleanup";
+
+export interface RuntimeArtifactDirectory {
+  name: string;
+  path: string;
+}
+
+export type CandidateArtifactRoots = Record<string, string>;
+
+const leases = new Map<string, RuntimeArtifactDirectory[]>();
+const MAX_ENTRIES = 16_384;
+
+/** These are relative candidate-home paths, never operator filesystem grants. */
+export function requireArtifactDirectories(
+  directories: RuntimeArtifactDirectory[],
+): void {
+  if (!Array.isArray(directories) || directories.length > 16)
+    throw new Error("runtime artifact directories exceed the limit");
+  const names = new Set<string>();
+  for (const directory of directories) {
+    requireArtifactDirectory(directory);
+    if (names.has(directory.name))
+      throw new Error("runtime artifact directory names overlap");
+    names.add(directory.name);
+  }
+}
+
+function requireArtifactDirectory(directory: RuntimeArtifactDirectory): void {
+  if (!/^[a-z][a-z0-9_]{0,63}$/.test(directory.name))
+    throw new Error("runtime artifact directory name is invalid");
+  requireUnreservedArtifactName(directory.name);
+  const path = directory.path;
+  if (!validArtifactPath(path))
+    throw new Error("runtime artifact directory path is invalid");
+  if (path.split("/").some((part) => !part || part === "." || part === ".."))
+    throw new Error(
+      "runtime artifact directory must be a bounded relative path",
+    );
+}
+
+function validArtifactPath(path: string): boolean {
+  if (typeof path !== "string" || path.length > 4096 || path.includes("\\"))
+    return false;
+  return !Array.from(path).some(
+    (character) =>
+      character.charCodeAt(0) < 0x20 || character.charCodeAt(0) === 0x7f,
+  );
+}
+
+/** Allocate before host execution so exceptions cannot lose cleanup ownership. */
+export async function allocateCandidateRuntime(
+  policy: RuntimePolicy | undefined,
+  retainHome = false,
+): Promise<string | undefined> {
+  const directories = policy?.artifactDirectories ?? [];
+  const codexState = codexStateEnabled(policy);
+  if (!candidateRuntimeRequired(directories, codexState, retainHome))
+    return undefined;
+  requireArtifactDirectories(directories);
+  const root = await allocateNativeState("candidate-runtime-");
+  leases.set(root, directories);
+  allocateCodexStateLease(root, codexState);
+  return root;
+}
+
+function candidateRuntimeRequired(
+  directories: RuntimeArtifactDirectory[],
+  codexState: boolean,
+  retainHome: boolean,
+): boolean {
+  return directories.length > 0 || codexState || retainHome;
+}
+
+export async function requireCandidateRuntime(
+  root: string | undefined,
+  role?: RuntimeRole,
+): Promise<void> {
+  if (root === undefined) return;
+  requireCandidateRole(role);
+  if (!leases.has(root) || (await realpath(root)) !== root)
+    throw new Error("candidate runtime must be owned by the current trial");
+}
+
+function requireCandidateRole(role: RuntimeRole | undefined): void {
+  if (role !== undefined && role !== "candidate")
+    throw new Error(
+      "retained candidate runtime cannot be used by another role",
+    );
+}
+
+export async function candidateArtifactRoots(
+  root: string | undefined,
+): Promise<CandidateArtifactRoots | undefined> {
+  if (root === undefined) return undefined;
+  const directories = leases.get(root);
+  if (!directories) throw new Error("candidate runtime lease is unavailable");
+  const roots = Object.fromEntries(
+    directories.map(({ name, path }) => [name, join(root, "home", path)]),
+  );
+  Object.assign(roots, candidateCodexArtifactRoots(root));
+  await requireCandidateArtifactRoots(roots);
+  return roots;
+}
+
+/** Validate the engine binding and refuse candidate-created filesystem aliases. */
+export async function requireCandidateArtifactRoots(
+  roots: CandidateArtifactRoots | undefined,
+): Promise<void> {
+  for (const [name, path] of Object.entries(roots ?? {})) {
+    const binding = artifactRootBinding(name, path);
+    await requireArtifactAncestors(binding.root, binding.directoryPath);
+    if (binding.file && !(await lstat(path)).isFile())
+      throw new Error(
+        "candidate Codex configuration must remain a regular file",
+      );
+    await requireArtifactTree(path);
+  }
+}
+
+function artifactRootBinding(name: string, path: string) {
+  const codex = candidateCodexArtifactBinding(name, path);
+  if (codex) return codex;
+  return homeArtifactBinding(name, path);
+}
+
+function homeArtifactBinding(name: string, path: string) {
+  const parts = relative(nativeStatePath(), path).split(sep);
+  const root = join(nativeStatePath(), parts[0] ?? "");
+  const directory = leases.get(root)?.find((item) => item.name === name);
+  if (!directory || path !== join(root, "home", directory.path))
+    throw new Error("candidate artifact grant is not bound to an active trial");
+  return { root, directoryPath: join("home", directory.path), file: false };
+}
+
+async function requireArtifactAncestors(
+  root: string,
+  path: string,
+): Promise<void> {
+  let current = root;
+  for (const part of ["", ...path.split(sep)]) {
+    current = join(current, part);
+    const info = await lstat(current);
+    if (!info.isDirectory() || (await realpath(current)) !== current)
+      throw new Error(
+        "candidate artifact ancestors must be canonical directories",
+      );
+  }
+}
+
+async function requireArtifactTree(root: string): Promise<void> {
+  const pending = [root];
+  for (const path of pending) {
+    if (pending.length > MAX_ENTRIES)
+      throw new Error("candidate artifact tree exceeds the entry limit");
+    pending.push(...(await artifactChildren(path)));
+  }
+}
+
+async function artifactChildren(path: string): Promise<string[]> {
+  const info = await lstat(path);
+  if (info.isDirectory())
+    return (await readdir(path)).map((name) => join(path, name));
+  if (!info.isFile() || info.nlink !== 1)
+    throw new Error("candidate artifact tree contains a link or special file");
+  return [];
+}
+
+export function candidateArtifactEnvironment(
+  roots: CandidateArtifactRoots | undefined,
+): Record<string, string> {
+  return Object.fromEntries(
+    Object.entries(roots ?? {}).map(([name, path]) => [
+      `SEVRO_CANDIDATE_ARTIFACT_${name.toUpperCase()}`,
+      path,
+    ]),
+  );
+}
+
+export function candidateArtifactReadRoots(
+  roots: CandidateArtifactRoots | undefined,
+): string[] {
+  return Object.values(roots ?? {});
+}
+
+export async function releaseCandidateRuntime(
+  root: string | undefined,
+): Promise<void> {
+  if (root === undefined) return;
+  await releaseCandidateCodexState(root);
+  await clearFixtureContents(root);
+  await rm(root, { recursive: true, force: true });
+  leases.delete(root);
+}

package/src/candidate-codex-state.ts

@@ -0,0 +1,185 @@
+import { lstat, realpath, rm } from "node:fs/promises";
+import { basename, dirname, join, relative } from "node:path";
+import { userInfo } from "node:os";
+import { nativeStatePath } from "./native-transcript-state";
+import { clearFixtureContents } from "./fixtures/cleanup";
+import type { RuntimePolicy } from "./runtime/config";
+
+interface CodexPaths {
+  codexHome: string;
+  parentHome: string;
+  commandHome: string;
+}
+interface ArtifactBinding {
+  root: string;
+  path: string;
+  directoryPath: string;
+  file?: boolean;
+}
+interface CodexLease {
+  state?: { stateRoot: string; artifacts: Record<string, ArtifactBinding> };
+}
+const leases = new Map<string, CodexLease>();
+const ARTIFACT_NAMES = [
+  "codex_config",
+  "codex_skills",
+  "agent_skills",
+  "command_agent_skills",
+  "command_codex_skills",
+];
+
+export function codexStateEnabled(policy: RuntimePolicy | undefined): boolean {
+  return policy?.codexState === true;
+}
+
+export function requireUnreservedArtifactName(name: string): void {
+  if (ARTIFACT_NAMES.includes(name))
+    throw new Error(
+      "runtime artifact directory name is reserved for Codex state",
+    );
+}
+
+export function allocateCodexStateLease(root: string, enabled: boolean): void {
+  if (enabled) leases.set(root, {});
+}
+
+/** Bind the actual host state to one trial before execution; never substitute a grader home. */
+export async function retainCandidateCodexState(
+  runtimeRoot: string | undefined,
+  stateRoot: string,
+  paths: CodexPaths,
+): Promise<void> {
+  if (runtimeRoot === undefined) return;
+  const lease = leases.get(runtimeRoot);
+  if (!lease) return;
+  if (lease.state)
+    throw new Error(
+      "candidate Codex state is already bound to a native request",
+    );
+  requireCodexLayout(stateRoot, runtimeRoot, paths);
+  await requirePrivateState(stateRoot);
+  await requirePrivateState(paths.codexHome);
+  lease.state = {
+    stateRoot,
+    artifacts: codexArtifacts(stateRoot, runtimeRoot, paths),
+  };
+}
+
+function requireCodexLayout(
+  stateRoot: string,
+  runtimeRoot: string,
+  paths: CodexPaths,
+): void {
+  if (
+    dirname(stateRoot) !== nativeStatePath() ||
+    !basename(stateRoot).startsWith("codex-")
+  )
+    throw new Error("candidate Codex state must be a native request directory");
+  requireCodexHomes(stateRoot, runtimeRoot, paths);
+}
+
+function requireCodexHomes(
+  stateRoot: string,
+  runtimeRoot: string,
+  paths: CodexPaths,
+): void {
+  if (
+    paths.codexHome !== join(stateRoot, "codex-home") ||
+    paths.parentHome !== join(stateRoot, "home") ||
+    paths.commandHome !== join(runtimeRoot, "home")
+  )
+    throw new Error("candidate Codex homes must belong to the current trial");
+}
+
+async function requirePrivateState(root: string): Promise<void> {
+  const info = await lstat(root);
+  if (
+    !info.isDirectory() ||
+    info.uid !== userInfo().uid ||
+    (info.mode & 0o077) !== 0 ||
+    (await realpath(root)) !== root
+  )
+    throw new Error(
+      "candidate Codex state must be owned, private and canonical",
+    );
+}
+
+function binding(root: string, path: string, file = false): ArtifactBinding {
+  return {
+    root,
+    path,
+    directoryPath: relative(root, file ? dirname(path) : path),
+    file,
+  };
+}
+
+function codexArtifacts(
+  stateRoot: string,
+  runtimeRoot: string,
+  paths: CodexPaths,
+): Record<string, ArtifactBinding> {
+  return {
+    codex_config: binding(
+      stateRoot,
+      join(paths.codexHome, "config.toml"),
+      true,
+    ),
+    codex_skills: binding(stateRoot, join(paths.codexHome, "skills")),
+    agent_skills: binding(
+      stateRoot,
+      join(paths.parentHome, ".agents", "skills"),
+    ),
+    command_agent_skills: binding(
+      runtimeRoot,
+      join(paths.commandHome, ".agents", "skills"),
+    ),
+    command_codex_skills: binding(
+      runtimeRoot,
+      join(paths.commandHome, ".codex", "skills"),
+    ),
+  };
+}
+
+export function candidateCodexArtifactRoots(
+  root: string,
+): Record<string, string> {
+  const lease = leases.get(root);
+  if (!lease) return {};
+  if (!lease.state) throw new Error("candidate Codex state is unavailable");
+  return Object.fromEntries(
+    Object.entries(lease.state.artifacts).map(([name, item]) => [
+      name,
+      item.path,
+    ]),
+  );
+}
+
+export function candidateCodexArtifactBinding(
+  name: string,
+  path: string,
+): ArtifactBinding | undefined {
+  for (const lease of leases.values()) {
+    const artifact = lease.state?.artifacts[name];
+    if (artifact?.path === path) return artifact;
+  }
+  return undefined;
+}
+
+export function candidateCodexStateRetained(
+  runtimeRoot: string | undefined,
+  stateRoot: string,
+): boolean {
+  return (
+    runtimeRoot !== undefined &&
+    leases.get(runtimeRoot)?.state?.stateRoot === stateRoot
+  );
+}
+
+export async function releaseCandidateCodexState(root: string): Promise<void> {
+  const state = leases.get(root)?.state;
+  if (state) {
+    await clearFixtureContents(state.stateRoot);
+    await rm(state.stateRoot, { recursive: true, force: true });
+  }
+  leases.delete(root);
+}

package/src/case-validation.ts

@@ -0,0 +1,105 @@
+import { type ResolvedCase } from "./engine";
+import { prepareGeneratedFixture } from "./fixtures/generated";
+import { prepareRepositoryFixture } from "./fixtures/repository";
+import { isOptionalNonblankString, isRecord } from "./value-guards";
+
+function fixtureDeclarationIssue(
+  value: Record<string, unknown>,
+): string | undefined {
+  try {
+    if (value.kind === "generated") prepareGeneratedFixture(value);
+    else prepareRepositoryFixture({ kind: "repository", ...value });
+    return undefined;
+  } catch (error) {
+    const detail =
+      error instanceof Error ? error.message : "invalid declaration";
+    return `fixture: ${detail}; correct the fixture declaration`;
+  }
+}
+
+export function validInlineFixture(value: Record<string, unknown>): boolean {
+  return (
+    !Object.hasOwn(value, "sourceRef") &&
+    Object.hasOwn(value, "files") &&
+    isRecord(value.files) &&
+    Object.values(value.files).every((content) => typeof content === "string")
+  );
+}
+
+function inlineFixtureIssue(
+  value: Record<string, unknown>,
+): string | undefined {
+  if (!Object.hasOwn(value, "files") || !isRecord(value.files))
+    return "fixture.files: expected an object mapping file paths to string contents";
+  const entry = Object.entries(value.files).find(
+    ([, content]) => typeof content !== "string",
+  );
+  return entry
+    ? `fixture.files[${JSON.stringify(entry[0])}]: expected string contents`
+    : undefined;
+}
+
+function fixtureIssue(value: unknown): string | undefined {
+  if (!isRecord(value)) return "fixture: expected an object";
+  if (value.kind === "generated") return fixtureDeclarationIssue(value);
+  if (value.kind !== undefined)
+    return "fixture.kind: omit for inline/repository fixtures, or use generated";
+  if (Object.hasOwn(value, "sourceRef")) return fixtureDeclarationIssue(value);
+  return inlineFixtureIssue(value);
+}
+
+function nonemptyString(value: unknown): value is string {
+  return typeof value === "string" && value.length > 0;
+}
+
+function caseDetailsIssue(value: Record<string, unknown>): string | undefined {
+  for (const field of ["id", "prompt"])
+    if (!nonemptyString(value[field]))
+      return `${field}: expected a nonempty string`;
+  if (!isOptionalNonblankString(value.followUpPrompt))
+    return "followUpPrompt: omit or supply a nonblank string";
+  return fixtureIssue(value.fixture);
+}
+
+function checkIssue(value: unknown, location: string): string | undefined {
+  if (!isRecord(value)) return `${location}: expected an object`;
+  for (const field of ["id", "grader"])
+    if (typeof value[field] !== "string")
+      return `${location}.${field}: expected a string`;
+  if (!isRecord(value.configuration))
+    return `${location}.configuration: expected an object`;
+  return undefined;
+}
+
+function checksIssue(value: unknown): string | undefined {
+  if (!Array.isArray(value))
+    return "checks: expected an array of check objects";
+  for (const [index, check] of value.entries()) {
+    const issue = checkIssue(check, `checks[${index}]`);
+    if (issue) return issue;
+  }
+  return undefined;
+}
+
+function requiredEvidenceIssue(value: unknown): string | undefined {
+  if (!Array.isArray(value))
+    return "requiredEvidence: expected an array of strings";
+  const index = value.findIndex((entry: unknown) => typeof entry !== "string");
+  return index < 0
+    ? undefined
+    : `requiredEvidence[${index}]: expected a string`;
+}
+
+/** Explain the first invalid field using the same checks as the case type guard. */
+export function resolvedCaseIssue(value: unknown): string | undefined {
+  if (!isRecord(value)) return "case: expected an object";
+  return (
+    caseDetailsIssue(value) ??
+    checksIssue(value.checks) ??
+    requiredEvidenceIssue(value.requiredEvidence)
+  );
+}
+
+export function isResolvedCase(value: unknown): value is ResolvedCase {
+  return resolvedCaseIssue(value) === undefined;
+}

package/src/cli/cancellation.ts

@@ -0,0 +1,20 @@
+export class CommandCancellation {
+  private readonly controller = new AbortController();
+  private readonly interrupt = () => {
+    this.controller.abort("SIGINT");
+  };
+  private readonly terminate = () => {
+    this.controller.abort("SIGTERM");
+  };
+  get signal(): AbortSignal {
+    return this.controller.signal;
+  }
+  constructor() {
+    process.on("SIGINT", this.interrupt);
+    process.on("SIGTERM", this.terminate);
+  }
+  dispose(): void {
+    process.off("SIGINT", this.interrupt);
+    process.off("SIGTERM", this.terminate);
+  }
+}

package/src/cli/command.ts

@@ -0,0 +1,81 @@
+import { prepareRunInvocation, type PreparedInvocation } from "./preparation";
+import { evaluateInvocation } from "./evaluation";
+import { CommandCancellation } from "./cancellation";
+import { display, displayFailure, runnerFailureCategory } from "./presentation";
+import { displayEffectiveSettings } from "./settings";
+import { jsonOutputRequested } from "./run-options";
+import { retainCliResult } from "./completion";
+
+type CommandRelease = (() => Promise<void>) | undefined;
+
+export async function main(
+  argv: string[],
+  release?: () => Promise<void>,
+): Promise<void> {
+  const json = jsonOutputRequested(argv);
+  let prepared: PreparedInvocation;
+  try {
+    prepared = await prepareRunInvocation(argv);
+  } catch (error) {
+    await releaseBeforeDisplay(release, 64);
+    displayFailure(error, json, 64, "invalid invocation");
+    return;
+  }
+  const cancellation = new CommandCancellation();
+  try {
+    displayEffectiveSettings(prepared);
+    const { result } = await evaluateInvocation(prepared, cancellation.signal);
+    const completion = await retainCompletion(
+      result,
+      prepared.invocation.resultsRoot,
+    );
+    await releaseBeforeDisplay(release, result.exitCode);
+    display(result, prepared.invocation.json, completion);
+    process.exitCode = result.exitCode;
+  } catch (error) {
+    await releaseBeforeDisplay(release, 70);
+    displayFailure(
+      error,
+      prepared.invocation.json,
+      runnerFailureCategory(error),
+      "runner failure",
+    );
+  } finally {
+    cancellation.dispose();
+  }
+}
+
+async function retainCompletion(
+  result: Parameters<typeof retainCliResult>[0],
+  resultsRoot: string,
+) {
+  try {
+    return await retainCliResult(result, resultsRoot);
+  } catch (cause) {
+    if (result.exitCode === 0)
+      throw new Error(
+        "CLI result/latest could not be retained; check --results-root permissions",
+        { cause },
+      );
+    process.stderr.write(
+      "sevro: CLI result/latest could not be retained; original run outcome preserved; inspect evidence directly\n",
+    );
+    return undefined;
+  }
+}
+
+async function releaseBeforeDisplay(
+  release: CommandRelease,
+  exitCode: number,
+): Promise<void> {
+  try {
+    await release?.();
+  } catch (error) {
+    if (exitCode === 0) throw error;
+    const detail =
+      error instanceof Error
+        ? error.message
+        : "temporary project could not be removed";
+    process.stderr.write(`sevro: demo cleanup failed: ${detail}\n`);
+  }
+}

package/src/cli/completion.ts

@@ -0,0 +1,15 @@
+import { dirname, join } from "node:path";
+import { atomicWriteJson } from "../storage";
+import type { CliResult } from "../engine";
+import { createReport } from "../report";
+import { withResultsLock } from "../results-lock";
+
+/** Latest means the last successfully retained CLI result, including dry/failing runs. */
+export async function retainCliResult(result: CliResult, resultsRoot: string) {
+  return withResultsLock(resultsRoot, async () => {
+    const resultFile = join(dirname(result.evidencePath), "result.json");
+    await atomicWriteJson(resultFile, result);
+    await atomicWriteJson(join(resultsRoot, "latest.json"), result);
+    return { resultFile, report: await createReport([resultFile]) };
+  });
+}

package/src/cli/demo.ts

@@ -0,0 +1,59 @@
+import { mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { main } from "./command";
+import { InvocationError } from "./invocation";
+import { displayFailure } from "./presentation";
+
+/** Run the packaged deterministic case without inheriting a caller's project policy. */
+export async function demoCommand(argv: string[]): Promise<void> {
+  const json = argv.includes("--json");
+  if (argv.some((arg) => arg !== "--json")) {
+    displayFailure(
+      new InvocationError("demo accepts only --json; run sevro demo --help"),
+      json,
+      64,
+      "invalid demo invocation",
+    );
+    return;
+  }
+  let projectRoot: string;
+  try {
+    projectRoot = await mkdtemp(join(tmpdir(), "sevro-demo-"));
+  } catch (cause) {
+    displayFailure(
+      new Error(
+        `demo temporary project could not be created under ${tmpdir()}; set TMPDIR to a writable directory`,
+        { cause },
+      ),
+      json,
+      70,
+      "demo preparation failed",
+    );
+    return;
+  }
+  await main(
+    [
+      "run",
+      ...(json ? ["--json"] : []),
+      "--case-file",
+      join(import.meta.dir, "..", "..", "examples", "basic", "graded.json"),
+      "--adapter-module",
+      join(import.meta.dir, "..", "..", "examples", "basic", "host.ts"),
+      "--project-root",
+      projectRoot,
+      "--results-root",
+      join(process.cwd(), ".sevro", "results"),
+    ],
+    demoCleanup(projectRoot),
+  );
+}
+
+function demoCleanup(projectRoot: string): () => Promise<void> {
+  let attempted = false;
+  return async () => {
+    if (attempted) return;
+    attempted = true;
+    await rm(projectRoot, { recursive: true, force: true });
+  };
+}