@bjoernrochel/sevro 0.1.0-rc.8 → 1.1.0

package/examples/templates/response/case.json

@@ -0,0 +1,13 @@
+{
+  "id": "response-check",
+  "prompt": "Return exactly ready.",
+  "fixture": { "files": { "README.md": "Response-check fixture.\n" } },
+  "checks": [
+    {
+      "id": "ready",
+      "grader": "sevro.regex",
+      "configuration": { "pattern": "^ready$" }
+    }
+  ],
+  "requiredEvidence": []
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bjoernrochel/sevro",
-  "version": "0.1.0-rc.8",
+  "version": "1.1.0",
   "license": "SEE LICENSE IN LICENSE",
   "description": "A standalone evaluation runner for agentic coding capabilities",
   "type": "module",
@@ -19,13 +19,21 @@
     "README.md",
     "CONTRIBUTING.md",
     "LICENSE",
-    "docs/",
+    "docs/*.md",
+    "!docs/*-validation.md",
+    "!docs/engine-slice.md",
+    "!docs/evidence.md",
+    "!docs/usability-study.md",
+    "docs/assets/README.md",
+    "docs/licenses/",
+    "docs/specs/",
     "examples/",
     "schemas/",
-    "src/"
+    "src/",
+    "dist/"
   ],
   "bin": {
-    "sevro": "./src/cli.ts"
+    "sevro": "./dist/src/cli.js"
   },
   "scripts": {
     "check:typescript": "bun scripts/check-typescript.ts",
@@ -42,14 +50,18 @@
     "test:docs-examples": "bun scripts/verify-doc-examples.ts",
     "guide:sync": "bun scripts/sync-guide.ts",
     "eval:guide": "bun scripts/eval-guide.ts",
-    "release:prepare": "bun scripts/prepare-release.ts",
+    "release:prepare": "bun run build && bun scripts/prepare-release.ts",
     "typecheck": "tsc --noEmit",
-    "format:check": "prettier --check README.md CONTRIBUTING.md AGENTS.md CLAUDE.md 'docs/**/*.md' '.agents/**/*.md' '.agents/**/*.json' '.claude/**/*.md' 'scripts/**/*.md' examples/**/*.json examples/**/*.ts schemas/*.json src/**/*.ts scripts/**/*.ts scripts/**/*.cjs tests/**/*.ts eslint.config.mjs package.json tsconfig.json typescript-sources.json .github/workflows/*.yml"
+    "format:check": "prettier --check README.md CONTRIBUTING.md AGENTS.md CLAUDE.md 'docs/**/*.md' '.agents/**/*.md' '.agents/**/*.json' '.claude/**/*.md' 'scripts/**/*.md' 'scripts/**/*.json' 'examples/**/*.json' 'examples/**/*.ts' 'schemas/*.json' 'src/**/*.ts' 'scripts/**/*.ts' 'scripts/**/*.cjs' 'tests/**/*.ts' eslint.config.mjs package.json tsconfig.json '.github/workflows/*.yml'",
+    "build": "bun scripts/build.ts",
+    "prepack": "bun run build",
+    "test:runtime-node": "bun scripts/verify-node-runtime.ts"
   },
   "dependencies": {
     "ajv": "8.20.0",
     "smol-toml": "1.8.0",
-    "typescript": "5.9.3"
+    "typescript": "5.9.3",
+    "yaml": "2.9.1"
   },
   "devDependencies": {
     "@babel/core": "8.0.6",
@@ -68,11 +80,13 @@
     "istanbul-lib-coverage": "3.2.2",
     "istanbul-lib-instrument": "6.0.3",
     "istanbul-lib-report": "3.0.1",
-    "istanbul-lib-source-maps": "5.0.6",
     "istanbul-reports": "3.2.0",
     "markdown-it": "14.2.0",
-    "prettier": "^3.9.9",
+    "prettier": "3.9.9",
     "shell-quote": "1.12.0",
     "typescript-eslint": "8.71.1"
+  },
+  "engines": {
+    "node": ">=24.13.0"
   }
 }

package/schemas/report-v1.schema.json

@@ -110,6 +110,15 @@
         "domainOutcomes"
       ],
       "properties": {
+        "nativeTranscripts": {
+          "type": "object",
+          "required": ["retained", "pruned"],
+          "properties": {
+            "retained": { "type": "integer", "minimum": 0 },
+            "pruned": { "type": "integer", "minimum": 0 }
+          },
+          "additionalProperties": false
+        },
         "resultFile": { "$ref": "#/$defs/path" },
         "evidencePath": { "type": ["string", "null"] },
         "runId": { "type": ["string", "null"] },

package/schemas/run-evidence-v1.schema.json

@@ -80,7 +80,8 @@
       "required": ["digest", "redacted"],
       "properties": {
         "digest": { "$ref": "#/$defs/digest" },
-        "redacted": { "type": "object" }
+        "redacted": { "type": "object" },
+        "resolved": { "type": "object" }
       },
       "additionalProperties": false
     },

package/schemas/runtime-v1.schema.json

@@ -5,7 +5,23 @@
   "type": "object",
   "additionalProperties": false,
   "required": ["format"],
+  "$defs": {
+    "runtimeRole": {
+      "$comment": "Generated from RUNTIME_ROLES by bun run schemas:generate.",
+      "enum": ["candidate", "checks", "hooks", "semantic", "advisory"]
+    }
+  },
   "properties": {
+    "codexState": {
+      "type": "boolean"
+    },
+    "network": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "localHttp": { "type": "boolean" }
+      }
+    },
     "nativeTranscripts": {
       "type": "boolean"
     },
@@ -52,6 +68,32 @@
             }
           }
         },
+        "sharedCacheDirectories": {
+          "type": "array",
+          "maxItems": 16,
+          "items": {
+            "type": "object",
+            "additionalProperties": false,
+            "required": ["source", "target", "roles"],
+            "properties": {
+              "source": { "type": "string", "minLength": 1, "maxLength": 4096 },
+              "target": {
+                "type": "string",
+                "pattern": "^[a-z][a-z0-9-]{0,63}$",
+                "not": { "enum": ["home", "tmp"] }
+              },
+              "roles": {
+                "type": "array",
+                "minItems": 1,
+                "maxItems": 5,
+                "uniqueItems": true,
+                "items": {
+                  "$ref": "#/$defs/runtimeRole"
+                }
+              }
+            }
+          }
+        },
         "seedDirectories": {
           "type": "array",
           "maxItems": 16,
@@ -78,13 +120,7 @@
                 "maxItems": 5,
                 "uniqueItems": true,
                 "items": {
-                  "enum": [
-                    "candidate",
-                    "checks",
-                    "hooks",
-                    "semantic",
-                    "advisory"
-                  ]
+                  "$ref": "#/$defs/runtimeRole"
                 }
               }
             }

package/src/advisory.ts

@@ -7,14 +7,14 @@ const DIMENSIONS = [
   "scopeDiscipline",
 ] as const;
 
-export interface AdvisoryAssessment {
+export type AdvisoryAssessment = {
   verdict: "pass" | "fail";
   overallScore: number;
   dimensions: Record<(typeof DIMENSIONS)[number], number>;
   strengths: string[];
   weaknesses: string[];
   summary: string;
-}
+};
 
 function object(value: unknown): Record<string, unknown> {
   if (!value || typeof value !== "object" || Array.isArray(value))

package/src/candidate-artifacts.ts

@@ -1,11 +1,20 @@
 import { lstat, readdir, realpath, rm } from "node:fs/promises";
+import { containsControl } from "./value-guards";
 import { join, relative, sep } from "node:path";
+import {
+  allocateCodexStateLease,
+  candidateCodexArtifactBinding,
+  candidateCodexArtifactRoots,
+  codexStateEnabled,
+  releaseCandidateCodexState,
+  requireUnreservedArtifactName,
+} from "./candidate-codex-state";
 import {
   allocateNativeState,
   nativeStatePath,
 } from "./native-transcript-state";
-import type { RuntimePolicy, RuntimeRole } from "./runtime-config";
-import { clearFixtureContents } from "./fixture-cleanup";
+import type { RuntimePolicy, RuntimeRole } from "./runtime/config";
+import { clearFixtureContents } from "./fixtures/cleanup";
 
 export interface RuntimeArtifactDirectory {
   name: string;
@@ -35,6 +44,7 @@ export function requireArtifactDirectories(
 function requireArtifactDirectory(directory: RuntimeArtifactDirectory): void {
   if (!/^[a-z][a-z0-9_]{0,63}$/.test(directory.name))
     throw new Error("runtime artifact directory name is invalid");
+  requireUnreservedArtifactName(directory.name);
   const path = directory.path;
   if (!validArtifactPath(path))
     throw new Error("runtime artifact directory path is invalid");
@@ -47,24 +57,33 @@ function requireArtifactDirectory(directory: RuntimeArtifactDirectory): void {
 function validArtifactPath(path: string): boolean {
   if (typeof path !== "string" || path.length > 4096 || path.includes("\\"))
     return false;
-  return !Array.from(path).some(
-    (character) =>
-      character.charCodeAt(0) < 0x20 || character.charCodeAt(0) === 0x7f,
-  );
+  return !containsControl(path);
 }
 
 /** Allocate before host execution so exceptions cannot lose cleanup ownership. */
 export async function allocateCandidateRuntime(
   policy: RuntimePolicy | undefined,
+  retainHome = false,
 ): Promise<string | undefined> {
   const directories = policy?.artifactDirectories ?? [];
-  if (!directories.length) return undefined;
+  const codexState = codexStateEnabled(policy);
+  if (!candidateRuntimeRequired(directories, codexState, retainHome))
+    return undefined;
   requireArtifactDirectories(directories);
   const root = await allocateNativeState("candidate-runtime-");
   leases.set(root, directories);
+  allocateCodexStateLease(root, codexState);
   return root;
 }
 
+function candidateRuntimeRequired(
+  directories: RuntimeArtifactDirectory[],
+  codexState: boolean,
+  retainHome: boolean,
+): boolean {
+  return directories.length > 0 || codexState || retainHome;
+}
+
 export async function requireCandidateRuntime(
   root: string | undefined,
   role?: RuntimeRole,
@@ -91,6 +110,7 @@ export async function candidateArtifactRoots(
   const roots = Object.fromEntries(
     directories.map(({ name, path }) => [name, join(root, "home", path)]),
   );
+  Object.assign(roots, candidateCodexArtifactRoots(root));
   await requireCandidateArtifactRoots(roots);
   return roots;
 }
@@ -100,19 +120,29 @@ export async function requireCandidateArtifactRoots(
   roots: CandidateArtifactRoots | undefined,
 ): Promise<void> {
   for (const [name, path] of Object.entries(roots ?? {})) {
-    const { root, directory } = artifactRootBinding(name, path);
-    await requireArtifactAncestors(root, directory.path);
+    const binding = artifactRootBinding(name, path);
+    await requireArtifactAncestors(binding.root, binding.directoryPath);
+    if (binding.file && !(await lstat(path)).isFile())
+      throw new Error(
+        "candidate Codex configuration must remain a regular file",
+      );
     await requireArtifactTree(path);
   }
 }
 
 function artifactRootBinding(name: string, path: string) {
+  const codex = candidateCodexArtifactBinding(name, path);
+  if (codex) return codex;
+  return homeArtifactBinding(name, path);
+}
+
+function homeArtifactBinding(name: string, path: string) {
   const parts = relative(nativeStatePath(), path).split(sep);
   const root = join(nativeStatePath(), parts[0] ?? "");
   const directory = leases.get(root)?.find((item) => item.name === name);
   if (!directory || path !== join(root, "home", directory.path))
     throw new Error("candidate artifact grant is not bound to an active trial");
-  return { root, directory };
+  return { root, directoryPath: join("home", directory.path), file: false };
 }
 
 async function requireArtifactAncestors(
@@ -120,7 +150,7 @@ async function requireArtifactAncestors(
   path: string,
 ): Promise<void> {
   let current = root;
-  for (const part of ["", "home", ...path.split("/")]) {
+  for (const part of ["", ...path.split(sep)]) {
     current = join(current, part);
     const info = await lstat(current);
     if (!info.isDirectory() || (await realpath(current)) !== current)
@@ -169,6 +199,7 @@ export async function releaseCandidateRuntime(
   root: string | undefined,
 ): Promise<void> {
   if (root === undefined) return;
+  await releaseCandidateCodexState(root);
   await clearFixtureContents(root);
   await rm(root, { recursive: true, force: true });
   leases.delete(root);

package/src/candidate-codex-state.ts

@@ -0,0 +1,185 @@
+import { lstat, realpath, rm } from "node:fs/promises";
+import { basename, dirname, join, relative } from "node:path";
+import { userInfo } from "node:os";
+import { nativeStatePath } from "./native-transcript-state";
+import { clearFixtureContents } from "./fixtures/cleanup";
+import type { RuntimePolicy } from "./runtime/config";
+
+interface CodexPaths {
+  codexHome: string;
+  parentHome: string;
+  commandHome: string;
+}
+interface ArtifactBinding {
+  root: string;
+  path: string;
+  directoryPath: string;
+  file?: boolean;
+}
+interface CodexLease {
+  state?: { stateRoot: string; artifacts: Record<string, ArtifactBinding> };
+}
+const leases = new Map<string, CodexLease>();
+const ARTIFACT_NAMES = [
+  "codex_config",
+  "codex_skills",
+  "agent_skills",
+  "command_agent_skills",
+  "command_codex_skills",
+];
+
+export function codexStateEnabled(policy: RuntimePolicy | undefined): boolean {
+  return policy?.codexState === true;
+}
+
+export function requireUnreservedArtifactName(name: string): void {
+  if (ARTIFACT_NAMES.includes(name))
+    throw new Error(
+      "runtime artifact directory name is reserved for Codex state",
+    );
+}
+
+export function allocateCodexStateLease(root: string, enabled: boolean): void {
+  if (enabled) leases.set(root, {});
+}
+
+/** Bind the actual host state to one trial before execution; never substitute a grader home. */
+export async function retainCandidateCodexState(
+  runtimeRoot: string | undefined,
+  stateRoot: string,
+  paths: CodexPaths,
+): Promise<void> {
+  if (runtimeRoot === undefined) return;
+  const lease = leases.get(runtimeRoot);
+  if (!lease) return;
+  if (lease.state)
+    throw new Error(
+      "candidate Codex state is already bound to a native request",
+    );
+  requireCodexLayout(stateRoot, runtimeRoot, paths);
+  await requirePrivateState(stateRoot);
+  await requirePrivateState(paths.codexHome);
+  lease.state = {
+    stateRoot,
+    artifacts: codexArtifacts(stateRoot, runtimeRoot, paths),
+  };
+}
+
+function requireCodexLayout(
+  stateRoot: string,
+  runtimeRoot: string,
+  paths: CodexPaths,
+): void {
+  if (
+    dirname(stateRoot) !== nativeStatePath() ||
+    !basename(stateRoot).startsWith("codex-")
+  )
+    throw new Error("candidate Codex state must be a native request directory");
+  requireCodexHomes(stateRoot, runtimeRoot, paths);
+}
+
+function requireCodexHomes(
+  stateRoot: string,
+  runtimeRoot: string,
+  paths: CodexPaths,
+): void {
+  if (
+    paths.codexHome !== join(stateRoot, "codex-home") ||
+    paths.parentHome !== join(stateRoot, "home") ||
+    paths.commandHome !== join(runtimeRoot, "home")
+  )
+    throw new Error("candidate Codex homes must belong to the current trial");
+}
+
+async function requirePrivateState(root: string): Promise<void> {
+  const info = await lstat(root);
+  if (
+    !info.isDirectory() ||
+    info.uid !== userInfo().uid ||
+    (info.mode & 0o077) !== 0 ||
+    (await realpath(root)) !== root
+  )
+    throw new Error(
+      "candidate Codex state must be owned, private and canonical",
+    );
+}
+
+function binding(root: string, path: string, file = false): ArtifactBinding {
+  return {
+    root,
+    path,
+    directoryPath: relative(root, file ? dirname(path) : path),
+    file,
+  };
+}
+
+function codexArtifacts(
+  stateRoot: string,
+  runtimeRoot: string,
+  paths: CodexPaths,
+): Record<string, ArtifactBinding> {
+  return {
+    codex_config: binding(
+      stateRoot,
+      join(paths.codexHome, "config.toml"),
+      true,
+    ),
+    codex_skills: binding(stateRoot, join(paths.codexHome, "skills")),
+    agent_skills: binding(
+      stateRoot,
+      join(paths.parentHome, ".agents", "skills"),
+    ),
+    command_agent_skills: binding(
+      runtimeRoot,
+      join(paths.commandHome, ".agents", "skills"),
+    ),
+    command_codex_skills: binding(
+      runtimeRoot,
+      join(paths.commandHome, ".codex", "skills"),
+    ),
+  };
+}
+
+export function candidateCodexArtifactRoots(
+  root: string,
+): Record<string, string> {
+  const lease = leases.get(root);
+  if (!lease) return {};
+  if (!lease.state) throw new Error("candidate Codex state is unavailable");
+  return Object.fromEntries(
+    Object.entries(lease.state.artifacts).map(([name, item]) => [
+      name,
+      item.path,
+    ]),
+  );
+}
+
+export function candidateCodexArtifactBinding(
+  name: string,
+  path: string,
+): ArtifactBinding | undefined {
+  for (const lease of leases.values()) {
+    const artifact = lease.state?.artifacts[name];
+    if (artifact?.path === path) return artifact;
+  }
+  return undefined;
+}
+
+export function candidateCodexStateRetained(
+  runtimeRoot: string | undefined,
+  stateRoot: string,
+): boolean {
+  return (
+    runtimeRoot !== undefined &&
+    leases.get(runtimeRoot)?.state?.stateRoot === stateRoot
+  );
+}
+
+export async function releaseCandidateCodexState(root: string): Promise<void> {
+  const state = leases.get(root)?.state;
+  if (state) {
+    await clearFixtureContents(state.stateRoot);
+    await rm(state.stateRoot, { recursive: true, force: true });
+  }
+  leases.delete(root);
+}

package/src/case-validation.ts

@@ -1,27 +1,19 @@
 import { type ResolvedCase } from "./engine";
-import { prepareGeneratedFixture } from "./generated-fixture";
-import { prepareRepositoryFixture } from "./repository-fixture";
-import {
-  isOptionalNonblankString,
-  isRecord,
-  isStringArray,
-} from "./value-guards";
+import { prepareGeneratedFixture } from "./fixtures/generated";
+import { prepareRepositoryFixture } from "./fixtures/repository";
+import { isOptionalNonblankString, isRecord } from "./value-guards";
 
-function validGeneratedFixture(value: Record<string, unknown>): boolean {
+function fixtureDeclarationIssue(
+  value: Record<string, unknown>,
+): string | undefined {
   try {
-    prepareGeneratedFixture(value);
-    return true;
-  } catch {
-    return false;
-  }
-}
-
-function validRepositoryFixture(value: Record<string, unknown>): boolean {
-  try {
-    prepareRepositoryFixture({ kind: "repository", ...value });
-    return true;
-  } catch {
-    return false;
+    if (value.kind === "generated") prepareGeneratedFixture(value);
+    else prepareRepositoryFixture({ kind: "repository", ...value });
+    return undefined;
+  } catch (error) {
+    const detail =
+      error instanceof Error ? error.message : "invalid declaration";
+    return `fixture: ${detail}; correct the fixture declaration`;
   }
 }
 
@@ -34,49 +26,80 @@ export function validInlineFixture(value: Record<string, unknown>): boolean {
   );
 }
 
-function fixtureFields(value: Record<string, unknown>): boolean {
-  if (Object.hasOwn(value, "sourceRef")) return validRepositoryFixture(value);
-  return validInlineFixture(value);
+function inlineFixtureIssue(
+  value: Record<string, unknown>,
+): string | undefined {
+  if (!Object.hasOwn(value, "files") || !isRecord(value.files))
+    return "fixture.files: expected an object mapping file paths to string contents";
+  const entry = Object.entries(value.files).find(
+    ([, content]) => typeof content !== "string",
+  );
+  return entry
+    ? `fixture.files[${JSON.stringify(entry[0])}]: expected string contents`
+    : undefined;
 }
 
-function validFixture(value: unknown): boolean {
-  if (!isRecord(value)) return false;
-  if (value.kind === "generated") return validGeneratedFixture(value);
-  if (value.kind !== undefined) return false;
-  return fixtureFields(value);
+function fixtureIssue(value: unknown): string | undefined {
+  if (!isRecord(value)) return "fixture: expected an object";
+  if (value.kind === "generated") return fixtureDeclarationIssue(value);
+  if (value.kind !== undefined)
+    return "fixture.kind: omit for inline/repository fixtures, or use generated";
+  if (Object.hasOwn(value, "sourceRef")) return fixtureDeclarationIssue(value);
+  return inlineFixtureIssue(value);
 }
 
 function nonemptyString(value: unknown): value is string {
   return typeof value === "string" && value.length > 0;
 }
 
-function validCaseDetails(value: Record<string, unknown>): boolean {
-  return (
-    nonemptyString(value.id) &&
-    nonemptyString(value.prompt) &&
-    isOptionalNonblankString(value.followUpPrompt) &&
-    validFixture(value.fixture)
-  );
+function caseDetailsIssue(value: Record<string, unknown>): string | undefined {
+  for (const field of ["id", "prompt"])
+    if (!nonemptyString(value[field]))
+      return `${field}: expected a nonempty string`;
+  if (!isOptionalNonblankString(value.followUpPrompt))
+    return "followUpPrompt: omit or supply a nonblank string";
+  return fixtureIssue(value.fixture);
 }
 
-function validCheck(value: unknown): boolean {
-  return (
-    isRecord(value) &&
-    typeof value.id === "string" &&
-    typeof value.grader === "string" &&
-    isRecord(value.configuration)
-  );
+function checkIssue(value: unknown, location: string): string | undefined {
+  if (!isRecord(value)) return `${location}: expected an object`;
+  for (const field of ["id", "grader"])
+    if (typeof value[field] !== "string")
+      return `${location}.${field}: expected a string`;
+  if (!isRecord(value.configuration))
+    return `${location}.configuration: expected an object`;
+  return undefined;
 }
 
-function validChecks(value: unknown): boolean {
-  return Array.isArray(value) && value.every(validCheck);
+function checksIssue(value: unknown): string | undefined {
+  if (!Array.isArray(value))
+    return "checks: expected an array of check objects";
+  for (const [index, check] of value.entries()) {
+    const issue = checkIssue(check, `checks[${index}]`);
+    if (issue) return issue;
+  }
+  return undefined;
 }
 
-export function isResolvedCase(value: unknown): value is ResolvedCase {
+function requiredEvidenceIssue(value: unknown): string | undefined {
+  if (!Array.isArray(value))
+    return "requiredEvidence: expected an array of strings";
+  const index = value.findIndex((entry: unknown) => typeof entry !== "string");
+  return index < 0
+    ? undefined
+    : `requiredEvidence[${index}]: expected a string`;
+}
+
+/** Explain the first invalid field using the same checks as the case type guard. */
+export function resolvedCaseIssue(value: unknown): string | undefined {
+  if (!isRecord(value)) return "case: expected an object";
   return (
-    isRecord(value) &&
-    validCaseDetails(value) &&
-    validChecks(value.checks) &&
-    isStringArray(value.requiredEvidence)
+    caseDetailsIssue(value) ??
+    checksIssue(value.checks) ??
+    requiredEvidenceIssue(value.requiredEvidence)
   );
 }
+
+export function isResolvedCase(value: unknown): value is ResolvedCase {
+  return resolvedCaseIssue(value) === undefined;
+}