@bjoernrochel/sevro 0.1.0-rc.8 → 1.0.0

package/examples/recipes/run-options.ts

@@ -0,0 +1,131 @@
+import { delimiter, isAbsolute, join } from "node:path";
+import { homedir } from "node:os";
+
+export const recipeNames = [
+  "coding-change",
+  "host-tools",
+  "node",
+  "python-uv",
+  "mise",
+  "semantic",
+  "lifecycle",
+  "repository-skill",
+  "plugins",
+];
+
+function validateSelection(
+  recipe: string,
+  route: string,
+  flags: string[],
+): void {
+  if (!recipeNames.includes(recipe))
+    throw new Error(`Choose a recipe: ${recipeNames.join(", ")}`);
+  if (!["deterministic", "codex", "claude"].includes(route))
+    throw new Error("Choose deterministic, codex, or claude");
+  if (flags.some((flag) => !["--dry", "--fail", "--goals-only"].includes(flag)))
+    throw new Error("Supported flags: --dry --fail --goals-only");
+}
+
+function validateNativeRecipe(recipe: string, route: string): void {
+  if (
+    ["repository-skill", "plugins"].includes(recipe) &&
+    route === "deterministic"
+  )
+    throw new Error(
+      "Skill and plugin recipes require codex or claude (add --dry to prepare without model calls)",
+    );
+}
+
+export function selectedOptions() {
+  const [recipe = "", route = "deterministic", ...flags] =
+    process.argv.slice(2);
+  validateSelection(recipe, route, flags);
+  validateNativeRecipe(recipe, route);
+  validateGoalsStep(flags);
+  return {
+    recipe,
+    route,
+    dry: flags.includes("--dry"),
+    fail: flags.includes("--fail"),
+    goalsOnly: flags.includes("--goals-only"),
+  };
+}
+export type RecipeOptions = ReturnType<typeof selectedOptions>;
+
+function validateGoalsStep(flags: string[]): void {
+  if (flags.includes("--goals-only") && !flags.includes("--dry"))
+    throw new Error(
+      "--goals-only is a dry preparation step; omit it for the complete hook recipe",
+    );
+}
+
+export function requireTool(name: string): string {
+  const path = Bun.which(name);
+  if (!path)
+    throw new Error(
+      `Recipe prerequisite missing: install ${name} and put it on PATH`,
+    );
+  return path;
+}
+
+export function runtimeFile(options: RecipeOptions): string {
+  const files: Record<string, string> = {
+    "coding-change": "path",
+    "host-tools": "path",
+    node: "path",
+    "python-uv": "python-uv",
+    mise: "mise",
+    plugins: options.goalsOnly ? "goals" : "plugins",
+  };
+  return join(
+    import.meta.dir,
+    "runtime",
+    `${files[options.recipe] ?? "empty"}.json`,
+  );
+}
+
+function languagePrerequisites(recipe: string): void {
+  if (recipe === "python-uv")
+    process.env.RECIPE_PYTHON ??= requireTool("python3");
+  if (recipe === "mise") selectMiseNode();
+  if (recipe === "host-tools") requireStandaloneRipgrep();
+}
+
+function requireStandaloneRipgrep(): void {
+  const roots = [
+    process.env.CODEX_HOME ?? join(homedir(), ".codex"),
+    process.env.CLAUDE_CONFIG_DIR ?? join(homedir(), ".claude"),
+  ];
+  const paths = (process.env.PATH ?? "")
+    .split(delimiter)
+    .filter((entry) => !roots.some((root) => entry.startsWith(root)));
+  if (!Bun.which("rg", { PATH: paths.join(delimiter) }))
+    throw new Error(
+      "Install standalone ripgrep outside Codex/Claude credential directories, then put it on PATH (for example brew install ripgrep or apt-get install ripgrep)",
+    );
+}
+
+function selectMiseNode(): void {
+  const result = Bun.spawnSync([requireTool("mise"), "where", "node@24.13.0"], {
+    stdout: "pipe",
+    stderr: "pipe",
+  });
+  const installation = result.stdout.toString().trim();
+  if (result.exitCode !== 0 || !isAbsolute(installation))
+    throw new Error(
+      "Install the recipe's Node with mise install node@24.13.0 before running it",
+    );
+  process.env.PATH = `${join(installation, "bin")}${delimiter}${process.env.PATH ?? "/usr/bin:/bin"}`;
+}
+
+export function runtimePrerequisites(recipe: string): void {
+  const tools: Record<string, string[]> = {
+    "coding-change": ["node"],
+    "host-tools": ["rg"],
+    node: ["node"],
+    "python-uv": ["uv", "python3"],
+    mise: ["mise"],
+  };
+  for (const tool of tools[recipe] ?? []) requireTool(tool);
+  languagePrerequisites(recipe);
+}

package/examples/recipes/run.ts

@@ -0,0 +1,63 @@
+import { mkdir, realpath, writeFile } from "node:fs/promises";
+import { join, resolve } from "node:path";
+import { hostArguments, recipeInputs, semanticArguments } from "./run-inputs";
+import {
+  runtimeFile,
+  runtimePrerequisites,
+  selectedOptions,
+} from "./run-options";
+
+const options = selectedOptions();
+runtimePrerequisites(options.recipe);
+const mode = options.dry ? "dry" : options.fail ? "fail" : "pass";
+const state = resolve(
+  ".sevro-recipes",
+  options.recipe,
+  options.route,
+  options.goalsOnly ? "goals" : mode,
+);
+await mkdir(join(state, "project"), { recursive: true });
+await mkdir(join(state, "config"), { recursive: true });
+const root = await realpath(state);
+const cli =
+  process.env.SEVRO_RECIPE_CLI ?? resolve(import.meta.dir, "../../src/cli.ts");
+const args = [
+  process.execPath,
+  cli,
+  "run",
+  "--json",
+  "--project-root",
+  join(root, "project"),
+  "--config-root",
+  join(root, "config"),
+  "--results-root",
+  join(root, "results"),
+  "--runtime-config-file",
+  runtimeFile(options),
+  "--condition",
+  "passive",
+  "--trials",
+  "1",
+  "--threshold",
+  "1",
+  "--shell-isolation",
+  "--protected-root",
+  import.meta.dir,
+  ...(await recipeInputs(options, root)),
+  ...(await hostArguments(options, root)),
+  ...(await semanticArguments(options, root)),
+  ...(options.dry ? ["--dry"] : []),
+];
+const child = Bun.spawn(args, {
+  env: { ...process.env, SEVRO_RECIPE_FAIL: options.fail ? "1" : "0" },
+  stdout: "pipe",
+  stderr: "inherit",
+});
+const [output, code] = await Promise.all([
+  new Response(child.stdout).text(),
+  child.exited,
+]);
+await writeFile(join(root, "result.json"), output);
+process.stderr.write(`Recipe result: ${join(root, "result.json")}\n`);
+process.stdout.write(output);
+process.exit(code);

package/examples/recipes/runtime/empty.json

@@ -0,0 +1 @@
+{ "format": "sevro.runtime.v1" }

package/examples/recipes/runtime/goals.json

@@ -0,0 +1 @@
+{ "format": "sevro.runtime.v1", "hooks": { "nativeGoal": true } }

package/examples/recipes/runtime/mise.json

@@ -0,0 +1,6 @@
+{
+  "format": "sevro.runtime.v1",
+  "environment": {
+    "inherit": ["PATH"]
+  }
+}

package/examples/recipes/runtime/path.json

@@ -0,0 +1 @@
+{ "format": "sevro.runtime.v1", "environment": { "inherit": ["PATH"] } }

package/examples/recipes/runtime/plugins.json

@@ -0,0 +1,4 @@
+{
+  "format": "sevro.runtime.v1",
+  "hooks": { "nativeGoal": true, "plugins": ["startup"] }
+}

package/examples/recipes/runtime/python-uv.json

@@ -0,0 +1,10 @@
+{
+  "format": "sevro.runtime.v1",
+  "environment": {
+    "inherit": ["PATH", "RECIPE_PYTHON"],
+    "set": {
+      "UV_CACHE_DIR": "{{sevro.runtime}}/uv-cache",
+      "UV_PYTHON_DOWNLOADS": "never"
+    }
+  }
+}

package/examples/recipes/semantic-judge.ts

@@ -0,0 +1,29 @@
+/** Protocol control only: use a real judge for qualitative model assessment. */
+const judge = {
+  id: "recipes.protocol-judge",
+  model: "exact-fixture-oracle",
+  effort: "none",
+  run({ prompt }: { prompt: string }) {
+    const match =
+      /<candidate-response-json>\s*([\s\S]*?)\s*<\/candidate-response-json>/.exec(
+        prompt,
+      );
+    if (!match?.[1]) throw new Error("Missing semantic candidate response");
+    const candidate: unknown = JSON.parse(match[1]);
+    const passed = candidate === "Run bun run dev on port 3000.";
+    return Promise.resolve({
+      complete: true,
+      finalMessage: JSON.stringify({
+        checks: [
+          {
+            id: "startup-meaning",
+            verdict: passed ? "pass" : "fail",
+            reason:
+              "Exact deterministic fixture oracle; no model judgment was made.",
+          },
+        ],
+      }),
+    });
+  },
+};
+export default judge;

package/examples/templates/coding/adapter.ts

@@ -0,0 +1,17 @@
+import { writeFile } from "node:fs/promises";
+import { join } from "node:path";
+
+/** Deterministic file editor; it exercises the evaluator, not model coding ability. */
+export default {
+  id: "template.coding",
+  model: "deterministic-editor",
+  effort: "none",
+  async run({ workspace }: { workspace: string }) {
+    if (process.env.SEVRO_TEMPLATE_FAIL !== "1")
+      await writeFile(
+        join(workspace, "add.sh"),
+        'add() { echo "$(( $1 + $2 ))"; }\n',
+      );
+    return { finalMessage: "Completed the coding task.", complete: true };
+  },
+};

package/examples/templates/coding/case.json

@@ -0,0 +1,15 @@
+{
+  "id": "coding-addition",
+  "prompt": "Fix add.sh so add computes integer addition. Preserve the add shell function interface.",
+  "fixture": { "files": { "add.sh": "add() { echo \"$(( $1 - $2 ))\"; }\n" } },
+  "checks": [
+    {
+      "id": "addition",
+      "grader": "sevro.shell",
+      "configuration": {
+        "run": ". ./add.sh; test \"$(add 2 3)\" = 5 && test \"$(add -2 -3)\" = -5 && test \"$(add 0 0)\" = 0"
+      }
+    }
+  ],
+  "requiredEvidence": []
+}

package/examples/templates/repository-skill/.agents/skills/startup-guide/SKILL.md

@@ -0,0 +1,6 @@
+---
+name: startup-guide
+description: Explain the sample application's start command and port.
+---
+
+Read README.md. State the exact start command and port. Do not change files.

package/examples/templates/repository-skill/extension-data.ts

@@ -0,0 +1,64 @@
+import { createHash } from "node:crypto";
+import { readFile } from "node:fs/promises";
+import { join } from "node:path";
+
+export const description = {
+  extension: { id: "template.startup", version: "1.0.0" },
+  protocols: ["sevro.extension.v1"],
+  requiredCapabilities: ["sevro.codex.repository-invocation"],
+  optionalCapabilities: [],
+  graders: [],
+  taskVerdictPolicies: [],
+};
+
+export const resolved = {
+  cases: [
+    {
+      id: "repository-skill",
+      prompt:
+        "{{sevro.skill_invocation}} Explain the application's start command and port. Do not change files.",
+      fixture: {
+        kind: "generated",
+        commits: [
+          {
+            message: "Initialize application",
+            files: { "README.md": "Run bun run dev on port 3000.\n" },
+          },
+        ],
+      },
+      checks: [
+        {
+          id: "start-command",
+          grader: "sevro.regex",
+          configuration: { pattern: "bun run dev" },
+        },
+        {
+          id: "port",
+          grader: "sevro.regex",
+          configuration: { pattern: "\\b3000\\b" },
+        },
+      ],
+      requiredEvidence: ["sevro.codex.explicit-invocation"],
+      extensionData: {},
+    },
+  ],
+};
+
+export async function prepare() {
+  const relativePath = ".agents/skills/startup-guide/SKILL.md";
+  const body = await readFile(join(import.meta.dir, relativePath));
+  return {
+    artifacts: [
+      {
+        id: "template.startup-skill",
+        relativePath,
+        sha256: createHash("sha256").update(body).digest("hex"),
+        contentBase64: body.toString("base64"),
+        gitExclude: true,
+      },
+    ],
+    requestedInstrumentation: [],
+    extensionData: {},
+    codexRepositorySkillInvocation: { skillName: "startup-guide" },
+  };
+}

package/examples/templates/repository-skill/extension.ts

@@ -0,0 +1,31 @@
+import { description, prepare, resolved } from "./extension-data";
+
+function record(value: unknown): Record<string, unknown> {
+  if (!value || typeof value !== "object" || Array.isArray(value))
+    throw new Error("Expected a protocol request object");
+  return value as Record<string, unknown>;
+}
+
+const request = record(JSON.parse(await Bun.stdin.text()) as unknown);
+async function result() {
+  switch (request.method) {
+    case "describe":
+      return description;
+    case "resolve":
+      return resolved;
+    case "prepare":
+      return await prepare();
+    case "evaluate":
+      return { checks: [], metrics: [] };
+    default:
+      throw new Error("Unsupported extension method");
+  }
+}
+process.stdout.write(
+  JSON.stringify({
+    protocol: request.protocol,
+    id: request.id,
+    method: request.method,
+    result: await result(),
+  }),
+);

package/examples/templates/response/adapter.ts

@@ -0,0 +1,12 @@
+/** Deterministic starter; replace with an explicit native route to evaluate a model. */
+export default {
+  id: "template.response",
+  model: "deterministic-example",
+  effort: "none",
+  run() {
+    return Promise.resolve({
+      finalMessage: process.env.SEVRO_TEMPLATE_FAIL === "1" ? "wrong" : "ready",
+      complete: true,
+    });
+  },
+};

package/examples/templates/response/case.json

@@ -0,0 +1,13 @@
+{
+  "id": "response-check",
+  "prompt": "Return exactly ready.",
+  "fixture": { "files": { "README.md": "Response-check fixture.\n" } },
+  "checks": [
+    {
+      "id": "ready",
+      "grader": "sevro.regex",
+      "configuration": { "pattern": "^ready$" }
+    }
+  ],
+  "requiredEvidence": []
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bjoernrochel/sevro",
-  "version": "0.1.0-rc.8",
+  "version": "1.0.0",
   "license": "SEE LICENSE IN LICENSE",
   "description": "A standalone evaluation runner for agentic coding capabilities",
   "type": "module",
@@ -19,7 +19,14 @@
     "README.md",
     "CONTRIBUTING.md",
     "LICENSE",
-    "docs/",
+    "docs/*.md",
+    "!docs/*-validation.md",
+    "!docs/engine-slice.md",
+    "!docs/evidence.md",
+    "!docs/usability-study.md",
+    "docs/assets/README.md",
+    "docs/licenses/",
+    "docs/specs/",
     "examples/",
     "schemas/",
     "src/"
@@ -44,7 +51,7 @@
     "eval:guide": "bun scripts/eval-guide.ts",
     "release:prepare": "bun scripts/prepare-release.ts",
     "typecheck": "tsc --noEmit",
-    "format:check": "prettier --check README.md CONTRIBUTING.md AGENTS.md CLAUDE.md 'docs/**/*.md' '.agents/**/*.md' '.agents/**/*.json' '.claude/**/*.md' 'scripts/**/*.md' examples/**/*.json examples/**/*.ts schemas/*.json src/**/*.ts scripts/**/*.ts scripts/**/*.cjs tests/**/*.ts eslint.config.mjs package.json tsconfig.json typescript-sources.json .github/workflows/*.yml"
+    "format:check": "prettier --check README.md CONTRIBUTING.md AGENTS.md CLAUDE.md 'docs/**/*.md' '.agents/**/*.md' '.agents/**/*.json' '.claude/**/*.md' 'scripts/**/*.md' 'scripts/**/*.json' 'examples/**/*.json' 'examples/**/*.ts' 'schemas/*.json' 'src/**/*.ts' 'scripts/**/*.ts' 'scripts/**/*.cjs' 'tests/**/*.ts' eslint.config.mjs package.json tsconfig.json '.github/workflows/*.yml'"
   },
   "dependencies": {
     "ajv": "8.20.0",
@@ -68,10 +75,9 @@
     "istanbul-lib-coverage": "3.2.2",
     "istanbul-lib-instrument": "6.0.3",
     "istanbul-lib-report": "3.0.1",
-    "istanbul-lib-source-maps": "5.0.6",
     "istanbul-reports": "3.2.0",
     "markdown-it": "14.2.0",
-    "prettier": "^3.9.9",
+    "prettier": "3.9.9",
     "shell-quote": "1.12.0",
     "typescript-eslint": "8.71.1"
   }

package/schemas/report-v1.schema.json

@@ -110,6 +110,15 @@
         "domainOutcomes"
       ],
       "properties": {
+        "nativeTranscripts": {
+          "type": "object",
+          "required": ["retained", "pruned"],
+          "properties": {
+            "retained": { "type": "integer", "minimum": 0 },
+            "pruned": { "type": "integer", "minimum": 0 }
+          },
+          "additionalProperties": false
+        },
         "resultFile": { "$ref": "#/$defs/path" },
         "evidencePath": { "type": ["string", "null"] },
         "runId": { "type": ["string", "null"] },

package/schemas/run-evidence-v1.schema.json

@@ -80,7 +80,8 @@
       "required": ["digest", "redacted"],
       "properties": {
         "digest": { "$ref": "#/$defs/digest" },
-        "redacted": { "type": "object" }
+        "redacted": { "type": "object" },
+        "resolved": { "type": "object" }
       },
       "additionalProperties": false
     },

package/schemas/runtime-v1.schema.json

@@ -5,7 +5,23 @@
   "type": "object",
   "additionalProperties": false,
   "required": ["format"],
+  "$defs": {
+    "runtimeRole": {
+      "$comment": "Generated from RUNTIME_ROLES by bun run schemas:generate.",
+      "enum": ["candidate", "checks", "hooks", "semantic", "advisory"]
+    }
+  },
   "properties": {
+    "codexState": {
+      "type": "boolean"
+    },
+    "network": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "localHttp": { "type": "boolean" }
+      }
+    },
     "nativeTranscripts": {
       "type": "boolean"
     },
@@ -52,6 +68,32 @@
             }
           }
         },
+        "sharedCacheDirectories": {
+          "type": "array",
+          "maxItems": 16,
+          "items": {
+            "type": "object",
+            "additionalProperties": false,
+            "required": ["source", "target", "roles"],
+            "properties": {
+              "source": { "type": "string", "minLength": 1, "maxLength": 4096 },
+              "target": {
+                "type": "string",
+                "pattern": "^[a-z][a-z0-9-]{0,63}$",
+                "not": { "enum": ["home", "tmp"] }
+              },
+              "roles": {
+                "type": "array",
+                "minItems": 1,
+                "maxItems": 5,
+                "uniqueItems": true,
+                "items": {
+                  "$ref": "#/$defs/runtimeRole"
+                }
+              }
+            }
+          }
+        },
         "seedDirectories": {
           "type": "array",
           "maxItems": 16,
@@ -78,13 +120,7 @@
                 "maxItems": 5,
                 "uniqueItems": true,
                 "items": {
-                  "enum": [
-                    "candidate",
-                    "checks",
-                    "hooks",
-                    "semantic",
-                    "advisory"
-                  ]
+                  "$ref": "#/$defs/runtimeRole"
                 }
               }
             }

package/src/advisory.ts

@@ -7,14 +7,14 @@ const DIMENSIONS = [
   "scopeDiscipline",
 ] as const;
 
-export interface AdvisoryAssessment {
+export type AdvisoryAssessment = {
   verdict: "pass" | "fail";
   overallScore: number;
   dimensions: Record<(typeof DIMENSIONS)[number], number>;
   strengths: string[];
   weaknesses: string[];
   summary: string;
-}
+};
 
 function object(value: unknown): Record<string, unknown> {
   if (!value || typeof value !== "object" || Array.isArray(value))