@bjoernrochel/sevro 0.1.0-rc.8 → 1.1.0

package/examples/recipes/extension.ts

@@ -0,0 +1,38 @@
+import { readFileSync } from "node:fs";
+import {
+  describe,
+  evaluate,
+  prepare,
+  record,
+  resolveCase,
+} from "./extension-methods";
+
+const mode = process.argv[2] ?? "lifecycle";
+const host = process.argv[3] ?? "deterministic";
+const fail = process.argv[4] === "fail";
+const request = record(JSON.parse(readFileSync(0, "utf8")) as unknown);
+
+async function result() {
+  switch (request.method) {
+    case "describe":
+      return describe(mode, host);
+    case "resolve":
+      return resolveCase(mode, host, fail);
+    case "prepare":
+      return await prepare(mode, host);
+    case "evaluate":
+      return evaluate(record(request.params));
+    default:
+      throw new Error("Unsupported extension method");
+  }
+}
+
+// One request per process; stdout contains only the matching protocol response.
+process.stdout.write(
+  JSON.stringify({
+    protocol: request.protocol,
+    id: request.id,
+    method: request.method,
+    result: await result(),
+  }),
+);

package/examples/recipes/modules.ts

@@ -0,0 +1,9 @@
+import { join } from "node:path";
+
+/** Locate source modules for Bun development or compiled modules for consumers. */
+export function recipeFile(...parts: string[]): string {
+  const path = join(import.meta.dirname, ...parts);
+  return import.meta.filename.endsWith(".ts")
+    ? path
+    : path.replace(/\.ts$/, ".js");
+}

package/examples/recipes/repository-skill/.agents/skills/startup-guide/SKILL.md

@@ -0,0 +1,7 @@
+---
+name: startup-guide
+description: Explain how to start this sample application and identify its port.
+---
+
+Read README.md. Explain the exact start command and port from that file.
+Do not change files. Do not invent a different command or port.

package/examples/recipes/repository-skill/.claude/skills/startup-guide/SKILL.md

@@ -0,0 +1,7 @@
+---
+name: startup-guide
+description: Explain how to start this sample application and identify its port.
+---
+
+Read README.md. Explain the exact start command and port from that file.
+Do not change files. Do not invent a different command or port.

package/examples/recipes/repository-skill/README.md

@@ -0,0 +1,3 @@
+# Sample application
+
+Run `bun run dev` to start the application on port `3000`.

package/examples/recipes/run-inputs.ts

@@ -0,0 +1,183 @@
+import { recipeFile } from "./modules";
+import { readFile, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { record } from "./extension-methods";
+import { requireTool, type RecipeOptions } from "./run-options";
+
+function shellQuote(text: string): string {
+  return `'${text.replaceAll("'", "'\\''")}'`;
+}
+
+function evaluatorCode(test: string): string {
+  if (import.meta.filename.endsWith(".js")) return test;
+  return new Bun.Transpiler({ loader: "ts" }).transformSync(test);
+}
+
+async function caseFile(
+  options: RecipeOptions,
+  state: string,
+): Promise<string[]> {
+  const value = record(
+    JSON.parse(
+      await readFile(
+        join(import.meta.dirname, "cases", `${options.recipe}.json`),
+        "utf8",
+      ),
+    ) as unknown,
+  );
+  if (options.recipe === "coding-change") {
+    const test = await readFile(recipeFile("coding-change/check.ts"), "utf8");
+    const checks: unknown[] = Array.isArray(value.checks) ? value.checks : [];
+    const check = record(checks[0]);
+    const javascript = evaluatorCode(test);
+    const protectedTest = shellQuote(recipeFile("coding-change/check.ts"));
+    record(check.configuration).run =
+      `test ! -r ${protectedTest}\nnode --input-type=module -e ${shellQuote(javascript)}`;
+  }
+  if (options.fail) negativeFixture(value, options.recipe);
+  const path = join(state, "case.json");
+  await writeFile(path, JSON.stringify(value));
+  return ["--case-file", path];
+}
+
+function negativeFixture(value: Record<string, unknown>, recipe: string): void {
+  if (["coding-change", "semantic"].includes(recipe)) {
+    value.prompt =
+      "Deliberate negative control: leave math.cjs unchanged if present and respond only: npm start on port 9000.";
+    return;
+  }
+  const files = record(record(value.fixture).files);
+  for (const [path, content] of Object.entries(files)) {
+    if (typeof content !== "string")
+      throw new Error("Expected inline fixture text");
+    files[path] = content.replaceAll("3000", "9000");
+  }
+}
+
+async function extensionFiles(
+  options: RecipeOptions,
+  state: string,
+): Promise<string[]> {
+  const command = join(state, "extension-command.json");
+  await writeFile(
+    command,
+    JSON.stringify([
+      process.execPath,
+      recipeFile("extension.ts"),
+      options.recipe,
+      options.route,
+      options.fail ? "fail" : "pass",
+    ]),
+  );
+  const sources = [
+    "extension.ts",
+    "extension-methods.ts",
+    "extension-artifacts.ts",
+    "repository-skill/.agents/skills/startup-guide/SKILL.md",
+    "repository-skill/.claude/skills/startup-guide/SKILL.md",
+  ];
+  return [
+    "--extension-command-file",
+    command,
+    "--case-id",
+    "recipe-extension",
+    ...sources.flatMap((path) => ["--extension-source-file", recipeFile(path)]),
+  ];
+}
+
+export async function recipeInputs(
+  options: RecipeOptions,
+  state: string,
+): Promise<string[]> {
+  if (["lifecycle", "repository-skill", "plugins"].includes(options.recipe))
+    return extensionFiles(options, state);
+  return caseFile(options, state);
+}
+
+async function codexAuthentication(
+  options: RecipeOptions,
+  state: string,
+): Promise<string> {
+  if (!options.dry) {
+    const path = process.env.SEVRO_RECIPE_CODEX_AUTH;
+    if (!path)
+      throw new Error(
+        "Set SEVRO_RECIPE_CODEX_AUTH to your private Codex auth.json path",
+      );
+    return path;
+  }
+  const path = join(state, "dry-auth.json");
+  await writeFile(path, "{}\n");
+  return path;
+}
+
+export async function hostArguments(
+  options: RecipeOptions,
+  state: string,
+): Promise<string[]> {
+  if (options.route === "deterministic")
+    return ["--adapter-module", recipeFile("candidate.ts")];
+  const model = selectedModel(options.dry, "SEVRO_RECIPE_MODEL");
+  if (!model)
+    throw new Error(
+      "Set SEVRO_RECIPE_MODEL to an available model for the selected native host",
+    );
+  const args = [
+    "--host",
+    options.route,
+    "--model",
+    model,
+    "--effort",
+    "medium",
+  ];
+  if (options.route === "codex")
+    return [
+      ...args,
+      "--codex-bin",
+      requireTool("codex"),
+      "--codex-auth-file",
+      await codexAuthentication(options, state),
+    ];
+  return [
+    ...args,
+    "--claude-bin",
+    requireTool("claude"),
+    ...(options.recipe === "repository-skill"
+      ? ["--claude-project-settings"]
+      : []),
+  ];
+}
+
+export async function semanticArguments(
+  options: RecipeOptions,
+  state: string,
+): Promise<string[]> {
+  if (options.recipe !== "semantic") return [];
+  if (options.route === "deterministic")
+    return ["--semantic-adapter-module", recipeFile("semantic-judge.ts")];
+  const model = selectedModel(options.dry, "SEVRO_RECIPE_JUDGE_MODEL");
+  if (!model)
+    throw new Error(
+      "Set SEVRO_RECIPE_JUDGE_MODEL for the separate Codex judge route",
+    );
+  const args = [
+    "--semantic-host",
+    "codex",
+    "--semantic-model",
+    model,
+    "--semantic-effort",
+    "medium",
+  ];
+  if (options.route === "codex") return args;
+  return [
+    ...args,
+    "--codex-bin",
+    requireTool("codex"),
+    "--codex-auth-file",
+    await codexAuthentication(options, state),
+  ];
+}
+
+function selectedModel(dry: boolean, variable: string): string | undefined {
+  return dry ? "dry-model" : process.env[variable];
+}

package/examples/recipes/run-options.ts

@@ -0,0 +1,132 @@
+import { spawnSync } from "node:child_process";
+import { which } from "./executable";
+import { delimiter, isAbsolute, join } from "node:path";
+import { homedir } from "node:os";
+
+export const recipeNames = [
+  "coding-change",
+  "host-tools",
+  "node",
+  "python-uv",
+  "mise",
+  "semantic",
+  "lifecycle",
+  "repository-skill",
+  "plugins",
+];
+
+function validateSelection(
+  recipe: string,
+  route: string,
+  flags: string[],
+): void {
+  if (!recipeNames.includes(recipe))
+    throw new Error(`Choose a recipe: ${recipeNames.join(", ")}`);
+  if (!["deterministic", "codex", "claude"].includes(route))
+    throw new Error("Choose deterministic, codex, or claude");
+  if (flags.some((flag) => !["--dry", "--fail", "--goals-only"].includes(flag)))
+    throw new Error("Supported flags: --dry --fail --goals-only");
+}
+
+function validateNativeRecipe(recipe: string, route: string): void {
+  if (
+    ["repository-skill", "plugins"].includes(recipe) &&
+    route === "deterministic"
+  )
+    throw new Error(
+      "Skill and plugin recipes require codex or claude (add --dry to prepare without model calls)",
+    );
+}
+
+export function selectedOptions() {
+  const [recipe = "", route = "deterministic", ...flags] =
+    process.argv.slice(2);
+  validateSelection(recipe, route, flags);
+  validateNativeRecipe(recipe, route);
+  validateGoalsStep(flags);
+  return {
+    recipe,
+    route,
+    dry: flags.includes("--dry"),
+    fail: flags.includes("--fail"),
+    goalsOnly: flags.includes("--goals-only"),
+  };
+}
+export type RecipeOptions = ReturnType<typeof selectedOptions>;
+
+function validateGoalsStep(flags: string[]): void {
+  if (flags.includes("--goals-only") && !flags.includes("--dry"))
+    throw new Error(
+      "--goals-only is a dry preparation step; omit it for the complete hook recipe",
+    );
+}
+
+export function requireTool(name: string): string {
+  const path = which(name);
+  if (!path)
+    throw new Error(
+      `Recipe prerequisite missing: install ${name} and put it on PATH`,
+    );
+  return path;
+}
+
+export function runtimeFile(options: RecipeOptions): string {
+  const files: Record<string, string> = {
+    "coding-change": "path",
+    "host-tools": "path",
+    node: "path",
+    "python-uv": "python-uv",
+    mise: "mise",
+    plugins: options.goalsOnly ? "goals" : "plugins",
+  };
+  return join(
+    import.meta.dirname,
+    "runtime",
+    `${files[options.recipe] ?? "empty"}.json`,
+  );
+}
+
+function languagePrerequisites(recipe: string): void {
+  if (recipe === "python-uv")
+    process.env.RECIPE_PYTHON ??= requireTool("python3");
+  if (recipe === "mise") selectMiseNode();
+  if (recipe === "host-tools") requireStandaloneRipgrep();
+}
+
+function requireStandaloneRipgrep(): void {
+  const roots = [
+    process.env.CODEX_HOME ?? join(homedir(), ".codex"),
+    process.env.CLAUDE_CONFIG_DIR ?? join(homedir(), ".claude"),
+  ];
+  const paths = (process.env.PATH ?? "")
+    .split(delimiter)
+    .filter((entry) => !roots.some((root) => entry.startsWith(root)));
+  if (!which("rg", paths.join(delimiter)))
+    throw new Error(
+      "Install standalone ripgrep outside Codex/Claude credential directories, then put it on PATH (for example brew install ripgrep or apt-get install ripgrep)",
+    );
+}
+
+function selectMiseNode(): void {
+  const result = spawnSync(requireTool("mise"), ["where", "node@24.13.0"], {
+    encoding: "utf8",
+  });
+  const installation = result.stdout.trim();
+  if (result.status !== 0 || !isAbsolute(installation))
+    throw new Error(
+      "Install the recipe's Node with mise install node@24.13.0 before running it",
+    );
+  process.env.PATH = `${join(installation, "bin")}${delimiter}${process.env.PATH ?? "/usr/bin:/bin"}`;
+}
+
+export function runtimePrerequisites(recipe: string): void {
+  const tools: Record<string, string[]> = {
+    "coding-change": ["node"],
+    "host-tools": ["rg"],
+    node: ["node"],
+    "python-uv": ["uv", "python3"],
+    mise: ["mise"],
+  };
+  for (const tool of tools[recipe] ?? []) requireTool(tool);
+  languagePrerequisites(recipe);
+}

package/examples/recipes/run.ts

@@ -0,0 +1,77 @@
+import { spawn } from "node:child_process";
+import { Readable } from "node:stream";
+import { mkdir, realpath, writeFile } from "node:fs/promises";
+import { join, resolve } from "node:path";
+import { hostArguments, recipeInputs, semanticArguments } from "./run-inputs";
+import {
+  runtimeFile,
+  runtimePrerequisites,
+  selectedOptions,
+} from "./run-options";
+
+const options = selectedOptions();
+runtimePrerequisites(options.recipe);
+const mode = options.dry ? "dry" : options.fail ? "fail" : "pass";
+const state = resolve(
+  ".sevro-recipes",
+  options.recipe,
+  options.route,
+  options.goalsOnly ? "goals" : mode,
+);
+await mkdir(join(state, "project"), { recursive: true });
+await mkdir(join(state, "config"), { recursive: true });
+const root = await realpath(state);
+const cli =
+  process.env.SEVRO_RECIPE_CLI ??
+  resolve(
+    import.meta.dirname,
+    import.meta.filename.endsWith(".ts")
+      ? "../../src/cli.ts"
+      : "../../src/cli.js",
+  );
+const args = [
+  process.execPath,
+  cli,
+  "run",
+  "--json",
+  "--project-root",
+  join(root, "project"),
+  "--config-root",
+  join(root, "config"),
+  "--results-root",
+  join(root, "results"),
+  "--runtime-config-file",
+  runtimeFile(options),
+  "--condition",
+  "passive",
+  "--trials",
+  "1",
+  "--threshold",
+  "1",
+  "--shell-isolation",
+  "--protected-root",
+  import.meta.dirname,
+  ...(await recipeInputs(options, root)),
+  ...(await hostArguments(options, root)),
+  ...(await semanticArguments(options, root)),
+  ...(options.dry ? ["--dry"] : []),
+];
+const child = spawn(process.execPath, args.slice(1), {
+  env: { ...process.env, SEVRO_RECIPE_FAIL: options.fail ? "1" : "0" },
+  stdio: ["ignore", "pipe", "inherit"],
+});
+const [output, code] = await Promise.all([
+  new Response(
+    Readable.toWeb(child.stdout) as unknown as ReadableStream<Uint8Array>,
+  ).text(),
+  new Promise<number>((resolve, reject) => {
+    child.once("error", reject);
+    child.once("close", (code) => {
+      resolve(code ?? 1);
+    });
+  }),
+]);
+await writeFile(join(root, "result.json"), output);
+process.stderr.write(`Recipe result: ${join(root, "result.json")}\n`);
+process.stdout.write(output);
+process.exit(code);

package/examples/recipes/runtime/empty.json

@@ -0,0 +1 @@
+{ "format": "sevro.runtime.v1" }

package/examples/recipes/runtime/goals.json

@@ -0,0 +1 @@
+{ "format": "sevro.runtime.v1", "hooks": { "nativeGoal": true } }

package/examples/recipes/runtime/mise.json

@@ -0,0 +1,6 @@
+{
+  "format": "sevro.runtime.v1",
+  "environment": {
+    "inherit": ["PATH"]
+  }
+}

package/examples/recipes/runtime/path.json

@@ -0,0 +1 @@
+{ "format": "sevro.runtime.v1", "environment": { "inherit": ["PATH"] } }

package/examples/recipes/runtime/plugins.json

@@ -0,0 +1,4 @@
+{
+  "format": "sevro.runtime.v1",
+  "hooks": { "nativeGoal": true, "plugins": ["startup"] }
+}

package/examples/recipes/runtime/python-uv.json

@@ -0,0 +1,10 @@
+{
+  "format": "sevro.runtime.v1",
+  "environment": {
+    "inherit": ["PATH", "RECIPE_PYTHON"],
+    "set": {
+      "UV_CACHE_DIR": "{{sevro.runtime}}/uv-cache",
+      "UV_PYTHON_DOWNLOADS": "never"
+    }
+  }
+}

package/examples/recipes/semantic-judge.ts

@@ -0,0 +1,29 @@
+/** Protocol control only: use a real judge for qualitative model assessment. */
+const judge = {
+  id: "recipes.protocol-judge",
+  model: "exact-fixture-oracle",
+  effort: "none",
+  run({ prompt }: { prompt: string }) {
+    const match =
+      /<candidate-response-json>\s*([\s\S]*?)\s*<\/candidate-response-json>/.exec(
+        prompt,
+      );
+    if (!match?.[1]) throw new Error("Missing semantic candidate response");
+    const candidate: unknown = JSON.parse(match[1]);
+    const passed = candidate === "Run bun run dev on port 3000.";
+    return Promise.resolve({
+      complete: true,
+      finalMessage: JSON.stringify({
+        checks: [
+          {
+            id: "startup-meaning",
+            verdict: passed ? "pass" : "fail",
+            reason:
+              "Exact deterministic fixture oracle; no model judgment was made.",
+          },
+        ],
+      }),
+    });
+  },
+};
+export default judge;

package/examples/templates/coding/adapter.ts

@@ -0,0 +1,17 @@
+import { writeFile } from "node:fs/promises";
+import { join } from "node:path";
+
+/** Deterministic file editor; it exercises the evaluator, not model coding ability. */
+export default {
+  id: "template.coding",
+  model: "deterministic-editor",
+  effort: "none",
+  async run({ workspace }: { workspace: string }) {
+    if (process.env.SEVRO_TEMPLATE_FAIL !== "1")
+      await writeFile(
+        join(workspace, "add.sh"),
+        'add() { echo "$(( $1 + $2 ))"; }\n',
+      );
+    return { finalMessage: "Completed the coding task.", complete: true };
+  },
+};

package/examples/templates/coding/case.json

@@ -0,0 +1,15 @@
+{
+  "id": "coding-addition",
+  "prompt": "Fix add.sh so add computes integer addition. Preserve the add shell function interface.",
+  "fixture": { "files": { "add.sh": "add() { echo \"$(( $1 - $2 ))\"; }\n" } },
+  "checks": [
+    {
+      "id": "addition",
+      "grader": "sevro.shell",
+      "configuration": {
+        "run": ". ./add.sh; test \"$(add 2 3)\" = 5 && test \"$(add -2 -3)\" = -5 && test \"$(add 0 0)\" = 0"
+      }
+    }
+  ],
+  "requiredEvidence": []
+}

package/examples/templates/repository-skill/.agents/skills/startup-guide/SKILL.md

@@ -0,0 +1,6 @@
+---
+name: startup-guide
+description: Explain the sample application's start command and port.
+---
+
+Read README.md. State the exact start command and port. Do not change files.

package/examples/templates/repository-skill/extension-data.ts

@@ -0,0 +1,64 @@
+import { createHash } from "node:crypto";
+import { readFile } from "node:fs/promises";
+import { join } from "node:path";
+
+export const description = {
+  extension: { id: "template.startup", version: "1.0.0" },
+  protocols: ["sevro.extension.v1"],
+  requiredCapabilities: ["sevro.codex.repository-invocation"],
+  optionalCapabilities: [],
+  graders: [],
+  taskVerdictPolicies: [],
+};
+
+export const resolved = {
+  cases: [
+    {
+      id: "repository-skill",
+      prompt:
+        "{{sevro.skill_invocation}} Explain the application's start command and port. Do not change files.",
+      fixture: {
+        kind: "generated",
+        commits: [
+          {
+            message: "Initialize application",
+            files: { "README.md": "Run bun run dev on port 3000.\n" },
+          },
+        ],
+      },
+      checks: [
+        {
+          id: "start-command",
+          grader: "sevro.regex",
+          configuration: { pattern: "bun run dev" },
+        },
+        {
+          id: "port",
+          grader: "sevro.regex",
+          configuration: { pattern: "\\b3000\\b" },
+        },
+      ],
+      requiredEvidence: ["sevro.codex.explicit-invocation"],
+      extensionData: {},
+    },
+  ],
+};
+
+export async function prepare() {
+  const relativePath = ".agents/skills/startup-guide/SKILL.md";
+  const body = await readFile(join(import.meta.dirname, relativePath));
+  return {
+    artifacts: [
+      {
+        id: "template.startup-skill",
+        relativePath,
+        sha256: createHash("sha256").update(body).digest("hex"),
+        contentBase64: body.toString("base64"),
+        gitExclude: true,
+      },
+    ],
+    requestedInstrumentation: [],
+    extensionData: {},
+    codexRepositorySkillInvocation: { skillName: "startup-guide" },
+  };
+}

package/examples/templates/repository-skill/extension.ts

@@ -0,0 +1,32 @@
+import { description, prepare, resolved } from "./extension-data";
+import { readFileSync } from "node:fs";
+
+function record(value: unknown): Record<string, unknown> {
+  if (!value || typeof value !== "object" || Array.isArray(value))
+    throw new Error("Expected a protocol request object");
+  return value as Record<string, unknown>;
+}
+
+const request = record(JSON.parse(readFileSync(0, "utf8")) as unknown);
+async function result() {
+  switch (request.method) {
+    case "describe":
+      return description;
+    case "resolve":
+      return resolved;
+    case "prepare":
+      return await prepare();
+    case "evaluate":
+      return { checks: [], metrics: [] };
+    default:
+      throw new Error("Unsupported extension method");
+  }
+}
+process.stdout.write(
+  JSON.stringify({
+    protocol: request.protocol,
+    id: request.id,
+    method: request.method,
+    result: await result(),
+  }),
+);

package/examples/templates/response/adapter.ts

@@ -0,0 +1,12 @@
+/** Deterministic starter; replace with an explicit native route to evaluate a model. */
+export default {
+  id: "template.response",
+  model: "deterministic-example",
+  effort: "none",
+  run() {
+    return Promise.resolve({
+      finalMessage: process.env.SEVRO_TEMPLATE_FAIL === "1" ? "wrong" : "ready",
+      complete: true,
+    });
+  },
+};