@bjoernrochel/sevro 0.1.0-rc.8 → 1.0.0

package/examples/recipes/cases/coding-change.json

@@ -0,0 +1,21 @@
+{
+  "id": "recipe-coding-change",
+  "prompt": "Fix add in math.cjs so it adds its two numeric arguments. Do not modify README.md. Explain how to start the application.",
+  "fixture": {
+    "files": {
+      "README.md": "Fix add in math.cjs. Start the application with bun run dev on port 3000.\n",
+      "math.cjs": "exports.add = (a, b) => a - b;\n"
+    }
+  },
+  "checks": [
+    {
+      "id": "addition",
+      "grader": "sevro.shell",
+      "configuration": {
+        "run": "{{evaluator-test}}",
+        "expectExact": "coding checks passed"
+      }
+    }
+  ],
+  "requiredEvidence": []
+}

package/examples/recipes/cases/host-tools.json

@@ -0,0 +1,21 @@
+{
+  "id": "recipe-host-tools",
+  "prompt": "Read app.env with rg, then explain the application's start command and port.",
+  "fixture": {
+    "files": {
+      "README.md": "Start with bun run dev on port 3000.\n",
+      "app.env": "port=3000\n"
+    }
+  },
+  "checks": [
+    {
+      "id": "rg-access",
+      "grader": "sevro.shell",
+      "configuration": {
+        "run": "rg '^port=3000$' app.env",
+        "expectExact": "port=3000"
+      }
+    }
+  ],
+  "requiredEvidence": []
+}

package/examples/recipes/cases/mise.json

@@ -0,0 +1,21 @@
+{
+  "id": "recipe-mise",
+  "prompt": "Use node from PATH to read app.json, then explain the application's start command and port. The operator has resolved mise's existing node@24.13.0 installation; do not download anything.",
+  "fixture": {
+    "files": {
+      "README.md": "Start with bun run dev on port 3000.\n",
+      "app.json": "{\"port\":3000}\n"
+    }
+  },
+  "checks": [
+    {
+      "id": "mise-node-access",
+      "grader": "sevro.shell",
+      "configuration": {
+        "run": "node -e 'if (process.version !== \"v24.13.0\") process.exit(1); const a = require(\"./app.json\"); if (a.port !== 3000) process.exit(1); console.log(a.port)'",
+        "expectExact": "3000"
+      }
+    }
+  ],
+  "requiredEvidence": []
+}

package/examples/recipes/cases/node.json

@@ -0,0 +1,21 @@
+{
+  "id": "recipe-node",
+  "prompt": "Read app.json with Node and explain the application's start command and port.",
+  "fixture": {
+    "files": {
+      "README.md": "Start with bun run dev on port 3000.\n",
+      "app.json": "{\"port\":3000}\n"
+    }
+  },
+  "checks": [
+    {
+      "id": "node-access",
+      "grader": "sevro.shell",
+      "configuration": {
+        "run": "node -e 'const a = require(\"./app.json\"); if (a.port !== 3000) process.exit(1); console.log(a.port)'",
+        "expectExact": "3000"
+      }
+    }
+  ],
+  "requiredEvidence": []
+}

package/examples/recipes/cases/python-uv.json

@@ -0,0 +1,21 @@
+{
+  "id": "recipe-python-uv",
+  "prompt": "Use uv run --offline --no-project --python \"$RECIPE_PYTHON\" python to read app.json, then explain the application's start command and port.",
+  "fixture": {
+    "files": {
+      "README.md": "Start with bun run dev on port 3000.\n",
+      "app.json": "{\"port\":3000}\n"
+    }
+  },
+  "checks": [
+    {
+      "id": "uv-python-access",
+      "grader": "sevro.shell",
+      "configuration": {
+        "run": "uv run --offline --no-project --python \"$RECIPE_PYTHON\" python -c 'import json; p=json.load(open(\"app.json\"))[\"port\"]; assert p == 3000; print(p)'",
+        "expectExact": "3000"
+      }
+    }
+  ],
+  "requiredEvidence": []
+}

package/examples/recipes/cases/semantic.json

@@ -0,0 +1,17 @@
+{
+  "id": "recipe-semantic",
+  "prompt": "Read README.md and explain how to start the application, including its port.",
+  "fixture": {
+    "files": { "README.md": "Start with bun run dev on port 3000.\n" }
+  },
+  "checks": [
+    {
+      "id": "startup-meaning",
+      "grader": "sevro.semantic",
+      "configuration": {
+        "proposition": "The response tells the reader to run bun run dev and says that the application listens on port 3000."
+      }
+    }
+  ],
+  "requiredEvidence": []
+}

package/examples/recipes/coding-change/check.ts

@@ -0,0 +1,23 @@
+import assert from "node:assert/strict";
+import { join } from "node:path";
+import { pathToFileURL } from "node:url";
+
+interface Addition {
+  add: (left: number, right: number) => number;
+}
+function isAddition(value: unknown): value is Addition {
+  return (
+    typeof value === "object" &&
+    value !== null &&
+    "add" in value &&
+    typeof value.add === "function"
+  );
+}
+const candidate: unknown = await import(
+  pathToFileURL(join(process.cwd(), "math.cjs")).href
+);
+assert.ok(isAddition(candidate), "Candidate must export add");
+assert.equal(candidate.add(2, 3), 5);
+assert.equal(candidate.add(-2, 3), 1);
+assert.equal(candidate.add(0, 0), 0);
+console.log("coding checks passed");

package/examples/recipes/extension-artifacts.ts

@@ -0,0 +1,65 @@
+import { createHash } from "node:crypto";
+import { readFile } from "node:fs/promises";
+import { join } from "node:path";
+
+export function artifact(relativePath: string, content: string) {
+  return {
+    id: `recipes.${createHash("sha256").update(relativePath).digest("hex").slice(0, 16)}`,
+    relativePath,
+    sha256: createHash("sha256").update(content).digest("hex"),
+    contentBase64: Buffer.from(content).toString("base64"),
+    gitExclude: true,
+  };
+}
+
+export async function repositoryArtifacts(host: string) {
+  const directory = host === "claude" ? ".claude" : ".agents";
+  const relative = `${directory}/skills/startup-guide/SKILL.md`;
+  const body = await readFile(
+    join(import.meta.dir, "repository-skill", relative),
+    "utf8",
+  );
+  return [artifact(relative, body)];
+}
+
+export function pluginArtifacts() {
+  const files = {
+    "marketplace/.claude-plugin/marketplace.json": JSON.stringify({
+      name: "recipes",
+      owner: { name: "Sevro examples" },
+      plugins: [
+        { name: "startup", source: "./plugin", description: "Startup recipe" },
+      ],
+    }),
+    "marketplace/plugin/.claude-plugin/plugin.json": JSON.stringify({
+      name: "startup",
+      version: "1.0.0",
+    }),
+    "marketplace/plugin/.codex-plugin/plugin.json": JSON.stringify({
+      name: "startup",
+      version: "1.0.0",
+      skills: "./skills/",
+    }),
+    "marketplace/plugin/skills/startup-guide/SKILL.md":
+      "---\nname: startup-guide\ndescription: Explain application startup.\n---\nRead README.md. Explain its command and port.\n",
+    "marketplace/plugin/hooks/hooks.json": JSON.stringify({
+      hooks: {
+        SessionStart: [
+          {
+            hooks: [
+              {
+                type: "command",
+                command: '/bin/sh "${CLAUDE_PLUGIN_ROOT}/hooks/start.sh"',
+              },
+            ],
+          },
+        ],
+      },
+    }),
+    "marketplace/plugin/hooks/start.sh":
+      "#!/bin/sh\nprintf 'hook ran\\n' > \"$PWD/hook-ran.txt\"\n",
+  };
+  return Object.entries(files).map(([path, content]) =>
+    artifact(path, content),
+  );
+}

package/examples/recipes/extension-methods.ts

@@ -0,0 +1,174 @@
+import {
+  artifact,
+  pluginArtifacts,
+  repositoryArtifacts,
+} from "./extension-artifacts";
+
+export function record(value: unknown): Record<string, unknown> {
+  if (value === null || typeof value !== "object" || Array.isArray(value))
+    throw new Error("Expected a JSON object");
+  return value as Record<string, unknown>;
+}
+
+export function describe(mode: string, host: string) {
+  const capabilities: Record<string, string[]> = {
+    lifecycle: [],
+    "repository-skill": [`sevro.${host}.repository-invocation`],
+    plugins:
+      host === "codex"
+        ? ["sevro.codex.plugin-marketplace", "sevro.codex.explicit-invocation"]
+        : ["sevro.claude.plugin-dirs", "sevro.claude.explicit-invocation"],
+  };
+  const requiredCapabilities = capabilities[mode];
+  if (!requiredCapabilities) throw new Error("Unknown extension recipe");
+  return {
+    extension: { id: "recipes.startup", version: "1.0.0" },
+    protocols: ["sevro.extension.v1"],
+    requiredCapabilities,
+    optionalCapabilities: [],
+    graders: ["recipes.startup"],
+    taskVerdictPolicies: [],
+  };
+}
+
+function requiredInvocation(mode: string, host: string): string[] {
+  if (mode !== "repository-skill") return [];
+  return [
+    host === "codex"
+      ? "sevro.codex.explicit-invocation"
+      : "sevro.claude.repository-invocation",
+  ];
+}
+
+export function resolveCase(mode: string, host: string, fail: boolean) {
+  const invocation = mode === "lifecycle" ? "" : "{{sevro.skill_invocation}} ";
+  const prompt = fail
+    ? "Deliberate negative control: return only wrong."
+    : "Read README.md. Explain the application's start command and port. Do not change files.";
+  const requiredEvidence = requiredInvocation(mode, host);
+  return {
+    cases: [
+      {
+        id: "recipe-extension",
+        prompt: invocation + prompt,
+        fixture: {
+          kind: "generated",
+          commits: [
+            {
+              message: "chore: initialize sample",
+              files: { "README.md": "Run bun run dev on port 3000.\n" },
+            },
+          ],
+        },
+        checks: [
+          {
+            id: "startup-command",
+            grader: "sevro.regex",
+            configuration: { pattern: "bun run dev" },
+          },
+          {
+            id: "prepared-input",
+            grader: "sevro.shell",
+            configuration: {
+              run: "test \"$(cat startup.txt)\" = 'prepared startup data'",
+            },
+          },
+          ...(mode === "plugins"
+            ? [
+                {
+                  id: "plugin-hook",
+                  grader: "sevro.shell",
+                  configuration: {
+                    run: "cat hook-ran.txt",
+                    expectExact: "hook ran",
+                  },
+                },
+              ]
+            : []),
+          {
+            id: "recipes.startup.answer",
+            grader: "recipes.startup",
+            configuration: {},
+          },
+        ],
+        requiredEvidence,
+        extensionData: { "recipes.startup": { mode, host } },
+      },
+    ],
+  };
+}
+
+export async function prepare(mode: string, host: string) {
+  const base = {
+    artifacts: [artifact("startup.txt", "prepared startup data\n")],
+    requestedInstrumentation: [],
+    extensionData: { "recipes.startup": { mode, host } },
+  };
+  switch (mode) {
+    case "repository-skill":
+      return {
+        ...base,
+        artifacts: [...base.artifacts, ...(await repositoryArtifacts(host))],
+        ...(host === "codex"
+          ? { codexRepositorySkillInvocation: { skillName: "startup-guide" } }
+          : {
+              claudeRepositorySkillInvocation: { skillName: "startup-guide" },
+            }),
+      };
+    case "plugins":
+      return {
+        ...base,
+        artifacts: [...base.artifacts, ...pluginArtifacts()],
+        ...(host === "codex"
+          ? {
+              codexMarketplace: {
+                artifactRoot: "marketplace",
+                marketplaceName: "recipes",
+                pluginNames: ["startup"],
+              },
+              codexSkillInvocation: {
+                pluginName: "startup",
+                skillName: "startup-guide",
+              },
+            }
+          : {
+              claudePluginDirs: { artifactRoots: ["marketplace/plugin"] },
+              claudeSkillInvocation: {
+                pluginName: "startup",
+                skillName: "startup-guide",
+              },
+            }),
+      };
+    default:
+      return base;
+  }
+}
+
+function answerStatus(observation: Record<string, unknown>) {
+  if (observation.completeness !== "complete") return "unavailable";
+  const text = record(observation.data).text;
+  return typeof text === "string" && /\b3000\b/.test(text)
+    ? "passed"
+    : "failed";
+}
+
+export function evaluate(params: Record<string, unknown>) {
+  if (!Array.isArray(params.observations))
+    throw new Error("Missing observations");
+  const observations: unknown[] = params.observations;
+  const observation = observations
+    .map(record)
+    .find((item) => item.id === "sevro.observation.final-message");
+  if (!observation) throw new Error("Missing final-message observation");
+  const status = answerStatus(observation);
+  return {
+    checks: [
+      {
+        id: "recipes.startup.answer",
+        status,
+        evidenceRefs: ["sevro.observation.final-message"],
+      },
+    ],
+    metrics: [],
+  };
+}

package/examples/recipes/extension.ts

@@ -0,0 +1,37 @@
+import {
+  describe,
+  evaluate,
+  prepare,
+  record,
+  resolveCase,
+} from "./extension-methods";
+
+const mode = process.argv[2] ?? "lifecycle";
+const host = process.argv[3] ?? "deterministic";
+const fail = process.argv[4] === "fail";
+const request = record(JSON.parse(await Bun.stdin.text()) as unknown);
+
+async function result() {
+  switch (request.method) {
+    case "describe":
+      return describe(mode, host);
+    case "resolve":
+      return resolveCase(mode, host, fail);
+    case "prepare":
+      return await prepare(mode, host);
+    case "evaluate":
+      return evaluate(record(request.params));
+    default:
+      throw new Error("Unsupported extension method");
+  }
+}
+
+// One request per process; stdout contains only the matching protocol response.
+process.stdout.write(
+  JSON.stringify({
+    protocol: request.protocol,
+    id: request.id,
+    method: request.method,
+    result: await result(),
+  }),
+);

package/examples/recipes/repository-skill/.agents/skills/startup-guide/SKILL.md

@@ -0,0 +1,7 @@
+---
+name: startup-guide
+description: Explain how to start this sample application and identify its port.
+---
+
+Read README.md. Explain the exact start command and port from that file.
+Do not change files. Do not invent a different command or port.

package/examples/recipes/repository-skill/.claude/skills/startup-guide/SKILL.md

@@ -0,0 +1,7 @@
+---
+name: startup-guide
+description: Explain how to start this sample application and identify its port.
+---
+
+Read README.md. Explain the exact start command and port from that file.
+Do not change files. Do not invent a different command or port.

package/examples/recipes/repository-skill/README.md

@@ -0,0 +1,3 @@
+# Sample application
+
+Run `bun run dev` to start the application on port `3000`.

package/examples/recipes/run-inputs.ts

@@ -0,0 +1,188 @@
+import { readFile, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { record } from "./extension-methods";
+import { requireTool, type RecipeOptions } from "./run-options";
+
+function shellQuote(text: string): string {
+  return `'${text.replaceAll("'", "'\\''")}'`;
+}
+
+async function caseFile(
+  options: RecipeOptions,
+  state: string,
+): Promise<string[]> {
+  const value = record(
+    JSON.parse(
+      await readFile(
+        join(import.meta.dir, "cases", `${options.recipe}.json`),
+        "utf8",
+      ),
+    ) as unknown,
+  );
+  if (options.recipe === "coding-change") {
+    const test = await readFile(
+      join(import.meta.dir, "coding-change/check.ts"),
+      "utf8",
+    );
+    const checks: unknown[] = Array.isArray(value.checks) ? value.checks : [];
+    const check = record(checks[0]);
+    const javascript = new Bun.Transpiler({ loader: "ts" }).transformSync(test);
+    const protectedTest = shellQuote(
+      join(import.meta.dir, "coding-change/check.ts"),
+    );
+    record(check.configuration).run =
+      `test ! -r ${protectedTest}\nnode --input-type=module -e ${shellQuote(javascript)}`;
+  }
+  if (options.fail) negativeFixture(value, options.recipe);
+  const path = join(state, "case.json");
+  await writeFile(path, JSON.stringify(value));
+  return ["--case-file", path];
+}
+
+function negativeFixture(value: Record<string, unknown>, recipe: string): void {
+  if (["coding-change", "semantic"].includes(recipe)) {
+    value.prompt =
+      "Deliberate negative control: leave math.cjs unchanged if present and respond only: npm start on port 9000.";
+    return;
+  }
+  const files = record(record(value.fixture).files);
+  for (const [path, content] of Object.entries(files)) {
+    if (typeof content !== "string")
+      throw new Error("Expected inline fixture text");
+    files[path] = content.replaceAll("3000", "9000");
+  }
+}
+
+async function extensionFiles(
+  options: RecipeOptions,
+  state: string,
+): Promise<string[]> {
+  const command = join(state, "extension-command.json");
+  await writeFile(
+    command,
+    JSON.stringify([
+      process.execPath,
+      join(import.meta.dir, "extension.ts"),
+      options.recipe,
+      options.route,
+      options.fail ? "fail" : "pass",
+    ]),
+  );
+  const sources = [
+    "extension.ts",
+    "extension-methods.ts",
+    "extension-artifacts.ts",
+    "repository-skill/.agents/skills/startup-guide/SKILL.md",
+    "repository-skill/.claude/skills/startup-guide/SKILL.md",
+  ];
+  return [
+    "--extension-command-file",
+    command,
+    "--case-id",
+    "recipe-extension",
+    ...sources.flatMap((path) => [
+      "--extension-source-file",
+      join(import.meta.dir, path),
+    ]),
+  ];
+}
+
+export async function recipeInputs(
+  options: RecipeOptions,
+  state: string,
+): Promise<string[]> {
+  if (["lifecycle", "repository-skill", "plugins"].includes(options.recipe))
+    return extensionFiles(options, state);
+  return caseFile(options, state);
+}
+
+async function codexAuthentication(
+  options: RecipeOptions,
+  state: string,
+): Promise<string> {
+  if (!options.dry) {
+    const path = process.env.SEVRO_RECIPE_CODEX_AUTH;
+    if (!path)
+      throw new Error(
+        "Set SEVRO_RECIPE_CODEX_AUTH to your private Codex auth.json path",
+      );
+    return path;
+  }
+  const path = join(state, "dry-auth.json");
+  await writeFile(path, "{}\n");
+  return path;
+}
+
+export async function hostArguments(
+  options: RecipeOptions,
+  state: string,
+): Promise<string[]> {
+  if (options.route === "deterministic")
+    return ["--adapter-module", join(import.meta.dir, "candidate.ts")];
+  const model = selectedModel(options.dry, "SEVRO_RECIPE_MODEL");
+  if (!model)
+    throw new Error(
+      "Set SEVRO_RECIPE_MODEL to an available model for the selected native host",
+    );
+  const args = [
+    "--host",
+    options.route,
+    "--model",
+    model,
+    "--effort",
+    "medium",
+  ];
+  if (options.route === "codex")
+    return [
+      ...args,
+      "--codex-bin",
+      requireTool("codex"),
+      "--codex-auth-file",
+      await codexAuthentication(options, state),
+    ];
+  return [
+    ...args,
+    "--claude-bin",
+    requireTool("claude"),
+    ...(options.recipe === "repository-skill"
+      ? ["--claude-project-settings"]
+      : []),
+  ];
+}
+
+export async function semanticArguments(
+  options: RecipeOptions,
+  state: string,
+): Promise<string[]> {
+  if (options.recipe !== "semantic") return [];
+  if (options.route === "deterministic")
+    return [
+      "--semantic-adapter-module",
+      join(import.meta.dir, "semantic-judge.ts"),
+    ];
+  const model = selectedModel(options.dry, "SEVRO_RECIPE_JUDGE_MODEL");
+  if (!model)
+    throw new Error(
+      "Set SEVRO_RECIPE_JUDGE_MODEL for the separate Codex judge route",
+    );
+  const args = [
+    "--semantic-host",
+    "codex",
+    "--semantic-model",
+    model,
+    "--semantic-effort",
+    "medium",
+  ];
+  if (options.route === "codex") return args;
+  return [
+    ...args,
+    "--codex-bin",
+    requireTool("codex"),
+    "--codex-auth-file",
+    await codexAuthentication(options, state),
+  ];
+}
+
+function selectedModel(dry: boolean, variable: string): string | undefined {
+  return dry ? "dry-model" : process.env[variable];
+}