@bjoernrochel/sevro 0.1.0-rc.2 → 0.1.0-rc.21

package/examples/recipes/extension-artifacts.ts

@@ -0,0 +1,65 @@
+import { createHash } from "node:crypto";
+import { readFile } from "node:fs/promises";
+import { join } from "node:path";
+
+export function artifact(relativePath: string, content: string) {
+  return {
+    id: `recipes.${createHash("sha256").update(relativePath).digest("hex").slice(0, 16)}`,
+    relativePath,
+    sha256: createHash("sha256").update(content).digest("hex"),
+    contentBase64: Buffer.from(content).toString("base64"),
+    gitExclude: true,
+  };
+}
+
+export async function repositoryArtifacts(host: string) {
+  const directory = host === "claude" ? ".claude" : ".agents";
+  const relative = `${directory}/skills/startup-guide/SKILL.md`;
+  const body = await readFile(
+    join(import.meta.dir, "repository-skill", relative),
+    "utf8",
+  );
+  return [artifact(relative, body)];
+}
+
+export function pluginArtifacts() {
+  const files = {
+    "marketplace/.claude-plugin/marketplace.json": JSON.stringify({
+      name: "recipes",
+      owner: { name: "Sevro examples" },
+      plugins: [
+        { name: "startup", source: "./plugin", description: "Startup recipe" },
+      ],
+    }),
+    "marketplace/plugin/.claude-plugin/plugin.json": JSON.stringify({
+      name: "startup",
+      version: "1.0.0",
+    }),
+    "marketplace/plugin/.codex-plugin/plugin.json": JSON.stringify({
+      name: "startup",
+      version: "1.0.0",
+      skills: "./skills/",
+    }),
+    "marketplace/plugin/skills/startup-guide/SKILL.md":
+      "---\nname: startup-guide\ndescription: Explain application startup.\n---\nRead README.md. Explain its command and port.\n",
+    "marketplace/plugin/hooks/hooks.json": JSON.stringify({
+      hooks: {
+        SessionStart: [
+          {
+            hooks: [
+              {
+                type: "command",
+                command: '/bin/sh "${CLAUDE_PLUGIN_ROOT}/hooks/start.sh"',
+              },
+            ],
+          },
+        ],
+      },
+    }),
+    "marketplace/plugin/hooks/start.sh":
+      "#!/bin/sh\nprintf 'hook ran\\n' > \"$PWD/hook-ran.txt\"\n",
+  };
+  return Object.entries(files).map(([path, content]) =>
+    artifact(path, content),
+  );
+}

package/examples/recipes/extension-methods.ts

@@ -0,0 +1,174 @@
+import {
+  artifact,
+  pluginArtifacts,
+  repositoryArtifacts,
+} from "./extension-artifacts";
+
+export function record(value: unknown): Record<string, unknown> {
+  if (value === null || typeof value !== "object" || Array.isArray(value))
+    throw new Error("Expected a JSON object");
+  return value as Record<string, unknown>;
+}
+
+export function describe(mode: string, host: string) {
+  const capabilities: Record<string, string[]> = {
+    lifecycle: [],
+    "repository-skill": [`sevro.${host}.repository-invocation`],
+    plugins:
+      host === "codex"
+        ? ["sevro.codex.plugin-marketplace", "sevro.codex.explicit-invocation"]
+        : ["sevro.claude.plugin-dirs", "sevro.claude.explicit-invocation"],
+  };
+  const requiredCapabilities = capabilities[mode];
+  if (!requiredCapabilities) throw new Error("Unknown extension recipe");
+  return {
+    extension: { id: "recipes.startup", version: "1.0.0" },
+    protocols: ["sevro.extension.v1"],
+    requiredCapabilities,
+    optionalCapabilities: [],
+    graders: ["recipes.startup"],
+    taskVerdictPolicies: [],
+  };
+}
+
+function requiredInvocation(mode: string, host: string): string[] {
+  if (mode !== "repository-skill") return [];
+  return [
+    host === "codex"
+      ? "sevro.codex.explicit-invocation"
+      : "sevro.claude.repository-invocation",
+  ];
+}
+
+export function resolveCase(mode: string, host: string, fail: boolean) {
+  const invocation = mode === "lifecycle" ? "" : "{{sevro.skill_invocation}} ";
+  const prompt = fail
+    ? "Deliberate negative control: return only wrong."
+    : "Read README.md. Explain the application's start command and port. Do not change files.";
+  const requiredEvidence = requiredInvocation(mode, host);
+  return {
+    cases: [
+      {
+        id: "recipe-extension",
+        prompt: invocation + prompt,
+        fixture: {
+          kind: "generated",
+          commits: [
+            {
+              message: "chore: initialize sample",
+              files: { "README.md": "Run bun run dev on port 3000.\n" },
+            },
+          ],
+        },
+        checks: [
+          {
+            id: "startup-command",
+            grader: "sevro.regex",
+            configuration: { pattern: "bun run dev" },
+          },
+          {
+            id: "prepared-input",
+            grader: "sevro.shell",
+            configuration: {
+              run: "test \"$(cat startup.txt)\" = 'prepared startup data'",
+            },
+          },
+          ...(mode === "plugins"
+            ? [
+                {
+                  id: "plugin-hook",
+                  grader: "sevro.shell",
+                  configuration: {
+                    run: "cat hook-ran.txt",
+                    expectExact: "hook ran",
+                  },
+                },
+              ]
+            : []),
+          {
+            id: "recipes.startup.answer",
+            grader: "recipes.startup",
+            configuration: {},
+          },
+        ],
+        requiredEvidence,
+        extensionData: { "recipes.startup": { mode, host } },
+      },
+    ],
+  };
+}
+
+export async function prepare(mode: string, host: string) {
+  const base = {
+    artifacts: [artifact("startup.txt", "prepared startup data\n")],
+    requestedInstrumentation: [],
+    extensionData: { "recipes.startup": { mode, host } },
+  };
+  switch (mode) {
+    case "repository-skill":
+      return {
+        ...base,
+        artifacts: [...base.artifacts, ...(await repositoryArtifacts(host))],
+        ...(host === "codex"
+          ? { codexRepositorySkillInvocation: { skillName: "startup-guide" } }
+          : {
+              claudeRepositorySkillInvocation: { skillName: "startup-guide" },
+            }),
+      };
+    case "plugins":
+      return {
+        ...base,
+        artifacts: [...base.artifacts, ...pluginArtifacts()],
+        ...(host === "codex"
+          ? {
+              codexMarketplace: {
+                artifactRoot: "marketplace",
+                marketplaceName: "recipes",
+                pluginNames: ["startup"],
+              },
+              codexSkillInvocation: {
+                pluginName: "startup",
+                skillName: "startup-guide",
+              },
+            }
+          : {
+              claudePluginDirs: { artifactRoots: ["marketplace/plugin"] },
+              claudeSkillInvocation: {
+                pluginName: "startup",
+                skillName: "startup-guide",
+              },
+            }),
+      };
+    default:
+      return base;
+  }
+}
+
+function answerStatus(observation: Record<string, unknown>) {
+  if (observation.completeness !== "complete") return "unavailable";
+  const text = record(observation.data).text;
+  return typeof text === "string" && /\b3000\b/.test(text)
+    ? "passed"
+    : "failed";
+}
+
+export function evaluate(params: Record<string, unknown>) {
+  if (!Array.isArray(params.observations))
+    throw new Error("Missing observations");
+  const observations: unknown[] = params.observations;
+  const observation = observations
+    .map(record)
+    .find((item) => item.id === "sevro.observation.final-message");
+  if (!observation) throw new Error("Missing final-message observation");
+  const status = answerStatus(observation);
+  return {
+    checks: [
+      {
+        id: "recipes.startup.answer",
+        status,
+        evidenceRefs: ["sevro.observation.final-message"],
+      },
+    ],
+    metrics: [],
+  };
+}

package/examples/recipes/extension.ts

@@ -0,0 +1,37 @@
+import {
+  describe,
+  evaluate,
+  prepare,
+  record,
+  resolveCase,
+} from "./extension-methods";
+
+const mode = process.argv[2] ?? "lifecycle";
+const host = process.argv[3] ?? "deterministic";
+const fail = process.argv[4] === "fail";
+const request = record(JSON.parse(await Bun.stdin.text()) as unknown);
+
+async function result() {
+  switch (request.method) {
+    case "describe":
+      return describe(mode, host);
+    case "resolve":
+      return resolveCase(mode, host, fail);
+    case "prepare":
+      return await prepare(mode, host);
+    case "evaluate":
+      return evaluate(record(request.params));
+    default:
+      throw new Error("Unsupported extension method");
+  }
+}
+
+// One request per process; stdout contains only the matching protocol response.
+process.stdout.write(
+  JSON.stringify({
+    protocol: request.protocol,
+    id: request.id,
+    method: request.method,
+    result: await result(),
+  }),
+);

package/examples/recipes/repository-skill/.agents/skills/startup-guide/SKILL.md

@@ -0,0 +1,7 @@
+---
+name: startup-guide
+description: Explain how to start this sample application and identify its port.
+---
+
+Read README.md. Explain the exact start command and port from that file.
+Do not change files. Do not invent a different command or port.

package/examples/recipes/repository-skill/.claude/skills/startup-guide/SKILL.md

@@ -0,0 +1,7 @@
+---
+name: startup-guide
+description: Explain how to start this sample application and identify its port.
+---
+
+Read README.md. Explain the exact start command and port from that file.
+Do not change files. Do not invent a different command or port.

package/examples/recipes/repository-skill/README.md

@@ -0,0 +1,3 @@
+# Sample application
+
+Run `bun run dev` to start the application on port `3000`.

package/examples/recipes/run-inputs.ts

@@ -0,0 +1,188 @@
+import { readFile, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { record } from "./extension-methods";
+import { requireTool, type RecipeOptions } from "./run-options";
+
+function shellQuote(text: string): string {
+  return `'${text.replaceAll("'", "'\\''")}'`;
+}
+
+async function caseFile(
+  options: RecipeOptions,
+  state: string,
+): Promise<string[]> {
+  const value = record(
+    JSON.parse(
+      await readFile(
+        join(import.meta.dir, "cases", `${options.recipe}.json`),
+        "utf8",
+      ),
+    ) as unknown,
+  );
+  if (options.recipe === "coding-change") {
+    const test = await readFile(
+      join(import.meta.dir, "coding-change/check.ts"),
+      "utf8",
+    );
+    const checks: unknown[] = Array.isArray(value.checks) ? value.checks : [];
+    const check = record(checks[0]);
+    const javascript = new Bun.Transpiler({ loader: "ts" }).transformSync(test);
+    const protectedTest = shellQuote(
+      join(import.meta.dir, "coding-change/check.ts"),
+    );
+    record(check.configuration).run =
+      `test ! -r ${protectedTest}\nnode --input-type=module -e ${shellQuote(javascript)}`;
+  }
+  if (options.fail) negativeFixture(value, options.recipe);
+  const path = join(state, "case.json");
+  await writeFile(path, JSON.stringify(value));
+  return ["--case-file", path];
+}
+
+function negativeFixture(value: Record<string, unknown>, recipe: string): void {
+  if (["coding-change", "semantic"].includes(recipe)) {
+    value.prompt =
+      "Deliberate negative control: leave math.cjs unchanged if present and respond only: npm start on port 9000.";
+    return;
+  }
+  const files = record(record(value.fixture).files);
+  for (const [path, content] of Object.entries(files)) {
+    if (typeof content !== "string")
+      throw new Error("Expected inline fixture text");
+    files[path] = content.replaceAll("3000", "9000");
+  }
+}
+
+async function extensionFiles(
+  options: RecipeOptions,
+  state: string,
+): Promise<string[]> {
+  const command = join(state, "extension-command.json");
+  await writeFile(
+    command,
+    JSON.stringify([
+      process.execPath,
+      join(import.meta.dir, "extension.ts"),
+      options.recipe,
+      options.route,
+      options.fail ? "fail" : "pass",
+    ]),
+  );
+  const sources = [
+    "extension.ts",
+    "extension-methods.ts",
+    "extension-artifacts.ts",
+    "repository-skill/.agents/skills/startup-guide/SKILL.md",
+    "repository-skill/.claude/skills/startup-guide/SKILL.md",
+  ];
+  return [
+    "--extension-command-file",
+    command,
+    "--case-id",
+    "recipe-extension",
+    ...sources.flatMap((path) => [
+      "--extension-source-file",
+      join(import.meta.dir, path),
+    ]),
+  ];
+}
+
+export async function recipeInputs(
+  options: RecipeOptions,
+  state: string,
+): Promise<string[]> {
+  if (["lifecycle", "repository-skill", "plugins"].includes(options.recipe))
+    return extensionFiles(options, state);
+  return caseFile(options, state);
+}
+
+async function codexAuthentication(
+  options: RecipeOptions,
+  state: string,
+): Promise<string> {
+  if (!options.dry) {
+    const path = process.env.SEVRO_RECIPE_CODEX_AUTH;
+    if (!path)
+      throw new Error(
+        "Set SEVRO_RECIPE_CODEX_AUTH to your private Codex auth.json path",
+      );
+    return path;
+  }
+  const path = join(state, "dry-auth.json");
+  await writeFile(path, "{}\n");
+  return path;
+}
+
+export async function hostArguments(
+  options: RecipeOptions,
+  state: string,
+): Promise<string[]> {
+  if (options.route === "deterministic")
+    return ["--adapter-module", join(import.meta.dir, "candidate.ts")];
+  const model = selectedModel(options.dry, "SEVRO_RECIPE_MODEL");
+  if (!model)
+    throw new Error(
+      "Set SEVRO_RECIPE_MODEL to an available model for the selected native host",
+    );
+  const args = [
+    "--host",
+    options.route,
+    "--model",
+    model,
+    "--effort",
+    "medium",
+  ];
+  if (options.route === "codex")
+    return [
+      ...args,
+      "--codex-bin",
+      requireTool("codex"),
+      "--codex-auth-file",
+      await codexAuthentication(options, state),
+    ];
+  return [
+    ...args,
+    "--claude-bin",
+    requireTool("claude"),
+    ...(options.recipe === "repository-skill"
+      ? ["--claude-project-settings"]
+      : []),
+  ];
+}
+
+export async function semanticArguments(
+  options: RecipeOptions,
+  state: string,
+): Promise<string[]> {
+  if (options.recipe !== "semantic") return [];
+  if (options.route === "deterministic")
+    return [
+      "--semantic-adapter-module",
+      join(import.meta.dir, "semantic-judge.ts"),
+    ];
+  const model = selectedModel(options.dry, "SEVRO_RECIPE_JUDGE_MODEL");
+  if (!model)
+    throw new Error(
+      "Set SEVRO_RECIPE_JUDGE_MODEL for the separate Codex judge route",
+    );
+  const args = [
+    "--semantic-host",
+    "codex",
+    "--semantic-model",
+    model,
+    "--semantic-effort",
+    "medium",
+  ];
+  if (options.route === "codex") return args;
+  return [
+    ...args,
+    "--codex-bin",
+    requireTool("codex"),
+    "--codex-auth-file",
+    await codexAuthentication(options, state),
+  ];
+}
+
+function selectedModel(dry: boolean, variable: string): string | undefined {
+  return dry ? "dry-model" : process.env[variable];
+}

package/examples/recipes/run-options.ts

@@ -0,0 +1,131 @@
+import { delimiter, isAbsolute, join } from "node:path";
+import { homedir } from "node:os";
+
+export const recipeNames = [
+  "coding-change",
+  "host-tools",
+  "node",
+  "python-uv",
+  "mise",
+  "semantic",
+  "lifecycle",
+  "repository-skill",
+  "plugins",
+];
+
+function validateSelection(
+  recipe: string,
+  route: string,
+  flags: string[],
+): void {
+  if (!recipeNames.includes(recipe))
+    throw new Error(`Choose a recipe: ${recipeNames.join(", ")}`);
+  if (!["deterministic", "codex", "claude"].includes(route))
+    throw new Error("Choose deterministic, codex, or claude");
+  if (flags.some((flag) => !["--dry", "--fail", "--goals-only"].includes(flag)))
+    throw new Error("Supported flags: --dry --fail --goals-only");
+}
+
+function validateNativeRecipe(recipe: string, route: string): void {
+  if (
+    ["repository-skill", "plugins"].includes(recipe) &&
+    route === "deterministic"
+  )
+    throw new Error(
+      "Skill and plugin recipes require codex or claude (add --dry to prepare without model calls)",
+    );
+}
+
+export function selectedOptions() {
+  const [recipe = "", route = "deterministic", ...flags] =
+    process.argv.slice(2);
+  validateSelection(recipe, route, flags);
+  validateNativeRecipe(recipe, route);
+  validateGoalsStep(flags);
+  return {
+    recipe,
+    route,
+    dry: flags.includes("--dry"),
+    fail: flags.includes("--fail"),
+    goalsOnly: flags.includes("--goals-only"),
+  };
+}
+export type RecipeOptions = ReturnType<typeof selectedOptions>;
+
+function validateGoalsStep(flags: string[]): void {
+  if (flags.includes("--goals-only") && !flags.includes("--dry"))
+    throw new Error(
+      "--goals-only is a dry preparation step; omit it for the complete hook recipe",
+    );
+}
+
+export function requireTool(name: string): string {
+  const path = Bun.which(name);
+  if (!path)
+    throw new Error(
+      `Recipe prerequisite missing: install ${name} and put it on PATH`,
+    );
+  return path;
+}
+
+export function runtimeFile(options: RecipeOptions): string {
+  const files: Record<string, string> = {
+    "coding-change": "path",
+    "host-tools": "path",
+    node: "path",
+    "python-uv": "python-uv",
+    mise: "mise",
+    plugins: options.goalsOnly ? "goals" : "plugins",
+  };
+  return join(
+    import.meta.dir,
+    "runtime",
+    `${files[options.recipe] ?? "empty"}.json`,
+  );
+}
+
+function languagePrerequisites(recipe: string): void {
+  if (recipe === "python-uv")
+    process.env.RECIPE_PYTHON ??= requireTool("python3");
+  if (recipe === "mise") selectMiseNode();
+  if (recipe === "host-tools") requireStandaloneRipgrep();
+}
+
+function requireStandaloneRipgrep(): void {
+  const roots = [
+    process.env.CODEX_HOME ?? join(homedir(), ".codex"),
+    process.env.CLAUDE_CONFIG_DIR ?? join(homedir(), ".claude"),
+  ];
+  const paths = (process.env.PATH ?? "")
+    .split(delimiter)
+    .filter((entry) => !roots.some((root) => entry.startsWith(root)));
+  if (!Bun.which("rg", { PATH: paths.join(delimiter) }))
+    throw new Error(
+      "Install standalone ripgrep outside Codex/Claude credential directories, then put it on PATH (for example brew install ripgrep or apt-get install ripgrep)",
+    );
+}
+
+function selectMiseNode(): void {
+  const result = Bun.spawnSync([requireTool("mise"), "where", "node@24.13.0"], {
+    stdout: "pipe",
+    stderr: "pipe",
+  });
+  const installation = result.stdout.toString().trim();
+  if (result.exitCode !== 0 || !isAbsolute(installation))
+    throw new Error(
+      "Install the recipe's Node with mise install node@24.13.0 before running it",
+    );
+  process.env.PATH = `${join(installation, "bin")}${delimiter}${process.env.PATH ?? "/usr/bin:/bin"}`;
+}
+
+export function runtimePrerequisites(recipe: string): void {
+  const tools: Record<string, string[]> = {
+    "coding-change": ["node"],
+    "host-tools": ["rg"],
+    node: ["node"],
+    "python-uv": ["uv", "python3"],
+    mise: ["mise"],
+  };
+  for (const tool of tools[recipe] ?? []) requireTool(tool);
+  languagePrerequisites(recipe);
+}

package/examples/recipes/run.ts

@@ -0,0 +1,63 @@
+import { mkdir, realpath, writeFile } from "node:fs/promises";
+import { join, resolve } from "node:path";
+import { hostArguments, recipeInputs, semanticArguments } from "./run-inputs";
+import {
+  runtimeFile,
+  runtimePrerequisites,
+  selectedOptions,
+} from "./run-options";
+
+const options = selectedOptions();
+runtimePrerequisites(options.recipe);
+const mode = options.dry ? "dry" : options.fail ? "fail" : "pass";
+const state = resolve(
+  ".sevro-recipes",
+  options.recipe,
+  options.route,
+  options.goalsOnly ? "goals" : mode,
+);
+await mkdir(join(state, "project"), { recursive: true });
+await mkdir(join(state, "config"), { recursive: true });
+const root = await realpath(state);
+const cli =
+  process.env.SEVRO_RECIPE_CLI ?? resolve(import.meta.dir, "../../src/cli.ts");
+const args = [
+  process.execPath,
+  cli,
+  "run",
+  "--json",
+  "--project-root",
+  join(root, "project"),
+  "--config-root",
+  join(root, "config"),
+  "--results-root",
+  join(root, "results"),
+  "--runtime-config-file",
+  runtimeFile(options),
+  "--condition",
+  "passive",
+  "--trials",
+  "1",
+  "--threshold",
+  "1",
+  "--shell-isolation",
+  "--protected-root",
+  import.meta.dir,
+  ...(await recipeInputs(options, root)),
+  ...(await hostArguments(options, root)),
+  ...(await semanticArguments(options, root)),
+  ...(options.dry ? ["--dry"] : []),
+];
+const child = Bun.spawn(args, {
+  env: { ...process.env, SEVRO_RECIPE_FAIL: options.fail ? "1" : "0" },
+  stdout: "pipe",
+  stderr: "inherit",
+});
+const [output, code] = await Promise.all([
+  new Response(child.stdout).text(),
+  child.exited,
+]);
+await writeFile(join(root, "result.json"), output);
+process.stderr.write(`Recipe result: ${join(root, "result.json")}\n`);
+process.stdout.write(output);
+process.exit(code);

package/examples/recipes/runtime/empty.json

@@ -0,0 +1 @@
+{ "format": "sevro.runtime.v1" }

package/examples/recipes/runtime/goals.json

@@ -0,0 +1 @@
+{ "format": "sevro.runtime.v1", "hooks": { "nativeGoal": true } }

package/examples/recipes/runtime/mise.json

@@ -0,0 +1,6 @@
+{
+  "format": "sevro.runtime.v1",
+  "environment": {
+    "inherit": ["PATH"]
+  }
+}

package/examples/recipes/runtime/path.json

@@ -0,0 +1 @@
+{ "format": "sevro.runtime.v1", "environment": { "inherit": ["PATH"] } }