@bjoernrochel/sevro 0.1.0-rc.2 → 0.1.0-rc.21

package/src/cli/evaluation.ts

@@ -0,0 +1,55 @@
+import { runEvaluation } from "../engine";
+import {
+  packageBuildDigest,
+  projectIdentityDigest,
+  projectProvenance,
+} from "../provenance";
+import { selectedCase } from "./extension";
+import type { PreparedInvocation } from "./preparation";
+import { assessmentSettings, effectiveSettings } from "./settings";
+import { displayProgress } from "./progress";
+
+export async function evaluateInvocation(
+  prepared: PreparedInvocation,
+  signal: AbortSignal,
+) {
+  const { invocation, preparationSources, host, semanticHost, advisoryHost } =
+    prepared;
+  const { caseData, extension } = await selectedCase(prepared, signal);
+  return runEvaluation({
+    onProgress: displayProgress,
+    resolvedRunSettings: effectiveSettings(prepared),
+    assessmentSettings: assessmentSettings(prepared),
+    runtimePolicy: prepared.runtimePolicy,
+    projectRoot: invocation.projectRoot,
+    resultsRoot: invocation.resultsRoot,
+    runStateRoot: invocation.runStateRoot,
+    case: caseData,
+    extension,
+    preparationSources,
+    host,
+    semanticHost,
+    advisoryHost,
+    advisoryExcludedPaths: invocation.advisoryExcludedPaths,
+    shellIsolation: invocation.shellIsolation,
+    runnerBuildDigest:
+      invocation.runnerBuildDigest ?? (await packageBuildDigest()),
+    runnerCheckoutRoot: invocation.runnerCheckoutRoot,
+    projectDigest:
+      invocation.projectDigest ??
+      (await projectIdentityDigest(
+        invocation.projectRoot,
+        await projectProvenance(invocation.projectRoot, [
+          invocation.resultsRoot,
+          invocation.runStateRoot,
+        ]),
+        [invocation.resultsRoot, invocation.runStateRoot],
+      )),
+    condition: invocation.condition,
+    trialCount: invocation.trialCount,
+    jobs: invocation.jobs,
+    passThreshold: invocation.passThreshold,
+    dry: invocation.dry,
+    signal,
+  });
+}

package/src/cli/extension.ts

@@ -0,0 +1,79 @@
+import { pathToFileURL } from "node:url";
+import type { ResolvedCase } from "../engine";
+import { openExtensionSession, type ExtensionCase } from "../extension-session";
+import { resolvedFixture } from "../resolved-case";
+import { InvocationError, type Invocation } from "./invocation";
+import { parseCase, loadExtensionOptions } from "./loading";
+import type { PreparedInvocation } from "./preparation";
+
+function selectExtensionCase(
+  cases: ExtensionCase[],
+  caseId: string,
+): { caseData: ResolvedCase; resolvedCase: ExtensionCase } {
+  const resolvedCase = cases.find((item) => item.id === caseId);
+  if (!resolvedCase)
+    throw new InvocationError("extension did not resolve the selected case");
+  const caseData = parseCase({
+    ...resolvedCase,
+    fixture: resolvedFixture(resolvedCase.fixture),
+  });
+  return { caseData, resolvedCase };
+}
+
+async function resolveSelectedExtension(
+  prepared: PreparedInvocation,
+  selected: NonNullable<Invocation["extension"]>,
+  signal: AbortSignal,
+) {
+  const { invocation, host } = prepared;
+  const options = await loadExtensionOptions(selected);
+  const hostCapabilities = [
+    ...(host.instrumentation ?? []).map((item) => item.id),
+    ...(host.hostCapabilities ?? []),
+  ];
+  const session = await openExtensionSession({
+    ...options,
+    sourceFiles: selected.sourceFiles,
+    engineCapabilities: [
+      "sevro.host.exec",
+      "sevro.fixture.setup",
+      "sevro.case.host-route",
+    ],
+    hostCapabilities,
+    taskVerdictPolicy: selected.taskVerdictPolicy,
+    replaceBuiltinGraders: selected.replaceBuiltinGraders,
+    signal,
+  });
+  const chosen = selectExtensionCase(
+    await session.resolve(
+      pathToFileURL(invocation.projectRoot).href,
+      { caseIds: [selected.caseId] },
+      {
+        id: host.id,
+        model: host.model,
+        effort: host.effort,
+        capabilities: hostCapabilities,
+      },
+    ),
+    selected.caseId,
+  );
+  return {
+    caseData: chosen.caseData,
+    extension: { session, resolvedCase: chosen.resolvedCase },
+  };
+}
+
+export async function selectedCase(
+  prepared: PreparedInvocation,
+  signal: AbortSignal,
+) {
+  if (prepared.invocation.extension)
+    return resolveSelectedExtension(
+      prepared,
+      prepared.invocation.extension,
+      signal,
+    );
+  if (!prepared.caseData)
+    throw new InvocationError("invalid resolved case file");
+  return { caseData: prepared.caseData, extension: undefined };
+}

package/src/cli/help.ts

@@ -0,0 +1,222 @@
+import { version } from "../../package.json";
+
+const TOP_LEVEL_HELP = `Usage: sevro <command> [options]
+
+Examples:
+  sevro demo
+  sevro init --template response --directory ./my-evaluation
+  sevro run --case-file /absolute/case.json --adapter-module /absolute/adapter.ts \\
+    --project-root /absolute/project --results-root /absolute/results \\
+    --condition passive --trials 1 --threshold 1
+  sevro run --help
+  sevro report --result-file /absolute/results/run/result.json
+  sevro prune
+
+Commands:
+  demo      Run the packaged credential-free assessed example; use demo --help.
+  init      Create a new evaluation project; use init --help.
+  run       Evaluate one resolved case or extension case; use run --help.
+  report    Summarize retained results; use report --help.
+  prune     Remove retained results using a retention policy; use prune --help.
+
+Options:
+  -h, --help     Show help without preparing a run.
+  -v, --version  Print the installed Sevro package version.
+`;
+
+const RUN_HELP = `Usage: sevro run [options]
+
+Examples:
+  sevro run --case-file /absolute/case.json --adapter-module /absolute/adapter.ts \\
+    --project-root /absolute/project --results-root /absolute/results \\
+    --condition passive --trials 1 --threshold 1
+  Add --dry to prepare without calling the candidate or graders.
+  Add --json to save a sevro.cli-result.v1 result from stdout.
+
+Required:
+  --case-file <path> OR --extension-command-file <path>  Select one case source.
+  --adapter-module <path> OR --host <codex|claude>      Select one candidate route.
+
+Common options and defaults:
+  --run-manifest-file <path>  Saved sevro.run.v1 settings; explicit flags override them.
+  Precedence: built-in defaults, saved settings, then explicit CLI options.
+  Manifest paths use its own directory; CLI paths use the current directory.
+  --no-dry, --no-shell-isolation and other --no-<boolean> flags override saved true.
+  --project-root <path> (default: current directory)  Project identity; does not copy its files into the fixture.
+  --results-root <path> (default: <project-root>/.sevro/results)  Retained evidence.
+  --condition <passive|enforced> (default: passive)  Bundled hosts use passive.
+  --trials <integer> (default: 1)  Positive number of trials.
+  --threshold <number> (default: 1)  All assessed trials must pass; range (0, 1].
+  Relative run paths resolve from the current directory and are canonicalized before protection checks.
+  --jobs <integer> (default: 3)  Maximum parallel trials, positive integer.
+  --config-root <path> (default: --project-root)  Native concurrency configuration.
+  --run-state-root <path> (default: --results-root)  Active records/checkpoints.
+  --runtime-config-file <path>  Runtime policy; otherwise discover project sevro.json.
+  --dry                       Prepare only (default: false).
+  --json                      One versioned JSON result on stdout (default: false).
+                              Human diagnostics go to stderr in both modes.
+  --retain-host-diagnostics   Retain private native diagnostics (default: false).
+  --shell-isolation          Isolate shell checks (default: false).
+  --protected-root <path>    Extra protected source/private root; repeatable.
+
+Bundled hosts:
+  --model <name> --effort <level>  Required for a bundled candidate host.
+  --codex-bin <path> (default: codex on PATH)  Explicit binary override.
+  --codex-auth-file <path>     Required for every Codex route.
+  --codex-entrypoint <exec|app-server>  Candidate transport (default: exec).
+  --codex-timeout-ms <integer>  1..1800000 milliseconds (default: 600000).
+  --claude-bin <path> (default: claude on PATH)  Explicit binary override.
+  Native timeout defaults to 600000 ms; shell checks expect exit 0 within 30000 ms.
+  --claude-credential-file <path>  Optional credential override; otherwise use native auth.
+  --claude-project-settings   Allow fixture project settings (default: false).
+
+Extension cases:
+  --case-id <id>              Required with --extension-command-file.
+  --extension-source-file <path>  Required source closure; repeatable.
+  --extension-configuration-file <path>  Private JSON object; supply with
+  --extension-redacted-configuration-file <path>  Redacted retained configuration.
+  --task-verdict-policy <id>  Select an advertised namespaced extension policy.
+  --replace-builtin-grader <id>  Explicit replacement; repeatable.
+  --case-source-root <path> --case-source-map-file <path>  Supply both for mapped sources.
+
+Optional grading routes (none selected by default):
+  --semantic-adapter-module <path> OR --semantic-host codex
+  --semantic-model <name> --semantic-effort <level>  Required for native semantic route.
+  --advisory-adapter-module <path> OR --advisory-host codex
+  --advisory-model <name> --advisory-effort <level>  Required for native advisory route.
+  --advisory-exclude <relative-path>  Exclude an evaluator path; repeatable.
+
+Provenance overrides (normally derived automatically):
+  --runner-build-digest <sha256> --project-digest <sha256>  64 lowercase hex characters.
+  --runner-checkout-root <path>  Record the running checkout's revision/dirty digest.
+
+Deprecated runtime options:
+  --toolchain-bin-dir <path> --claude-uv-cache-dir <path>
+  Migrate to sevro.json; these cannot be combined with a runtime policy.
+
+  -h, --help                 Show this help without preparing a run.
+Reference: docs/development-cli.md, docs/run-settings-v1.md and docs/results-v1.md in the installed package.
+`;
+
+const DEMO_HELP = `Usage: sevro demo [--json]
+
+Examples:
+  sevro demo
+  sevro demo --json
+
+Run the packaged deterministic fixture and adapter without model credentials.
+Shows an assessed verdict and retains evidence in .sevro/results beneath the current directory.
+Caller project files, sevro.json and native configuration are not loaded.
+
+Options:
+  --json       Emit one sevro.cli-result.v1 document on stdout.
+  -h, --help   Show this help without preparing a run.
+`;
+
+const REPORT_HELP = `Usage: sevro report [options]
+
+Examples:
+  sevro report --latest
+  sevro report --result-file /absolute/results/run/result.json
+  sevro report --result-file /absolute/results/first/result.json \\
+    --result-file /absolute/results/second/result.json --json
+
+Required:
+  --latest OR --result-file <path>  A retained result; repeat paths for distinct runs.
+                       Relative or absolute paths use the current directory.
+                       Keep each result's referenced evidence available and unchanged.
+
+Options and defaults:
+  --latest     Read the last retained CLI result, including dry or failed runs.
+  --results-root <path>  With --latest, override <cwd>/.sevro/results.
+  --json       Emit sevro.report.v1 JSON; Markdown by default.
+  -h, --help   Show this help without reading result files.
+`;
+
+const PRUNE_HELP = `Usage: sevro prune [options]
+
+Examples:
+  sevro prune
+  sevro prune --results-root ./other-results --keep 5
+  sevro prune --older-than 30d
+  sevro prune --latest
+  sevro prune --native-transcripts --older-than 30d
+
+Defaults: keep the newest 10 completed results under <cwd>/.sevro/results.
+Select at most one explicit retention policy:
+  --keep <positive-integer>  Keep this many newest results.
+  --latest                   Keep only the newest result (same as --keep 1).
+  --older-than <duration>    Remove results older than a positive integer plus ms, s, m, h, d or w.
+  --results-root <path>      Relative or absolute results root, resolved from cwd.
+  --native-transcripts       Remove only native transcript bundles; keep results and hashes.
+  -h, --help                 Show help without reading or deleting results.
+
+Age and ordering use result.json modification times; equal times sort by path.
+Each retained attempt counts separately, including dry, failed and interrupted runs.
+Active/unrecognized attempts and directories without result.json are preserved.
+Evidence is permanently deleted. Remaining results stay usable by sevro report;
+latest.json is updated, or removed when no completed results remain.
+With --native-transcripts, no policy removes bundles from all completed results;
+--keep/--latest preserve bundles for the newest results, and --older-than selects older results.
+Result/evidence files and latest.json stay unchanged. Reports label removed bundles as pruned.
+Missing or empty results roots are a successful no-op. Invalid or unreadable inputs exit 64.
+Reference: docs/reading-results.md in the installed package.
+`;
+
+const INIT_HELP = `Usage: sevro init [options]
+
+Examples:
+  sevro init --template response --directory ./my-evaluation
+  cd my-evaluation
+  sevro run --run-manifest-file run.json --dry
+  sevro run --run-manifest-file run.json
+  sevro report --latest
+
+Options and defaults:
+  --template <response|repository-skill|coding>  Default: response.
+  --directory <new-path>  Default: ./sevro-evaluation; refuses existing paths.
+  -h, --help  Show this help without creating files.
+
+Response and coding use deterministic adapters and real assertions, including negative controls.
+Coding checks need sandbox-exec (macOS) or bubblewrap/socat (Linux).
+Repository-skill uses native Codex and requires explicit --model, --effort and --codex-auth-file.
+Read the generated README.md for prerequisites and live commands; dry preparation is not an assessment.
+Reference: docs/evaluation-workflow.md in the installed package.
+`;
+
+const HELP_REQUESTS: Record<string, string> = {
+  "": TOP_LEVEL_HELP,
+  "--help": TOP_LEVEL_HELP,
+  "-h": TOP_LEVEL_HELP,
+  help: TOP_LEVEL_HELP,
+  "run --help": RUN_HELP,
+  "run -h": RUN_HELP,
+  "help run": RUN_HELP,
+  "demo --help": DEMO_HELP,
+  "demo -h": DEMO_HELP,
+  "help demo": DEMO_HELP,
+  "init --help": INIT_HELP,
+  "init -h": INIT_HELP,
+  "help init": INIT_HELP,
+  "report --help": REPORT_HELP,
+  "report -h": REPORT_HELP,
+  "help report": REPORT_HELP,
+  "prune --help": PRUNE_HELP,
+  "prune -h": PRUNE_HELP,
+  "help prune": PRUNE_HELP,
+};
+
+/** Handle informational commands before loading inputs or creating run state. */
+export function informationCommand(argv: string[]): boolean {
+  const request = argv.join(" ");
+  if (["--version", "-v"].includes(request)) {
+    process.stdout.write(`sevro ${version}\n`);
+    return true;
+  }
+  const help = Object.hasOwn(HELP_REQUESTS, request)
+    ? HELP_REQUESTS[request]
+    : undefined;
+  if (help === undefined) return false;
+  process.stdout.write(help);
+  return true;
+}

package/src/cli/hosts.ts

@@ -0,0 +1,59 @@
+import type { HostAdapter } from "../engine";
+import { createClaudeHost } from "../hosts/claude";
+import { createCodexHost } from "../hosts/codex";
+import { codexAgentConcurrency } from "../hosts/codex-configuration";
+import { InvocationError, type Invocation } from "./invocation";
+import { loadHost } from "./loading";
+
+export async function selectedConcurrency(
+  invocation: Invocation,
+  configRoot: string,
+): Promise<number | null> {
+  return [
+    invocation.codex,
+    invocation.semanticCodex,
+    invocation.advisoryCodex,
+  ].some(Boolean)
+    ? codexAgentConcurrency(configRoot)
+    : null;
+}
+
+export async function candidateHost(
+  invocation: Invocation,
+  agentConcurrencyLimit: number | null,
+): Promise<HostAdapter> {
+  if (invocation.codex)
+    return retainExecutable(
+      createCodexHost({ ...invocation.codex, agentConcurrencyLimit }),
+      invocation.codex.binary,
+    );
+  if (invocation.claude)
+    return retainExecutable(
+      createClaudeHost(invocation.claude),
+      invocation.claude.binary,
+    );
+  if (!invocation.adapterModule)
+    throw new InvocationError("missing --adapter-module or --host");
+  return loadHost(invocation.adapterModule);
+}
+
+export async function secondaryHost(
+  module: string | undefined,
+  codex: Invocation["semanticCodex"],
+  agentConcurrencyLimit: number | null,
+): Promise<HostAdapter | undefined> {
+  if (module) return loadHost(module);
+  if (codex)
+    return retainExecutable(
+      createCodexHost({ ...codex, agentConcurrencyLimit }),
+      codex.binary,
+    );
+  return undefined;
+}
+
+function retainExecutable(host: HostAdapter, binary: string): HostAdapter {
+  return {
+    ...host,
+    configuration: { ...host.configuration, "sevro.host.executable": binary },
+  };
+}

package/src/cli/init-settings.ts

@@ -0,0 +1,30 @@
+import { join } from "node:path";
+
+export function templateSettings(template: string) {
+  if (template === "repository-skill")
+    return {
+      "project-root": ".",
+      "extension-command-file": "command.json",
+      "extension-source-file": [
+        "extension.ts",
+        "extension-data.ts",
+        ".agents/skills/startup-guide/SKILL.md",
+      ],
+      "case-id": "repository-skill",
+      host: "codex",
+    };
+  return {
+    "case-file": "case.json",
+    "adapter-module": "adapter.ts",
+    "project-root": ".",
+    ...(template === "coding" ? { "shell-isolation": true } : {}),
+  };
+}
+
+export function templateCommand(destination: string) {
+  return [process.execPath, join(destination, "extension.ts")];
+}
+
+export function nativeReadme() {
+  return `# Repository skill evaluation\n\nThis template mounts the actual startup-guide skill and requires Codex's explicit invocation evidence plus assertions for the start command and port. Missing evidence remains not_assessed.\n\nPrerequisites: Bun, Git, Codex on PATH, supported native isolation (sandbox-exec on macOS or bubblewrap/socat on Linux), and a readable Codex authentication file. Choose the exact model and effort explicitly. Do not store credentials in run.json.\n\n\`\`\`sh\nsevro run --run-manifest-file run.json --model YOUR_MODEL --effort YOUR_EFFORT --codex-auth-file /absolute/auth.json --dry\nsevro run --run-manifest-file run.json --model YOUR_MODEL --effort YOUR_EFFORT --codex-auth-file /absolute/auth.json\nsevro report --latest\n\`\`\`\n\nDry preparation checks mounts and declarations without calling the model. No authenticated model execution or skill dispatch is established by dry preparation. Live execution may incur provider charges. To create a negative control, change the expected port in extension-data.ts to 9000 while leaving the fixture at 3000; rerun and inspect the failed port assertion. Model compliance and measurements are unknown until actually executed.\n\nEdit the copied skill and assertions, prepare again, then rerun the same model/effort/condition. Keep private .sevro evidence out of Git. command.json uses the initializer's absolute Bun and extension paths; regenerate it after moving this project.\n`;
+}

package/src/cli/init.ts

@@ -0,0 +1,111 @@
+import { cp, mkdir, rm, writeFile } from "node:fs/promises";
+import { dirname, join, resolve } from "node:path";
+import { parseArgs } from "node:util";
+import {
+  nativeReadme,
+  templateCommand,
+  templateSettings,
+} from "./init-settings";
+
+function initOptions(argv: string[]) {
+  const { values, positionals } = parseArgs({
+    args: argv,
+    strict: true,
+    allowPositionals: true,
+    options: {
+      template: { type: "string", default: "response" },
+      directory: { type: "string", default: "sevro-evaluation" },
+    },
+  });
+  if (
+    positionals.length ||
+    !["response", "coding", "repository-skill"].includes(values.template)
+  )
+    throw new Error(
+      "use --template response, coding or repository-skill; run sevro init --help",
+    );
+  if (!values.directory.trim())
+    throw new Error("--directory must name a new directory");
+  return { template: values.template, destination: resolve(values.directory) };
+}
+
+async function reserveDirectory(destination: string): Promise<void> {
+  await mkdir(dirname(destination), { recursive: true });
+  try {
+    await mkdir(destination);
+  } catch (cause) {
+    throw new Error(
+      `init destination already exists or cannot be created: ${destination}; choose a new --directory`,
+      { cause },
+    );
+  }
+}
+
+function starterReadme(template: string) {
+  if (template === "repository-skill") return nativeReadme();
+  const assertion =
+    template === "coding"
+      ? "positive, negative and zero addition assertions against edited add.sh"
+      : "an exact response assertion";
+  const prerequisites =
+    template === "coding"
+      ? "Bun and sandbox-exec on macOS, or bubblewrap/socat on Linux"
+      : "Bun";
+  return `# ${template} evaluation\n\nThis deterministic adapter exercises ${assertion}, without model credentials. Prerequisites: ${prerequisites}.\n\n\`\`\`sh\nsevro run --run-manifest-file run.json --dry\nsevro run --run-manifest-file run.json\nsevro report --latest\nSEVRO_TEMPLATE_FAIL=1 sevro run --run-manifest-file run.json\n\`\`\`\n\nThe negative control must report task=failed and exit 1. Dry preparation is not an assessment.\nEdit case.json and adapter.ts, prepare, then run again. Keep private .sevro evidence out of Git.\nFor a native model, replace the adapter route explicitly with --host, --model and --effort; Codex also requires --codex-auth-file. No model execution or native skill dispatch is established by this adapter.\n`;
+}
+
+async function initialize(template: string, destination: string) {
+  await reserveDirectory(destination);
+  try {
+    await cp(
+      join(import.meta.dir, "..", "..", "examples", "templates", template),
+      destination,
+      { recursive: true, errorOnExist: true, force: false },
+    );
+    await writeFile(
+      join(destination, "run.json"),
+      JSON.stringify(
+        { format: "sevro.run.v1", settings: templateSettings(template) },
+        null,
+        2,
+      ) + "\n",
+      { flag: "wx" },
+    );
+    if (template === "repository-skill")
+      await writeFile(
+        join(destination, "command.json"),
+        JSON.stringify(templateCommand(destination)) + "\n",
+        { flag: "wx" },
+      );
+    await writeFile(join(destination, ".gitignore"), ".sevro/\n", {
+      flag: "wx",
+    });
+    await writeFile(join(destination, "README.md"), starterReadme(template), {
+      flag: "wx",
+    });
+  } catch (cause) {
+    await rm(destination, { recursive: true, force: true });
+    throw cause;
+  }
+}
+
+export async function initCommand(argv: string[]): Promise<number> {
+  try {
+    const { template, destination } = initOptions(argv);
+    await initialize(template, destination);
+    const quoted = `'${destination.replaceAll("'", "'\\''")}'`;
+    const native =
+      template === "repository-skill"
+        ? " --model YOUR_MODEL --effort YOUR_EFFORT --codex-auth-file /absolute/auth.json"
+        : "";
+    process.stdout.write(
+      `Created ${template} evaluation at ${destination}\nOpen README.md, then:\n  cd ${quoted}\n  sevro run --run-manifest-file run.json${native} --dry\n  sevro run --run-manifest-file run.json${native}\n`,
+    );
+    return 0;
+  } catch (error) {
+    process.stderr.write(
+      `sevro: ${error instanceof Error ? error.message : "initialization failed"}\n`,
+    );
+    return 64;
+  }
+}