@bjoernrochel/sevro 0.1.0-rc.8 → 1.1.0

package/src/cli/cancellation.ts

@@ -0,0 +1,20 @@
+export class CommandCancellation {
+  private readonly controller = new AbortController();
+  private readonly interrupt = () => {
+    this.controller.abort("SIGINT");
+  };
+  private readonly terminate = () => {
+    this.controller.abort("SIGTERM");
+  };
+  get signal(): AbortSignal {
+    return this.controller.signal;
+  }
+  constructor() {
+    process.on("SIGINT", this.interrupt);
+    process.on("SIGTERM", this.terminate);
+  }
+  dispose(): void {
+    process.off("SIGINT", this.interrupt);
+    process.off("SIGTERM", this.terminate);
+  }
+}

package/src/cli/command.ts

@@ -0,0 +1,87 @@
+import { prepareRunInvocation, type PreparedInvocation } from "./preparation";
+import { evaluateInvocation } from "./evaluation";
+import { CommandCancellation } from "./cancellation";
+import { display, displayFailure, runnerFailureCategory } from "./presentation";
+import { displayEffectiveSettings } from "./settings";
+import { jsonOutputRequested } from "./run-options";
+import { retainCliResult } from "./completion";
+
+type CommandRelease = (() => Promise<void>) | undefined;
+
+export async function main(
+  argv: string[],
+  release?: () => Promise<void>,
+): Promise<void> {
+  const json = jsonOutputRequested(argv);
+  let prepared: PreparedInvocation;
+  try {
+    prepared = await prepareRunInvocation(argv);
+  } catch (error) {
+    await releaseBeforeDisplay(release, 64);
+    displayFailure(
+      error,
+      json,
+      64,
+      "invalid invocation",
+      argv[0] === "run" ? "run" : "",
+    );
+    return;
+  }
+  const cancellation = new CommandCancellation();
+  try {
+    displayEffectiveSettings(prepared);
+    const { result } = await evaluateInvocation(prepared, cancellation.signal);
+    const completion = await retainCompletion(
+      result,
+      prepared.invocation.resultsRoot,
+    );
+    await releaseBeforeDisplay(release, result.exitCode);
+    display(result, prepared.invocation.json, completion);
+    process.exitCode = result.exitCode;
+  } catch (error) {
+    await releaseBeforeDisplay(release, 70);
+    displayFailure(
+      error,
+      prepared.invocation.json,
+      runnerFailureCategory(error),
+      "runner failure",
+    );
+  } finally {
+    cancellation.dispose();
+  }
+}
+
+async function retainCompletion(
+  result: Parameters<typeof retainCliResult>[0],
+  resultsRoot: string,
+) {
+  try {
+    return await retainCliResult(result, resultsRoot);
+  } catch (cause) {
+    if (result.exitCode === 0)
+      throw new Error(
+        "CLI result/latest could not be retained; check --results-root permissions",
+        { cause },
+      );
+    process.stderr.write(
+      "sevro: CLI result/latest could not be retained; original run outcome preserved; inspect evidence directly\n",
+    );
+    return undefined;
+  }
+}
+
+async function releaseBeforeDisplay(
+  release: CommandRelease,
+  exitCode: number,
+): Promise<void> {
+  try {
+    await release?.();
+  } catch (error) {
+    if (exitCode === 0) throw error;
+    const detail =
+      error instanceof Error
+        ? error.message
+        : "temporary project could not be removed";
+    process.stderr.write(`sevro: demo cleanup failed: ${detail}\n`);
+  }
+}

package/src/cli/completion.ts

@@ -0,0 +1,15 @@
+import { dirname, join } from "node:path";
+import { atomicWriteJson } from "../storage";
+import type { CliResult } from "../engine";
+import { createReport } from "../report";
+import { withResultsLock } from "../results-lock";
+
+/** Latest means the last successfully retained CLI result, including dry/failing runs. */
+export async function retainCliResult(result: CliResult, resultsRoot: string) {
+  return withResultsLock(resultsRoot, async () => {
+    const resultFile = join(dirname(result.evidencePath), "result.json");
+    await atomicWriteJson(resultFile, result);
+    await atomicWriteJson(join(resultsRoot, "latest.json"), result);
+    return { resultFile, report: await createReport([resultFile]) };
+  });
+}

package/src/cli/demo.ts

@@ -0,0 +1,61 @@
+import { examplePath } from "../runner-paths";
+import { mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { main } from "./command";
+import { InvocationError } from "./invocation";
+import { displayFailure } from "./presentation";
+
+/** Run the packaged deterministic case without inheriting a caller's project policy. */
+export async function demoCommand(argv: string[]): Promise<void> {
+  const json = argv.includes("--json");
+  if (argv.some((arg) => arg !== "--json")) {
+    displayFailure(
+      new InvocationError("demo accepts only --json; run sevro demo --help"),
+      json,
+      64,
+      "invalid demo invocation",
+      "demo",
+    );
+    return;
+  }
+  let projectRoot: string;
+  try {
+    projectRoot = await mkdtemp(join(tmpdir(), "sevro-demo-"));
+  } catch (cause) {
+    displayFailure(
+      new Error(
+        `demo temporary project could not be created under ${tmpdir()}; set TMPDIR to a writable directory`,
+        { cause },
+      ),
+      json,
+      70,
+      "demo preparation failed",
+    );
+    return;
+  }
+  await main(
+    [
+      "run",
+      ...(json ? ["--json"] : []),
+      "--case-file",
+      examplePath("basic", "graded.json"),
+      "--adapter-module",
+      examplePath("basic", "host.ts"),
+      "--project-root",
+      projectRoot,
+      "--results-root",
+      join(process.cwd(), ".sevro", "results"),
+    ],
+    demoCleanup(projectRoot),
+  );
+}
+
+function demoCleanup(projectRoot: string): () => Promise<void> {
+  let attempted = false;
+  return async () => {
+    if (attempted) return;
+    attempted = true;
+    await rm(projectRoot, { recursive: true, force: true });
+  };
+}

package/src/cli/evaluation.ts

@@ -0,0 +1,55 @@
+import { runEvaluation } from "../engine";
+import {
+  packageBuildDigest,
+  projectIdentityDigest,
+  projectProvenance,
+} from "../provenance";
+import { selectedCase } from "./extension";
+import type { PreparedInvocation } from "./preparation";
+import { assessmentSettings, effectiveSettings } from "./settings";
+import { displayProgress } from "./progress";
+
+export async function evaluateInvocation(
+  prepared: PreparedInvocation,
+  signal: AbortSignal,
+) {
+  const { invocation, preparationSources, host, semanticHost, advisoryHost } =
+    prepared;
+  const { caseData, extension } = await selectedCase(prepared, signal);
+  return runEvaluation({
+    onProgress: displayProgress,
+    resolvedRunSettings: effectiveSettings(prepared),
+    assessmentSettings: assessmentSettings(prepared),
+    runtimePolicy: prepared.runtimePolicy,
+    projectRoot: invocation.projectRoot,
+    resultsRoot: invocation.resultsRoot,
+    runStateRoot: invocation.runStateRoot,
+    case: caseData,
+    extension,
+    preparationSources,
+    host,
+    semanticHost,
+    advisoryHost,
+    advisoryExcludedPaths: invocation.advisoryExcludedPaths,
+    shellIsolation: invocation.shellIsolation,
+    runnerBuildDigest:
+      invocation.runnerBuildDigest ?? (await packageBuildDigest()),
+    runnerCheckoutRoot: invocation.runnerCheckoutRoot,
+    projectDigest:
+      invocation.projectDigest ??
+      (await projectIdentityDigest(
+        invocation.projectRoot,
+        await projectProvenance(invocation.projectRoot, [
+          invocation.resultsRoot,
+          invocation.runStateRoot,
+        ]),
+        [invocation.resultsRoot, invocation.runStateRoot],
+      )),
+    condition: invocation.condition,
+    trialCount: invocation.trialCount,
+    jobs: invocation.jobs,
+    passThreshold: invocation.passThreshold,
+    dry: invocation.dry,
+    signal,
+  });
+}

package/src/cli/extension.ts

@@ -0,0 +1,79 @@
+import { pathToFileURL } from "node:url";
+import type { ResolvedCase } from "../engine";
+import { openExtensionSession, type ExtensionCase } from "../extension-session";
+import { resolvedFixture } from "../resolved-case";
+import { InvocationError, type Invocation } from "./invocation";
+import { parseCase, loadExtensionOptions } from "./loading";
+import type { PreparedInvocation } from "./preparation";
+
+function selectExtensionCase(
+  cases: ExtensionCase[],
+  caseId: string,
+): { caseData: ResolvedCase; resolvedCase: ExtensionCase } {
+  const resolvedCase = cases.find((item) => item.id === caseId);
+  if (!resolvedCase)
+    throw new InvocationError("extension did not resolve the selected case");
+  const caseData = parseCase({
+    ...resolvedCase,
+    fixture: resolvedFixture(resolvedCase.fixture),
+  });
+  return { caseData, resolvedCase };
+}
+
+async function resolveSelectedExtension(
+  prepared: PreparedInvocation,
+  selected: NonNullable<Invocation["extension"]>,
+  signal: AbortSignal,
+) {
+  const { invocation, host } = prepared;
+  const options = await loadExtensionOptions(selected);
+  const hostCapabilities = [
+    ...(host.instrumentation ?? []).map((item) => item.id),
+    ...(host.hostCapabilities ?? []),
+  ];
+  const session = await openExtensionSession({
+    ...options,
+    sourceFiles: selected.sourceFiles,
+    engineCapabilities: [
+      "sevro.host.exec",
+      "sevro.fixture.setup",
+      "sevro.case.host-route",
+    ],
+    hostCapabilities,
+    taskVerdictPolicy: selected.taskVerdictPolicy,
+    replaceBuiltinGraders: selected.replaceBuiltinGraders,
+    signal,
+  });
+  const chosen = selectExtensionCase(
+    await session.resolve(
+      pathToFileURL(invocation.projectRoot).href,
+      { caseIds: [selected.caseId] },
+      {
+        id: host.id,
+        model: host.model,
+        effort: host.effort,
+        capabilities: hostCapabilities,
+      },
+    ),
+    selected.caseId,
+  );
+  return {
+    caseData: chosen.caseData,
+    extension: { session, resolvedCase: chosen.resolvedCase },
+  };
+}
+
+export async function selectedCase(
+  prepared: PreparedInvocation,
+  signal: AbortSignal,
+) {
+  if (prepared.invocation.extension)
+    return resolveSelectedExtension(
+      prepared,
+      prepared.invocation.extension,
+      signal,
+    );
+  if (!prepared.caseData)
+    throw new InvocationError("invalid resolved case file");
+  return { caseData: prepared.caseData, extension: undefined };
+}

package/src/cli/help.ts

@@ -0,0 +1,199 @@
+import { RUN_OPTIONS, optionsWith, type RunOptionName } from "./run-options";
+import packageJson from "../../package.json" with { type: "json" };
+const { version } = packageJson;
+
+const TOP_LEVEL_HELP = `Usage: sevro <command> [options]
+
+Examples:
+  sevro demo
+  sevro init --template response --directory ./my-evaluation
+  sevro run --case-file /absolute/case.json --adapter-module /absolute/adapter.js \\
+    --project-root /absolute/project --results-root /absolute/results \\
+    --condition passive --trials 1 --threshold 1
+  sevro run --help
+  sevro report --result-file /absolute/results/run/result.json
+  sevro prune
+
+Commands:
+  demo      Run the packaged credential-free assessed example; use demo --help.
+  init      Create a new evaluation project; use init --help.
+  run       Evaluate one resolved case or extension case; use run --help.
+  report    Summarize retained results; use report --help.
+  prune     Remove retained results using a retention policy; use prune --help.
+
+Options:
+  -h, --help     Show help without preparing a run.
+  -v, --version  Print the installed Sevro package version.
+`;
+
+const RUN_HELP = `Usage: sevro run [options]
+
+Examples:
+  sevro run --case-file /absolute/case.json --adapter-module /absolute/adapter.js \\
+    --project-root /absolute/project --results-root /absolute/results \\
+    --condition passive --trials 1 --threshold 1
+  Add --dry to prepare without calling the candidate or graders.
+  Add --json to save a sevro.cli-result.v1 result from stdout.
+
+Required:
+${runOptionHelp("required")}
+
+Common options and defaults:
+${runOptionHelp("common")}
+  Precedence: built-in defaults, saved settings, then explicit CLI options.
+  Manifest paths use its own directory; CLI paths use the current directory.
+  --no-dry, --no-shell-isolation and other --no-<boolean> flags override saved true.
+  Relative run paths resolve from the current directory and are canonicalized before protection checks.
+
+Bundled hosts:
+${runOptionHelp("native")}
+  Native timeout defaults to 600000 ms; shell checks expect exit 0 within 30000 ms.
+
+Extension cases:
+${runOptionHelp("extension")}
+
+Optional grading routes (none selected by default):
+${runOptionHelp("grading")}
+
+Provenance overrides (normally derived automatically):
+${runOptionHelp("provenance")}
+
+Deprecated runtime options:
+${runOptionHelp("deprecated")}
+
+  -h, --help                 Show this help without preparing a run.
+Reference: docs/development-cli.md, docs/run-settings-v1.md and docs/results-v1.md in the installed package.
+`;
+
+function runOptionHelp(
+  section: (typeof RUN_OPTIONS)[RunOptionName]["section"],
+): string {
+  return optionsWith("section", section)
+    .map((name) => `  --${name} ${RUN_OPTIONS[name].help}`)
+    .join("\n");
+}
+
+const DEMO_HELP = `Usage: sevro demo [--json]
+
+Examples:
+  sevro demo
+  sevro demo --json
+
+Run the packaged deterministic fixture and adapter without model credentials.
+Shows an assessed verdict and retains evidence in .sevro/results beneath the current directory.
+Caller project files, sevro.json and native configuration are not loaded.
+
+Options:
+  --json       Emit one sevro.cli-result.v1 document on stdout.
+  -h, --help   Show this help without preparing a run.
+`;
+
+const REPORT_HELP = `Usage: sevro report [options]
+
+Examples:
+  sevro report --latest
+  sevro report --result-file /absolute/results/run/result.json
+  sevro report --result-file /absolute/results/first/result.json \\
+    --result-file /absolute/results/second/result.json --json
+
+Required:
+  --latest OR --result-file <path>  A retained result; repeat paths for distinct runs.
+                       Relative or absolute paths use the current directory.
+                       Keep each result's referenced evidence available and unchanged.
+
+Options and defaults:
+  --latest     Read the last retained CLI result, including dry or failed runs.
+  --results-root <path>  With --latest, override <cwd>/.sevro/results.
+  --json       Emit sevro.report.v1 JSON; Markdown by default.
+  -h, --help   Show this help without reading result files.
+`;
+
+const PRUNE_HELP = `Usage: sevro prune [options]
+
+Examples:
+  sevro prune
+  sevro prune --results-root ./other-results --keep 5
+  sevro prune --older-than 30d
+  sevro prune --latest
+  sevro prune --native-transcripts --older-than 30d
+
+Defaults: keep the newest 10 completed results under <cwd>/.sevro/results.
+Select at most one explicit retention policy:
+  --keep <positive-integer>  Keep this many newest results.
+  --latest                   Keep only the newest result (same as --keep 1).
+  --older-than <duration>    Remove results older than a positive integer plus ms, s, m, h, d or w.
+  --results-root <path>      Relative or absolute results root, resolved from cwd.
+  --native-transcripts       Remove only native transcript bundles; keep results and hashes.
+  -h, --help                 Show help without reading or deleting results.
+
+Age and ordering use result.json modification times; equal times sort by path.
+Each retained attempt counts separately, including dry, failed and interrupted runs.
+Active/unrecognized attempts and directories without result.json are preserved.
+Evidence is permanently deleted. Remaining results stay usable by sevro report;
+latest.json is updated, or removed when no completed results remain.
+With --native-transcripts, no policy removes bundles from all completed results;
+--keep/--latest preserve bundles for the newest results, and --older-than selects older results.
+Result/evidence files and latest.json stay unchanged. Reports label removed bundles as pruned.
+Missing or empty results roots are a successful no-op. Invalid or unreadable inputs exit 64.
+Reference: docs/reading-results.md in the installed package.
+`;
+
+const INIT_HELP = `Usage: sevro init [options]
+
+Examples:
+  sevro init --template response --directory ./my-evaluation
+  cd my-evaluation
+  sevro run --run-manifest-file run.json --dry
+  sevro run --run-manifest-file run.json
+  sevro report --latest
+
+Options and defaults:
+  --template <response|repository-skill|coding>  Default: response.
+  --directory <new-path>  Default: ./sevro-evaluation; refuses existing paths.
+  -h, --help  Show this help without creating files.
+
+Response and coding use deterministic adapters and real assertions, including negative controls.
+Coding checks need sandbox-exec (macOS) or bubblewrap/socat (Linux).
+Repository-skill uses native Codex and requires explicit --model, --effort and --codex-auth-file.
+Read the generated README.md for prerequisites and live commands; dry preparation is not an assessment.
+Reference: docs/evaluation-workflow.md in the installed package.
+`;
+
+const HELP_REQUESTS: Record<string, string> = {
+  "": TOP_LEVEL_HELP,
+  "--help": TOP_LEVEL_HELP,
+  "-h": TOP_LEVEL_HELP,
+  help: TOP_LEVEL_HELP,
+  "run --help": RUN_HELP,
+  "run -h": RUN_HELP,
+  "help run": RUN_HELP,
+  "demo --help": DEMO_HELP,
+  "demo -h": DEMO_HELP,
+  "help demo": DEMO_HELP,
+  "init --help": INIT_HELP,
+  "init -h": INIT_HELP,
+  "help init": INIT_HELP,
+  "report --help": REPORT_HELP,
+  "report -h": REPORT_HELP,
+  "help report": REPORT_HELP,
+  "prune --help": PRUNE_HELP,
+  "prune -h": PRUNE_HELP,
+  "help prune": PRUNE_HELP,
+};
+
+/** Handle informational commands before loading inputs or creating run state. */
+export function informationCommand(argv: string[]): boolean {
+  const request = argv.slice(1).some((arg) => ["--help", "-h"].includes(arg))
+    ? [argv[0], "--help"].join(" ")
+    : argv.join(" ");
+  if (["--version", "-v"].includes(request)) {
+    process.stdout.write(`sevro ${version}\n`);
+    return true;
+  }
+  const help = Object.hasOwn(HELP_REQUESTS, request)
+    ? HELP_REQUESTS[request]
+    : undefined;
+  if (help === undefined) return false;
+  process.stdout.write(help);
+  return true;
+}

package/src/cli/hosts.ts

@@ -0,0 +1,55 @@
+import type { HostAdapter } from "../engine";
+import { createClaudeHost } from "../hosts/claude";
+import { createCodexHost } from "../hosts/codex";
+import { codexAgentConcurrency } from "../hosts/codex-configuration";
+import { InvocationError, nativeRoutes, type Invocation } from "./invocation";
+import { loadHost } from "./loading";
+
+export async function selectedConcurrency(
+  invocation: Invocation,
+  configRoot: string,
+): Promise<number | null> {
+  return nativeRoutes({ ...invocation, claude: undefined }).length
+    ? codexAgentConcurrency(configRoot)
+    : null;
+}
+
+export async function candidateHost(
+  invocation: Invocation,
+  agentConcurrencyLimit: number | null,
+): Promise<HostAdapter> {
+  if (invocation.codex)
+    return retainExecutable(
+      createCodexHost({ ...invocation.codex, agentConcurrencyLimit }),
+      invocation.codex.binary,
+    );
+  if (invocation.claude)
+    return retainExecutable(
+      createClaudeHost(invocation.claude),
+      invocation.claude.binary,
+    );
+  if (!invocation.adapterModule)
+    throw new InvocationError("missing --adapter-module or --host");
+  return loadHost(invocation.adapterModule);
+}
+
+export async function secondaryHost(
+  module: string | undefined,
+  codex: Invocation["semanticCodex"],
+  agentConcurrencyLimit: number | null,
+): Promise<HostAdapter | undefined> {
+  if (module) return loadHost(module);
+  if (codex)
+    return retainExecutable(
+      createCodexHost({ ...codex, agentConcurrencyLimit }),
+      codex.binary,
+    );
+  return undefined;
+}
+
+function retainExecutable(host: HostAdapter, binary: string): HostAdapter {
+  return {
+    ...host,
+    configuration: { ...host.configuration, "sevro.host.executable": binary },
+  };
+}

package/src/cli/init-settings.ts

@@ -0,0 +1,31 @@
+import { join } from "node:path";
+import { runtimeModule } from "../runner-paths";
+
+export function templateSettings(template: string) {
+  if (template === "repository-skill")
+    return {
+      "project-root": ".",
+      "extension-command-file": "command.json",
+      "extension-source-file": [
+        "extension.ts",
+        "extension-data.ts",
+        ".agents/skills/startup-guide/SKILL.md",
+      ].map(runtimeModule),
+      "case-id": "repository-skill",
+      host: "codex",
+    };
+  return {
+    "case-file": "case.json",
+    "adapter-module": runtimeModule("adapter.ts"),
+    "project-root": ".",
+    ...(template === "coding" ? { "shell-isolation": true } : {}),
+  };
+}
+
+export function templateCommand(destination: string) {
+  return [process.execPath, join(destination, runtimeModule("extension.ts"))];
+}
+
+export function nativeReadme() {
+  return `# Repository skill evaluation\n\nThis template mounts the actual startup-guide skill and requires Codex's explicit invocation evidence plus assertions for the start command and port. Missing evidence remains not_assessed.\n\nPrerequisites: Node.js 24.13.0 or newer, or Bun 1.3.13, plus Git, Codex on PATH, supported native isolation (sandbox-exec on macOS or bubblewrap/socat on Linux), and a readable Codex authentication file. Choose the exact model and effort explicitly. Do not store credentials in run.json.\n\n\`\`\`sh\nsevro run --run-manifest-file run.json --model YOUR_MODEL --effort YOUR_EFFORT --codex-auth-file /absolute/auth.json --dry\nsevro run --run-manifest-file run.json --model YOUR_MODEL --effort YOUR_EFFORT --codex-auth-file /absolute/auth.json\nsevro report --latest\n\`\`\`\n\nDry preparation checks mounts and declarations without calling the model. No authenticated model execution or skill dispatch is established by dry preparation. Live execution may incur provider charges. To create a negative control, change the expected port in the extension data file to 9000 while leaving the fixture at 3000; rerun and inspect the failed port assertion. Model compliance and measurements are unknown until actually executed.\n\nEdit the copied skill and assertions, prepare again, then rerun the same model/effort/condition. Keep private .sevro evidence out of Git. command.json uses the initializer's absolute selected runtime and extension paths; regenerate it after moving this project.\n`;
+}