@bjoernrochel/sevro 0.1.0-rc.8 → 1.1.0

package/src/cli/run-options.ts

@@ -0,0 +1,403 @@
+import { parseArgs } from "node:util";
+
+type RunOption = {
+  type: "string" | "boolean";
+  kind: "string" | "boolean" | "number" | "path" | "path-list";
+  multiple?: boolean;
+  cliOnly?: boolean;
+  private?: boolean;
+  extension?: boolean;
+  route?: "codex" | "claude";
+  section:
+    | "required"
+    | "common"
+    | "native"
+    | "extension"
+    | "grading"
+    | "provenance"
+    | "deprecated";
+  help: string;
+};
+
+/** Parsing, saved-setting semantics, protection and help share one inventory. */
+export const RUN_OPTIONS = {
+  json: {
+    type: "boolean",
+    kind: "boolean",
+    cliOnly: true,
+    section: "common",
+    help: "One versioned JSON result on stdout (default: false); human diagnostics go to stderr.",
+  },
+  "run-manifest-file": {
+    type: "string",
+    kind: "path",
+    cliOnly: true,
+    private: true,
+    section: "common",
+    help: "<path>  Saved sevro.run.v1 settings; explicit flags override them.",
+  },
+  "retain-host-diagnostics": {
+    type: "boolean",
+    kind: "boolean",
+    section: "common",
+    help: "Retain private native diagnostics (default: false).",
+  },
+  dry: {
+    type: "boolean",
+    kind: "boolean",
+    section: "common",
+    help: "Prepare only (default: false).",
+  },
+  "case-file": {
+    type: "string",
+    kind: "path",
+    private: true,
+    section: "required",
+    help: "<path>  Select exactly one of --case-file or --extension-command-file.",
+  },
+  "case-id": {
+    type: "string",
+    kind: "string",
+    extension: true,
+    section: "extension",
+    help: "<id>  Required with --extension-command-file.",
+  },
+  "extension-command-file": {
+    type: "string",
+    kind: "path",
+    extension: true,
+    private: true,
+    section: "required",
+    help: "<path>  Select one extension case instead of --case-file.",
+  },
+  "extension-source-file": {
+    type: "string",
+    kind: "path-list",
+    multiple: true,
+    extension: true,
+    private: true,
+    section: "extension",
+    help: "<path>  Required source closure; repeatable.",
+  },
+  "extension-configuration-file": {
+    type: "string",
+    kind: "path",
+    extension: true,
+    private: true,
+    section: "extension",
+    help: "<path>  Private JSON object; supply with --extension-redacted-configuration-file.",
+  },
+  "extension-redacted-configuration-file": {
+    type: "string",
+    kind: "path",
+    extension: true,
+    private: true,
+    section: "extension",
+    help: "<path>  Redacted retained configuration; supply with --extension-configuration-file.",
+  },
+  "task-verdict-policy": {
+    type: "string",
+    kind: "string",
+    extension: true,
+    section: "extension",
+    help: "<id>  Select an advertised namespaced extension policy.",
+  },
+  "replace-builtin-grader": {
+    type: "string",
+    kind: "string",
+    multiple: true,
+    extension: true,
+    section: "extension",
+    help: "<id>  Explicit replacement; repeatable.",
+  },
+  "case-source-root": {
+    type: "string",
+    kind: "path",
+    private: true,
+    section: "extension",
+    help: "<path>  Supply with --case-source-map-file for mapped sources.",
+  },
+  "case-source-map-file": {
+    type: "string",
+    kind: "path",
+    private: true,
+    section: "extension",
+    help: "<path>  Supply with --case-source-root for mapped sources.",
+  },
+  "adapter-module": {
+    type: "string",
+    kind: "path",
+    section: "required",
+    help: "<path>  Select exactly one of --adapter-module or --host <codex|claude>.",
+  },
+  "semantic-adapter-module": {
+    type: "string",
+    kind: "path",
+    private: true,
+    section: "grading",
+    help: "<path>  Select this or --semantic-host codex.",
+  },
+  "semantic-host": {
+    type: "string",
+    kind: "string",
+    section: "grading",
+    help: "<codex>  Native semantic route; none selected by default.",
+  },
+  "semantic-model": {
+    type: "string",
+    kind: "string",
+    section: "grading",
+    help: "<name>  Required for native semantic route.",
+  },
+  "semantic-effort": {
+    type: "string",
+    kind: "string",
+    section: "grading",
+    help: "<level>  Required for native semantic route.",
+  },
+  "advisory-adapter-module": {
+    type: "string",
+    kind: "path",
+    private: true,
+    section: "grading",
+    help: "<path>  Select this or --advisory-host codex.",
+  },
+  "advisory-host": {
+    type: "string",
+    kind: "string",
+    section: "grading",
+    help: "<codex>  Native advisory route; none selected by default.",
+  },
+  "advisory-model": {
+    type: "string",
+    kind: "string",
+    section: "grading",
+    help: "<name>  Required for native advisory route.",
+  },
+  "advisory-effort": {
+    type: "string",
+    kind: "string",
+    section: "grading",
+    help: "<level>  Required for native advisory route.",
+  },
+  "advisory-exclude": {
+    type: "string",
+    kind: "string",
+    multiple: true,
+    section: "grading",
+    help: "<relative-path>  Exclude an evaluator path; repeatable.",
+  },
+  host: {
+    type: "string",
+    kind: "string",
+    section: "required",
+    help: "<codex|claude>  Bundled candidate route instead of --adapter-module.",
+  },
+  "codex-bin": {
+    type: "string",
+    kind: "path",
+    route: "codex",
+    section: "native",
+    help: "<path> (default: codex on PATH)  Explicit binary override.",
+  },
+  "codex-entrypoint": {
+    type: "string",
+    kind: "string",
+    section: "native",
+    help: "<exec|app-server>  Candidate transport (default: exec).",
+  },
+  "codex-timeout-ms": {
+    type: "string",
+    kind: "number",
+    route: "codex",
+    section: "native",
+    help: "<integer>  1..1800000 milliseconds (default: 600000).",
+  },
+  "codex-auth-file": {
+    type: "string",
+    kind: "path",
+    private: true,
+    route: "codex",
+    section: "native",
+    help: "<path>  Required for every Codex route.",
+  },
+  "claude-bin": {
+    type: "string",
+    kind: "path",
+    route: "claude",
+    section: "native",
+    help: "<path> (default: claude on PATH)  Explicit binary override.",
+  },
+  "claude-credential-file": {
+    type: "string",
+    kind: "path",
+    private: true,
+    route: "claude",
+    section: "native",
+    help: "<path>  Optional credential override; otherwise use native auth.",
+  },
+  "claude-uv-cache-dir": {
+    type: "string",
+    kind: "path",
+    route: "claude",
+    section: "deprecated",
+    help: "<path>  Migrate to sevro.json; cannot be combined with a runtime policy.",
+  },
+  "claude-project-settings": {
+    type: "boolean",
+    kind: "boolean",
+    route: "claude",
+    section: "native",
+    help: "Allow fixture project settings (default: false).",
+  },
+  "toolchain-bin-dir": {
+    type: "string",
+    kind: "path",
+    section: "deprecated",
+    help: "<path>  Migrate to sevro.json; cannot be combined with a runtime policy.",
+  },
+  model: {
+    type: "string",
+    kind: "string",
+    section: "native",
+    help: "<name>  Required for a bundled candidate host.",
+  },
+  effort: {
+    type: "string",
+    kind: "string",
+    section: "native",
+    help: "<level>  Required for a bundled candidate host.",
+  },
+  "shell-isolation": {
+    type: "boolean",
+    kind: "boolean",
+    section: "common",
+    help: "Isolate shell checks (default: false).",
+  },
+  "protected-root": {
+    type: "string",
+    kind: "path-list",
+    multiple: true,
+    section: "common",
+    help: "<path>  Extra protected source/private root; repeatable.",
+  },
+  "project-root": {
+    type: "string",
+    kind: "path",
+    section: "common",
+    help: "<path> (default: current directory)  Project identity; does not copy its files into the fixture.",
+  },
+  "config-root": {
+    type: "string",
+    kind: "path",
+    section: "common",
+    help: "<path> (default: --project-root)  Native concurrency configuration.",
+  },
+  "runtime-config-file": {
+    type: "string",
+    kind: "path",
+    private: true,
+    section: "common",
+    help: "<path>  Runtime policy; otherwise discover project sevro.json.",
+  },
+  "results-root": {
+    type: "string",
+    kind: "path",
+    section: "common",
+    help: "<path> (default: <project-root>/.sevro/results)  Retained evidence.",
+  },
+  "run-state-root": {
+    type: "string",
+    kind: "path",
+    section: "common",
+    help: "<path> (default: --results-root)  Active records/checkpoints.",
+  },
+  "runner-build-digest": {
+    type: "string",
+    kind: "string",
+    section: "provenance",
+    help: "<sha256>  64 lowercase hex characters; normally derived automatically.",
+  },
+  "runner-checkout-root": {
+    type: "string",
+    kind: "path",
+    section: "provenance",
+    help: "<path>  Record the running checkout's revision/dirty digest.",
+  },
+  "project-digest": {
+    type: "string",
+    kind: "string",
+    section: "provenance",
+    help: "<sha256>  64 lowercase hex characters; normally derived automatically.",
+  },
+  condition: {
+    type: "string",
+    kind: "string",
+    section: "common",
+    help: "<passive|enforced> (default: passive)  Bundled hosts use passive.",
+  },
+  trials: {
+    type: "string",
+    kind: "number",
+    section: "common",
+    help: "<integer> (default: 1)  Positive number of trials.",
+  },
+  jobs: {
+    type: "string",
+    kind: "number",
+    section: "common",
+    help: "<integer> (default: 3)  Maximum parallel trials, positive integer.",
+  },
+  threshold: {
+    type: "string",
+    kind: "number",
+    section: "common",
+    help: "<number> (default: 1)  All assessed trials must pass; range (0, 1].",
+  },
+} as const satisfies Record<string, RunOption>;
+
+export type RunOptionName = keyof typeof RUN_OPTIONS;
+
+type MatchingOptions<K extends keyof RunOption, V> = {
+  [N in RunOptionName]: (typeof RUN_OPTIONS)[N] extends Record<K, V>
+    ? N
+    : never;
+}[RunOptionName];
+
+export function optionsWith<
+  K extends keyof RunOption,
+  const V extends RunOption[K],
+>(key: K, value: V): MatchingOptions<K, V>[] {
+  return (Object.keys(RUN_OPTIONS) as RunOptionName[]).filter(
+    (name) => (RUN_OPTIONS[name] as RunOption)[key] === value,
+  ) as MatchingOptions<K, V>[];
+}
+
+export const PATH_OPTIONS = optionsWith("kind", "path");
+export const PATH_LIST_OPTIONS = optionsWith("kind", "path-list");
+export const NUMBER_OPTIONS = new Set<string>(optionsWith("kind", "number"));
+export const LIST_OPTIONS = new Set<string>(optionsWith("multiple", true));
+export const CLI_ONLY = new Set<string>(optionsWith("cliOnly", true));
+export const EXTENSION_OPTIONS = optionsWith("extension", true);
+
+export function runArguments(argv: string[]) {
+  return parseArgs({
+    args: argv,
+    allowPositionals: true,
+    allowNegative: true,
+    strict: true,
+    options: RUN_OPTIONS,
+  });
+}
+
+export type RunValues = ReturnType<typeof runArguments>["values"];
+
+/** Preserve the last explicit output selection even when parsing fails. */
+export function jsonOutputRequested(argv: string[]): boolean {
+  let json = false;
+  for (const argument of argv) {
+    if (argument === "--json") json = true;
+    if (argument === "--no-json") json = false;
+  }
+  return json;
+}

package/src/cli/run-overrides.ts

@@ -0,0 +1,69 @@
+import { EXTENSION_OPTIONS, optionsWith, type RunValues } from "./run-options";
+
+type Key = keyof RunValues;
+
+function removeSaved(saved: RunValues, keys: readonly Key[]): void {
+  for (const key of keys) Reflect.deleteProperty(saved, key);
+}
+
+const CANDIDATE_OPTIONS = [
+  "host",
+  "model",
+  "effort",
+  "codex-entrypoint",
+] as const;
+
+function overrideCandidate(saved: RunValues, explicit: RunValues): void {
+  if (explicit["adapter-module"] !== undefined)
+    removeSaved(saved, CANDIDATE_OPTIONS);
+  if (explicit.host === undefined) return;
+  delete saved["adapter-module"];
+  if (saved.host !== explicit.host)
+    removeSaved(saved, ["model", "effort", "codex-entrypoint"]);
+}
+
+function overrideSecondary(
+  saved: RunValues,
+  explicit: RunValues,
+  role: "semantic" | "advisory",
+): void {
+  if (explicit[`${role}-host`] !== undefined)
+    Reflect.deleteProperty(saved, `${role}-adapter-module`);
+  if (explicit[`${role}-adapter-module`] !== undefined)
+    removeSaved(saved, [`${role}-host`, `${role}-model`, `${role}-effort`]);
+}
+
+function removeUnusedNativeOptions(saved: RunValues, merged: RunValues): void {
+  if (
+    ![merged.host, merged["semantic-host"], merged["advisory-host"]].includes(
+      "codex",
+    )
+  )
+    removeSaved(saved, optionsWith("route", "codex"));
+  if (merged.host !== "claude")
+    removeSaved(saved, optionsWith("route", "claude"));
+}
+
+export function overrideRunSelections(
+  saved: RunValues,
+  explicit: RunValues,
+): void {
+  if (explicit["case-file"] !== undefined)
+    removeSaved(saved, EXTENSION_OPTIONS);
+  if (explicit["extension-command-file"] !== undefined)
+    delete saved["case-file"];
+  overrideCandidate(saved, explicit);
+  overrideSecondary(saved, explicit, "semantic");
+  overrideSecondary(saved, explicit, "advisory");
+  if (
+    [
+      "host",
+      "adapter-module",
+      "semantic-host",
+      "semantic-adapter-module",
+      "advisory-host",
+      "advisory-adapter-module",
+    ].some((key) => explicit[key as Key] !== undefined)
+  )
+    removeUnusedNativeOptions(saved, { ...saved, ...explicit });
+}

package/src/cli/settings.ts

@@ -0,0 +1,154 @@
+import { join } from "node:path";
+import type { HostAdapter } from "../engine";
+import type { PreparedInvocation } from "./preparation";
+import { nativeRoutes } from "./invocation";
+
+type NativeRoute = { binary: string; timeoutMs?: number } | undefined;
+
+function nativeSettings(native: NativeRoute) {
+  if (!native) return { executable: null, timeoutMs: null };
+  return { executable: native.binary, timeoutMs: native.timeoutMs ?? 600_000 };
+}
+
+function routeSettings(host: HostAdapter | undefined, native: NativeRoute) {
+  if (!host) return null;
+  return {
+    host: host.id,
+    model: host.model,
+    effort: host.effort,
+    ...nativeSettings(native),
+  };
+}
+
+/** A whitelist of resolved operator settings; never copy credentials or runtime environment. */
+export function effectiveSettings(prepared: PreparedInvocation) {
+  const { invocation, host, semanticHost, advisoryHost } = prepared;
+  return {
+    format: "sevro.resolved-run-settings.v1",
+    ...sourceSettings(prepared),
+    ...extensionSettings(invocation.extension),
+    ...adapterSettings(invocation),
+    ...legacyRuntimeSettings(invocation),
+    protectedRoots: invocation.protectedRoots,
+    condition: invocation.condition,
+    trials: invocation.trialCount,
+    threshold: invocation.passThreshold,
+    jobs: invocation.jobs,
+    dry: invocation.dry,
+    projectRoot: invocation.projectRoot,
+    configRoot: invocation.configRoot,
+    resultsRoot: invocation.resultsRoot,
+    runStateRoot: invocation.runStateRoot,
+    runtimeConfigFile: selectedRuntimeFile(prepared),
+    candidate: routeSettings(host, invocation.codex ?? invocation.claude),
+    semantic: routeSettings(semanticHost, invocation.semanticCodex),
+    advisory: routeSettings(advisoryHost, invocation.advisoryCodex),
+    codexEntrypoint: selectedEntrypoint(invocation),
+    agentConcurrencyLimit: prepared.agentConcurrencyLimit,
+    shellIsolation: Boolean(invocation.shellIsolation),
+    retainHostDiagnostics: retainedDiagnostics(invocation),
+    claudeProjectSettings: projectSettings(invocation),
+    shellCheckDefaults: { expectedExitCode: 0, timeoutMs: 30_000 },
+  };
+}
+
+function selectedCaseId(prepared: PreparedInvocation): string | null {
+  if (prepared.caseData) return prepared.caseData.id;
+  return prepared.invocation.extension?.caseId ?? null;
+}
+
+function sourceSettings(prepared: PreparedInvocation) {
+  const { invocation } = prepared;
+  return {
+    runManifestFile: invocation.runManifestFile ?? null,
+    caseFile: invocation.caseFile ?? null,
+    caseId: selectedCaseId(prepared),
+    caseSourceMapping: Boolean(invocation.caseSourceRoot),
+  };
+}
+
+function extensionSettings(
+  extension: PreparedInvocation["invocation"]["extension"],
+) {
+  if (!extension)
+    return {
+      extensionCommandFile: null,
+      extensionSourceFileCount: 0,
+      taskVerdictPolicy: null,
+      replaceBuiltinGraders: [],
+    };
+  return {
+    extensionCommandFile: extension.commandFile,
+    extensionSourceFileCount: extension.sourceFiles.length,
+    taskVerdictPolicy: extension.taskVerdictPolicy ?? null,
+    replaceBuiltinGraders: extension.replaceBuiltinGraders,
+  };
+}
+
+function adapterSettings(invocation: PreparedInvocation["invocation"]) {
+  return {
+    adapterModule: invocation.adapterModule ?? null,
+    semanticAdapterModule: invocation.semanticAdapterModule ?? null,
+    advisoryAdapterModule: invocation.advisoryAdapterModule ?? null,
+    advisoryExcludedPaths: invocation.advisoryExcludedPaths,
+  };
+}
+
+function selectedEntrypoint(invocation: PreparedInvocation["invocation"]) {
+  return invocation.codex ? (invocation.codex.entrypoint ?? "exec") : null;
+}
+
+function projectSettings(
+  invocation: PreparedInvocation["invocation"],
+): boolean {
+  return invocation.claude?.projectSettings ?? false;
+}
+
+function legacyRuntimeSettings(invocation: PreparedInvocation["invocation"]) {
+  return {
+    toolchainBinDir: selectedToolchain(invocation),
+    claudeUvCacheDir: invocation.claude?.uvCacheDir ?? null,
+  };
+}
+
+function selectedToolchain(invocation: PreparedInvocation["invocation"]) {
+  return (
+    (invocation.shellIsolation ?? invocation.claude)?.toolchainBinDir ?? null
+  );
+}
+
+function retainedDiagnostics(
+  invocation: PreparedInvocation["invocation"],
+): boolean {
+  return nativeRoutes(invocation).some((host) => host.retainHostDiagnostics);
+}
+
+/** Assessment-affecting CLI settings absent from the engine's other identity inputs. */
+export function assessmentSettings(prepared: PreparedInvocation) {
+  const { invocation } = prepared;
+  return {
+    shellIsolation: Boolean(invocation.shellIsolation),
+    protectedRoots: invocation.protectedRoots,
+    ...legacyRuntimeSettings(invocation),
+    candidateTimeoutMs: nativeSettings(invocation.codex ?? invocation.claude)
+      .timeoutMs,
+    semanticTimeoutMs: nativeSettings(invocation.semanticCodex).timeoutMs,
+    advisoryTimeoutMs: nativeSettings(invocation.advisoryCodex).timeoutMs,
+    claudeProjectSettings: invocation.claude?.projectSettings ?? false,
+  };
+}
+
+function selectedRuntimeFile(prepared: PreparedInvocation): string | null {
+  return (
+    prepared.invocation.runtimeConfigFile ??
+    (prepared.runtimePolicy
+      ? join(prepared.invocation.projectRoot, "sevro.json")
+      : null)
+  );
+}
+
+export function displayEffectiveSettings(prepared: PreparedInvocation): void {
+  process.stderr.write(
+    `sevro: effective settings ${JSON.stringify(effectiveSettings(prepared))}\n`,
+  );
+}

package/src/cli/summary.ts

@@ -0,0 +1,43 @@
+import type { CliResultData, ReportData } from "../schema-types";
+
+function line(value: string): string {
+  return value.replaceAll(/\p{Cc}/gu, " ").slice(0, 4096);
+}
+
+function metric(value: number | null): string {
+  return value === null ? "unknown" : String(value);
+}
+
+export function displayTrialSummary(result: CliResultData): void {
+  for (const selected of result.cases)
+    for (const trial of selected.trials) {
+      process.stdout.write(
+        `trial case=${line(selected.caseId)} trial=${trial.trial} execution=${trial.execution.status} grading=${trial.grading.status} task=${trial.task.verdict}\n`,
+      );
+      displayFailedChecks(trial.checks);
+    }
+}
+
+function displayFailedChecks(
+  checks: CliResultData["cases"][number]["trials"][number]["checks"],
+): void {
+  for (const check of checks)
+    if (check.status !== "passed")
+      process.stdout.write(
+        `  check=${line(check.id)} status=${check.status} reason=${line(check.detail ?? "unknown")}\n`,
+      );
+}
+
+export function displayMeasurements(report: ReportData): void {
+  for (const row of report.rows) {
+    const duration =
+      row.candidateDurationMs === null
+        ? "unknown"
+        : `${row.candidateDurationMs.toFixed(2)}ms`;
+    const cost =
+      row.costUsd === null ? "unknown" : `$${row.costUsd.toFixed(4)}`;
+    process.stdout.write(
+      `case=${line(row.caseId)} candidate duration=${duration} candidate tokens=${metric(row.inputTokens)}/${metric(row.outputTokens)} cost=${cost}\n`,
+    );
+  }
+}