@bjoernrochel/sevro 0.1.0-rc.2 → 0.1.0-rc.21

package/src/cli/loading.ts

@@ -0,0 +1,151 @@
+import { readFile } from "node:fs/promises";
+import { isAbsolute } from "node:path";
+import { pathToFileURL } from "node:url";
+import { isResolvedCase, resolvedCaseIssue } from "../case-validation";
+import type { HostAdapter, ResolvedCase } from "../engine";
+import { prepareInstrumentation } from "../instrumentation";
+import { isRecord, isStringArray } from "../value-guards";
+import { InvocationError, type Invocation } from "./invocation";
+
+export function parseCase(value: unknown): ResolvedCase {
+  if (!isResolvedCase(value))
+    throw new InvocationError(
+      `invalid resolved case file: ${resolvedCaseIssue(value) ?? "invalid case"}`,
+    );
+  return value;
+}
+
+export async function loadCase(path: string): Promise<ResolvedCase> {
+  try {
+    return parseCase(JSON.parse(await readFile(path, "utf8")));
+  } catch (cause) {
+    const detail = cause instanceof InvocationError ? `: ${cause.message}` : "";
+    throw new InvocationError(
+      `resolved case file is unreadable or invalid at ${path}${detail}; supply --case-file with readable JSON and correct the named field`,
+      {
+        cause,
+      },
+    );
+  }
+}
+
+async function loadJson(path: string, label: string): Promise<unknown> {
+  try {
+    return JSON.parse(await readFile(path, "utf8"));
+  } catch (cause) {
+    throw new InvocationError(`${label} is unreadable or invalid`, { cause });
+  }
+}
+
+function validExtensionCommand(value: unknown): value is string[] {
+  return (
+    isStringArray(value) &&
+    value.length > 0 &&
+    value.every((part) => part.length > 0) &&
+    isAbsolute(value[0] ?? "")
+  );
+}
+
+async function loadExtensionCommand(path: string): Promise<string[]> {
+  const command = await loadJson(path, "extension command file");
+  if (!validExtensionCommand(command))
+    throw new InvocationError(
+      "extension command must be an absolute argv array",
+    );
+  return command;
+}
+
+export async function loadExtensionOptions(
+  selected: NonNullable<Invocation["extension"]>,
+) {
+  const command = await loadExtensionCommand(selected.commandFile);
+  const configuration = selected.configurationFile
+    ? await loadJson(selected.configurationFile, "extension configuration file")
+    : {};
+  const redactedConfiguration = selected.redactedConfigurationFile
+    ? await loadJson(
+        selected.redactedConfigurationFile,
+        "redacted extension configuration file",
+      )
+    : {};
+  if (!isRecord(configuration) || !isRecord(redactedConfiguration))
+    throw new InvocationError("extension configuration must be JSON objects");
+  return { command, configuration, redactedConfiguration };
+}
+
+export async function loadPreparationSources(selected: Invocation) {
+  if (!selected.caseSourceRoot || !selected.caseSourceMapFile) return undefined;
+  const refs = await loadJson(
+    selected.caseSourceMapFile,
+    "case source map file",
+  );
+  if (
+    !isRecord(refs) ||
+    !Object.values(refs).every(
+      (value) => typeof value === "string" && value.startsWith("file:///"),
+    )
+  )
+    throw new InvocationError("case source map must contain file URLs");
+  return {
+    root: selected.caseSourceRoot,
+    refs: refs as Record<string, string>,
+  };
+}
+
+async function adapterModule(path: string): Promise<unknown> {
+  try {
+    return await import(pathToFileURL(path).href);
+  } catch (cause) {
+    throw new InvocationError(
+      `host adapter module could not be loaded at ${path}; check that the module exists and imports successfully, then correct --adapter-module or the selected grading adapter option`,
+      { cause },
+    );
+  }
+}
+
+export async function loadHost(path: string): Promise<HostAdapter> {
+  const module = await adapterModule(path);
+  const host = isRecord(module) ? module.default : null;
+  if (!validHostAdapter(host))
+    throw new InvocationError(
+      `host adapter module has no valid default adapter at ${path}; export a default adapter with string id, model, effort and a run function`,
+    );
+  validateHostInstrumentation(host);
+  validateHostCapabilities(host.hostCapabilities);
+  return host as unknown as HostAdapter;
+}
+
+function validateHostInstrumentation(host: Record<string, unknown>): void {
+  try {
+    prepareInstrumentation(
+      [],
+      (host.instrumentation as HostAdapter["instrumentation"]) ?? [],
+      [],
+      "passive",
+    );
+  } catch (cause) {
+    throw new InvocationError("host adapter instrumentation is invalid", {
+      cause,
+    });
+  }
+}
+
+function validHostAdapter(value: unknown): value is Record<string, unknown> {
+  return (
+    isRecord(value) &&
+    typeof value.id === "string" &&
+    typeof value.model === "string" &&
+    typeof value.effort === "string" &&
+    typeof value.run === "function"
+  );
+}
+
+function validateHostCapabilities(value: unknown): void {
+  if (value === undefined) return;
+  if (
+    !isStringArray(value) ||
+    value.some((id) => !/^[a-z][a-z0-9]*(?:[.-][a-z0-9]+)+$/.test(id)) ||
+    new Set(value).size !== value.length
+  )
+    throw new InvocationError("host adapter capabilities are invalid");
+}

package/src/cli/preparation.ts

@@ -0,0 +1,163 @@
+import { join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { configurationRoot } from "../hosts/codex-configuration";
+import {
+  InvocationError,
+  parseInvocation,
+  type Invocation,
+} from "./invocation";
+import {
+  loadRuntimeConfiguration,
+  runtimeNativeGoalEnabled,
+} from "../runtime/config";
+import {
+  canonicalRuntimeRoot,
+  requireRuntimeReadRoots,
+} from "../runtime/paths";
+import { loadCase, loadPreparationSources } from "./loading";
+import { candidateHost, secondaryHost, selectedConcurrency } from "./hosts";
+
+type PreparationSources = Awaited<ReturnType<typeof loadPreparationSources>>;
+
+function protectMappedSources(
+  invocation: Invocation,
+  preparationSources: PreparationSources,
+): void {
+  const sourceRoots = mappedSourceRoots(preparationSources);
+  for (const native of [
+    invocation.codex,
+    invocation.claude,
+    invocation.semanticCodex,
+    invocation.advisoryCodex,
+  ]) {
+    if (native)
+      native.additionalProtectedRoots = [
+        ...native.additionalProtectedRoots,
+        ...sourceRoots,
+      ];
+  }
+  invocation.shellIsolation?.protectedRoots.push(...sourceRoots);
+}
+
+export async function prepareRunInvocation(argv: string[]) {
+  const invocation = await parseInvocation(argv);
+  warnDeprecatedRuntimeOptions(invocation);
+  const preparationSources = await loadPreparationSources(invocation);
+  protectMappedSources(invocation, preparationSources);
+  const runtimePolicy = await loadRuntimeConfiguration(
+    invocation.projectRoot,
+    invocation.runtimeConfigFile,
+    await invocationProtectedRuntimeRoots(invocation),
+  );
+  requireRuntimeOptionCompatibility(invocation, runtimePolicy);
+  selectRuntimeTransport(invocation, runtimePolicy);
+  const configRoot = await configurationRoot(invocation.configRoot);
+  await validateInvocationRuntime(invocation, runtimePolicy);
+  const agentConcurrencyLimit = await selectedConcurrency(
+    invocation,
+    configRoot,
+  );
+  const caseData = invocation.caseFile
+    ? await loadCase(invocation.caseFile)
+    : undefined;
+  const host = await candidateHost(invocation, agentConcurrencyLimit);
+  const semanticHost = await secondaryHost(
+    invocation.semanticAdapterModule,
+    invocation.semanticCodex,
+    agentConcurrencyLimit,
+  );
+  const advisoryHost = await secondaryHost(
+    invocation.advisoryAdapterModule,
+    invocation.advisoryCodex,
+    agentConcurrencyLimit,
+  );
+  return {
+    invocation,
+    runtimePolicy,
+    preparationSources,
+    caseData,
+    host,
+    semanticHost,
+    advisoryHost,
+    agentConcurrencyLimit,
+  };
+}
+
+function selectRuntimeTransport(
+  invocation: Invocation,
+  policy: Awaited<ReturnType<typeof loadRuntimeConfiguration>>,
+): void {
+  if (!runtimeNativeGoalEnabled(policy)) return;
+  if (invocation.codex) selectCodexGoalTransport(invocation.codex);
+}
+
+function selectCodexGoalTransport(
+  codex: NonNullable<Invocation["codex"]>,
+): void {
+  if (codex.entrypoint === "exec")
+    throw new InvocationError(
+      "native-goal runtime policy requires Codex app-server",
+    );
+  codex.entrypoint = "app-server";
+}
+
+function warnDeprecatedRuntimeOptions(invocation: Invocation): void {
+  for (const option of invocation.deprecatedRuntimeOptions)
+    console.warn(
+      `--${option} is deprecated; declare the runtime in sevro.json or --runtime-config-file instead`,
+    );
+}
+
+function requireRuntimeOptionCompatibility(
+  invocation: Invocation,
+  policy: Awaited<ReturnType<typeof loadRuntimeConfiguration>>,
+): void {
+  if (policy && invocation.deprecatedRuntimeOptions.length)
+    throw new InvocationError(
+      "runtime configuration cannot be combined with deprecated runtime options",
+    );
+}
+
+export type PreparedInvocation = Awaited<
+  ReturnType<typeof prepareRunInvocation>
+>;
+
+async function validateInvocationRuntime(
+  invocation: Invocation,
+  policy: Awaited<ReturnType<typeof loadRuntimeConfiguration>>,
+): Promise<void> {
+  if (!policy) return;
+  const protectedRoots = await invocationProtectedRuntimeRoots(invocation);
+  requireRuntimeReadRoots(policy.readOnlyRoots, protectedRoots);
+  requireRuntimeReadRoots(
+    (policy.seeds ?? []).map((seed) => seed.source),
+    protectedRoots,
+  );
+}
+
+async function invocationProtectedRuntimeRoots(
+  invocation: Invocation,
+): Promise<string[]> {
+  const native = [
+    invocation.codex,
+    invocation.claude,
+    invocation.semanticCodex,
+    invocation.advisoryCodex,
+  ];
+  const protectedRoots = [
+    join(import.meta.dir, "..", ".."),
+    invocation.projectRoot,
+    invocation.configRoot,
+    invocation.resultsRoot,
+    invocation.runStateRoot,
+    ...native.flatMap((host) => host?.additionalProtectedRoots ?? []),
+    ...(invocation.shellIsolation?.protectedRoots ?? []),
+  ];
+  return Promise.all(protectedRoots.map(canonicalRuntimeRoot));
+}
+
+function mappedSourceRoots(preparationSources: PreparationSources): string[] {
+  return Object.values(preparationSources?.refs ?? {}).map((url) =>
+    fileURLToPath(url),
+  );
+}

package/src/cli/presentation.ts

@@ -0,0 +1,91 @@
+import { EvaluationConfigurationError } from "../engine";
+import { InvocationError } from "./invocation";
+import { assertCliResult } from "../schema";
+import type { ReportData } from "../schema-types";
+import { displayMeasurements, displayTrialSummary } from "./summary";
+
+function failure(code: 64 | 70, message: string) {
+  const result = {
+    format: "sevro.cli-result.v1",
+    runId: null,
+    execution: { status: "not_run" },
+    grading: { status: "not_requested" },
+    task: { verdict: "not_assessed" },
+    exitCode: code,
+    evidencePath: null,
+    cases: [],
+    diagnostic: {
+      code: code === 64 ? "sevro.invocation.invalid" : "sevro.runner.error",
+      message: message.slice(0, 4096),
+    },
+  };
+  assertCliResult(result);
+  return result;
+}
+
+export function display(
+  result: unknown,
+  json: boolean,
+  completion?: { resultFile: string; report: ReportData },
+): void {
+  assertCliResult(result);
+  if (result.diagnostic)
+    process.stderr.write(`sevro: ${result.diagnostic.message}\n`);
+  if (json) {
+    process.stdout.write(`${JSON.stringify(result)}\n`);
+    return;
+  }
+  process.stdout.write(
+    `execution=${result.execution.status} grading=${result.grading.status} task=${result.task.verdict}\n`,
+  );
+  displayTrialSummary(result);
+  displayDomainOutcomes(result.cases);
+  if (completion) {
+    displayMeasurements(completion.report);
+    process.stdout.write(`result=${completion.resultFile}\n`);
+  }
+  if (result.evidencePath)
+    process.stdout.write(`evidence=${result.evidencePath}\n`);
+}
+
+function displayDomainOutcomes(
+  cases: Array<{
+    caseId: string;
+    trials: Array<{
+      trial: number;
+      domainOutcomes?: Array<{ id: string; status: string }>;
+    }>;
+  }>,
+): void {
+  for (const selected of cases)
+    for (const trial of selected.trials)
+      for (const outcome of trial.domainOutcomes ?? [])
+        process.stdout.write(
+          `domain case=${selected.caseId} trial=${trial.trial} outcome=${outcome.id} status=${outcome.status}\n`,
+        );
+}
+
+export function runnerFailureCategory(error: unknown): 64 | 70 {
+  return error instanceof EvaluationConfigurationError ||
+    error instanceof InvocationError
+    ? 64
+    : 70;
+}
+
+export function displayFailure(
+  error: unknown,
+  json: boolean,
+  code: 64 | 70,
+  fallback: string,
+): void {
+  display(
+    failure(code, error instanceof Error ? error.message : fallback),
+    json,
+  );
+  process.stderr.write(
+    code === 64
+      ? "Correct the named option or input and retry. Run sevro run --help for examples and required options.\n"
+      : "Check the reported path and ensure --results-root and --run-state-root are writable directories, then retry.\n",
+  );
+  process.exitCode = code;
+}

package/src/cli/progress.ts

@@ -0,0 +1,6 @@
+import type { EvaluationProgress } from "../evaluation/types";
+
+export function displayProgress(event: EvaluationProgress): void {
+  const trial = event.trial === undefined ? "" : ` trial=${event.trial}`;
+  process.stderr.write(`sevro: ${event.stage} ${event.status}${trial}\n`);
+}

package/src/cli/run-manifest.ts

@@ -0,0 +1,133 @@
+import { lstat, readFile, realpath } from "node:fs/promises";
+import { dirname, resolve } from "node:path";
+import { isRecord, isStringArray } from "../value-guards";
+import { PATH_OPTIONS, RUN_OPTIONS, type RunValues } from "./run-options";
+import { overrideRunSelections } from "./run-overrides";
+
+const NUMBER_OPTIONS = new Set([
+  "trials",
+  "jobs",
+  "threshold",
+  "codex-timeout-ms",
+]);
+const CLI_ONLY = new Set(["json", "run-manifest-file"]);
+const LIST_OPTIONS = new Set([
+  "extension-source-file",
+  "replace-builtin-grader",
+  "advisory-exclude",
+  "protected-root",
+]);
+const FILE_LIMIT = 64 * 1024;
+
+function fieldError(field: string, requirement: string): never {
+  throw new Error(`run manifest ${field}: ${requirement}`);
+}
+
+function manifestSettings(value: unknown): Record<string, unknown> {
+  if (!isRecord(value)) fieldError("$", "expected an object");
+  validateManifestFields(value);
+  if (value.format !== "sevro.run.v1")
+    fieldError("format", 'expected "sevro.run.v1"');
+  if (!isRecord(value.settings))
+    fieldError("settings", "expected an object of run options");
+  return value.settings;
+}
+
+function validateManifestFields(value: Record<string, unknown>): void {
+  for (const key of Object.keys(value))
+    if (!["format", "settings"].includes(key))
+      fieldError(key, "unknown field; use format and settings");
+}
+
+function finiteNumber(value: unknown): boolean {
+  return typeof value === "number" && Number.isFinite(value);
+}
+
+function nonemptyString(value: unknown): boolean {
+  return typeof value === "string" && Boolean(value.trim());
+}
+
+function nonemptyStringList(value: unknown): boolean {
+  return isStringArray(value) && value.every(nonemptyString);
+}
+
+function validateSetting(name: string, value: unknown): void {
+  if (!Object.hasOwn(RUN_OPTIONS, name) || CLI_ONLY.has(name))
+    fieldError(
+      `settings.${name}`,
+      "unknown saved option; see sevro run --help",
+    );
+  if (NUMBER_OPTIONS.has(name)) {
+    if (!finiteNumber(value))
+      fieldError(`settings.${name}`, "expected a finite number");
+    return;
+  }
+  validateOrdinarySetting(name, value);
+}
+
+function validateOrdinarySetting(name: string, value: unknown): void {
+  if (LIST_OPTIONS.has(name)) {
+    if (!nonemptyStringList(value))
+      fieldError(`settings.${name}`, "expected an array of nonempty strings");
+    return;
+  }
+  const option = RUN_OPTIONS[name as keyof typeof RUN_OPTIONS];
+  validateScalar(name, value, option.type);
+}
+
+function validateScalar(name: string, value: unknown, type: string): void {
+  if (type === "boolean") {
+    if (typeof value !== "boolean")
+      fieldError(`settings.${name}`, "expected a boolean");
+  } else if (!nonemptyString(value))
+    fieldError(`settings.${name}`, "expected a nonempty string");
+}
+
+function resolvedSetting(name: string, value: unknown, base: string) {
+  const pathOption = (PATH_OPTIONS as readonly string[]).includes(name);
+  if (pathOption) return resolve(base, value as string);
+  if (["extension-source-file", "protected-root"].includes(name))
+    return (value as string[]).map((path) => resolve(base, path));
+  return NUMBER_OPTIONS.has(name) ? String(value) : value;
+}
+
+async function readManifest(path: string): Promise<RunValues> {
+  const stat = await lstat(path);
+  if (!stat.isFile() || stat.size > FILE_LIMIT)
+    fieldError("$", "use a regular, non-symlink JSON file of at most 64 KiB");
+  const source = await readFile(path, "utf8");
+  let value: unknown;
+  try {
+    value = JSON.parse(source);
+  } catch (cause) {
+    throw new Error("run manifest $: invalid JSON; check the JSON syntax", {
+      cause,
+    });
+  }
+  const settings = manifestSettings(value);
+  const base = dirname(await realpath(path));
+  const resolved: Record<string, unknown> = {};
+  for (const [name, setting] of Object.entries(settings)) {
+    validateSetting(name, setting);
+    resolved[name] = resolvedSetting(name, setting, base);
+  }
+  return resolved;
+}
+
+/** Explicit flags replace saved selections and lists; defaults are applied later. */
+export async function savedRunValues(explicit: RunValues): Promise<RunValues> {
+  const file = explicit["run-manifest-file"];
+  if (!file) return explicit;
+  const path = resolve(file);
+  try {
+    const saved = await readManifest(path);
+    overrideRunSelections(saved, explicit);
+    return { ...saved, ...explicit };
+  } catch (cause) {
+    const detail = cause instanceof Error ? cause.message : "cannot read file";
+    throw new Error(
+      `--run-manifest-file ${path}: ${detail}; correct the named field or select a readable manifest`,
+      { cause },
+    );
+  }
+}

package/src/cli/run-options.ts

@@ -0,0 +1,101 @@
+import { parseArgs } from "node:util";
+
+export const RUN_OPTIONS = {
+  json: { type: "boolean" },
+  "run-manifest-file": { type: "string" },
+  "retain-host-diagnostics": { type: "boolean" },
+  dry: { type: "boolean" },
+  "case-file": { type: "string" },
+  "case-id": { type: "string" },
+  "extension-command-file": { type: "string" },
+  "extension-source-file": { type: "string", multiple: true },
+  "extension-configuration-file": { type: "string" },
+  "extension-redacted-configuration-file": { type: "string" },
+  "task-verdict-policy": { type: "string" },
+  "replace-builtin-grader": { type: "string", multiple: true },
+  "case-source-root": { type: "string" },
+  "case-source-map-file": { type: "string" },
+  "adapter-module": { type: "string" },
+  "semantic-adapter-module": { type: "string" },
+  "semantic-host": { type: "string" },
+  "semantic-model": { type: "string" },
+  "semantic-effort": { type: "string" },
+  "advisory-adapter-module": { type: "string" },
+  "advisory-host": { type: "string" },
+  "advisory-model": { type: "string" },
+  "advisory-effort": { type: "string" },
+  "advisory-exclude": { type: "string", multiple: true },
+  host: { type: "string" },
+  "codex-bin": { type: "string" },
+  "codex-entrypoint": { type: "string" },
+  "codex-timeout-ms": { type: "string" },
+  "codex-auth-file": { type: "string" },
+  "claude-bin": { type: "string" },
+  "claude-credential-file": { type: "string" },
+  "claude-uv-cache-dir": { type: "string" },
+  "claude-project-settings": { type: "boolean" },
+  "toolchain-bin-dir": { type: "string" },
+  model: { type: "string" },
+  effort: { type: "string" },
+  "shell-isolation": { type: "boolean" },
+  "protected-root": { type: "string", multiple: true },
+  "project-root": { type: "string" },
+  "config-root": { type: "string" },
+  "runtime-config-file": { type: "string" },
+  "results-root": { type: "string" },
+  "run-state-root": { type: "string" },
+  "runner-build-digest": { type: "string" },
+  "runner-checkout-root": { type: "string" },
+  "project-digest": { type: "string" },
+  condition: { type: "string" },
+  trials: { type: "string" },
+  jobs: { type: "string" },
+  threshold: { type: "string" },
+} as const;
+
+export const PATH_OPTIONS = [
+  "run-manifest-file",
+  "case-file",
+  "extension-command-file",
+  "extension-configuration-file",
+  "extension-redacted-configuration-file",
+  "case-source-root",
+  "case-source-map-file",
+  "adapter-module",
+  "semantic-adapter-module",
+  "advisory-adapter-module",
+  "codex-bin",
+  "codex-auth-file",
+  "claude-bin",
+  "claude-credential-file",
+  "claude-uv-cache-dir",
+  "toolchain-bin-dir",
+  "project-root",
+  "config-root",
+  "runtime-config-file",
+  "results-root",
+  "run-state-root",
+  "runner-checkout-root",
+] as const;
+
+export function runArguments(argv: string[]) {
+  return parseArgs({
+    args: argv,
+    allowPositionals: true,
+    allowNegative: true,
+    strict: true,
+    options: RUN_OPTIONS,
+  });
+}
+
+export type RunValues = ReturnType<typeof runArguments>["values"];
+
+/** Preserve the last explicit output selection even when parsing fails. */
+export function jsonOutputRequested(argv: string[]): boolean {
+  let json = false;
+  for (const argument of argv) {
+    if (argument === "--json") json = true;
+    if (argument === "--no-json") json = false;
+  }
+  return json;
+}