@bjoernrochel/sevro 0.1.0-rc.2 → 0.1.0-rc.21

package/src/cli/run-overrides.ts

@@ -0,0 +1,83 @@
+import type { RunValues } from "./run-options";
+
+type Key = keyof RunValues;
+
+function removeSaved(saved: RunValues, keys: readonly Key[]): void {
+  for (const key of keys) Reflect.deleteProperty(saved, key);
+}
+
+const EXTENSION_OPTIONS = [
+  "extension-command-file",
+  "case-id",
+  "extension-source-file",
+  "extension-configuration-file",
+  "extension-redacted-configuration-file",
+  "task-verdict-policy",
+  "replace-builtin-grader",
+] as const;
+const CANDIDATE_OPTIONS = [
+  "host",
+  "model",
+  "effort",
+  "codex-entrypoint",
+] as const;
+
+function overrideCandidate(saved: RunValues, explicit: RunValues): void {
+  if (explicit["adapter-module"] !== undefined)
+    removeSaved(saved, CANDIDATE_OPTIONS);
+  if (explicit.host === undefined) return;
+  delete saved["adapter-module"];
+  if (saved.host !== explicit.host)
+    removeSaved(saved, ["model", "effort", "codex-entrypoint"]);
+}
+
+function overrideSecondary(
+  saved: RunValues,
+  explicit: RunValues,
+  role: "semantic" | "advisory",
+): void {
+  if (explicit[`${role}-host`] !== undefined)
+    Reflect.deleteProperty(saved, `${role}-adapter-module`);
+  if (explicit[`${role}-adapter-module`] !== undefined)
+    removeSaved(saved, [`${role}-host`, `${role}-model`, `${role}-effort`]);
+}
+
+function removeUnusedNativeOptions(saved: RunValues, merged: RunValues): void {
+  if (
+    ![merged.host, merged["semantic-host"], merged["advisory-host"]].includes(
+      "codex",
+    )
+  )
+    removeSaved(saved, ["codex-bin", "codex-auth-file", "codex-timeout-ms"]);
+  if (merged.host !== "claude")
+    removeSaved(saved, [
+      "claude-bin",
+      "claude-credential-file",
+      "claude-uv-cache-dir",
+      "claude-project-settings",
+    ]);
+}
+
+export function overrideRunSelections(
+  saved: RunValues,
+  explicit: RunValues,
+): void {
+  if (explicit["case-file"] !== undefined)
+    removeSaved(saved, EXTENSION_OPTIONS);
+  if (explicit["extension-command-file"] !== undefined)
+    delete saved["case-file"];
+  overrideCandidate(saved, explicit);
+  overrideSecondary(saved, explicit, "semantic");
+  overrideSecondary(saved, explicit, "advisory");
+  if (
+    [
+      "host",
+      "adapter-module",
+      "semantic-host",
+      "semantic-adapter-module",
+      "advisory-host",
+      "advisory-adapter-module",
+    ].some((key) => explicit[key as Key] !== undefined)
+  )
+    removeUnusedNativeOptions(saved, { ...saved, ...explicit });
+}

package/src/cli/settings.ts

@@ -0,0 +1,160 @@
+import { join } from "node:path";
+import type { HostAdapter } from "../engine";
+import type { PreparedInvocation } from "./preparation";
+
+type NativeRoute = { binary: string; timeoutMs?: number } | undefined;
+
+function nativeSettings(native: NativeRoute) {
+  if (!native) return { executable: null, timeoutMs: null };
+  return { executable: native.binary, timeoutMs: native.timeoutMs ?? 600_000 };
+}
+
+function routeSettings(host: HostAdapter | undefined, native: NativeRoute) {
+  if (!host) return null;
+  return {
+    host: host.id,
+    model: host.model,
+    effort: host.effort,
+    ...nativeSettings(native),
+  };
+}
+
+/** A whitelist of resolved operator settings; never copy credentials or runtime environment. */
+export function effectiveSettings(prepared: PreparedInvocation) {
+  const { invocation, host, semanticHost, advisoryHost } = prepared;
+  return {
+    format: "sevro.resolved-run-settings.v1",
+    ...sourceSettings(prepared),
+    ...extensionSettings(invocation.extension),
+    ...adapterSettings(invocation),
+    ...legacyRuntimeSettings(invocation),
+    protectedRoots: invocation.protectedRoots,
+    condition: invocation.condition,
+    trials: invocation.trialCount,
+    threshold: invocation.passThreshold,
+    jobs: invocation.jobs,
+    dry: invocation.dry,
+    projectRoot: invocation.projectRoot,
+    configRoot: invocation.configRoot,
+    resultsRoot: invocation.resultsRoot,
+    runStateRoot: invocation.runStateRoot,
+    runtimeConfigFile: selectedRuntimeFile(prepared),
+    candidate: routeSettings(host, invocation.codex ?? invocation.claude),
+    semantic: routeSettings(semanticHost, invocation.semanticCodex),
+    advisory: routeSettings(advisoryHost, invocation.advisoryCodex),
+    codexEntrypoint: selectedEntrypoint(invocation),
+    agentConcurrencyLimit: prepared.agentConcurrencyLimit,
+    shellIsolation: Boolean(invocation.shellIsolation),
+    retainHostDiagnostics: retainedDiagnostics(invocation),
+    claudeProjectSettings: projectSettings(invocation),
+    shellCheckDefaults: { expectedExitCode: 0, timeoutMs: 30_000 },
+  };
+}
+
+function selectedCaseId(prepared: PreparedInvocation): string | null {
+  if (prepared.caseData) return prepared.caseData.id;
+  return prepared.invocation.extension?.caseId ?? null;
+}
+
+function sourceSettings(prepared: PreparedInvocation) {
+  const { invocation } = prepared;
+  return {
+    runManifestFile: invocation.runManifestFile ?? null,
+    caseFile: invocation.caseFile ?? null,
+    caseId: selectedCaseId(prepared),
+    caseSourceMapping: Boolean(invocation.caseSourceRoot),
+  };
+}
+
+function extensionSettings(
+  extension: PreparedInvocation["invocation"]["extension"],
+) {
+  if (!extension)
+    return {
+      extensionCommandFile: null,
+      extensionSourceFileCount: 0,
+      taskVerdictPolicy: null,
+      replaceBuiltinGraders: [],
+    };
+  return {
+    extensionCommandFile: extension.commandFile,
+    extensionSourceFileCount: extension.sourceFiles.length,
+    taskVerdictPolicy: extension.taskVerdictPolicy ?? null,
+    replaceBuiltinGraders: extension.replaceBuiltinGraders,
+  };
+}
+
+function adapterSettings(invocation: PreparedInvocation["invocation"]) {
+  return {
+    adapterModule: invocation.adapterModule ?? null,
+    semanticAdapterModule: invocation.semanticAdapterModule ?? null,
+    advisoryAdapterModule: invocation.advisoryAdapterModule ?? null,
+    advisoryExcludedPaths: invocation.advisoryExcludedPaths,
+  };
+}
+
+function selectedEntrypoint(invocation: PreparedInvocation["invocation"]) {
+  return invocation.codex ? (invocation.codex.entrypoint ?? "exec") : null;
+}
+
+function projectSettings(
+  invocation: PreparedInvocation["invocation"],
+): boolean {
+  return invocation.claude?.projectSettings ?? false;
+}
+
+function legacyRuntimeSettings(invocation: PreparedInvocation["invocation"]) {
+  return {
+    toolchainBinDir: selectedToolchain(invocation),
+    claudeUvCacheDir: invocation.claude?.uvCacheDir ?? null,
+  };
+}
+
+function selectedToolchain(invocation: PreparedInvocation["invocation"]) {
+  return (
+    invocation.shellIsolation?.toolchainBinDir ??
+    invocation.claude?.toolchainBinDir ??
+    null
+  );
+}
+
+function retainedDiagnostics(
+  invocation: PreparedInvocation["invocation"],
+): boolean {
+  return [
+    invocation.codex,
+    invocation.claude,
+    invocation.semanticCodex,
+    invocation.advisoryCodex,
+  ].some((host) => host?.retainHostDiagnostics === true);
+}
+
+/** Assessment-affecting CLI settings absent from the engine's other identity inputs. */
+export function assessmentSettings(prepared: PreparedInvocation) {
+  const { invocation } = prepared;
+  return {
+    shellIsolation: Boolean(invocation.shellIsolation),
+    protectedRoots: invocation.protectedRoots,
+    ...legacyRuntimeSettings(invocation),
+    candidateTimeoutMs: nativeSettings(invocation.codex ?? invocation.claude)
+      .timeoutMs,
+    semanticTimeoutMs: nativeSettings(invocation.semanticCodex).timeoutMs,
+    advisoryTimeoutMs: nativeSettings(invocation.advisoryCodex).timeoutMs,
+    claudeProjectSettings: invocation.claude?.projectSettings ?? false,
+  };
+}
+
+function selectedRuntimeFile(prepared: PreparedInvocation): string | null {
+  return (
+    prepared.invocation.runtimeConfigFile ??
+    (prepared.runtimePolicy
+      ? join(prepared.invocation.projectRoot, "sevro.json")
+      : null)
+  );
+}
+
+export function displayEffectiveSettings(prepared: PreparedInvocation): void {
+  process.stderr.write(
+    `sevro: effective settings ${JSON.stringify(effectiveSettings(prepared))}\n`,
+  );
+}

package/src/cli/summary.ts

@@ -0,0 +1,43 @@
+import type { CliResultData, ReportData } from "../schema-types";
+
+function line(value: string): string {
+  return value.replaceAll(/\p{Cc}/gu, " ").slice(0, 4096);
+}
+
+function metric(value: number | null): string {
+  return value === null ? "unknown" : String(value);
+}
+
+export function displayTrialSummary(result: CliResultData): void {
+  for (const selected of result.cases)
+    for (const trial of selected.trials) {
+      process.stdout.write(
+        `trial case=${line(selected.caseId)} trial=${trial.trial} execution=${trial.execution.status} grading=${trial.grading.status} task=${trial.task.verdict}\n`,
+      );
+      displayFailedChecks(trial.checks);
+    }
+}
+
+function displayFailedChecks(
+  checks: CliResultData["cases"][number]["trials"][number]["checks"],
+): void {
+  for (const check of checks)
+    if (check.status !== "passed")
+      process.stdout.write(
+        `  check=${line(check.id)} status=${check.status} reason=${line(check.detail ?? "unknown")}\n`,
+      );
+}
+
+export function displayMeasurements(report: ReportData): void {
+  for (const row of report.rows) {
+    const duration =
+      row.candidateDurationMs === null
+        ? "unknown"
+        : `${row.candidateDurationMs.toFixed(2)}ms`;
+    const cost =
+      row.costUsd === null ? "unknown" : `$${row.costUsd.toFixed(4)}`;
+    process.stdout.write(
+      `case=${line(row.caseId)} candidate duration=${duration} candidate tokens=${metric(row.inputTokens)}/${metric(row.outputTokens)} cost=${cost}\n`,
+    );
+  }
+}