@bjoernrochel/sevro 0.1.0-rc.8 → 1.1.0

package/src/evaluation/resources.ts

@@ -0,0 +1,241 @@
+import { randomUUID } from "node:crypto";
+import { mkdir, realpath, rm, writeFile } from "node:fs/promises";
+import { dirname, join, resolve } from "node:path";
+import { pathToFileURL } from "node:url";
+import {
+  allocateWorkspace,
+  retainWorkspace,
+  removeWorkspace,
+} from "../fixtures/workspace-state";
+import { createFixture } from "../fixtures/create";
+import { fixtureBinDirectory } from "../fixtures/bin";
+import { fixtureParts } from "../fixtures/preparation";
+import { startRunOwner, type RunOwner } from "../run-owner";
+import { EvaluationConfigurationError, type EvaluationOptions } from "./types";
+import type { PreparedFixture, PreparedInvocation } from "./plan";
+import {
+  INVOCATION_PLACEHOLDER,
+  LEGACY_CODEX_PLACEHOLDER,
+} from "./invocation-plan";
+import type { ArtifactReference as PreparationArtifactReference } from "../contract-types";
+export type { ArtifactReference as PreparationArtifactReference } from "../contract-types";
+
+/** Reservations outlive completed contents until all admitted trials drain. */
+export class RunWorkspaceReservations {
+  private readonly candidates: string[] = [];
+  private readonly semantics: string[] = [];
+  private readonly retained = new Set<string>();
+
+  async reserve(input: {
+    count: number;
+    semantic: boolean;
+    signal?: AbortSignal;
+  }): Promise<void> {
+    for (
+      let trial = 1;
+      trial <= input.count && !input.signal?.aborted;
+      trial++
+    ) {
+      this.candidates.push(await allocateWorkspace());
+      if (input.semantic) this.semantics.push(await allocateWorkspace());
+    }
+  }
+
+  candidate(trial: number): string {
+    const reservation = this.candidates[trial - 1];
+    if (reservation === undefined)
+      throw new Error(`trial ${trial} workspace was not reserved`);
+    return reservation;
+  }
+
+  semantic(trial: number): string | undefined {
+    return this.semantics[trial - 1];
+  }
+
+  retain(workspace: string): void {
+    this.retained.add(workspace);
+  }
+
+  completed(reservation: string): void {
+    this.retained.delete(reservation);
+  }
+
+  async cleanup(): Promise<void> {
+    for (const workspace of this.retained) await retainWorkspace(workspace);
+    await Promise.all(
+      [...this.candidates, ...this.semantics]
+        .filter((workspace) => !this.retained.has(workspace))
+        .map(async (workspace) => {
+          try {
+            await removeWorkspace(workspace);
+          } catch {
+            console.warn(`fixture cleanup failed; retained at ${workspace}`);
+          }
+        }),
+    );
+  }
+}
+
+export interface RunResources {
+  readonly runId: string;
+  readonly runDir: string;
+  readonly runStateRoot: string;
+  readonly stateDir: string;
+  readonly evidencePath: string;
+  readonly owner: RunOwner;
+  readonly workspaces: RunWorkspaceReservations;
+}
+
+export async function allocateRunResources(
+  options: Pick<EvaluationOptions, "resultsRoot" | "runStateRoot">,
+  evaluationDigest: string,
+): Promise<RunResources> {
+  const runId = randomUUID();
+  await mkdir(options.resultsRoot, { recursive: true, mode: 0o700 });
+  const runDir = resolve(await realpath(options.resultsRoot), runId);
+  await mkdir(runDir, { mode: 0o700 });
+  const runStateRoot = options.runStateRoot ?? options.resultsRoot;
+  await mkdir(runStateRoot, { recursive: true, mode: 0o700 });
+  const stateRoot = await realpath(runStateRoot);
+  const stateDir = join(stateRoot, runId);
+  if (stateDir !== runDir) await mkdir(stateDir, { mode: 0o700 });
+  const activePath = join(stateRoot, "active", `${runId}.json`);
+  const checkpointPath = join(stateDir, "checkpoint.json");
+  const evidencePath = join(runDir, "run.json");
+  let owner: RunOwner;
+  try {
+    owner = startRunOwner({
+      stateRoot,
+      evaluationDigest,
+      attemptId: runId,
+      activeRunPath: activePath,
+      checkpointPath,
+      evidenceDirectory: stateDir,
+      artifactPath: evidencePath,
+    });
+  } catch (error) {
+    await rm(runDir, { recursive: true, force: true });
+    if (stateDir !== runDir)
+      await rm(stateDir, { recursive: true, force: true });
+    throw error;
+  }
+  return {
+    runId,
+    runDir,
+    runStateRoot,
+    stateDir,
+    evidencePath,
+    owner,
+    workspaces: new RunWorkspaceReservations(),
+  };
+}
+
+export async function retainPreparationArtifacts(
+  artifacts: PreparedFixture["inlineArtifacts"],
+  runDir: string,
+): Promise<PreparationArtifactReference[]> {
+  const artifactRefs: PreparationArtifactReference[] = [];
+  for (const artifact of artifacts) {
+    const retainedPath = join(
+      runDir,
+      "prepared",
+      ...fixtureParts(artifact.relativePath),
+    );
+    await mkdir(dirname(retainedPath), { recursive: true, mode: 0o700 });
+    await writeFile(retainedPath, artifact.bytes, {
+      flag: "wx",
+      mode: artifact.executable ? 0o700 : 0o600,
+    });
+    artifactRefs.push({
+      id: artifact.id,
+      path: pathToFileURL(retainedPath).href,
+      sha256: artifact.sha256,
+      ...(artifact.gitExclude ? { gitExclude: true } : {}),
+      ...(artifact.executable ? { executable: true } : {}),
+    });
+  }
+  return artifactRefs;
+}
+
+function renderTrialPrompt(
+  prompt: string,
+  workspace: string,
+  invocation: PreparedInvocation,
+): string {
+  return prompt
+    .replaceAll("{{sevro.workspace}}", workspace)
+    .replaceAll(
+      INVOCATION_PLACEHOLDER,
+      invocation.invocationToken ?? INVOCATION_PLACEHOLDER,
+    )
+    .replaceAll(
+      LEGACY_CODEX_PLACEHOLDER,
+      invocation.invocationToken ?? LEGACY_CODEX_PLACEHOLDER,
+    );
+}
+
+type TrialPreparationOptions = Pick<
+  EvaluationOptions,
+  "case" | "preparationSources" | "signal"
+>;
+
+export async function prepareTrialWorkspace(
+  options: TrialPreparationOptions,
+  fixture: PreparedFixture,
+  invocation: PreparedInvocation,
+  workspaces: RunWorkspaceReservations,
+  trial: number,
+  candidateRuntimeRoot?: string,
+) {
+  const reservation = workspaces.candidate(trial);
+  workspaces.retain(reservation);
+  const workspace = await createFixture(fixture.source, {
+    artifacts: fixture.inlineArtifacts,
+    sources: options.preparationSources,
+    setup: fixture.fixtureSetup,
+    projectRoot: fixture.projectRoot,
+    signal: options.signal,
+    reservedWorkspace: reservation,
+    candidateRuntimeRoot,
+  }).catch((error: unknown) => {
+    if (
+      !options.signal?.aborted ||
+      error instanceof EvaluationConfigurationError
+    )
+      throw error;
+    return null;
+  });
+  const details = await trialWorkspaceDetails(
+    options.case,
+    invocation,
+    reservation,
+    workspace,
+  );
+  return { ...details, candidateRuntimeRoot };
+}
+
+async function trialWorkspaceDetails(
+  caseData: EvaluationOptions["case"],
+  invocation: PreparedInvocation,
+  reservation: string,
+  preparedWorkspace: string | null,
+) {
+  const workspace = preparedWorkspace ?? reservation;
+  const fixtureBinDir =
+    preparedWorkspace === null
+      ? undefined
+      : ((await fixtureBinDirectory(workspace)) ?? undefined);
+  const trialPrompt = renderTrialPrompt(caseData.prompt, workspace, invocation);
+  const trialFollowUpPrompt =
+    caseData.followUpPrompt === undefined
+      ? undefined
+      : renderTrialPrompt(caseData.followUpPrompt, workspace, invocation);
+  return {
+    reservation,
+    workspace,
+    preparationCancelled: preparedWorkspace === null,
+    fixtureBinDir,
+    trialPrompt,
+    trialFollowUpPrompt,
+  };
+}

package/src/evaluation/run.ts

@@ -0,0 +1,206 @@
+import { prepareEvaluation, type PreparedEvaluation } from "./plan";
+import { type CliResult, type EvaluationOptions } from "./types";
+import { canonicalJson, hashJson } from "../identity";
+import { exitCodeFor, summarizeCases } from "../results";
+import { runtimePolicySnapshot } from "../runtime/config";
+import { cleanupExpiredWorkspaces } from "../fixtures/workspace-state";
+import { assertCliResult, assertRunEvidence } from "../schema";
+import { atomicWriteJson } from "../storage";
+import {
+  allocateRunResources,
+  retainPreparationArtifacts,
+  type RunResources,
+} from "./resources";
+import type { EvaluationContext } from "./context";
+import { ResultCollector, type CollectedResults } from "./collection";
+import { executeEvaluationTrials } from "./trial";
+import { evaluationProgress } from "./progress";
+
+function evaluationSignal(
+  options: EvaluationOptions,
+): "SIGINT" | "SIGTERM" | undefined {
+  const reason: unknown = options.signal?.aborted
+    ? options.signal.reason
+    : null;
+  return reason === "SIGINT" || reason === "SIGTERM" ? reason : undefined;
+}
+
+function finalCliResult(
+  options: EvaluationOptions,
+  resources: RunResources,
+  collected: CollectedResults,
+): CliResult {
+  const caseResult = collected.caseSummary;
+  const assessment = summarizeCases([caseResult]);
+  const diagnostic = collected.diagnostic;
+  return {
+    format: "sevro.cli-result.v1",
+    runId: resources.runId,
+    ...assessment,
+    exitCode: exitCodeFor(assessment, {
+      dryRun: options.dry,
+      signal: evaluationSignal(options),
+    }),
+    evidencePath: resources.evidencePath,
+    cases: [caseResult],
+    ...(diagnostic ? { diagnostic } : {}),
+  };
+}
+
+function finalConditionEvidence(
+  options: Pick<EvaluationOptions, "condition" | "trialCount">,
+  requestedInstrumentation: PreparedEvaluation["requestedInstrumentation"],
+  trials: CollectedResults["trials"],
+) {
+  const actualConditions = [
+    ...new Set(trials.map((entry) => entry.condition.actual)),
+  ];
+  const commonAppliedInstrumentation =
+    trials.length === options.trialCount &&
+    trials.every(
+      (entry) =>
+        canonicalJson(entry.condition.appliedInstrumentation) ===
+        canonicalJson(requestedInstrumentation),
+    )
+      ? requestedInstrumentation
+      : [];
+  return {
+    requested: options.condition,
+    actual: actualConditions.length === 1 ? actualConditions[0] : "unknown",
+    requestedInstrumentation,
+    appliedInstrumentation: commonAppliedInstrumentation,
+  };
+}
+
+function finalExtensionEvidence(
+  options: EvaluationOptions,
+  replacedBuiltinGraders: string[],
+) {
+  const extension = options.extension;
+  if (!extension) return null;
+  const identity = extension.session.identity;
+  return {
+    id: identity.id,
+    version: identity.version,
+    sourceDigest: identity.sourceDigest,
+    configurationDigest: identity.configurationDigest,
+    protocol: identity.protocol,
+    capabilities: identity.capabilities,
+    replacements: {
+      graders: replacedBuiltinGraders,
+      taskVerdictPolicy: identity.selectedTaskVerdictPolicy,
+    },
+  };
+}
+
+function finalRunEvidence(
+  options: EvaluationOptions,
+  prepared: PreparedEvaluation,
+  resources: RunResources,
+  collected: CollectedResults,
+  result: CliResult,
+) {
+  const condition = finalConditionEvidence(
+    options,
+    prepared.requestedInstrumentation,
+    collected.trials,
+  );
+  const diagnostic = collected.diagnostic;
+  return {
+    format: "sevro.run-evidence.v1",
+    runId: resources.runId,
+    evaluationIdentity: prepared.evaluationIdentity,
+    configuration: {
+      digest: hashJson(prepared.redactedConfig),
+      redacted: prepared.redactedConfig,
+      ...(options.resolvedRunSettings
+        ? { resolved: options.resolvedRunSettings }
+        : {}),
+    },
+    runner: prepared.runner,
+    project: prepared.project,
+    extension: finalExtensionEvidence(
+      options,
+      prepared.checks.replacedBuiltinGraders,
+    ),
+    condition,
+    graders: {
+      active: prepared.activeGraders,
+      replacedDefaults: prepared.checks.replacedBuiltinGraders,
+    },
+    routes: prepared.routes,
+    result,
+    trials: collected.trials,
+    ...(diagnostic ? { diagnostic } : {}),
+  };
+}
+
+async function finishEvaluationRun(
+  options: EvaluationOptions,
+  context: EvaluationContext,
+) {
+  const collected = context.results.snapshot({
+    caseId: options.case.id,
+    passThreshold: options.passThreshold,
+    cancelled: Boolean(options.signal?.aborted),
+  });
+  const result = finalCliResult(options, context.resources, collected);
+  const evidence = finalRunEvidence(
+    options,
+    context.prepared,
+    context.resources,
+    collected,
+    result,
+  );
+  assertCliResult(result);
+  assertRunEvidence(evidence);
+  evaluationProgress(options, "evidence", "started");
+  await atomicWriteJson(context.resources.evidencePath, evidence);
+  context.results.checkpoint(
+    options.signal?.aborted ? "interrupted" : "complete",
+  );
+  evaluationProgress(options, "evidence", "completed");
+  return { result };
+}
+
+/** Execute resolved evaluator data through an injected host and retain every trial. */
+export async function runEvaluation(
+  options: EvaluationOptions,
+): Promise<{ result: CliResult }> {
+  options = {
+    ...options,
+    runtimePolicy: runtimePolicySnapshot(options.runtimePolicy),
+  };
+  evaluationProgress(options, "preparation", "started");
+  await cleanupExpiredWorkspaces();
+  const prepared = await prepareEvaluation(options);
+  const resources = await allocateRunResources(
+    options,
+    prepared.evaluationIdentity.digest,
+  );
+  const results = new ResultCollector(resources.owner);
+  try {
+    const preparationArtifacts = await retainPreparationArtifacts(
+      prepared.fixture.inlineArtifacts,
+      resources.runDir,
+    );
+    await resources.workspaces.reserve({
+      count: options.trialCount,
+      semantic: prepared.checks.preparedSemantic.length > 0,
+      signal: options.signal,
+    });
+    const context: EvaluationContext = {
+      prepared,
+      resources,
+      results,
+      preparationArtifacts,
+    };
+    await executeEvaluationTrials(options, context);
+    return await finishEvaluationRun(options, context);
+  } catch (error) {
+    results.diagnoseFailure();
+    throw error;
+  } finally {
+    await resources.workspaces.cleanup();
+  }
+}

package/src/{evaluation-semantic.ts → evaluation/semantic.ts}

@@ -1,56 +1,52 @@
-import { clearFixtureContents } from "./fixture-cleanup";
-import type { CandidateArtifactRoots } from "./candidate-artifacts";
+import { clearFixtureContents } from "../fixtures/cleanup";
+import type { CandidateArtifactRoots } from "../candidate-artifacts";
+import { createFixture } from "../fixtures/create";
+import { hostArtifacts } from "../evidence/host-results";
+import { usage } from "../evidence/usage";
+import { gradingRuntimePolicy, scopedHostArtifacts } from "./grading-host";
 import {
-  CodexProcessFailure,
-  codexFailureMessage,
-} from "./hosts/codex-process";
-import { join } from "node:path";
-import { pathToFileURL } from "node:url";
-import { createFixture, hostArtifacts, usage } from "./evaluation-fixture";
-import {
-  gradingRuntimePolicy,
-  scopedHostArtifacts,
-} from "./evaluation-grading-host";
-import type { HostAdapter, HostResult, ResolvedCase } from "./evaluation-types";
-import { readSemanticArtifact } from "./graders/artifact";
+  HostExecutionFailure,
+  type HostAdapter,
+  type HostResult,
+} from "./types";
+import { readSemanticArtifact } from "../graders/artifact";
 import {
   parseSemanticVerdicts,
   semanticCheckGroups,
   semanticPrompt,
   type PreparedSemanticCheck,
   type SemanticVerdict,
-} from "./graders/semantic";
-import { sha256 } from "./identity";
-import type { CheckOutcome } from "./results";
-import type { RuntimePolicy } from "./runtime-config";
+} from "../graders/semantic";
+import { sha256 } from "../identity";
+import type { CheckOutcome } from "../results";
+import type {
+  TrialEvidenceWriter,
+  TrialObservation,
+} from "../evidence/trial-writer";
+import type { GradingFailure, GradingResult } from "./grading";
+import type { RuntimePolicy } from "../runtime/config";
+import { HostDiagnosticPersistenceError } from "../host-diagnostics";
 
-interface SemanticObservation {
-  id: string;
-  source: string;
-  completeness: "complete" | "partial";
-  data: Record<string, unknown>;
-}
 interface SemanticGradingInput {
-  checks: PreparedSemanticCheck[];
-  declaredChecks: ResolvedCase["checks"];
-  host: HostAdapter | undefined;
-  message: string | null;
-  candidateWorkspace: string;
-  candidateTranscriptRoot?: string;
-  candidateArtifactRoots?: CandidateArtifactRoots;
-  projectRoot: string;
-  reservedWorkspace?: string;
-  runDir: string;
-  trial: number;
-  runtimePolicy?: RuntimePolicy;
-  signal?: AbortSignal;
+  readonly checks: readonly PreparedSemanticCheck[];
+  readonly declaredChecks: readonly { id: string }[];
+  readonly host: HostAdapter | undefined;
+  readonly message: string | null;
+  readonly candidateWorkspace: string;
+  readonly candidateTranscriptRoot?: string;
+  readonly candidateArtifactRoots?: CandidateArtifactRoots;
+  readonly projectRoot: string;
+  readonly reservedWorkspace?: string;
+  readonly runtimePolicy?: RuntimePolicy;
+  readonly signal?: AbortSignal;
 }
 interface SemanticGradingEffects {
-  artifactRefs: { id: string; path: string; sha256: string }[];
-  canGrade(): boolean;
-  failGrading(message: string, cancelable?: boolean): void;
-  retainArtifacts(artifacts: ReturnType<typeof hostArtifacts>): Promise<void>;
-  writeTrialFile(path: string, bytes: string | Uint8Array): Promise<void>;
+  readonly evidence: Pick<
+    TrialEvidenceWriter,
+    "artifactRefs" | "retainArtifacts" | "retainSemanticVerdicts"
+  >;
+  /** Close admission at detection, before retaining or cleaning auxiliary state. */
+  onFailure(failure: GradingFailure): void;
   retainWorkspace(workspace: string): Promise<void>;
 }
 type SemanticGroup = ReturnType<typeof semanticCheckGroups>[number];
@@ -101,16 +97,25 @@ function semanticHostPrompt(
   );
 }
 class SemanticGrading {
-  private observations: SemanticObservation[] = [];
+  private observations: TrialObservation[] = [];
   private checks: CheckOutcome[] = [];
+  private failure: GradingFailure | null = null;
   constructor(
     private readonly input: SemanticGradingInput,
     private readonly effects: SemanticGradingEffects,
   ) {}
-  async run() {
-    for (const group of semanticCheckGroups(this.input.checks))
+  async run(): Promise<GradingResult> {
+    for (const group of semanticCheckGroups([...this.input.checks]))
       await this.gradeSemanticGroup(group);
-    return { checks: this.checks, observations: this.observations };
+    return {
+      checks: this.checks,
+      observations: this.observations,
+      failure: this.failure,
+    };
+  }
+  private failGrading(message: string, cancelable = true): void {
+    this.failure = { message, cancelable };
+    this.effects.onFailure(this.failure);
   }
   private async semanticInput(
     pattern: string | undefined,
@@ -122,16 +127,12 @@ class SemanticGrading {
 
   private async semanticWorkspace(): Promise<string> {
     return createFixture(
-      { files: {} },
-      [],
-      undefined,
-      null,
-      null,
-      null,
-      null,
-      this.input.projectRoot,
-      this.input.signal,
-      this.input.reservedWorkspace,
+      { kind: "inline", files: {} },
+      {
+        projectRoot: this.input.projectRoot,
+        signal: this.input.signal,
+        reservedWorkspace: this.input.reservedWorkspace,
+      },
     );
   }
 
@@ -150,6 +151,13 @@ class SemanticGrading {
       if (!host) throw new Error("semantic host is unavailable");
       const response = await host
         .run({
+          retainHostDiagnostics: (bytes) =>
+            this.effects.evidence.retainArtifacts([
+              {
+                id: `sevro.semantic${suffix}.sevro.host.diagnostics`,
+                bytes: Buffer.from(bytes),
+              },
+            ]),
           candidateTranscriptRoot: this.input.candidateTranscriptRoot,
           candidateArtifactRoots: this.input.candidateArtifactRoots,
           prompt: semanticHostPrompt(message, group, artifact),
@@ -163,45 +171,24 @@ class SemanticGrading {
           signal: this.input.signal,
         })
         .catch((cause: unknown) => {
-          if (cause instanceof CodexProcessFailure) return cause.result;
+          if (cause instanceof HostExecutionFailure) return cause.result;
           throw cause;
         });
       const artifacts = scopedHostArtifacts(
         response,
         `sevro.semantic${suffix}.`,
-        this.effects.artifactRefs,
+        this.effects.evidence.artifactRefs,
         this.input.declaredChecks,
         `sevro.semantic.verdicts${suffix}`,
       );
       return { result: response, artifacts, source, hostId: host.id };
-    } catch {
-      this.effects.failGrading("semantic grading did not complete");
+    } catch (cause) {
+      if (cause instanceof HostDiagnosticPersistenceError) throw cause;
+      this.failGrading("semantic grading did not complete");
       return { result: null, artifacts: [], source, hostId: null };
     }
   }
 
-  private async retainSemanticVerdicts(
-    result: HostResult,
-    suffix: string,
-  ): Promise<void> {
-    if (
-      result.finalMessage !== null &&
-      Buffer.byteLength(result.finalMessage, "utf8") <= 1024 * 1024
-    ) {
-      const bytes = Buffer.from(result.finalMessage, "utf8");
-      const path = join(
-        this.input.runDir,
-        `trial-${this.input.trial}-semantic-verdicts${suffix}.json`,
-      );
-      await this.effects.writeTrialFile(path, bytes);
-      this.effects.artifactRefs.push({
-        id: `sevro.semantic.verdicts${suffix}`,
-        path: pathToFileURL(path).href,
-        sha256: sha256(bytes),
-      });
-    }
-  }
-
   private recordSemanticVerdict(
     verdict: SemanticVerdict,
     pattern: string | undefined,
@@ -242,8 +229,8 @@ class SemanticGrading {
         this.recordSemanticVerdict(verdict, pattern, source, hostId);
       return true;
     } catch {
-      this.effects.failGrading(
-        codexFailureMessage(result) ?? "semantic grading did not complete",
+      this.failGrading(
+        result.failureMessage ?? "semantic grading did not complete",
         result.processFailure?.reason === "cancelled",
       );
       return false;
@@ -259,7 +246,7 @@ class SemanticGrading {
   private async gradeSemanticGroup(group: SemanticGroup): Promise<void> {
     const pattern = group[0].artifactPath;
     const suffix = semanticGroupSuffix(pattern);
-    if (!this.effects.canGrade()) return;
+    if (this.failure) return;
     const message = this.input.message;
     if (message === null) {
       this.checks.push(
@@ -283,14 +270,17 @@ class SemanticGrading {
         message,
       );
       if (outcome.result === null) return;
-      await this.effects.retainArtifacts(outcome.artifacts);
+      await this.effects.evidence.retainArtifacts(outcome.artifacts);
       this.observations.push({
         id: `sevro.observation.semantic.usage${suffix}`,
         source: outcome.hostId,
         completeness: outcome.result.usageComplete ? "complete" : "partial",
         data: usage(outcome.result),
       });
-      await this.retainSemanticVerdicts(outcome.result, suffix);
+      await this.effects.evidence.retainSemanticVerdicts(
+        outcome.result.finalMessage,
+        suffix,
+      );
       successful = this.parseSemanticResults(
         outcome.result,
         group,