@bjoernrochel/sevro 0.1.0-rc.2 → 0.1.0-rc.21

package/src/evaluation/progress.ts

@@ -0,0 +1,19 @@
+import type { EvaluationOptions, EvaluationProgress } from "./types";
+
+/** Observation is best-effort and cannot change assessment or persistence. */
+export function evaluationProgress(
+  options: EvaluationOptions,
+  stage: EvaluationProgress["stage"],
+  status: EvaluationProgress["status"],
+  trial?: number,
+): void {
+  try {
+    options.onProgress?.({
+      stage,
+      status,
+      ...(trial === undefined ? {} : { trial }),
+    });
+  } catch {
+    // An observer has no authority over the evaluation lifecycle.
+  }
+}

package/src/evaluation/resources.ts

@@ -0,0 +1,241 @@
+import { randomUUID } from "node:crypto";
+import { mkdir, realpath, rm, writeFile } from "node:fs/promises";
+import { dirname, join, resolve } from "node:path";
+import { pathToFileURL } from "node:url";
+import {
+  allocateWorkspace,
+  retainWorkspace,
+  removeWorkspace,
+} from "../fixtures/workspace-state";
+import { createFixture } from "../fixtures/create";
+import { fixtureBinDirectory } from "../fixtures/bin";
+import { fixtureParts } from "../fixtures/preparation";
+import { startRunOwner, type RunOwner } from "../run-owner";
+import { EvaluationConfigurationError, type EvaluationOptions } from "./types";
+import type { PreparedFixture, PreparedInvocation } from "./plan";
+import type { ArtifactReference as PreparationArtifactReference } from "../contract-types";
+export type { ArtifactReference as PreparationArtifactReference } from "../contract-types";
+
+/** Reservations outlive completed contents until all admitted trials drain. */
+export class RunWorkspaceReservations {
+  private readonly candidates: string[] = [];
+  private readonly semantics: string[] = [];
+  private readonly retained = new Set<string>();
+
+  async reserve(input: {
+    count: number;
+    semantic: boolean;
+    signal?: AbortSignal;
+  }): Promise<void> {
+    for (
+      let trial = 1;
+      trial <= input.count && !input.signal?.aborted;
+      trial++
+    ) {
+      this.candidates.push(await allocateWorkspace());
+      if (input.semantic) this.semantics.push(await allocateWorkspace());
+    }
+  }
+
+  candidate(trial: number): string {
+    const reservation = this.candidates[trial - 1];
+    if (reservation === undefined)
+      throw new Error(`trial ${trial} workspace was not reserved`);
+    return reservation;
+  }
+
+  semantic(trial: number): string | undefined {
+    return this.semantics[trial - 1];
+  }
+
+  retain(workspace: string): void {
+    this.retained.add(workspace);
+  }
+
+  completed(reservation: string): void {
+    this.retained.delete(reservation);
+  }
+
+  async cleanup(): Promise<void> {
+    for (const workspace of this.retained) await retainWorkspace(workspace);
+    await Promise.all(
+      [...this.candidates, ...this.semantics]
+        .filter((workspace) => !this.retained.has(workspace))
+        .map(async (workspace) => {
+          try {
+            await removeWorkspace(workspace);
+          } catch {
+            console.warn(`fixture cleanup failed; retained at ${workspace}`);
+          }
+        }),
+    );
+  }
+}
+
+export interface RunResources {
+  readonly runId: string;
+  readonly runDir: string;
+  readonly runStateRoot: string;
+  readonly stateDir: string;
+  readonly evidencePath: string;
+  readonly owner: RunOwner;
+  readonly workspaces: RunWorkspaceReservations;
+}
+
+export async function allocateRunResources(
+  options: Pick<EvaluationOptions, "resultsRoot" | "runStateRoot">,
+  evaluationDigest: string,
+): Promise<RunResources> {
+  const runId = randomUUID();
+  await mkdir(options.resultsRoot, { recursive: true, mode: 0o700 });
+  const runDir = resolve(await realpath(options.resultsRoot), runId);
+  await mkdir(runDir, { mode: 0o700 });
+  const runStateRoot = options.runStateRoot ?? options.resultsRoot;
+  await mkdir(runStateRoot, { recursive: true, mode: 0o700 });
+  const stateRoot = await realpath(runStateRoot);
+  const stateDir = join(stateRoot, runId);
+  if (stateDir !== runDir) await mkdir(stateDir, { mode: 0o700 });
+  const activePath = join(stateRoot, "active", `${runId}.json`);
+  const checkpointPath = join(stateDir, "checkpoint.json");
+  const evidencePath = join(runDir, "run.json");
+  let owner: RunOwner;
+  try {
+    owner = startRunOwner({
+      stateRoot,
+      evaluationDigest,
+      attemptId: runId,
+      activeRunPath: activePath,
+      checkpointPath,
+      evidenceDirectory: stateDir,
+      artifactPath: evidencePath,
+    });
+  } catch (error) {
+    await rm(runDir, { recursive: true, force: true });
+    if (stateDir !== runDir)
+      await rm(stateDir, { recursive: true, force: true });
+    throw error;
+  }
+  return {
+    runId,
+    runDir,
+    runStateRoot,
+    stateDir,
+    evidencePath,
+    owner,
+    workspaces: new RunWorkspaceReservations(),
+  };
+}
+
+export async function retainPreparationArtifacts(
+  artifacts: PreparedFixture["inlineArtifacts"],
+  runDir: string,
+): Promise<PreparationArtifactReference[]> {
+  const artifactRefs: PreparationArtifactReference[] = [];
+  for (const artifact of artifacts) {
+    const retainedPath = join(
+      runDir,
+      "prepared",
+      ...fixtureParts(artifact.relativePath),
+    );
+    await mkdir(dirname(retainedPath), { recursive: true, mode: 0o700 });
+    await writeFile(retainedPath, artifact.bytes, {
+      flag: "wx",
+      mode: artifact.executable ? 0o700 : 0o600,
+    });
+    artifactRefs.push({
+      id: artifact.id,
+      path: pathToFileURL(retainedPath).href,
+      sha256: artifact.sha256,
+      ...(artifact.gitExclude ? { gitExclude: true } : {}),
+      ...(artifact.executable ? { executable: true } : {}),
+    });
+  }
+  return artifactRefs;
+}
+
+function renderTrialPrompt(
+  prompt: string,
+  workspace: string,
+  invocation: PreparedInvocation,
+): string {
+  return prompt
+    .replaceAll("{{sevro.workspace}}", workspace)
+    .replaceAll(
+      invocation.invocationPlaceholder,
+      invocation.invocationToken ?? invocation.invocationPlaceholder,
+    )
+    .replaceAll(
+      invocation.legacyCodexPlaceholder,
+      invocation.invocationToken ?? invocation.legacyCodexPlaceholder,
+    );
+}
+
+type TrialPreparationOptions = Pick<
+  EvaluationOptions,
+  "case" | "preparationSources" | "signal"
+>;
+
+export async function prepareTrialWorkspace(
+  options: TrialPreparationOptions,
+  fixture: PreparedFixture,
+  invocation: PreparedInvocation,
+  workspaces: RunWorkspaceReservations,
+  trial: number,
+  candidateRuntimeRoot?: string,
+) {
+  const reservation = workspaces.candidate(trial);
+  workspaces.retain(reservation);
+  const workspace = await createFixture(
+    options.case.fixture,
+    fixture.inlineArtifacts,
+    options.preparationSources,
+    fixture.repository,
+    fixture.repositoryFixture,
+    fixture.generated,
+    fixture.fixtureSetup,
+    fixture.projectRoot,
+    options.signal,
+    reservation,
+    candidateRuntimeRoot,
+  ).catch((error: unknown) => {
+    if (
+      !options.signal?.aborted ||
+      error instanceof EvaluationConfigurationError
+    )
+      throw error;
+    return null;
+  });
+  const details = await trialWorkspaceDetails(
+    options.case,
+    invocation,
+    reservation,
+    workspace,
+  );
+  return { ...details, candidateRuntimeRoot };
+}
+
+async function trialWorkspaceDetails(
+  caseData: EvaluationOptions["case"],
+  invocation: PreparedInvocation,
+  reservation: string,
+  preparedWorkspace: string | null,
+) {
+  const workspace = preparedWorkspace ?? reservation;
+  const fixtureBinDir =
+    preparedWorkspace === null
+      ? undefined
+      : ((await fixtureBinDirectory(workspace)) ?? undefined);
+  const trialPrompt = renderTrialPrompt(caseData.prompt, workspace, invocation);
+  const trialFollowUpPrompt =
+    caseData.followUpPrompt === undefined
+      ? undefined
+      : renderTrialPrompt(caseData.followUpPrompt, workspace, invocation);
+  return {
+    reservation,
+    workspace,
+    preparationCancelled: preparedWorkspace === null,
+    fixtureBinDir,
+    trialPrompt,
+    trialFollowUpPrompt,
+  };
+}

package/src/evaluation/run.ts

@@ -0,0 +1,206 @@
+import { prepareEvaluation, type PreparedEvaluation } from "./plan";
+import { type CliResult, type EvaluationOptions } from "./types";
+import { canonicalJson, hashJson } from "../identity";
+import { exitCodeFor, summarizeCases } from "../results";
+import { runtimePolicySnapshot } from "../runtime/config";
+import { cleanupExpiredWorkspaces } from "../fixtures/workspace-state";
+import { assertCliResult, assertRunEvidence } from "../schema";
+import { atomicWriteJson } from "../storage";
+import {
+  allocateRunResources,
+  retainPreparationArtifacts,
+  type RunResources,
+} from "./resources";
+import type { EvaluationContext } from "./context";
+import { ResultCollector, type CollectedResults } from "./collection";
+import { executeEvaluationTrials } from "./trial";
+import { evaluationProgress } from "./progress";
+
+function evaluationSignal(
+  options: EvaluationOptions,
+): "SIGINT" | "SIGTERM" | undefined {
+  const reason: unknown = options.signal?.aborted
+    ? options.signal.reason
+    : null;
+  return reason === "SIGINT" || reason === "SIGTERM" ? reason : undefined;
+}
+
+function finalCliResult(
+  options: EvaluationOptions,
+  resources: RunResources,
+  collected: CollectedResults,
+): CliResult {
+  const caseResult = collected.caseSummary;
+  const assessment = summarizeCases([caseResult]);
+  const diagnostic = collected.diagnostic;
+  return {
+    format: "sevro.cli-result.v1",
+    runId: resources.runId,
+    ...assessment,
+    exitCode: exitCodeFor(assessment, {
+      dryRun: options.dry,
+      signal: evaluationSignal(options),
+    }),
+    evidencePath: resources.evidencePath,
+    cases: [caseResult],
+    ...(diagnostic ? { diagnostic } : {}),
+  };
+}
+
+function finalConditionEvidence(
+  options: Pick<EvaluationOptions, "condition" | "trialCount">,
+  requestedInstrumentation: PreparedEvaluation["requestedInstrumentation"],
+  trials: CollectedResults["trials"],
+) {
+  const actualConditions = [
+    ...new Set(trials.map((entry) => entry.condition.actual)),
+  ];
+  const commonAppliedInstrumentation =
+    trials.length === options.trialCount &&
+    trials.every(
+      (entry) =>
+        canonicalJson(entry.condition.appliedInstrumentation) ===
+        canonicalJson(requestedInstrumentation),
+    )
+      ? requestedInstrumentation
+      : [];
+  return {
+    requested: options.condition,
+    actual: actualConditions.length === 1 ? actualConditions[0] : "unknown",
+    requestedInstrumentation,
+    appliedInstrumentation: commonAppliedInstrumentation,
+  };
+}
+
+function finalExtensionEvidence(
+  options: EvaluationOptions,
+  replacedBuiltinGraders: string[],
+) {
+  const extension = options.extension;
+  if (!extension) return null;
+  const identity = extension.session.identity;
+  return {
+    id: identity.id,
+    version: identity.version,
+    sourceDigest: identity.sourceDigest,
+    configurationDigest: identity.configurationDigest,
+    protocol: identity.protocol,
+    capabilities: identity.capabilities,
+    replacements: {
+      graders: replacedBuiltinGraders,
+      taskVerdictPolicy: identity.selectedTaskVerdictPolicy,
+    },
+  };
+}
+
+function finalRunEvidence(
+  options: EvaluationOptions,
+  prepared: PreparedEvaluation,
+  resources: RunResources,
+  collected: CollectedResults,
+  result: CliResult,
+) {
+  const condition = finalConditionEvidence(
+    options,
+    prepared.requestedInstrumentation,
+    collected.trials,
+  );
+  const diagnostic = collected.diagnostic;
+  return {
+    format: "sevro.run-evidence.v1",
+    runId: resources.runId,
+    evaluationIdentity: prepared.evaluationIdentity,
+    configuration: {
+      digest: hashJson(prepared.redactedConfig),
+      redacted: prepared.redactedConfig,
+      ...(options.resolvedRunSettings
+        ? { resolved: options.resolvedRunSettings }
+        : {}),
+    },
+    runner: prepared.runner,
+    project: prepared.project,
+    extension: finalExtensionEvidence(
+      options,
+      prepared.checks.replacedBuiltinGraders,
+    ),
+    condition,
+    graders: {
+      active: prepared.activeGraders,
+      replacedDefaults: prepared.checks.replacedBuiltinGraders,
+    },
+    routes: prepared.routes,
+    result,
+    trials: collected.trials,
+    ...(diagnostic ? { diagnostic } : {}),
+  };
+}
+
+async function finishEvaluationRun(
+  options: EvaluationOptions,
+  context: EvaluationContext,
+) {
+  const collected = context.results.snapshot({
+    caseId: options.case.id,
+    passThreshold: options.passThreshold,
+    cancelled: Boolean(options.signal?.aborted),
+  });
+  const result = finalCliResult(options, context.resources, collected);
+  const evidence = finalRunEvidence(
+    options,
+    context.prepared,
+    context.resources,
+    collected,
+    result,
+  );
+  assertCliResult(result);
+  assertRunEvidence(evidence);
+  evaluationProgress(options, "evidence", "started");
+  await atomicWriteJson(context.resources.evidencePath, evidence);
+  context.results.checkpoint(
+    options.signal?.aborted ? "interrupted" : "complete",
+  );
+  evaluationProgress(options, "evidence", "completed");
+  return { result };
+}
+
+/** Execute resolved evaluator data through an injected host and retain every trial. */
+export async function runEvaluation(
+  options: EvaluationOptions,
+): Promise<{ result: CliResult }> {
+  options = {
+    ...options,
+    runtimePolicy: runtimePolicySnapshot(options.runtimePolicy),
+  };
+  evaluationProgress(options, "preparation", "started");
+  await cleanupExpiredWorkspaces();
+  const prepared = await prepareEvaluation(options);
+  const resources = await allocateRunResources(
+    options,
+    prepared.evaluationIdentity.digest,
+  );
+  const results = new ResultCollector(resources.owner);
+  try {
+    const preparationArtifacts = await retainPreparationArtifacts(
+      prepared.fixture.inlineArtifacts,
+      resources.runDir,
+    );
+    await resources.workspaces.reserve({
+      count: options.trialCount,
+      semantic: prepared.checks.preparedSemantic.length > 0,
+      signal: options.signal,
+    });
+    const context: EvaluationContext = {
+      prepared,
+      resources,
+      results,
+      preparationArtifacts,
+    };
+    await executeEvaluationTrials(options, context);
+    return await finishEvaluationRun(options, context);
+  } catch (error) {
+    results.diagnoseFailure();
+    throw error;
+  } finally {
+    await resources.workspaces.cleanup();
+  }
+}

package/src/{trial-scheduler.ts → evaluation/scheduler.ts}

@@ -9,8 +9,10 @@ export async function scheduleTrials(
     stopped = true;
   };
   let failure: { error: unknown } | undefined;
+  const admitting = () =>
+    !stopped && !options.signal?.aborted && nextTrial <= options.count;
   const worker = async () => {
-    while (!stopped && !options.signal?.aborted && nextTrial <= options.count) {
+    while (admitting()) {
       const trial = nextTrial++;
       try {
         if (!(await run(trial, stopAdmission))) stopAdmission();