@bjoernrochel/sevro 0.1.0-rc.7 → 1.0.0

package/src/engine.ts

@@ -1,11 +1,19 @@
-export { EvaluationConfigurationError } from "./evaluation-types";
+export {
+  EvaluationConfigurationError,
+  HostExecutionFailure,
+} from "./evaluation/types";
 export type {
   HostResult,
   HostAdapter,
   ResolvedCase,
   EvaluationOptions,
+  EvaluationProgress,
   CliResult,
-} from "./evaluation-types";
-export { runEvaluation } from "./evaluation-run";
-export { loadRuntimeConfiguration } from "./runtime-config";
-export type { RuntimeConfiguration, RuntimePolicy } from "./runtime-config";
+} from "./evaluation/types";
+export { runEvaluation } from "./evaluation/run";
+export { loadRuntimeConfiguration } from "./runtime/config";
+export type { RuntimeConfiguration, RuntimePolicy } from "./runtime/config";
+export type {
+  RuntimeArtifactDirectory,
+  CandidateArtifactRoots,
+} from "./candidate-artifacts";

package/src/evaluation/advisory.ts

@@ -0,0 +1,150 @@
+import {
+  advisoryPrompt,
+  parseAdvisoryAssessment,
+  type AdvisoryAssessment,
+} from "../advisory";
+import type { CandidateArtifactRoots } from "../candidate-artifacts";
+import type { TrialEvidenceWriter } from "../evidence/trial-writer";
+import { usage } from "../evidence/usage";
+import { buildBlindAdvisoryFixture } from "../fixtures/advisory";
+import { removeWorkspace } from "../fixtures/workspace-state";
+import { HostDiagnosticPersistenceError } from "../host-diagnostics";
+import type { CheckOutcome } from "../results";
+import type { AdvisoryReviewData } from "../schema-types";
+import type { RuntimePolicy } from "../runtime/config";
+import { gradingRuntimePolicy, scopedHostArtifacts } from "./grading-host";
+import {
+  TrialPersistenceError,
+  type HostAdapter,
+  type HostResult,
+} from "./types";
+
+export type AdvisoryReview = AdvisoryReviewData;
+
+interface AdvisoryInput {
+  readonly host: HostAdapter;
+  readonly candidateCompleted: boolean;
+  readonly candidateWorkspace: string;
+  readonly candidateTranscriptRoot?: string;
+  readonly candidateArtifactRoots?: CandidateArtifactRoots;
+  readonly baseRevision: string | null;
+  readonly excludedPaths?: string[];
+  readonly task: string;
+  readonly checks: readonly CheckOutcome[];
+  readonly declaredChecks: { id: string }[];
+  readonly runtimePolicy?: RuntimePolicy;
+  readonly signal?: AbortSignal;
+}
+
+interface AdvisoryEffects {
+  readonly evidence: Pick<
+    TrialEvidenceWriter,
+    "artifactRefs" | "retainArtifacts" | "retainAdvisoryResponse"
+  >;
+  retainWorkspace(workspace: string): Promise<void>;
+}
+
+function defaultAdvisoryReview(host: HostAdapter): AdvisoryReview {
+  return {
+    status: "not_run",
+    assessment: null,
+    usage: usage(null),
+    rawResult: { source: host.id, path: null, sha256: null },
+  };
+}
+
+function completeAdvisoryAssessment(response: HostResult): AdvisoryAssessment {
+  if (
+    response.executionFailed ||
+    !response.complete ||
+    response.finalMessage === null
+  )
+    throw new Error("advisory response is incomplete");
+  return parseAdvisoryAssessment(response.finalMessage);
+}
+
+async function cleanupAdvisoryWorkspace(
+  workspace: string | null,
+  successful: boolean,
+  effects: AdvisoryEffects,
+): Promise<void> {
+  if (!workspace) return;
+  if (!successful) return effects.retainWorkspace(workspace);
+  try {
+    await removeWorkspace(workspace);
+  } catch {
+    console.warn(`advisory fixture cleanup failed; retained at ${workspace}`);
+  }
+}
+
+async function executeAdvisoryReview(
+  input: AdvisoryInput,
+  effects: AdvisoryEffects,
+  review: AdvisoryReview,
+): Promise<void> {
+  let workspace: string | null = null;
+  try {
+    if (input.baseRevision === null)
+      throw new Error("advisory base revision is unavailable");
+    workspace = await buildBlindAdvisoryFixture(input.candidateWorkspace, {
+      baseRevision: input.baseRevision,
+      excludedPaths: input.excludedPaths,
+    });
+    const response = await input.host.run({
+      retainHostDiagnostics: (bytes) =>
+        effects.evidence.retainArtifacts([
+          {
+            id: "sevro.advisory.sevro.host.diagnostics",
+            bytes: Buffer.from(bytes),
+          },
+        ]),
+      candidateTranscriptRoot: input.candidateTranscriptRoot,
+      candidateArtifactRoots: input.candidateArtifactRoots,
+      prompt: advisoryPrompt(input.task, [...input.checks]),
+      runtimePolicy: gradingRuntimePolicy(input.runtimePolicy),
+      runtimeRole: "advisory",
+      workspace,
+      condition: "passive",
+      signal: input.signal,
+    });
+    review.usage = usage(response);
+    const artifacts = scopedHostArtifacts(
+      response,
+      "sevro.advisory.",
+      effects.evidence.artifactRefs,
+      input.declaredChecks,
+      "sevro.advisory.response",
+    );
+    await effects.evidence.retainArtifacts(artifacts);
+    review.rawResult = await effects.evidence.retainAdvisoryResponse(
+      input.host.id,
+      response.finalMessage,
+    );
+    review.assessment = completeAdvisoryAssessment(response);
+    review.status = "completed";
+  } catch (error) {
+    if (
+      error instanceof HostDiagnosticPersistenceError ||
+      error instanceof TrialPersistenceError
+    )
+      throw error;
+    review.status = "failed";
+  } finally {
+    await cleanupAdvisoryWorkspace(
+      workspace,
+      review.status === "completed",
+      effects,
+    );
+  }
+}
+
+/** Advisory execution depends on candidate completion, independently of task grading. */
+export async function runAdvisoryReview(
+  input: AdvisoryInput,
+  effects: AdvisoryEffects,
+): Promise<AdvisoryReview> {
+  const review = defaultAdvisoryReview(input.host);
+  if (!input.candidateCompleted || input.signal?.aborted) return review;
+  await executeAdvisoryReview(input, effects, review);
+  return review;
+}

package/src/{evaluation-check-plan.ts → evaluation/check-plan.ts}

@@ -1,40 +1,83 @@
 import { isAbsolute } from "node:path";
-import { validInlineFixture } from "./case-validation";
+import { validQualifiedId } from "../value-guards";
+import { validInlineFixture } from "../case-validation";
 import {
   EvaluationConfigurationError,
   type EvaluationOptions,
   type HostAdapter,
   type ResolvedCase,
-} from "./evaluation-types";
+} from "./types";
 import {
   prepareGeneratedFixture,
   type GeneratedFixture,
-} from "./generated-fixture";
+} from "../fixtures/generated";
 import {
   prepareGitHeadChecks,
   type GitHeadCheckDeclaration,
-} from "./graders/git-head";
+  type PreparedGitHeadCheck,
+} from "../graders/git-head";
 import {
   isOutputGrader,
   prepareOutputChecks,
   type OutputCheckDeclaration,
-} from "./graders/output";
+  type PreparedOutputCheck,
+} from "../graders/output";
 import {
   prepareSemanticChecks,
   type SemanticCheckDeclaration,
-} from "./graders/semantic";
+  type PreparedSemanticCheck,
+} from "../graders/semantic";
 import {
   prepareShellChecks,
   type ShellCheckDeclaration,
-} from "./graders/shell";
-import { hashJson } from "./identity";
-import { fixtureParts } from "./preparation";
+  type PreparedShellCheck,
+} from "../graders/shell";
+import { hashJson } from "../identity";
+import { fixtureParts } from "../fixtures/preparation";
 import {
   prepareRepositoryFixture,
   type RepositoryFixture,
-} from "./repository-fixture";
-import { resolvedFixture } from "./resolved-case";
-import { isOptionalNonblankString, isRecord } from "./value-guards";
+} from "../fixtures/repository";
+import { resolvedFixture } from "../resolved-case";
+import { isOptionalNonblankString, isRecord } from "../value-guards";
+import type { FixtureSource } from "../fixtures/create";
+
+interface CaseCheckDeclarations {
+  allBuiltinDeclarations: OutputCheckDeclaration[];
+  allShellDeclarations: ShellCheckDeclaration[];
+  allGitHeadDeclarations: GitHeadCheckDeclaration[];
+  allSemanticDeclarations: SemanticCheckDeclaration[];
+  extensionDeclarations: ResolvedCase["checks"];
+  replacedBuiltinGraders: string[];
+  replaced: Set<string>;
+}
+
+interface ActiveCheckDeclarations {
+  readonly builtinDeclarations: OutputCheckDeclaration[];
+  readonly shellDeclarations: ShellCheckDeclaration[];
+  readonly gitHeadDeclarations: GitHeadCheckDeclaration[];
+  readonly semanticDeclarations: SemanticCheckDeclaration[];
+}
+
+interface BuiltInCheckPlan {
+  readonly prepared: PreparedOutputCheck[];
+  readonly preparedShell: PreparedShellCheck[];
+  readonly preparedGitHead: PreparedGitHeadCheck[];
+  readonly preparedSemantic: PreparedSemanticCheck[];
+}
+
+export interface PreparedChecks
+  extends ActiveCheckDeclarations, BuiltInCheckPlan {
+  readonly extensionDeclarations: ResolvedCase["checks"];
+  readonly replacedBuiltinGraders: string[];
+  readonly replaced: ReadonlySet<string>;
+}
+
+export interface EvaluationCheckPlan {
+  readonly source: FixtureSource<null>;
+  readonly jobs: number;
+  readonly checks: PreparedChecks;
+}
 
 function validateRootOptions(options: EvaluationOptions): void {
   if (
@@ -139,23 +182,23 @@ function validateInlineEvaluationFixture(
     );
 }
 
-function prepareEvaluationFixture(options: EvaluationOptions): {
-  generated: GeneratedFixture | null;
-  repositoryFixture: RepositoryFixture | null;
-} {
+function prepareEvaluationFixture(
+  options: EvaluationOptions,
+): FixtureSource<null> {
   const fixture = evaluationFixtureObject(options.case.fixture);
   if ("kind" in fixture)
     return {
-      generated: generatedEvaluationFixture(fixture),
-      repositoryFixture: null,
+      kind: "generated",
+      fixture: generatedEvaluationFixture(fixture),
     };
   if (Object.hasOwn(fixture, "sourceRef"))
     return {
-      generated: null,
-      repositoryFixture: repositoryEvaluationFixture(fixture),
+      kind: "repository",
+      fixture: repositoryEvaluationFixture(fixture),
+      repository: null,
     };
   validateInlineEvaluationFixture(fixture);
-  return { generated: null, repositoryFixture: null };
+  return { kind: "inline", files: options.case.fixture.files };
 }
 
 function validateTrialInteger(value: number, message: string): void {
@@ -173,15 +216,12 @@ function validatePassThreshold(value: number): void {
 function validateRequiredEvidence(ids: string[]): void {
   if (
     new Set(ids).size !== ids.length ||
-    ids.some((id) => !/^[a-z][a-z0-9]*(?:[.-][a-z0-9]+)+$/.test(id))
+    ids.some((id) => !validQualifiedId(id))
   )
     throw new EvaluationConfigurationError("invalid required evidence IDs");
 }
 
-function prepareTrialParameters(
-  options: EvaluationOptions,
-  state: ReturnType<typeof prepareEvaluationFixture>,
-) {
+function prepareTrialParameters(options: EvaluationOptions): number {
   validateTrialInteger(
     options.trialCount,
     "trial count must be a positive integer",
@@ -190,10 +230,10 @@ function prepareTrialParameters(
   validateTrialInteger(jobs, "jobs must be a positive integer");
   validatePassThreshold(options.passThreshold);
   validateRequiredEvidence(options.case.requiredEvidence);
-  return { ...state, jobs };
+  return jobs;
 }
 
-function declaredCaseChecks(options: EvaluationOptions) {
+function declaredCaseChecks(options: EvaluationOptions): CaseCheckDeclarations {
   const allBuiltinDeclarations = options.case.checks.filter((check) =>
     isOutputGrader(check.grader),
   ) as OutputCheckDeclaration[];
@@ -227,8 +267,6 @@ function declaredCaseChecks(options: EvaluationOptions) {
   };
 }
 
-type CaseCheckDeclarations = ReturnType<typeof declaredCaseChecks>;
-
 function validateBuiltInReplacements(context: CaseCheckDeclarations): void {
   const {
     allBuiltinDeclarations,
@@ -260,7 +298,9 @@ function validateBuiltInReplacements(context: CaseCheckDeclarations): void {
     );
 }
 
-function activeCheckDeclarations(context: CaseCheckDeclarations) {
+function activeCheckDeclarations(
+  context: CaseCheckDeclarations,
+): ActiveCheckDeclarations {
   const {
     allBuiltinDeclarations,
     allShellDeclarations,
@@ -288,10 +328,6 @@ function activeCheckDeclarations(context: CaseCheckDeclarations) {
   };
 }
 
-type CaseCheckPlan = ReturnType<typeof prepareTrialParameters> &
-  CaseCheckDeclarations &
-  ReturnType<typeof activeCheckDeclarations>;
-
 function validateSemanticRoute(
   options: EvaluationOptions,
   checks: SemanticCheckDeclaration[],
@@ -306,11 +342,8 @@ function validateSemanticRoute(
     );
 }
 
-function validateAdvisoryGitFixture(
-  options: EvaluationOptions,
-  generated: GeneratedFixture | null,
-): void {
-  if (!generated && !options.case.fixture.sourceRef)
+function validateAdvisoryGitFixture(source: FixtureSource<null>): void {
+  if (source.kind === "inline")
     throw new EvaluationConfigurationError(
       "advisory review requires a Git fixture",
     );
@@ -335,7 +368,7 @@ function validateAdvisoryExclusions(paths: string[]): void {
 
 function validateAdvisoryRoute(
   options: EvaluationOptions,
-  generated: GeneratedFixture | null,
+  source: FixtureSource<null>,
 ): void {
   const host = options.advisoryHost;
   if (!host) {
@@ -345,7 +378,7 @@ function validateAdvisoryRoute(
       );
     return;
   }
-  validateAdvisoryGitFixture(options, generated);
+  validateAdvisoryGitFixture(source);
   validateAdvisoryIdentity(host);
   validateAdvisoryExclusions(options.advisoryExcludedPaths ?? []);
 }
@@ -367,12 +400,11 @@ function validateShellRoute(
     );
 }
 
-function validateGitHeadRoute(context: CaseCheckPlan): void {
-  if (
-    context.gitHeadDeclarations.length &&
-    !context.generated &&
-    !context.repositoryFixture
-  )
+function validateGitHeadRoute(
+  source: FixtureSource<null>,
+  checks: GitHeadCheckDeclaration[],
+): void {
+  if (checks.length && source.kind === "inline")
     throw new EvaluationConfigurationError(
       "Git HEAD checks require a Git fixture",
     );
@@ -443,8 +475,8 @@ function validateInlinePaths(files: ResolvedCase["fixture"]["files"]): void {
 
 function prepareBuiltInChecks(
   options: EvaluationOptions,
-  context: CaseCheckPlan,
-) {
+  context: CaseCheckDeclarations,
+): BuiltInCheckPlan {
   const {
     allBuiltinDeclarations,
     allShellDeclarations,
@@ -483,28 +515,32 @@ function prepareBuiltInChecks(
 
 function prepareCheckPlan(
   options: EvaluationOptions,
-  state: ReturnType<typeof prepareTrialParameters>,
-) {
+  source: FixtureSource<null>,
+): PreparedChecks {
   const declarations = declaredCaseChecks(options);
   validateBuiltInReplacements(declarations);
-  const context = {
-    ...state,
-    ...declarations,
-    ...activeCheckDeclarations(declarations),
-  };
-  validateSemanticRoute(options, context.semanticDeclarations);
-  validateAdvisoryRoute(options, context.generated);
-  validateShellRoute(options, context.shellDeclarations);
-  validateGitHeadRoute(context);
+  const active = activeCheckDeclarations(declarations);
+  validateSemanticRoute(options, active.semanticDeclarations);
+  validateAdvisoryRoute(options, source);
+  validateShellRoute(options, active.shellDeclarations);
+  validateGitHeadRoute(source, active.gitHeadDeclarations);
   validateCaseCheckIds(options.case.checks);
-  validateExtensionGraders(options, context.extensionDeclarations);
+  validateExtensionGraders(options, declarations.extensionDeclarations);
   validateResolvedExtensionCase(options);
-  return { ...context, ...prepareBuiltInChecks(options, context) };
+  return {
+    ...active,
+    ...prepareBuiltInChecks(options, declarations),
+    extensionDeclarations: declarations.extensionDeclarations,
+    replacedBuiltinGraders: declarations.replacedBuiltinGraders,
+    replaced: declarations.replaced,
+  };
 }
 
-export function prepareEvaluationChecks(options: EvaluationOptions) {
+export function prepareEvaluationChecks(
+  options: EvaluationOptions,
+): EvaluationCheckPlan {
   validateEvaluationInputs(options);
-  const fixture = prepareEvaluationFixture(options);
-  const trials = prepareTrialParameters(options, fixture);
-  return prepareCheckPlan(options, trials);
+  const source = prepareEvaluationFixture(options);
+  const jobs = prepareTrialParameters(options);
+  return { source, jobs, checks: prepareCheckPlan(options, source) };
 }

package/src/evaluation/collection.ts

@@ -0,0 +1,88 @@
+import { checkpointRunOwner, type RunOwner } from "../run-owner";
+import { summarizeAssessments, type Assessment } from "../results";
+import type { Diagnostic } from "../contract-types";
+import type { RunEvidenceDataTrialEvidence } from "../schema-types";
+import type { CaseSummary, TrialSummary } from "./types";
+
+type TrialEvidence = RunEvidenceDataTrialEvidence;
+
+export interface CompletedTrial {
+  readonly summary: TrialSummary;
+  readonly evidence: TrialEvidence;
+  readonly diagnostic?: Diagnostic;
+}
+
+export interface CollectedResults {
+  readonly caseSummary: CaseSummary;
+  readonly trials: readonly TrialEvidence[];
+  readonly diagnostic: Diagnostic | undefined;
+}
+
+/** Registers retained trials before checkpointing; owns their final ordering. */
+export class ResultCollector {
+  private readonly summaries: TrialSummary[] = [];
+  private readonly evidence: TrialEvidence[] = [];
+  private readonly diagnostics: { trial: number; diagnostic: Diagnostic }[] =
+    [];
+
+  constructor(private readonly owner: RunOwner) {}
+
+  complete(record: CompletedTrial): void {
+    this.summaries.push(record.summary);
+    this.summaries.sort((left, right) => left.trial - right.trial);
+    this.evidence.push(record.evidence);
+    this.evidence.sort((left, right) => left.trial - right.trial);
+    if (record.diagnostic)
+      this.diagnostics.push({
+        trial: record.summary.trial,
+        diagnostic: record.diagnostic,
+      });
+    this.checkpoint("active");
+  }
+
+  snapshot(input: {
+    caseId: string;
+    passThreshold: number;
+    cancelled: boolean;
+  }): CollectedResults {
+    const assessment: Assessment =
+      !this.summaries.length && input.cancelled
+        ? {
+            execution: { status: "cancelled" },
+            grading: { status: "not_requested" },
+            task: { verdict: "not_assessed" },
+          }
+        : summarizeAssessments(this.summaries, input.passThreshold);
+    const diagnostic = this.diagnostics.sort(
+      (left, right) => left.trial - right.trial,
+    )[0]?.diagnostic;
+    return {
+      caseSummary: {
+        caseId: input.caseId,
+        ...assessment,
+        trials: this.summaries,
+      },
+      trials: this.evidence,
+      diagnostic,
+    };
+  }
+
+  checkpoint(status: "active" | "complete" | "interrupted"): void {
+    checkpointRunOwner(this.owner, this.checkpointTrials(), status);
+  }
+
+  diagnoseFailure(): void {
+    try {
+      checkpointRunOwner(this.owner, this.checkpointTrials(), "diagnostic");
+    } catch {
+      /* Preserve the original failure and retained artifacts. */
+    }
+  }
+
+  private checkpointTrials() {
+    return this.summaries.map((trial) => ({
+      trial: trial.trial,
+      artifactPath: trial.artifactPath,
+    }));
+  }
+}

package/src/evaluation/context.ts

@@ -0,0 +1,10 @@
+import type { ResultCollector } from "./collection";
+import type { PreparedEvaluation } from "./plan";
+import type { PreparationArtifactReference, RunResources } from "./resources";
+
+export interface EvaluationContext {
+  readonly prepared: PreparedEvaluation;
+  readonly resources: RunResources;
+  readonly results: ResultCollector;
+  readonly preparationArtifacts: readonly PreparationArtifactReference[];
+}

package/src/{evaluation-grading-host.ts → evaluation/grading-host.ts}

@@ -1,11 +1,11 @@
-import { hostArtifacts } from "./evaluation-fixture";
-import type { HostResult, ResolvedCase } from "./evaluation-types";
-import { runtimeSeedSources, type RuntimePolicy } from "./runtime-config";
+import { hostArtifacts } from "../evidence/host-results";
+import type { HostResult } from "./types";
+import { runtimeSeedSources, type RuntimePolicy } from "../runtime/config";
 export function scopedHostArtifacts(
   response: HostResult,
   prefix: string,
-  artifacts: { id: string }[],
-  checks: ResolvedCase["checks"],
+  artifacts: readonly { id: string }[],
+  checks: readonly { id: string }[],
   reservedId: string,
 ) {
   return hostArtifacts(