@bjoernrochel/sevro 0.1.0-rc.8 → 1.0.0

package/src/{evaluation-plan.ts → evaluation/plan.ts}

@@ -1,49 +1,61 @@
-import { randomUUID } from "node:crypto";
 import { realpath } from "node:fs/promises";
 import {
   EvaluationConfigurationError,
   type EvaluationOptions,
   type HostAdapter,
   type ResolvedCase,
-} from "./evaluation-types";
-import { prepareFixtureSetup, type FixtureSetup } from "./fixture-setup";
-import { type GeneratedFixture } from "./generated-fixture";
-import { type SemanticCheckDeclaration } from "./graders/semantic";
-import { canonicalJson, createEvaluationIdentity, hashJson } from "./identity";
+} from "./types";
+import { prepareFixtureSetup, type FixtureSetup } from "../fixtures/setup";
+import type { FixtureSource } from "../fixtures/create";
+import { type SemanticCheckDeclaration } from "../graders/semantic";
+import { canonicalJson, createEvaluationIdentity, hashJson } from "../identity";
 import {
   prepareInstrumentation,
   type InstrumentationRequest,
-} from "./instrumentation";
+} from "../instrumentation";
 import {
   fixtureParts,
   prepareArtifacts,
   type InlineArtifact,
-} from "./preparation";
-import { projectProvenance, runnerProvenance } from "./provenance";
+} from "../fixtures/preparation";
+import { projectProvenance, runnerProvenance } from "../provenance";
+import { resolveRepositorySource } from "../fixtures/repository";
 import {
-  resolveRepositorySource,
-  type RepositorySource,
-} from "./repository-fixture";
-import { prepareEvaluationChecks } from "./evaluation-check-plan";
-import { prepareInvocationPlan } from "./evaluation-invocation-plan";
+  prepareEvaluationChecks,
+  type EvaluationCheckPlan,
+  type PreparedChecks,
+} from "./check-plan";
+import {
+  prepareInvocationPlan,
+  type PreparedInvocation,
+} from "./invocation-plan";
+import type { PreparationResult } from "../extension-session";
+
+export type { PreparedChecks } from "./check-plan";
+export type { PreparedInvocation } from "./invocation-plan";
+
+interface EnvironmentPlan {
+  readonly projectRoot: string;
+  readonly source: FixtureSource;
+  readonly runner: Awaited<ReturnType<typeof runnerProvenance>>;
+  readonly project: Awaited<ReturnType<typeof projectProvenance>>;
+  readonly extensionPreparation: PreparationResult | null;
+}
+
+interface ArtifactPlan {
+  readonly fixtureSetup: FixtureSetup | null;
+  readonly requestedInstrumentation: InstrumentationRequest[];
+  readonly inlineArtifacts: InlineArtifact[];
+}
 
 async function prepareEvaluationEnvironment(
   options: EvaluationOptions,
-  state: ReturnType<typeof prepareEvaluationChecks>,
-) {
+  source: FixtureSource<null>,
+): Promise<EnvironmentPlan> {
   const projectRoot = await realpath(options.projectRoot).catch(() => {
     throw new EvaluationConfigurationError("project root is unreadable");
   });
-  const repository = options.case.fixture.sourceRef
-    ? await resolveRepositorySource(
-        options.case.fixture.sourceRef,
-        options.preparationSources,
-      ).catch((error: unknown) => {
-        throw new EvaluationConfigurationError(
-          error instanceof Error ? error.message : "invalid repository source",
-        );
-      })
-    : null;
+  const resolvedSource = await resolveFixtureSource(options, source);
   const [runner, project] = await Promise.all([
     runnerProvenance(
       options.runnerBuildDigest,
@@ -53,7 +65,7 @@ async function prepareEvaluationEnvironment(
         error instanceof Error ? error.message : "invalid runner provenance",
       );
     }),
-    projectProvenance(projectRoot),
+    projectProvenance(projectRoot, evaluationStorageRoots(options)),
   ]);
   const extensionPreparation = options.extension
     ? await options.extension.session.prepare(
@@ -69,16 +81,33 @@ async function prepareEvaluationEnvironment(
       )
     : null;
   return {
-    ...state,
     projectRoot,
-    repository,
+    source: resolvedSource,
     runner,
     project,
     extensionPreparation,
   };
 }
 
-type EnvironmentPlan = Awaited<ReturnType<typeof prepareEvaluationEnvironment>>;
+async function resolveFixtureSource(
+  options: EvaluationOptions,
+  source: FixtureSource<null>,
+): Promise<FixtureSource> {
+  if (source.kind !== "repository") return source;
+  const repository = await resolveRepositorySource(
+    source.fixture.sourceRef,
+    options.preparationSources,
+  ).catch((error: unknown) => {
+    throw new EvaluationConfigurationError(
+      error instanceof Error ? error.message : "invalid repository source",
+    );
+  });
+  return { ...source, repository };
+}
+
+function evaluationStorageRoots(options: EvaluationOptions): string[] {
+  return [options.resultsRoot, options.runStateRoot ?? options.resultsRoot];
+}
 
 function preparedFixtureSetup(
   preparation: EnvironmentPlan["extensionPreparation"],
@@ -110,10 +139,9 @@ function validateSetupCapability(
 
 function validateSetupFixture(
   setup: FixtureSetup | null,
-  generated: GeneratedFixture | null,
-  repository: RepositorySource | null,
+  source: FixtureSource,
 ): void {
-  if (setup && !generated && !repository)
+  if (setup && source.kind === "inline")
     throw new EvaluationConfigurationError(
       "fixture setup requires a Git fixture",
     );
@@ -148,17 +176,22 @@ function requestedInstrumentationPlan(
   }
 }
 
-function fixturePreparationPaths(
-  fixture: ResolvedCase["fixture"],
-  generated: GeneratedFixture | null,
-): string[] {
-  if (!generated) return Object.keys(fixture.files ?? {});
-  return [
-    ...new Set([
-      ...generated.commits.flatMap((commit) => Object.keys(commit.files)),
-      ...Object.keys(generated.files ?? {}),
-    ]),
-  ];
+function fixturePreparationPaths(source: FixtureSource): string[] {
+  switch (source.kind) {
+    case "inline":
+      return Object.keys(source.files ?? {});
+    case "repository":
+      return Object.keys(source.fixture.files ?? {});
+    case "generated":
+      return [
+        ...new Set([
+          ...source.fixture.commits.flatMap((commit) =>
+            Object.keys(commit.files),
+          ),
+          ...Object.keys(source.fixture.files ?? {}),
+        ]),
+      ];
+  }
 }
 
 async function preparedEvaluationArtifacts(
@@ -168,7 +201,7 @@ async function preparedEvaluationArtifacts(
   try {
     return await prepareArtifacts(
       context.extensionPreparation?.artifacts ?? [],
-      fixturePreparationPaths(options.case.fixture, context.generated),
+      fixturePreparationPaths(context.source),
       options.preparationSources,
     );
   } catch (cause) {
@@ -181,11 +214,10 @@ async function preparedEvaluationArtifacts(
 
 function validateArtifactMetadata(
   artifacts: InlineArtifact[],
-  generated: GeneratedFixture | null,
-  repository: RepositorySource | null,
+  source: FixtureSource,
 ): void {
   if (
-    (repository || generated) &&
+    source.kind !== "inline" &&
     artifacts.some((artifact) =>
       fixtureParts(artifact.relativePath).some(
         (part) => part.toLowerCase() === ".git",
@@ -199,12 +231,10 @@ function validateArtifactMetadata(
 
 function validateArtifactExclusions(
   artifacts: InlineArtifact[],
-  generated: GeneratedFixture | null,
-  repository: RepositorySource | null,
+  source: FixtureSource,
 ): void {
   if (
-    !repository &&
-    !generated &&
+    source.kind === "inline" &&
     artifacts.some((artifact) => artifact.gitExclude)
   )
     throw new EvaluationConfigurationError(
@@ -228,27 +258,25 @@ function validateSemanticArtifactId(
 async function prepareEvaluationArtifacts(
   options: EvaluationOptions,
   state: EnvironmentPlan,
-) {
+  checks: EvaluationCheckPlan,
+): Promise<ArtifactPlan> {
   const fixtureSetup = preparedFixtureSetup(state.extensionPreparation);
   validateSetupCapability(options, fixtureSetup);
-  validateSetupFixture(fixtureSetup, state.generated, state.repository);
+  validateSetupFixture(fixtureSetup, state.source);
   const requestedInstrumentation = requestedInstrumentationPlan(
     options,
     state.extensionPreparation,
   );
   const inlineArtifacts = await preparedEvaluationArtifacts(options, state);
-  validateArtifactMetadata(inlineArtifacts, state.generated, state.repository);
-  validateArtifactExclusions(
+  validateArtifactMetadata(inlineArtifacts, state.source);
+  validateArtifactExclusions(inlineArtifacts, state.source);
+  validateSemanticArtifactId(
     inlineArtifacts,
-    state.generated,
-    state.repository,
+    checks.checks.semanticDeclarations,
   );
-  validateSemanticArtifactId(inlineArtifacts, state.semanticDeclarations);
-  return { ...state, fixtureSetup, requestedInstrumentation, inlineArtifacts };
+  return { fixtureSetup, requestedInstrumentation, inlineArtifacts };
 }
 
-type ArtifactPlan = Awaited<ReturnType<typeof prepareEvaluationArtifacts>>;
-type InvocationPlan = ReturnType<typeof prepareInvocationPlan<ArtifactPlan>>;
 type EvaluationRouteRole = "candidate" | "semantic" | "advisory";
 interface ActiveGrader {
   id: string;
@@ -256,9 +284,35 @@ interface ActiveGrader {
   version: string;
 }
 
+export interface PreparedFixture {
+  readonly projectRoot: string;
+  readonly source: FixtureSource;
+  readonly fixtureSetup: ArtifactPlan["fixtureSetup"];
+  readonly inlineArtifacts: InlineArtifact[];
+}
+
+interface PreparedEvaluationInputs {
+  readonly fixture: PreparedFixture;
+  readonly checks: PreparedChecks;
+  readonly invocation: PreparedInvocation;
+  readonly jobs: number;
+  readonly requestedInstrumentation: InstrumentationRequest[];
+  readonly runner: EnvironmentPlan["runner"];
+  readonly project: EnvironmentPlan["project"];
+}
+
+export interface PreparedEvaluation extends PreparedEvaluationInputs {
+  readonly extensionData: Record<string, unknown>;
+  readonly route: ReturnType<typeof evaluationRoute>;
+  readonly routes: ReturnType<typeof evaluationRoutes>;
+  readonly redactedConfig: ReturnType<typeof redactedEvaluationConfiguration>;
+  readonly activeGraders: ActiveGrader[];
+  readonly evaluationIdentity: ReturnType<typeof createEvaluationIdentity>;
+}
+
 function extensionPreparationData(
   options: EvaluationOptions,
-  preparation: InvocationPlan["extensionPreparation"],
+  preparation: EnvironmentPlan["extensionPreparation"],
 ): Record<string, unknown> {
   return options.extension
     ? {
@@ -327,36 +381,16 @@ function redactedExtensionConfiguration(options: EvaluationOptions) {
   };
 }
 
-function skillInvocationConfiguration(state: InvocationPlan) {
+function redactedPreparationConfiguration(state: PreparedEvaluationInputs) {
   return {
-    ...(state.codexSkillInvocation
-      ? { codexSkillInvocation: state.codexSkillInvocation }
+    ...(state.fixture.fixtureSetup
+      ? { fixtureSetupDigest: hashJson(state.fixture.fixtureSetup) }
       : {}),
-    ...(state.codexRepositorySkillInvocation
-      ? { codexRepositorySkillInvocation: state.codexRepositorySkillInvocation }
+    ...(state.invocation.codexMarketplace
+      ? { codexMarketplace: state.invocation.codexMarketplace }
       : {}),
-    ...(state.claudeSkillInvocation
-      ? { claudeSkillInvocation: state.claudeSkillInvocation }
-      : {}),
-    ...(state.claudeRepositorySkillInvocation
-      ? {
-          claudeRepositorySkillInvocation:
-            state.claudeRepositorySkillInvocation,
-        }
-      : {}),
-  };
-}
-
-function redactedPreparationConfiguration(state: InvocationPlan) {
-  return {
-    ...(state.fixtureSetup
-      ? { fixtureSetupDigest: hashJson(state.fixtureSetup) }
-      : {}),
-    ...(state.codexMarketplace
-      ? { codexMarketplace: state.codexMarketplace }
-      : {}),
-    ...(state.claudePluginDirs
-      ? { claudePluginDirs: state.claudePluginDirs }
+    ...(state.invocation.claudePluginDirs
+      ? { claudePluginDirs: state.invocation.claudePluginDirs }
       : {}),
   };
 }
@@ -373,7 +407,7 @@ function redactedAdvisoryConfiguration(options: EvaluationOptions) {
 
 function redactedEvaluationConfiguration(
   options: EvaluationOptions,
-  state: InvocationPlan,
+  state: PreparedEvaluationInputs,
 ) {
   return {
     condition: options.condition,
@@ -382,17 +416,23 @@ function redactedEvaluationConfiguration(
     trialCount: options.trialCount,
     jobs: state.jobs,
     passThreshold: options.passThreshold,
-    ...redactedHostConfiguration(options, state.preparedSemantic.length > 0),
+    ...(options.assessmentSettings
+      ? { runSettings: options.assessmentSettings }
+      : {}),
+    ...redactedHostConfiguration(
+      options,
+      state.checks.preparedSemantic.length > 0,
+    ),
     ...redactedExtensionConfiguration(options),
     ...redactedPreparationConfiguration(state),
-    ...skillInvocationConfiguration(state),
+    ...state.invocation.configuration,
     ...redactedAdvisoryConfiguration(options),
   };
 }
 
 function activeEvaluationGraders(
   options: EvaluationOptions,
-  state: InvocationPlan,
+  state: PreparedChecks,
 ): ActiveGrader[] {
   const graders: ActiveGrader[] = [
     ...new Set(
@@ -440,14 +480,15 @@ function caseIdentityData(
   };
 }
 
-function baseFixtureIdentity(
-  state: InvocationPlan,
-  fixture: ResolvedCase["fixture"],
-) {
-  if (state.generated) return state.generated;
-  if (state.repository)
-    return { ...state.repositoryFixture, revision: state.repository.revision };
-  return { files: fixture.files };
+function baseFixtureIdentity(source: FixtureSource) {
+  switch (source.kind) {
+    case "generated":
+      return source.fixture;
+    case "repository":
+      return { ...source.fixture, revision: source.repository.revision };
+    case "inline":
+      return { files: source.files };
+  }
 }
 
 function artifactIdentityData(artifacts: InlineArtifact[]) {
@@ -462,21 +503,20 @@ function artifactIdentityData(artifacts: InlineArtifact[]) {
   );
 }
 
-function fixtureIdentityData(
-  state: InvocationPlan,
-  fixture: ResolvedCase["fixture"],
-) {
+function fixtureIdentityData(state: PreparedEvaluationInputs) {
   return {
-    ...baseFixtureIdentity(state, fixture),
-    artifacts: artifactIdentityData(state.inlineArtifacts),
-    ...(state.fixtureSetup ? { fixtureSetup: state.fixtureSetup } : {}),
-    ...(state.codexMarketplace
-      ? { codexMarketplace: state.codexMarketplace }
+    ...baseFixtureIdentity(state.fixture.source),
+    artifacts: artifactIdentityData(state.fixture.inlineArtifacts),
+    ...(state.fixture.fixtureSetup
+      ? { fixtureSetup: state.fixture.fixtureSetup }
+      : {}),
+    ...(state.invocation.codexMarketplace
+      ? { codexMarketplace: state.invocation.codexMarketplace }
       : {}),
-    ...(state.claudePluginDirs
-      ? { claudePluginDirs: state.claudePluginDirs }
+    ...(state.invocation.claudePluginDirs
+      ? { claudePluginDirs: state.invocation.claudePluginDirs }
       : {}),
-    ...skillInvocationConfiguration(state),
+    ...state.invocation.configuration,
   };
 }
 
@@ -491,7 +531,7 @@ function evaluatorIdentityData(
   };
 }
 
-type IdentityPlan = InvocationPlan & {
+type IdentityMetadata = {
   extensionData: Record<string, unknown>;
   route: ReturnType<typeof evaluationRoute>;
   routes: ReturnType<typeof evaluationRoutes>;
@@ -501,7 +541,8 @@ type IdentityPlan = InvocationPlan & {
 
 function evaluationComparisonIdentity(
   options: EvaluationOptions,
-  context: IdentityPlan,
+  prepared: PreparedEvaluationInputs,
+  context: IdentityMetadata,
 ) {
   try {
     return createEvaluationIdentity({
@@ -512,9 +553,7 @@ function evaluationComparisonIdentity(
       caseDigest: hashJson(
         caseIdentityData(options.case, context.extensionData),
       ),
-      fixtureDigest: hashJson(
-        fixtureIdentityData(context, options.case.fixture),
-      ),
+      fixtureDigest: hashJson(fixtureIdentityData(prepared)),
       checksDigest: hashJson(options.case.checks),
       requiredEvidenceDigest: hashJson(options.case.requiredEvidence),
       evaluatorDigest: hashJson(
@@ -522,8 +561,8 @@ function evaluationComparisonIdentity(
       ),
       graderDigest: hashJson(context.activeGraders),
       instrumentationDigest: hashJson({
-        requested: context.requestedInstrumentation,
-        applied: context.requestedInstrumentation,
+        requested: prepared.requestedInstrumentation,
+        applied: prepared.requestedInstrumentation,
       }),
       routeDigest: hashJson(
         context.routes.length > 1 ? context.routes : context.route,
@@ -542,36 +581,36 @@ function evaluationComparisonIdentity(
 
 function prepareEvaluationIdentity(
   options: EvaluationOptions,
-  state: InvocationPlan,
-) {
-  const extensionData = extensionPreparationData(
-    options,
-    state.extensionPreparation,
-  );
-  const runId = randomUUID();
+  state: PreparedEvaluationInputs,
+  preparation: EnvironmentPlan["extensionPreparation"],
+): PreparedEvaluation {
+  const extensionData = extensionPreparationData(options, preparation);
   const route = evaluationRoute(options.host, "candidate");
   const routes = evaluationRoutes(
     options,
-    state.preparedSemantic.length > 0,
+    state.checks.preparedSemantic.length > 0,
     route,
   );
   const redactedConfig = snapshotEvaluationConfiguration(options, state);
-  const activeGraders = activeEvaluationGraders(options, state);
+  const activeGraders = activeEvaluationGraders(options, state.checks);
   const context = {
-    ...state,
     extensionData,
     route,
     routes,
     redactedConfig,
     activeGraders,
   };
-  const evaluationIdentity = evaluationComparisonIdentity(options, context);
-  return { ...context, runId, evaluationIdentity };
+  const evaluationIdentity = evaluationComparisonIdentity(
+    options,
+    state,
+    context,
+  );
+  return { ...state, ...context, evaluationIdentity };
 }
 
 function snapshotEvaluationConfiguration(
   options: EvaluationOptions,
-  state: InvocationPlan,
+  state: PreparedEvaluationInputs,
 ): ReturnType<typeof redactedEvaluationConfiguration> {
   try {
     return JSON.parse(
@@ -585,12 +624,40 @@ function snapshotEvaluationConfiguration(
   }
 }
 
-export async function prepareEvaluation(options: EvaluationOptions) {
+export async function prepareEvaluation(
+  options: EvaluationOptions,
+): Promise<PreparedEvaluation> {
   const checks = prepareEvaluationChecks(options);
-  const environment = await prepareEvaluationEnvironment(options, checks);
-  const artifacts = await prepareEvaluationArtifacts(options, environment);
-  const invocation = prepareInvocationPlan(options, artifacts);
-  return prepareEvaluationIdentity(options, invocation);
+  const environment = await prepareEvaluationEnvironment(
+    options,
+    checks.source,
+  );
+  const artifacts = await prepareEvaluationArtifacts(
+    options,
+    environment,
+    checks,
+  );
+  const invocation = prepareInvocationPlan(options, {
+    extensionPreparation: environment.extensionPreparation,
+    inlineArtifacts: artifacts.inlineArtifacts,
+  });
+  const prepared: PreparedEvaluationInputs = {
+    fixture: {
+      projectRoot: environment.projectRoot,
+      source: environment.source,
+      fixtureSetup: artifacts.fixtureSetup,
+      inlineArtifacts: artifacts.inlineArtifacts,
+    },
+    checks: checks.checks,
+    invocation,
+    jobs: checks.jobs,
+    requestedInstrumentation: artifacts.requestedInstrumentation,
+    runner: environment.runner,
+    project: environment.project,
+  };
+  return prepareEvaluationIdentity(
+    options,
+    prepared,
+    environment.extensionPreparation,
+  );
 }
-
-export type EvaluationPlan = Awaited<ReturnType<typeof prepareEvaluation>>;

package/src/evaluation/progress.ts

@@ -0,0 +1,19 @@
+import type { EvaluationOptions, EvaluationProgress } from "./types";
+
+/** Observation is best-effort and cannot change assessment or persistence. */
+export function evaluationProgress(
+  options: EvaluationOptions,
+  stage: EvaluationProgress["stage"],
+  status: EvaluationProgress["status"],
+  trial?: number,
+): void {
+  try {
+    options.onProgress?.({
+      stage,
+      status,
+      ...(trial === undefined ? {} : { trial }),
+    });
+  } catch {
+    // An observer has no authority over the evaluation lifecycle.
+  }
+}