@bjoernrochel/sevro 0.1.0-rc.7 → 1.0.0

package/src/evaluation/grading.ts

@@ -0,0 +1,235 @@
+import {
+  assessGitHeadCheck,
+  gitHeadState,
+  type PreparedGitHeadCheck,
+} from "../graders/git-head";
+import { gradeOutput, type PreparedOutputCheck } from "../graders/output";
+import {
+  assessShellCheck,
+  runShellCheck,
+  type PreparedShellCheck,
+} from "../graders/shell";
+import { evaluationProtectedRoots } from "../hosts/isolation-roots";
+import { sha256 } from "../identity";
+import type { TrialObservation } from "../evidence/trial-writer";
+import {
+  ExtensionSourceChangedError,
+  type EvaluationRequest,
+  type EvaluationResult,
+} from "../extension-session";
+import type { CheckOutcome } from "../results";
+
+export interface GradingFailure {
+  readonly message: string;
+  readonly cancelable: boolean;
+}
+
+export interface GradingResult {
+  checks: CheckOutcome[];
+  observations: TrialObservation[];
+  failure: GradingFailure | null;
+}
+
+function failedGrading(
+  checks: CheckOutcome[],
+  observations: TrialObservation[],
+  message: string,
+): GradingResult {
+  const failure = { message, cancelable: true };
+  return { checks, observations, failure };
+}
+
+export function gradeCandidateOutput(input: {
+  readonly result: { finalMessage: string | null; complete: boolean } | null;
+  readonly checks: readonly PreparedOutputCheck[];
+  readonly evidenceRefs: string[];
+}): GradingResult {
+  try {
+    if (!input.result) throw new Error("host result is unavailable");
+    const checks = gradeOutput(
+      input.result.finalMessage,
+      input.result.complete,
+      [...input.checks],
+    ).map((check) => ({ ...check, evidenceRefs: [...input.evidenceRefs] }));
+    return { checks, observations: [], failure: null };
+  } catch {
+    return {
+      checks: [],
+      observations: [],
+      failure: {
+        message: "output grading did not complete",
+        cancelable: false,
+      },
+    };
+  }
+}
+
+interface ShellGradingInput {
+  readonly checks: readonly PreparedShellCheck[];
+  readonly protectedRoots:
+    Parameters<typeof evaluationProtectedRoots>[0] | null;
+  readonly execution: Omit<
+    Parameters<typeof runShellCheck>[1],
+    "protectedRoots"
+  >;
+}
+
+async function gradeShellCheck(
+  check: PreparedShellCheck,
+  execution: Parameters<typeof runShellCheck>[1],
+): Promise<{ check: CheckOutcome; observation: TrialObservation }> {
+  const result = await runShellCheck(check, execution);
+  const graded = assessShellCheck(check, result);
+  const id = `sevro.observation.shell.${check.id}`;
+  return {
+    observation: {
+      id,
+      source: "sevro.shell",
+      completeness: "complete",
+      data: {
+        exitCode: result.exitCode,
+        expectedExitCode: check.expectedExitCode,
+        ...(result.stdout === null
+          ? {}
+          : {
+              stdoutSha256: sha256(result.stdout),
+              stdoutByteLength: Buffer.byteLength(result.stdout, "utf8"),
+            }),
+      },
+    },
+    check: {
+      id: check.id,
+      grader: "sevro.shell",
+      status: graded.passed ? "passed" : "failed",
+      detail: graded.detail,
+      evidenceRefs: [id],
+    },
+  };
+}
+
+export async function gradeShellChecks(
+  input: ShellGradingInput,
+): Promise<GradingResult> {
+  const checks: CheckOutcome[] = [];
+  const observations: TrialObservation[] = [];
+  try {
+    if (!input.protectedRoots)
+      throw new Error("shell checks require explicit protected source roots");
+    const protectedRoots = await evaluationProtectedRoots(input.protectedRoots);
+    for (const check of input.checks) {
+      const graded = await gradeShellCheck(check, {
+        ...input.execution,
+        protectedRoots,
+        protectedRootsCanonical: true,
+      });
+      checks.push(graded.check);
+      observations.push(graded.observation);
+    }
+    return { checks, observations, failure: null };
+  } catch {
+    return failedGrading(
+      checks,
+      observations,
+      "shell grading did not complete",
+    );
+  }
+}
+
+export async function gradeGitHeadChecks(input: {
+  readonly checks: readonly PreparedGitHeadCheck[];
+  readonly base: string | null;
+  readonly workspace: string;
+  readonly signal?: AbortSignal;
+}): Promise<GradingResult> {
+  const observations: TrialObservation[] = [];
+  try {
+    const base = input.base;
+    if (base === null) throw new Error("Git HEAD base revision is unavailable");
+    const state = await gitHeadState(input.workspace, base, input.signal);
+    const id = "sevro.observation.git-head";
+    observations.push({
+      id,
+      source: "sevro.git-head",
+      completeness: "complete",
+      data: {
+        baseRevision: base,
+        currentRevision: state.currentRevision,
+        baseAncestor: state.baseAncestor,
+      },
+    });
+    const checks = input.checks.map((check): CheckOutcome => {
+      const graded = assessGitHeadCheck(check, base, state);
+      return {
+        id: check.id,
+        grader: "sevro.git-head",
+        status: graded.passed ? "passed" : "failed",
+        detail: graded.detail,
+        evidenceRefs: [id],
+      };
+    });
+    return { checks, observations, failure: null };
+  } catch {
+    return failedGrading([], observations, "Git HEAD grading did not complete");
+  }
+}
+
+function extensionCheckResult(
+  check: EvaluationResult["checks"][number],
+  declared: ReadonlyMap<string, string>,
+): CheckOutcome {
+  const grader = declared.get(check.id);
+  if (grader === undefined)
+    throw new Error("extension returned an undeclared check");
+  return {
+    id: check.id,
+    grader,
+    status: check.status,
+    ...(check.detail ? { detail: check.detail } : {}),
+    evidenceRefs: check.evidenceRefs,
+  };
+}
+
+export interface ExtensionGradingResult {
+  checks: CheckOutcome[];
+  metrics: EvaluationResult["metrics"];
+  domainOutcomes: NonNullable<EvaluationResult["domainOutcomes"]>;
+  recommendation: "passed" | "failed" | "not_assessed" | null;
+  failure: GradingFailure | null;
+}
+
+export async function gradeExtensionChecks(input: {
+  readonly evaluate: (request: EvaluationRequest) => Promise<EvaluationResult>;
+  readonly request: EvaluationRequest;
+  readonly declarations: readonly { id: string; grader: string }[];
+}): Promise<ExtensionGradingResult> {
+  try {
+    const result = await input.evaluate(input.request);
+    const declared = new Map(
+      input.declarations.map((check) => [check.id, check.grader]),
+    );
+    return {
+      checks: result.checks.map((check) =>
+        extensionCheckResult(check, declared),
+      ),
+      metrics: result.metrics,
+      domainOutcomes: result.domainOutcomes ?? [],
+      recommendation: result.taskVerdictRecommendation ?? null,
+      failure: null,
+    };
+  } catch (error) {
+    const failure = {
+      message:
+        error instanceof ExtensionSourceChangedError
+          ? "extension source inputs changed during evaluation; assessment was refused. Finish changing evaluator inputs, then start a fresh evaluation with inputs held stable."
+          : "extension grading did not complete",
+      cancelable: true,
+    };
+    return {
+      checks: [],
+      metrics: [],
+      domainOutcomes: [],
+      recommendation: null,
+      failure,
+    };
+  }
+}

package/src/{evaluation-invocation-plan.ts → evaluation/invocation-plan.ts}

@@ -1,11 +1,12 @@
 import {
   EvaluationConfigurationError,
   type EvaluationOptions,
+  type HostAdapter,
   type ResolvedCase,
-} from "./evaluation-types";
-import { fixtureParts, type InlineArtifact } from "./preparation";
-import { isStringArray } from "./value-guards";
-import type { PreparationResult } from "./extension-session";
+} from "./types";
+import { fixtureParts, type InlineArtifact } from "../fixtures/preparation";
+import { isStringArray } from "../value-guards";
+import type { PreparationResult } from "../extension-session";
 
 interface InvocationArtifacts {
   extensionPreparation: PreparationResult | null;
@@ -13,12 +14,39 @@ interface InvocationArtifacts {
 }
 
 type ClaudePluginDirectories = NonNullable<
-  NonNullable<InvocationArtifacts["extensionPreparation"]>["claudePluginDirs"]
+  PreparationResult["claudePluginDirs"]
 >;
-type CodexMarketplace = NonNullable<
-  NonNullable<InvocationArtifacts["extensionPreparation"]>["codexMarketplace"]
+type CodexMarketplace = NonNullable<PreparationResult["codexMarketplace"]>;
+
+interface PackageDeclarations {
+  readonly codexMarketplace: PreparationResult["codexMarketplace"];
+  readonly claudePluginDirs: PreparationResult["claudePluginDirs"];
+}
+
+interface PackagePlan extends InvocationArtifacts, PackageDeclarations {}
+
+interface SkillDeclarations {
+  readonly codexSkillInvocation: PreparationResult["codexSkillInvocation"];
+  readonly codexRepositorySkillInvocation: PreparationResult["codexRepositorySkillInvocation"];
+  readonly claudeSkillInvocation: PreparationResult["claudeSkillInvocation"];
+  readonly claudeRepositorySkillInvocation: PreparationResult["claudeRepositorySkillInvocation"];
+}
+
+type CandidatePackageRequest = Pick<
+  Parameters<HostAdapter["run"]>[0],
+  "codexMarketplace" | "claudePluginDirs" | "explicitSkillInvocation"
 >;
 
+export interface PreparedInvocation extends PackageDeclarations {
+  readonly selected: SelectedSkillInvocation;
+  readonly invocationToken: string | null;
+  readonly configuration: Partial<SkillDeclarations>;
+  readonly hostRequest: CandidatePackageRequest;
+}
+
+export const INVOCATION_PLACEHOLDER = "{{sevro.skill_invocation}}";
+export const LEGACY_CODEX_PLACEHOLDER = "{{sevro.codex.skill_invocation}}";
+
 function uniquePluginRoots(value: unknown): value is string[] {
   return (
     isStringArray(value) &&
@@ -144,10 +172,10 @@ function validateCodexMarketplacePackage(
   );
 }
 
-function preparePackageArtifacts<T extends InvocationArtifacts>(
+function preparePackageArtifacts(
   options: EvaluationOptions,
-  state: T,
-) {
+  state: InvocationArtifacts,
+): PackagePlan {
   const codexMarketplace = state.extensionPreparation?.codexMarketplace;
   const claudePluginDirs = state.extensionPreparation?.claudePluginDirs;
   validateClaudePluginPackage(options, claudePluginDirs, state.inlineArtifacts);
@@ -159,15 +187,9 @@ function preparePackageArtifacts<T extends InvocationArtifacts>(
   return { ...state, codexMarketplace, claudePluginDirs };
 }
 
-type PackagePlan = ReturnType<
-  typeof preparePackageArtifacts<InvocationArtifacts>
->;
-const INVOCATION_PLACEHOLDER = "{{sevro.skill_invocation}}";
-const LEGACY_CODEX_PLACEHOLDER = "{{sevro.codex.skill_invocation}}";
-
 function skillInvocationDeclarations(
-  preparation: PackagePlan["extensionPreparation"],
-) {
+  preparation: PreparationResult | null,
+): SkillDeclarations {
   return {
     codexSkillInvocation: preparation?.codexSkillInvocation,
     codexRepositorySkillInvocation: preparation?.codexRepositorySkillInvocation,
@@ -177,7 +199,6 @@ function skillInvocationDeclarations(
   };
 }
 
-type SkillDeclarations = ReturnType<typeof skillInvocationDeclarations>;
 type PluginSkillInvocation = NonNullable<
   SkillDeclarations["codexSkillInvocation"]
 >;
@@ -211,26 +232,96 @@ function selectedSkillInvocation(
   return { kind: "none" };
 }
 
-function invocationTokenFor(declarations: SkillDeclarations): string | null {
-  if (declarations.codexRepositorySkillInvocation)
-    return `$${declarations.codexRepositorySkillInvocation.skillName}`;
-  if (declarations.claudeRepositorySkillInvocation)
-    return `/${declarations.claudeRepositorySkillInvocation.skillName}`;
-  if (declarations.codexSkillInvocation)
-    return `$${declarations.codexSkillInvocation.pluginName}:${declarations.codexSkillInvocation.skillName}`;
-  if (declarations.claudeSkillInvocation)
-    return `/${declarations.claudeSkillInvocation.pluginName}:${declarations.claudeSkillInvocation.skillName}`;
-  return null;
+function skillInvocationConfiguration(
+  selected: SelectedSkillInvocation,
+): Partial<SkillDeclarations> {
+  switch (selected.kind) {
+    case "codex-plugin":
+      return { codexSkillInvocation: selected.value };
+    case "codex-repository":
+      return { codexRepositorySkillInvocation: selected.value };
+    case "claude-plugin":
+      return { claudeSkillInvocation: selected.value };
+    case "claude-repository":
+      return { claudeRepositorySkillInvocation: selected.value };
+    case "none":
+      return {};
+  }
 }
 
-function invocationMetadata(declarations: SkillDeclarations) {
-  const repositoryInvocation =
-    declarations.codexRepositorySkillInvocation ??
-    declarations.claudeRepositorySkillInvocation;
-  const invocation = repositoryInvocation
-    ? { ...repositoryInvocation, scope: "repository" as const }
-    : (declarations.codexSkillInvocation ?? declarations.claudeSkillInvocation);
-  return { repositoryInvocation, invocation };
+function candidateInvocationRequest(
+  selected: SelectedSkillInvocation,
+): CandidatePackageRequest {
+  switch (selected.kind) {
+    case "codex-plugin":
+      return {
+        explicitSkillInvocation: {
+          ...selected.value,
+          token: `$${selected.value.pluginName}:${selected.value.skillName}`,
+        },
+      };
+    case "claude-plugin":
+      return {
+        explicitSkillInvocation: {
+          ...selected.value,
+          token: `/${selected.value.pluginName}:${selected.value.skillName}`,
+        },
+      };
+    case "codex-repository":
+      return {
+        explicitSkillInvocation: {
+          ...selected.value,
+          scope: "repository",
+          token: `$${selected.value.skillName}`,
+        },
+      };
+    case "claude-repository":
+      return {
+        explicitSkillInvocation: {
+          ...selected.value,
+          scope: "repository",
+          token: `/${selected.value.skillName}`,
+        },
+      };
+    case "none":
+      return {};
+  }
+}
+
+function candidateMarketplaceRequest(
+  context: PackagePlan,
+): CandidatePackageRequest {
+  const marketplace = context.codexMarketplace;
+  if (!marketplace) return {};
+  return {
+    codexMarketplace: {
+      ...marketplace,
+      artifactPaths: context.inlineArtifacts
+        .filter((artifact) =>
+          artifact.relativePath.startsWith(`${marketplace.artifactRoot}/`),
+        )
+        .map((artifact) => artifact.relativePath),
+    },
+  };
+}
+
+function candidatePluginDirectoriesRequest(
+  context: PackagePlan,
+): CandidatePackageRequest {
+  const directories = context.claudePluginDirs;
+  if (!directories) return {};
+  return {
+    claudePluginDirs: {
+      ...directories,
+      artifactPaths: context.inlineArtifacts
+        .filter((artifact) =>
+          directories.artifactRoots.some((root) =>
+            artifact.relativePath.startsWith(`${root}/`),
+          ),
+        )
+        .map((artifact) => artifact.relativePath),
+    },
+  };
 }
 
 function promptPlaceholderCount(
@@ -509,38 +600,36 @@ function validateSelectedSkillInvocation(
   }
 }
 
-function prepareSkillInvocation<T extends PackagePlan>(
+function prepareSkillInvocation(
   options: EvaluationOptions,
-  state: T,
-) {
+  state: PackagePlan,
+): PreparedInvocation {
   const declarations = skillInvocationDeclarations(state.extensionPreparation);
-  const metadata = invocationMetadata(declarations);
-  const invocationToken = invocationTokenFor(declarations);
   const placeholderCount =
     promptPlaceholderCount(options.case, INVOCATION_PLACEHOLDER) +
     promptPlaceholderCount(options.case, LEGACY_CODEX_PLACEHOLDER);
   validateOnlyOneInvocation(declarations);
-  validateSelectedSkillInvocation(
-    options,
-    state,
-    selectedSkillInvocation(declarations),
-    placeholderCount,
-  );
+  const selected = selectedSkillInvocation(declarations);
+  validateSelectedSkillInvocation(options, state, selected, placeholderCount);
+  const hostRequest = {
+    ...candidateMarketplaceRequest(state),
+    ...candidatePluginDirectoriesRequest(state),
+    ...candidateInvocationRequest(selected),
+  };
   return {
-    ...state,
-    ...declarations,
-    ...metadata,
-    invocationToken,
-    invocationPlaceholder: INVOCATION_PLACEHOLDER,
-    legacyCodexPlaceholder: LEGACY_CODEX_PLACEHOLDER,
-    placeholderCount,
+    codexMarketplace: state.codexMarketplace,
+    claudePluginDirs: state.claudePluginDirs,
+    selected,
+    invocationToken: hostRequest.explicitSkillInvocation?.token ?? null,
+    configuration: skillInvocationConfiguration(selected),
+    hostRequest,
   };
 }
 
-export function prepareInvocationPlan<T extends InvocationArtifacts>(
+export function prepareInvocationPlan(
   options: EvaluationOptions,
-  state: T,
-) {
+  state: InvocationArtifacts,
+): PreparedInvocation {
   return prepareSkillInvocation(
     options,
     preparePackageArtifacts(options, state),