@bjoernrochel/sevro 1.0.0 → 1.1.0

package/dist/src/cli.js

@@ -0,0 +1,20 @@
+#!/usr/bin/env node
+import { main } from "./cli/command.js";
+import { reportCommand } from "./report.js";
+import { informationCommand } from "./cli/help.js";
+import { demoCommand } from "./cli/demo.js";
+import { initCommand } from "./cli/init.js";
+import { pruneCommand } from "./prune.js";
+if (informationCommand(process.argv.slice(2)))
+    process.exitCode = 0;
+else if (process.argv[2] === "demo")
+    await demoCommand(process.argv.slice(3));
+else if (process.argv[2] === "init")
+    process.exitCode = await initCommand(process.argv.slice(3));
+else if (process.argv[2] === "report")
+    process.exitCode = await reportCommand(process.argv.slice(2));
+else if (process.argv[2] === "prune")
+    process.exitCode = await pruneCommand(process.argv.slice(3));
+else
+    await main(process.argv.slice(2));
+//# sourceMappingURL=cli.js.map

package/dist/src/cli.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.js","sourceRoot":"","sources":["cli.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,IAAI,EAAE,yBAAsB;AACrC,OAAO,EAAE,aAAa,EAAE,oBAAiB;AACzC,OAAO,EAAE,kBAAkB,EAAE,sBAAmB;AAChD,OAAO,EAAE,WAAW,EAAE,sBAAmB;AACzC,OAAO,EAAE,WAAW,EAAE,sBAAmB;AACzC,OAAO,EAAE,YAAY,EAAE,mBAAgB;AAEvC,IAAI,kBAAkB,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAAE,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;KAC/D,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,MAAM;IAAE,MAAM,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;KACzE,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,MAAM;IACjC,OAAO,CAAC,QAAQ,GAAG,MAAM,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;KACzD,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,QAAQ;IACnC,OAAO,CAAC,QAAQ,GAAG,MAAM,aAAa,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;KAC3D,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,OAAO;IAClC,OAAO,CAAC,QAAQ,GAAG,MAAM,YAAY,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;;IAC1D,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC","sourcesContent":["#!/usr/bin/env node\nimport { main } from \"./cli/command\";\nimport { reportCommand } from \"./report\";\nimport { informationCommand } from \"./cli/help\";\nimport { demoCommand } from \"./cli/demo\";\nimport { initCommand } from \"./cli/init\";\nimport { pruneCommand } from \"./prune\";\n\nif (informationCommand(process.argv.slice(2))) process.exitCode = 0;\nelse if (process.argv[2] === \"demo\") await demoCommand(process.argv.slice(3));\nelse if (process.argv[2] === \"init\")\n  process.exitCode = await initCommand(process.argv.slice(3));\nelse if (process.argv[2] === \"report\")\n  process.exitCode = await reportCommand(process.argv.slice(2));\nelse if (process.argv[2] === \"prune\")\n  process.exitCode = await pruneCommand(process.argv.slice(3));\nelse await main(process.argv.slice(2));\n"]}

package/dist/src/contract-types.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=contract-types.js.map

package/dist/src/contract-types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"contract-types.js","sourceRoot":"","sources":["contract-types.ts"],"names":[],"mappings":"","sourcesContent":["/** Shared structures; public schemas and runtime guards still validate external data. */\nexport type ExecutionStatus = \"completed\" | \"failed\" | \"cancelled\" | \"not_run\";\nexport type GradingStatus =\n  \"completed\" | \"error\" | \"unavailable\" | \"not_requested\";\nexport type TaskVerdict = \"passed\" | \"failed\" | \"not_assessed\";\nexport type OutcomeStatus = \"passed\" | \"failed\" | \"unavailable\";\nexport type ExitCode = 0 | 1 | 2 | 3 | 4 | 64 | 70 | 130 | 143;\n\nexport interface Assessment {\n  execution: { status: ExecutionStatus };\n  grading: { status: GradingStatus };\n  task: { verdict: TaskVerdict };\n}\n\nexport type CheckDeclaration = {\n  id: string;\n  grader: string;\n  configuration: Record<string, unknown>;\n};\n\n/** Runner check outcomes include their grader and may lack evidence references. */\nexport interface CheckOutcome {\n  id: string;\n  grader: string;\n  status: OutcomeStatus;\n  detail?: string;\n  evidenceRefs?: string[];\n}\n\n/** Extension checks require references; their grader comes from the declaration. */\nexport type ExtensionCheckOutcome = {\n  id: string;\n  status: OutcomeStatus;\n  detail?: string;\n  evidenceRefs: string[];\n};\n\n/** Domain outcomes are retained independently of task checks and verdicts. */\nexport type DomainOutcome = {\n  id: string;\n  status: OutcomeStatus;\n  detail?: string;\n  evidenceRefs: string[];\n  data?: Record<string, unknown>;\n};\n\nexport type ObservationCompleteness = \"complete\" | \"partial\" | \"unavailable\";\n\nexport type ObservationEvidence = {\n  id: string;\n  source: string;\n  completeness: ObservationCompleteness;\n  data: Record<string, unknown>;\n};\n\nexport type MetricEvidence = {\n  id: string;\n  value: number | null;\n  unit: string;\n};\n\n/** Retained measurements keep unknown values and completeness separate. */\nexport type UsageEvidence = {\n  inputTokens: number | null;\n  outputTokens: number | null;\n  costUsd: number | null;\n  complete: boolean;\n};\n\nexport type RawResultReference = {\n  source: string;\n  path: string | null;\n  sha256: string | null;\n};\n\nexport type ArtifactReference = {\n  id: string;\n  path: string;\n  sha256: string;\n  gitExclude?: boolean;\n  executable?: boolean;\n};\n\nexport type EvidenceRoute = {\n  role: \"candidate\" | \"semantic\" | \"advisory\";\n  host: string;\n  model: string;\n  effort: string;\n};\n\nexport type Diagnostic = {\n  code: string;\n  message: string;\n};\n"]}

package/dist/src/engine.js

@@ -0,0 +1,4 @@
+export { EvaluationConfigurationError, HostExecutionFailure, } from "./evaluation/types.js";
+export { runEvaluation } from "./evaluation/run.js";
+export { loadRuntimeConfiguration } from "./runtime/config.js";
+//# sourceMappingURL=engine.js.map

package/dist/src/engine.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"engine.js","sourceRoot":"","sources":["engine.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,4BAA4B,EAC5B,oBAAoB,GACrB,8BAA2B;AAS5B,OAAO,EAAE,aAAa,EAAE,4BAAyB;AACjD,OAAO,EAAE,wBAAwB,EAAE,4BAAyB","sourcesContent":["export {\n  EvaluationConfigurationError,\n  HostExecutionFailure,\n} from \"./evaluation/types\";\nexport type {\n  HostResult,\n  HostAdapter,\n  ResolvedCase,\n  EvaluationOptions,\n  EvaluationProgress,\n  CliResult,\n} from \"./evaluation/types\";\nexport { runEvaluation } from \"./evaluation/run\";\nexport { loadRuntimeConfiguration } from \"./runtime/config\";\nexport type { RuntimeConfiguration, RuntimePolicy } from \"./runtime/config\";\nexport type {\n  RuntimeArtifactDirectory,\n  CandidateArtifactRoots,\n} from \"./candidate-artifacts\";\n"]}

package/dist/src/evaluation/advisory.js

@@ -0,0 +1,85 @@
+import { advisoryPrompt, parseAdvisoryAssessment, } from "../advisory.js";
+import { usage } from "../evidence/usage.js";
+import { buildBlindAdvisoryFixture } from "../fixtures/advisory.js";
+import { removeWorkspace } from "../fixtures/workspace-state.js";
+import { HostDiagnosticPersistenceError } from "../host-diagnostics.js";
+import { gradingRuntimePolicy, scopedHostArtifacts } from "./grading-host.js";
+import { TrialPersistenceError, } from "./types.js";
+function defaultAdvisoryReview(host) {
+    return {
+        status: "not_run",
+        assessment: null,
+        usage: usage(null),
+        rawResult: { source: host.id, path: null, sha256: null },
+    };
+}
+function completeAdvisoryAssessment(response) {
+    if (response.executionFailed ||
+        !response.complete ||
+        response.finalMessage === null)
+        throw new Error("advisory response is incomplete");
+    return parseAdvisoryAssessment(response.finalMessage);
+}
+async function cleanupAdvisoryWorkspace(workspace, successful, effects) {
+    if (!workspace)
+        return;
+    if (!successful)
+        return effects.retainWorkspace(workspace);
+    try {
+        await removeWorkspace(workspace);
+    }
+    catch {
+        console.warn(`advisory fixture cleanup failed; retained at ${workspace}`);
+    }
+}
+async function executeAdvisoryReview(input, effects, review) {
+    let workspace = null;
+    try {
+        if (input.baseRevision === null)
+            throw new Error("advisory base revision is unavailable");
+        workspace = await buildBlindAdvisoryFixture(input.candidateWorkspace, {
+            baseRevision: input.baseRevision,
+            excludedPaths: input.excludedPaths,
+        });
+        const response = await input.host.run({
+            retainHostDiagnostics: (bytes) => effects.evidence.retainArtifacts([
+                {
+                    id: "sevro.advisory.sevro.host.diagnostics",
+                    bytes: Buffer.from(bytes),
+                },
+            ]),
+            candidateTranscriptRoot: input.candidateTranscriptRoot,
+            candidateArtifactRoots: input.candidateArtifactRoots,
+            prompt: advisoryPrompt(input.task, [...input.checks]),
+            runtimePolicy: gradingRuntimePolicy(input.runtimePolicy),
+            runtimeRole: "advisory",
+            workspace,
+            condition: "passive",
+            signal: input.signal,
+        });
+        review.usage = usage(response);
+        const artifacts = scopedHostArtifacts(response, "sevro.advisory.", effects.evidence.artifactRefs, input.declaredChecks, "sevro.advisory.response");
+        await effects.evidence.retainArtifacts(artifacts);
+        review.rawResult = await effects.evidence.retainAdvisoryResponse(input.host.id, response.finalMessage);
+        review.assessment = completeAdvisoryAssessment(response);
+        review.status = "completed";
+    }
+    catch (error) {
+        if (error instanceof HostDiagnosticPersistenceError ||
+            error instanceof TrialPersistenceError)
+            throw error;
+        review.status = "failed";
+    }
+    finally {
+        await cleanupAdvisoryWorkspace(workspace, review.status === "completed", effects);
+    }
+}
+/** Advisory execution depends on candidate completion, independently of task grading. */
+export async function runAdvisoryReview(input, effects) {
+    const review = defaultAdvisoryReview(input.host);
+    if (!input.candidateCompleted || input.signal?.aborted)
+        return review;
+    await executeAdvisoryReview(input, effects, review);
+    return review;
+}
+//# sourceMappingURL=advisory.js.map

package/dist/src/evaluation/advisory.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"advisory.js","sourceRoot":"","sources":["advisory.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,cAAc,EACd,uBAAuB,GAExB,uBAAoB;AAGrB,OAAO,EAAE,KAAK,EAAE,6BAA0B;AAC1C,OAAO,EAAE,yBAAyB,EAAE,gCAA6B;AACjE,OAAO,EAAE,eAAe,EAAE,uCAAoC;AAC9D,OAAO,EAAE,8BAA8B,EAAE,+BAA4B;AAIrE,OAAO,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,0BAAuB;AAC3E,OAAO,EACL,qBAAqB,GAGtB,mBAAgB;AA2BjB,SAAS,qBAAqB,CAAC,IAAiB;IAC9C,OAAO;QACL,MAAM,EAAE,SAAS;QACjB,UAAU,EAAE,IAAI;QAChB,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC;QAClB,SAAS,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE;KACzD,CAAC;AACJ,CAAC;AAED,SAAS,0BAA0B,CAAC,QAAoB;IACtD,IACE,QAAQ,CAAC,eAAe;QACxB,CAAC,QAAQ,CAAC,QAAQ;QAClB,QAAQ,CAAC,YAAY,KAAK,IAAI;QAE9B,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;IACrD,OAAO,uBAAuB,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;AACxD,CAAC;AAED,KAAK,UAAU,wBAAwB,CACrC,SAAwB,EACxB,UAAmB,EACnB,OAAwB;IAExB,IAAI,CAAC,SAAS;QAAE,OAAO;IACvB,IAAI,CAAC,UAAU;QAAE,OAAO,OAAO,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC;IAC3D,IAAI,CAAC;QACH,MAAM,eAAe,CAAC,SAAS,CAAC,CAAC;IACnC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,CAAC,IAAI,CAAC,gDAAgD,SAAS,EAAE,CAAC,CAAC;IAC5E,CAAC;AACH,CAAC;AAED,KAAK,UAAU,qBAAqB,CAClC,KAAoB,EACpB,OAAwB,EACxB,MAAsB;IAEtB,IAAI,SAAS,GAAkB,IAAI,CAAC;IACpC,IAAI,CAAC;QACH,IAAI,KAAK,CAAC,YAAY,KAAK,IAAI;YAC7B,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAC3D,SAAS,GAAG,MAAM,yBAAyB,CAAC,KAAK,CAAC,kBAAkB,EAAE;YACpE,YAAY,EAAE,KAAK,CAAC,YAAY;YAChC,aAAa,EAAE,KAAK,CAAC,aAAa;SACnC,CAAC,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC;YACpC,qBAAqB,EAAE,CAAC,KAAK,EAAE,EAAE,CAC/B,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC;gBAC/B;oBACE,EAAE,EAAE,uCAAuC;oBAC3C,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC;iBAC1B;aACF,CAAC;YACJ,uBAAuB,EAAE,KAAK,CAAC,uBAAuB;YACtD,sBAAsB,EAAE,KAAK,CAAC,sBAAsB;YACpD,MAAM,EAAE,cAAc,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;YACrD,aAAa,EAAE,oBAAoB,CAAC,KAAK,CAAC,aAAa,CAAC;YACxD,WAAW,EAAE,UAAU;YACvB,SAAS;YACT,SAAS,EAAE,SAAS;YACpB,MAAM,EAAE,KAAK,CAAC,MAAM;SACrB,CAAC,CAAC;QACH,MAAM,CAAC,KAAK,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC;QAC/B,MAAM,SAAS,GAAG,mBAAmB,CACnC,QAAQ,EACR,iBAAiB,EACjB,OAAO,CAAC,QAAQ,CAAC,YAAY,EAC7B,KAAK,CAAC,cAAc,EACpB,yBAAyB,CAC1B,CAAC;QACF,MAAM,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC;QAClD,MAAM,CAAC,SAAS,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAC9D,KAAK,CAAC,IAAI,CAAC,EAAE,EACb,QAAQ,CAAC,YAAY,CACtB,CAAC;QACF,MAAM,CAAC,UAAU,GAAG,0BAA0B,CAAC,QAAQ,CAAC,CAAC;QACzD,MAAM,CAAC,MAAM,GAAG,WAAW,CAAC;IAC9B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IACE,KAAK,YAAY,8BAA8B;YAC/C,KAAK,YAAY,qBAAqB;YAEtC,MAAM,KAAK,CAAC;QACd,MAAM,CAAC,MAAM,GAAG,QAAQ,CAAC;IAC3B,CAAC;YAAS,CAAC;QACT,MAAM,wBAAwB,CAC5B,SAAS,EACT,MAAM,CAAC,MAAM,KAAK,WAAW,EAC7B,OAAO,CACR,CAAC;IACJ,CAAC;AACH,CAAC;AAED,yFAAyF;AACzF,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,KAAoB,EACpB,OAAwB;IAExB,MAAM,MAAM,GAAG,qBAAqB,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACjD,IAAI,CAAC,KAAK,CAAC,kBAAkB,IAAI,KAAK,CAAC,MAAM,EAAE,OAAO;QAAE,OAAO,MAAM,CAAC;IACtE,MAAM,qBAAqB,CAAC,KAAK,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;IACpD,OAAO,MAAM,CAAC;AAChB,CAAC","sourcesContent":["import {\n  advisoryPrompt,\n  parseAdvisoryAssessment,\n  type AdvisoryAssessment,\n} from \"../advisory\";\nimport type { CandidateArtifactRoots } from \"../candidate-artifacts\";\nimport type { TrialEvidenceWriter } from \"../evidence/trial-writer\";\nimport { usage } from \"../evidence/usage\";\nimport { buildBlindAdvisoryFixture } from \"../fixtures/advisory\";\nimport { removeWorkspace } from \"../fixtures/workspace-state\";\nimport { HostDiagnosticPersistenceError } from \"../host-diagnostics\";\nimport type { CheckOutcome } from \"../results\";\nimport type { AdvisoryReviewData } from \"../schema-types\";\nimport type { RuntimePolicy } from \"../runtime/config\";\nimport { gradingRuntimePolicy, scopedHostArtifacts } from \"./grading-host\";\nimport {\n  TrialPersistenceError,\n  type HostAdapter,\n  type HostResult,\n} from \"./types\";\n\nexport type AdvisoryReview = AdvisoryReviewData;\n\ninterface AdvisoryInput {\n  readonly host: HostAdapter;\n  readonly candidateCompleted: boolean;\n  readonly candidateWorkspace: string;\n  readonly candidateTranscriptRoot?: string;\n  readonly candidateArtifactRoots?: CandidateArtifactRoots;\n  readonly baseRevision: string | null;\n  readonly excludedPaths?: string[];\n  readonly task: string;\n  readonly checks: readonly CheckOutcome[];\n  readonly declaredChecks: { id: string }[];\n  readonly runtimePolicy?: RuntimePolicy;\n  readonly signal?: AbortSignal;\n}\n\ninterface AdvisoryEffects {\n  readonly evidence: Pick<\n    TrialEvidenceWriter,\n    \"artifactRefs\" | \"retainArtifacts\" | \"retainAdvisoryResponse\"\n  >;\n  retainWorkspace(workspace: string): Promise<void>;\n}\n\nfunction defaultAdvisoryReview(host: HostAdapter): AdvisoryReview {\n  return {\n    status: \"not_run\",\n    assessment: null,\n    usage: usage(null),\n    rawResult: { source: host.id, path: null, sha256: null },\n  };\n}\n\nfunction completeAdvisoryAssessment(response: HostResult): AdvisoryAssessment {\n  if (\n    response.executionFailed ||\n    !response.complete ||\n    response.finalMessage === null\n  )\n    throw new Error(\"advisory response is incomplete\");\n  return parseAdvisoryAssessment(response.finalMessage);\n}\n\nasync function cleanupAdvisoryWorkspace(\n  workspace: string | null,\n  successful: boolean,\n  effects: AdvisoryEffects,\n): Promise<void> {\n  if (!workspace) return;\n  if (!successful) return effects.retainWorkspace(workspace);\n  try {\n    await removeWorkspace(workspace);\n  } catch {\n    console.warn(`advisory fixture cleanup failed; retained at ${workspace}`);\n  }\n}\n\nasync function executeAdvisoryReview(\n  input: AdvisoryInput,\n  effects: AdvisoryEffects,\n  review: AdvisoryReview,\n): Promise<void> {\n  let workspace: string | null = null;\n  try {\n    if (input.baseRevision === null)\n      throw new Error(\"advisory base revision is unavailable\");\n    workspace = await buildBlindAdvisoryFixture(input.candidateWorkspace, {\n      baseRevision: input.baseRevision,\n      excludedPaths: input.excludedPaths,\n    });\n    const response = await input.host.run({\n      retainHostDiagnostics: (bytes) =>\n        effects.evidence.retainArtifacts([\n          {\n            id: \"sevro.advisory.sevro.host.diagnostics\",\n            bytes: Buffer.from(bytes),\n          },\n        ]),\n      candidateTranscriptRoot: input.candidateTranscriptRoot,\n      candidateArtifactRoots: input.candidateArtifactRoots,\n      prompt: advisoryPrompt(input.task, [...input.checks]),\n      runtimePolicy: gradingRuntimePolicy(input.runtimePolicy),\n      runtimeRole: \"advisory\",\n      workspace,\n      condition: \"passive\",\n      signal: input.signal,\n    });\n    review.usage = usage(response);\n    const artifacts = scopedHostArtifacts(\n      response,\n      \"sevro.advisory.\",\n      effects.evidence.artifactRefs,\n      input.declaredChecks,\n      \"sevro.advisory.response\",\n    );\n    await effects.evidence.retainArtifacts(artifacts);\n    review.rawResult = await effects.evidence.retainAdvisoryResponse(\n      input.host.id,\n      response.finalMessage,\n    );\n    review.assessment = completeAdvisoryAssessment(response);\n    review.status = \"completed\";\n  } catch (error) {\n    if (\n      error instanceof HostDiagnosticPersistenceError ||\n      error instanceof TrialPersistenceError\n    )\n      throw error;\n    review.status = \"failed\";\n  } finally {\n    await cleanupAdvisoryWorkspace(\n      workspace,\n      review.status === \"completed\",\n      effects,\n    );\n  }\n}\n\n/** Advisory execution depends on candidate completion, independently of task grading. */\nexport async function runAdvisoryReview(\n  input: AdvisoryInput,\n  effects: AdvisoryEffects,\n): Promise<AdvisoryReview> {\n  const review = defaultAdvisoryReview(input.host);\n  if (!input.candidateCompleted || input.signal?.aborted) return review;\n  await executeAdvisoryReview(input, effects, review);\n  return review;\n}\n"]}

package/dist/src/evaluation/check-plan.js

@@ -0,0 +1,292 @@
+import { isAbsolute } from "node:path";
+import { validQualifiedId } from "../value-guards.js";
+import { validInlineFixture } from "../case-validation.js";
+import { EvaluationConfigurationError, } from "./types.js";
+import { prepareGeneratedFixture, } from "../fixtures/generated.js";
+import { prepareGitHeadChecks, } from "../graders/git-head.js";
+import { isOutputGrader, prepareOutputChecks, } from "../graders/output.js";
+import { prepareSemanticChecks, } from "../graders/semantic.js";
+import { prepareShellChecks, } from "../graders/shell.js";
+import { hashJson } from "../identity.js";
+import { fixtureParts } from "../fixtures/preparation.js";
+import { prepareRepositoryFixture, } from "../fixtures/repository.js";
+import { resolvedFixture } from "../resolved-case.js";
+import { isOptionalNonblankString, isRecord } from "../value-guards.js";
+function validateRootOptions(options) {
+    if (!isAbsolute(options.projectRoot) ||
+        !isAbsolute(options.resultsRoot) ||
+        (options.runStateRoot !== undefined && !isAbsolute(options.runStateRoot)))
+        throw new EvaluationConfigurationError("project, results, and run-state roots must be absolute");
+}
+function hostIdentityComplete(host) {
+    return Boolean(host.id && host.model && host.effort);
+}
+function validateEvaluationIdentity(options) {
+    if (!options.case.id ||
+        !options.case.prompt ||
+        !hostIdentityComplete(options.host))
+        throw new EvaluationConfigurationError("case and host identities must be nonempty");
+}
+function validateFollowUpPrompt(value) {
+    if (!isOptionalNonblankString(value))
+        throw new EvaluationConfigurationError("follow-up prompt must be nonempty");
+}
+function validateContinuationHost(options) {
+    if (options.case.followUpPrompt !== undefined &&
+        !options.host.hostCapabilities?.includes("sevro.host.continuation"))
+        throw new EvaluationConfigurationError("selected host does not support continuation");
+}
+function validateContinuationNegotiation(options) {
+    if (options.case.followUpPrompt !== undefined &&
+        options.extension &&
+        !options.extension.session.identity.capabilities.includes("sevro.host.continuation"))
+        throw new EvaluationConfigurationError("continuation capability was not negotiated");
+}
+function validateEvaluationInputs(options) {
+    validateRootOptions(options);
+    validateEvaluationIdentity(options);
+    validateFollowUpPrompt(options.case.followUpPrompt);
+    validateContinuationHost(options);
+    validateContinuationNegotiation(options);
+}
+function evaluationFixtureObject(value) {
+    if (!isRecord(value))
+        throw new EvaluationConfigurationError("invalid fixture declaration");
+    return value;
+}
+function generatedEvaluationFixture(fixture) {
+    try {
+        return prepareGeneratedFixture(fixture);
+    }
+    catch (cause) {
+        throw new EvaluationConfigurationError(cause instanceof Error ? cause.message : "invalid generated fixture", { cause });
+    }
+}
+function repositoryEvaluationFixture(fixture) {
+    try {
+        return prepareRepositoryFixture({ kind: "repository", ...fixture });
+    }
+    catch (cause) {
+        throw new EvaluationConfigurationError(cause instanceof Error ? cause.message : "invalid repository fixture", { cause });
+    }
+}
+function validateInlineEvaluationFixture(fixture) {
+    if (!validInlineFixture(fixture))
+        throw new EvaluationConfigurationError("fixture must declare inline files or a repository source");
+}
+function prepareEvaluationFixture(options) {
+    const fixture = evaluationFixtureObject(options.case.fixture);
+    if ("kind" in fixture)
+        return {
+            kind: "generated",
+            fixture: generatedEvaluationFixture(fixture),
+        };
+    if (Object.hasOwn(fixture, "sourceRef"))
+        return {
+            kind: "repository",
+            fixture: repositoryEvaluationFixture(fixture),
+            repository: null,
+        };
+    validateInlineEvaluationFixture(fixture);
+    return { kind: "inline", files: options.case.fixture.files };
+}
+function validateTrialInteger(value, message) {
+    if (!Number.isSafeInteger(value) || value < 1)
+        throw new EvaluationConfigurationError(message);
+}
+function validatePassThreshold(value) {
+    if (!Number.isFinite(value) || value <= 0 || value > 1)
+        throw new EvaluationConfigurationError("pass threshold must be greater than zero and at most one");
+}
+function validateRequiredEvidence(ids) {
+    if (new Set(ids).size !== ids.length ||
+        ids.some((id) => !validQualifiedId(id)))
+        throw new EvaluationConfigurationError("invalid required evidence IDs");
+}
+function prepareTrialParameters(options) {
+    validateTrialInteger(options.trialCount, "trial count must be a positive integer");
+    const jobs = options.jobs ?? 3;
+    validateTrialInteger(jobs, "jobs must be a positive integer");
+    validatePassThreshold(options.passThreshold);
+    validateRequiredEvidence(options.case.requiredEvidence);
+    return jobs;
+}
+function declaredCaseChecks(options) {
+    const allBuiltinDeclarations = options.case.checks.filter((check) => isOutputGrader(check.grader));
+    const allShellDeclarations = options.case.checks.filter((check) => check.grader === "sevro.shell");
+    const allGitHeadDeclarations = options.case.checks.filter((check) => check.grader === "sevro.git-head");
+    const allSemanticDeclarations = options.case.checks.filter((check) => check.grader === "sevro.semantic");
+    const extensionDeclarations = options.case.checks.filter((check) => !isOutputGrader(check.grader) &&
+        check.grader !== "sevro.shell" &&
+        check.grader !== "sevro.git-head" &&
+        check.grader !== "sevro.semantic");
+    const replacedBuiltinGraders = options.extension?.session.identity.replacedBuiltinGraders ?? [];
+    const replaced = new Set(replacedBuiltinGraders);
+    return {
+        allBuiltinDeclarations,
+        allShellDeclarations,
+        allGitHeadDeclarations,
+        allSemanticDeclarations,
+        extensionDeclarations,
+        replacedBuiltinGraders,
+        replaced,
+    };
+}
+function validateBuiltInReplacements(context) {
+    const { allBuiltinDeclarations, allShellDeclarations, allGitHeadDeclarations, allSemanticDeclarations, extensionDeclarations, replacedBuiltinGraders, replaced, } = context;
+    const declaredGraders = new Set([
+        ...allBuiltinDeclarations,
+        ...allShellDeclarations,
+        ...allGitHeadDeclarations,
+        ...allSemanticDeclarations,
+    ].map((check) => check.grader));
+    if (replaced.size !== replacedBuiltinGraders.length ||
+        replacedBuiltinGraders.some((id) => !declaredGraders.has(id)))
+        throw new EvaluationConfigurationError("unknown or duplicate built-in grader replacement");
+    if (replaced.size && !extensionDeclarations.length)
+        throw new EvaluationConfigurationError("built-in grader replacement requires an extension check");
+}
+function activeCheckDeclarations(context) {
+    const { allBuiltinDeclarations, allShellDeclarations, allGitHeadDeclarations, allSemanticDeclarations, replaced, } = context;
+    const builtinDeclarations = allBuiltinDeclarations.filter((check) => !replaced.has(check.grader));
+    const shellDeclarations = allShellDeclarations.filter((check) => !replaced.has(check.grader));
+    const gitHeadDeclarations = allGitHeadDeclarations.filter((check) => !replaced.has(check.grader));
+    const semanticDeclarations = allSemanticDeclarations.filter((check) => !replaced.has(check.grader));
+    return {
+        builtinDeclarations,
+        shellDeclarations,
+        gitHeadDeclarations,
+        semanticDeclarations,
+    };
+}
+function validateSemanticRoute(options, checks) {
+    if (checks.length && !options.semanticHost)
+        throw new EvaluationConfigurationError("semantic checks require an explicit semantic host");
+    if (options.semanticHost && !hostIdentityComplete(options.semanticHost))
+        throw new EvaluationConfigurationError("semantic host identity is incomplete");
+}
+function validateAdvisoryGitFixture(source) {
+    if (source.kind === "inline")
+        throw new EvaluationConfigurationError("advisory review requires a Git fixture");
+}
+function validateAdvisoryExclusions(paths) {
+    try {
+        for (const path of paths) {
+            fixtureParts(path);
+            if (path.toLowerCase() === ".git" ||
+                path.toLowerCase().startsWith(".git/"))
+                throw new Error("repository metadata cannot be an advisory exclusion");
+        }
+    }
+    catch (cause) {
+        throw new EvaluationConfigurationError("invalid advisory exclusion path", {
+            cause,
+        });
+    }
+}
+function validateAdvisoryRoute(options, source) {
+    const host = options.advisoryHost;
+    if (!host) {
+        if (options.advisoryExcludedPaths?.length)
+            throw new EvaluationConfigurationError("advisory exclusions require an advisory host");
+        return;
+    }
+    validateAdvisoryGitFixture(source);
+    validateAdvisoryIdentity(host);
+    validateAdvisoryExclusions(options.advisoryExcludedPaths ?? []);
+}
+function validateAdvisoryIdentity(host) {
+    if (!hostIdentityComplete(host))
+        throw new EvaluationConfigurationError("advisory host identity is incomplete");
+}
+function validateShellRoute(options, checks) {
+    if (checks.length && !options.shellIsolation)
+        throw new EvaluationConfigurationError("shell checks require explicit protected source roots");
+}
+function validateGitHeadRoute(source, checks) {
+    if (checks.length && source.kind === "inline")
+        throw new EvaluationConfigurationError("Git HEAD checks require a Git fixture");
+}
+function validateCaseCheckIds(checks) {
+    if (new Set(checks.map((check) => check.id)).size !== checks.length)
+        throw new EvaluationConfigurationError("case check IDs must be unique");
+}
+function validateExtensionGraders(options, declarations) {
+    if (!declarations.length)
+        return;
+    const extension = options.extension;
+    if (!extension)
+        throw new EvaluationConfigurationError("case declares an unavailable extension grader");
+    if (declarations.some((check) => !extension.session.identity.graders.includes(check.grader)))
+        throw new EvaluationConfigurationError("case declares an unavailable extension grader");
+}
+function comparisonCase(caseData) {
+    return {
+        id: caseData.id,
+        prompt: caseData.prompt,
+        ...(caseData.followUpPrompt !== undefined
+            ? { followUpPrompt: caseData.followUpPrompt }
+            : {}),
+        fixture: caseData.fixture,
+        checks: caseData.checks,
+        requiredEvidence: caseData.requiredEvidence,
+    };
+}
+function validateResolvedExtensionCase(options) {
+    if (!options.extension)
+        return;
+    const resolved = options.extension.resolvedCase;
+    const comparable = comparisonCase({
+        ...resolved,
+        fixture: resolvedFixture(resolved.fixture),
+    });
+    if (hashJson(comparable) !== hashJson(comparisonCase(options.case)))
+        throw new EvaluationConfigurationError("resolved extension case does not match the selected case");
+}
+function activePreparedChecks(checks, replaced, grader) {
+    return replaced.has(grader) ? [] : checks;
+}
+function validateInlinePaths(files) {
+    for (const path of Object.keys(files ?? {}))
+        fixtureParts(path);
+}
+function prepareBuiltInChecks(options, context) {
+    const { allBuiltinDeclarations, allShellDeclarations, allGitHeadDeclarations, allSemanticDeclarations, replaced, } = context;
+    try {
+        const prepared = prepareOutputChecks(allBuiltinDeclarations).filter((check) => !replaced.has(check.grader));
+        const preparedShell = activePreparedChecks(prepareShellChecks(allShellDeclarations), replaced, "sevro.shell");
+        const preparedGitHead = activePreparedChecks(prepareGitHeadChecks(allGitHeadDeclarations), replaced, "sevro.git-head");
+        const preparedSemantic = activePreparedChecks(prepareSemanticChecks(allSemanticDeclarations), replaced, "sevro.semantic");
+        validateInlinePaths(options.case.fixture.files);
+        return { prepared, preparedShell, preparedGitHead, preparedSemantic };
+    }
+    catch (cause) {
+        throw new EvaluationConfigurationError(cause instanceof Error ? cause.message : "invalid case configuration", { cause });
+    }
+}
+function prepareCheckPlan(options, source) {
+    const declarations = declaredCaseChecks(options);
+    validateBuiltInReplacements(declarations);
+    const active = activeCheckDeclarations(declarations);
+    validateSemanticRoute(options, active.semanticDeclarations);
+    validateAdvisoryRoute(options, source);
+    validateShellRoute(options, active.shellDeclarations);
+    validateGitHeadRoute(source, active.gitHeadDeclarations);
+    validateCaseCheckIds(options.case.checks);
+    validateExtensionGraders(options, declarations.extensionDeclarations);
+    validateResolvedExtensionCase(options);
+    return {
+        ...active,
+        ...prepareBuiltInChecks(options, declarations),
+        extensionDeclarations: declarations.extensionDeclarations,
+        replacedBuiltinGraders: declarations.replacedBuiltinGraders,
+        replaced: declarations.replaced,
+    };
+}
+export function prepareEvaluationChecks(options) {
+    validateEvaluationInputs(options);
+    const source = prepareEvaluationFixture(options);
+    const jobs = prepareTrialParameters(options);
+    return { source, jobs, checks: prepareCheckPlan(options, source) };
+}
+//# sourceMappingURL=check-plan.js.map

package/dist/src/evaluation/check-plan.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"check-plan.js","sourceRoot":"","sources":["check-plan.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,kBAAkB;AACvC,OAAO,EAAE,gBAAgB,EAAE,2BAAwB;AACnD,OAAO,EAAE,kBAAkB,EAAE,8BAA2B;AACxD,OAAO,EACL,4BAA4B,GAI7B,mBAAgB;AACjB,OAAO,EACL,uBAAuB,GAExB,iCAA8B;AAC/B,OAAO,EACL,oBAAoB,GAGrB,+BAA4B;AAC7B,OAAO,EACL,cAAc,EACd,mBAAmB,GAGpB,6BAA0B;AAC3B,OAAO,EACL,qBAAqB,GAGtB,+BAA4B;AAC7B,OAAO,EACL,kBAAkB,GAGnB,4BAAyB;AAC1B,OAAO,EAAE,QAAQ,EAAE,uBAAoB;AACvC,OAAO,EAAE,YAAY,EAAE,mCAAgC;AACvD,OAAO,EACL,wBAAwB,GAEzB,kCAA+B;AAChC,OAAO,EAAE,eAAe,EAAE,4BAAyB;AACnD,OAAO,EAAE,wBAAwB,EAAE,QAAQ,EAAE,2BAAwB;AAwCrE,SAAS,mBAAmB,CAAC,OAA0B;IACrD,IACE,CAAC,UAAU,CAAC,OAAO,CAAC,WAAW,CAAC;QAChC,CAAC,UAAU,CAAC,OAAO,CAAC,WAAW,CAAC;QAChC,CAAC,OAAO,CAAC,YAAY,KAAK,SAAS,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;QAEzE,MAAM,IAAI,4BAA4B,CACpC,wDAAwD,CACzD,CAAC;AACN,CAAC;AAED,SAAS,oBAAoB,CAAC,IAAiB;IAC7C,OAAO,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC,CAAC;AACvD,CAAC;AAED,SAAS,0BAA0B,CAAC,OAA0B;IAC5D,IACE,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE;QAChB,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM;QACpB,CAAC,oBAAoB,CAAC,OAAO,CAAC,IAAI,CAAC;QAEnC,MAAM,IAAI,4BAA4B,CACpC,2CAA2C,CAC5C,CAAC;AACN,CAAC;AAED,SAAS,sBAAsB,CAAC,KAAc;IAC5C,IAAI,CAAC,wBAAwB,CAAC,KAAK,CAAC;QAClC,MAAM,IAAI,4BAA4B,CAAC,mCAAmC,CAAC,CAAC;AAChF,CAAC;AAED,SAAS,wBAAwB,CAAC,OAA0B;IAC1D,IACE,OAAO,CAAC,IAAI,CAAC,cAAc,KAAK,SAAS;QACzC,CAAC,OAAO,CAAC,IAAI,CAAC,gBAAgB,EAAE,QAAQ,CAAC,yBAAyB,CAAC;QAEnE,MAAM,IAAI,4BAA4B,CACpC,6CAA6C,CAC9C,CAAC;AACN,CAAC;AAED,SAAS,+BAA+B,CAAC,OAA0B;IACjE,IACE,OAAO,CAAC,IAAI,CAAC,cAAc,KAAK,SAAS;QACzC,OAAO,CAAC,SAAS;QACjB,CAAC,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,QAAQ,CACvD,yBAAyB,CAC1B;QAED,MAAM,IAAI,4BAA4B,CACpC,4CAA4C,CAC7C,CAAC;AACN,CAAC;AAED,SAAS,wBAAwB,CAAC,OAA0B;IAC1D,mBAAmB,CAAC,OAAO,CAAC,CAAC;IAC7B,0BAA0B,CAAC,OAAO,CAAC,CAAC;IACpC,sBAAsB,CAAC,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;IACpD,wBAAwB,CAAC,OAAO,CAAC,CAAC;IAClC,+BAA+B,CAAC,OAAO,CAAC,CAAC;AAC3C,CAAC;AAED,SAAS,uBAAuB,CAAC,KAAc;IAC7C,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;QAClB,MAAM,IAAI,4BAA4B,CAAC,6BAA6B,CAAC,CAAC;IACxE,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,0BAA0B,CACjC,OAAgC;IAEhC,IAAI,CAAC;QACH,OAAO,uBAAuB,CAAC,OAAO,CAAC,CAAC;IAC1C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,4BAA4B,CACpC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,2BAA2B,EACpE,EAAE,KAAK,EAAE,CACV,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,2BAA2B,CAClC,OAAgC;IAEhC,IAAI,CAAC;QACH,OAAO,wBAAwB,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IACtE,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,4BAA4B,CACpC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,4BAA4B,EACrE,EAAE,KAAK,EAAE,CACV,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,+BAA+B,CACtC,OAAgC;IAEhC,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC;QAC9B,MAAM,IAAI,4BAA4B,CACpC,0DAA0D,CAC3D,CAAC;AACN,CAAC;AAED,SAAS,wBAAwB,CAC/B,OAA0B;IAE1B,MAAM,OAAO,GAAG,uBAAuB,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC9D,IAAI,MAAM,IAAI,OAAO;QACnB,OAAO;YACL,IAAI,EAAE,WAAW;YACjB,OAAO,EAAE,0BAA0B,CAAC,OAAO,CAAC;SAC7C,CAAC;IACJ,IAAI,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,WAAW,CAAC;QACrC,OAAO;YACL,IAAI,EAAE,YAAY;YAClB,OAAO,EAAE,2BAA2B,CAAC,OAAO,CAAC;YAC7C,UAAU,EAAE,IAAI;SACjB,CAAC;IACJ,+BAA+B,CAAC,OAAO,CAAC,CAAC;IACzC,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;AAC/D,CAAC;AAED,SAAS,oBAAoB,CAAC,KAAa,EAAE,OAAe;IAC1D,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC;QAC3C,MAAM,IAAI,4BAA4B,CAAC,OAAO,CAAC,CAAC;AACpD,CAAC;AAED,SAAS,qBAAqB,CAAC,KAAa;IAC1C,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,IAAI,KAAK,GAAG,CAAC;QACpD,MAAM,IAAI,4BAA4B,CACpC,0DAA0D,CAC3D,CAAC;AACN,CAAC;AAED,SAAS,wBAAwB,CAAC,GAAa;IAC7C,IACE,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,KAAK,GAAG,CAAC,MAAM;QAChC,GAAG,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,gBAAgB,CAAC,EAAE,CAAC,CAAC;QAEvC,MAAM,IAAI,4BAA4B,CAAC,+BAA+B,CAAC,CAAC;AAC5E,CAAC;AAED,SAAS,sBAAsB,CAAC,OAA0B;IACxD,oBAAoB,CAClB,OAAO,CAAC,UAAU,EAClB,wCAAwC,CACzC,CAAC;IACF,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,CAAC,CAAC;IAC/B,oBAAoB,CAAC,IAAI,EAAE,iCAAiC,CAAC,CAAC;IAC9D,qBAAqB,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;IAC7C,wBAAwB,CAAC,OAAO,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;IACxD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,kBAAkB,CAAC,OAA0B;IACpD,MAAM,sBAAsB,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAClE,cAAc,CAAC,KAAK,CAAC,MAAM,CAAC,CACD,CAAC;IAC9B,MAAM,oBAAoB,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CACrD,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,KAAK,aAAa,CACf,CAAC;IAC7B,MAAM,sBAAsB,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CACvD,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,KAAK,gBAAgB,CAChB,CAAC;IAC/B,MAAM,uBAAuB,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CACxD,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,KAAK,gBAAgB,CACf,CAAC;IAChC,MAAM,qBAAqB,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CACtD,CAAC,KAAK,EAAE,EAAE,CACR,CAAC,cAAc,CAAC,KAAK,CAAC,MAAM,CAAC;QAC7B,KAAK,CAAC,MAAM,KAAK,aAAa;QAC9B,KAAK,CAAC,MAAM,KAAK,gBAAgB;QACjC,KAAK,CAAC,MAAM,KAAK,gBAAgB,CACpC,CAAC;IACF,MAAM,sBAAsB,GAC1B,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC,QAAQ,CAAC,sBAAsB,IAAI,EAAE,CAAC;IACnE,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,sBAAsB,CAAC,CAAC;IACjD,OAAO;QACL,sBAAsB;QACtB,oBAAoB;QACpB,sBAAsB;QACtB,uBAAuB;QACvB,qBAAqB;QACrB,sBAAsB;QACtB,QAAQ;KACT,CAAC;AACJ,CAAC;AAED,SAAS,2BAA2B,CAAC,OAA8B;IACjE,MAAM,EACJ,sBAAsB,EACtB,oBAAoB,EACpB,sBAAsB,EACtB,uBAAuB,EACvB,qBAAqB,EACrB,sBAAsB,EACtB,QAAQ,GACT,GAAG,OAAO,CAAC;IACZ,MAAM,eAAe,GAAG,IAAI,GAAG,CAC7B;QACE,GAAG,sBAAsB;QACzB,GAAG,oBAAoB;QACvB,GAAG,sBAAsB;QACzB,GAAG,uBAAuB;KAC3B,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAC/B,CAAC;IACF,IACE,QAAQ,CAAC,IAAI,KAAK,sBAAsB,CAAC,MAAM;QAC/C,sBAAsB,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,eAAe,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAE7D,MAAM,IAAI,4BAA4B,CACpC,kDAAkD,CACnD,CAAC;IACJ,IAAI,QAAQ,CAAC,IAAI,IAAI,CAAC,qBAAqB,CAAC,MAAM;QAChD,MAAM,IAAI,4BAA4B,CACpC,yDAAyD,CAC1D,CAAC;AACN,CAAC;AAED,SAAS,uBAAuB,CAC9B,OAA8B;IAE9B,MAAM,EACJ,sBAAsB,EACtB,oBAAoB,EACpB,sBAAsB,EACtB,uBAAuB,EACvB,QAAQ,GACT,GAAG,OAAO,CAAC;IACZ,MAAM,mBAAmB,GAAG,sBAAsB,CAAC,MAAM,CACvD,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CACvC,CAAC;IACF,MAAM,iBAAiB,GAAG,oBAAoB,CAAC,MAAM,CACnD,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CACvC,CAAC;IACF,MAAM,mBAAmB,GAAG,sBAAsB,CAAC,MAAM,CACvD,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CACvC,CAAC;IACF,MAAM,oBAAoB,GAAG,uBAAuB,CAAC,MAAM,CACzD,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CACvC,CAAC;IACF,OAAO;QACL,mBAAmB;QACnB,iBAAiB;QACjB,mBAAmB;QACnB,oBAAoB;KACrB,CAAC;AACJ,CAAC;AAED,SAAS,qBAAqB,CAC5B,OAA0B,EAC1B,MAAkC;IAElC,IAAI,MAAM,CAAC,MAAM,IAAI,CAAC,OAAO,CAAC,YAAY;QACxC,MAAM,IAAI,4BAA4B,CACpC,mDAAmD,CACpD,CAAC;IACJ,IAAI,OAAO,CAAC,YAAY,IAAI,CAAC,oBAAoB,CAAC,OAAO,CAAC,YAAY,CAAC;QACrE,MAAM,IAAI,4BAA4B,CACpC,sCAAsC,CACvC,CAAC;AACN,CAAC;AAED,SAAS,0BAA0B,CAAC,MAA2B;IAC7D,IAAI,MAAM,CAAC,IAAI,KAAK,QAAQ;QAC1B,MAAM,IAAI,4BAA4B,CACpC,wCAAwC,CACzC,CAAC;AACN,CAAC;AAED,SAAS,0BAA0B,CAAC,KAAe;IACjD,IAAI,CAAC;QACH,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,YAAY,CAAC,IAAI,CAAC,CAAC;YACnB,IACE,IAAI,CAAC,WAAW,EAAE,KAAK,MAAM;gBAC7B,IAAI,CAAC,WAAW,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC;gBAEtC,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAC;QAC3E,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,4BAA4B,CAAC,iCAAiC,EAAE;YACxE,KAAK;SACN,CAAC,CAAC;IACL,CAAC;AACH,CAAC;AAED,SAAS,qBAAqB,CAC5B,OAA0B,EAC1B,MAA2B;IAE3B,MAAM,IAAI,GAAG,OAAO,CAAC,YAAY,CAAC;IAClC,IAAI,CAAC,IAAI,EAAE,CAAC;QACV,IAAI,OAAO,CAAC,qBAAqB,EAAE,MAAM;YACvC,MAAM,IAAI,4BAA4B,CACpC,8CAA8C,CAC/C,CAAC;QACJ,OAAO;IACT,CAAC;IACD,0BAA0B,CAAC,MAAM,CAAC,CAAC;IACnC,wBAAwB,CAAC,IAAI,CAAC,CAAC;IAC/B,0BAA0B,CAAC,OAAO,CAAC,qBAAqB,IAAI,EAAE,CAAC,CAAC;AAClE,CAAC;AAED,SAAS,wBAAwB,CAAC,IAAiB;IACjD,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC;QAC7B,MAAM,IAAI,4BAA4B,CACpC,sCAAsC,CACvC,CAAC;AACN,CAAC;AAED,SAAS,kBAAkB,CACzB,OAA0B,EAC1B,MAA+B;IAE/B,IAAI,MAAM,CAAC,MAAM,IAAI,CAAC,OAAO,CAAC,cAAc;QAC1C,MAAM,IAAI,4BAA4B,CACpC,sDAAsD,CACvD,CAAC;AACN,CAAC;AAED,SAAS,oBAAoB,CAC3B,MAA2B,EAC3B,MAAiC;IAEjC,IAAI,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,IAAI,KAAK,QAAQ;QAC3C,MAAM,IAAI,4BAA4B,CACpC,uCAAuC,CACxC,CAAC;AACN,CAAC;AAED,SAAS,oBAAoB,CAAC,MAA8B;IAC1D,IAAI,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,MAAM;QACjE,MAAM,IAAI,4BAA4B,CAAC,+BAA+B,CAAC,CAAC;AAC5E,CAAC;AAED,SAAS,wBAAwB,CAC/B,OAA0B,EAC1B,YAAoC;IAEpC,IAAI,CAAC,YAAY,CAAC,MAAM;QAAE,OAAO;IACjC,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;IACpC,IAAI,CAAC,SAAS;QACZ,MAAM,IAAI,4BAA4B,CACpC,+CAA+C,CAChD,CAAC;IACJ,IACE,YAAY,CAAC,IAAI,CACf,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,CACtE;QAED,MAAM,IAAI,4BAA4B,CACpC,+CAA+C,CAChD,CAAC;AACN,CAAC;AAED,SAAS,cAAc,CAAC,QAAsB;IAC5C,OAAO;QACL,EAAE,EAAE,QAAQ,CAAC,EAAE;QACf,MAAM,EAAE,QAAQ,CAAC,MAAM;QACvB,GAAG,CAAC,QAAQ,CAAC,cAAc,KAAK,SAAS;YACvC,CAAC,CAAC,EAAE,cAAc,EAAE,QAAQ,CAAC,cAAc,EAAE;YAC7C,CAAC,CAAC,EAAE,CAAC;QACP,OAAO,EAAE,QAAQ,CAAC,OAAO;QACzB,MAAM,EAAE,QAAQ,CAAC,MAAM;QACvB,gBAAgB,EAAE,QAAQ,CAAC,gBAAgB;KAC5C,CAAC;AACJ,CAAC;AAED,SAAS,6BAA6B,CAAC,OAA0B;IAC/D,IAAI,CAAC,OAAO,CAAC,SAAS;QAAE,OAAO;IAC/B,MAAM,QAAQ,GAAG,OAAO,CAAC,SAAS,CAAC,YAAY,CAAC;IAChD,MAAM,UAAU,GAAG,cAAc,CAAC;QAChC,GAAG,QAAQ;QACX,OAAO,EAAE,eAAe,CAAC,QAAQ,CAAC,OAAO,CAAC;KAC3C,CAAC,CAAC;IACH,IAAI,QAAQ,CAAC,UAAU,CAAC,KAAK,QAAQ,CAAC,cAAc,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACjE,MAAM,IAAI,4BAA4B,CACpC,0DAA0D,CAC3D,CAAC;AACN,CAAC;AAED,SAAS,oBAAoB,CAC3B,MAAW,EACX,QAAqB,EACrB,MAAc;IAEd,OAAO,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;AAC5C,CAAC;AAED,SAAS,mBAAmB,CAAC,KAAuC;IAClE,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;QAAE,YAAY,CAAC,IAAI,CAAC,CAAC;AAClE,CAAC;AAED,SAAS,oBAAoB,CAC3B,OAA0B,EAC1B,OAA8B;IAE9B,MAAM,EACJ,sBAAsB,EACtB,oBAAoB,EACpB,sBAAsB,EACtB,uBAAuB,EACvB,QAAQ,GACT,GAAG,OAAO,CAAC;IACZ,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,mBAAmB,CAAC,sBAAsB,CAAC,CAAC,MAAM,CACjE,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CACvC,CAAC;QACF,MAAM,aAAa,GAAG,oBAAoB,CACxC,kBAAkB,CAAC,oBAAoB,CAAC,EACxC,QAAQ,EACR,aAAa,CACd,CAAC;QACF,MAAM,eAAe,GAAG,oBAAoB,CAC1C,oBAAoB,CAAC,sBAAsB,CAAC,EAC5C,QAAQ,EACR,gBAAgB,CACjB,CAAC;QACF,MAAM,gBAAgB,GAAG,oBAAoB,CAC3C,qBAAqB,CAAC,uBAAuB,CAAC,EAC9C,QAAQ,EACR,gBAAgB,CACjB,CAAC;QACF,mBAAmB,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAChD,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,eAAe,EAAE,gBAAgB,EAAE,CAAC;IACxE,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,4BAA4B,CACpC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,4BAA4B,EACrE,EAAE,KAAK,EAAE,CACV,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,gBAAgB,CACvB,OAA0B,EAC1B,MAA2B;IAE3B,MAAM,YAAY,GAAG,kBAAkB,CAAC,OAAO,CAAC,CAAC;IACjD,2BAA2B,CAAC,YAAY,CAAC,CAAC;IAC1C,MAAM,MAAM,GAAG,uBAAuB,CAAC,YAAY,CAAC,CAAC;IACrD,qBAAqB,CAAC,OAAO,EAAE,MAAM,CAAC,oBAAoB,CAAC,CAAC;IAC5D,qBAAqB,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IACvC,kBAAkB,CAAC,OAAO,EAAE,MAAM,CAAC,iBAAiB,CAAC,CAAC;IACtD,oBAAoB,CAAC,MAAM,EAAE,MAAM,CAAC,mBAAmB,CAAC,CAAC;IACzD,oBAAoB,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC1C,wBAAwB,CAAC,OAAO,EAAE,YAAY,CAAC,qBAAqB,CAAC,CAAC;IACtE,6BAA6B,CAAC,OAAO,CAAC,CAAC;IACvC,OAAO;QACL,GAAG,MAAM;QACT,GAAG,oBAAoB,CAAC,OAAO,EAAE,YAAY,CAAC;QAC9C,qBAAqB,EAAE,YAAY,CAAC,qBAAqB;QACzD,sBAAsB,EAAE,YAAY,CAAC,sBAAsB;QAC3D,QAAQ,EAAE,YAAY,CAAC,QAAQ;KAChC,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,uBAAuB,CACrC,OAA0B;IAE1B,wBAAwB,CAAC,OAAO,CAAC,CAAC;IAClC,MAAM,MAAM,GAAG,wBAAwB,CAAC,OAAO,CAAC,CAAC;IACjD,MAAM,IAAI,GAAG,sBAAsB,CAAC,OAAO,CAAC,CAAC;IAC7C,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,gBAAgB,CAAC,OAAO,EAAE,MAAM,CAAC,EAAE,CAAC;AACrE,CAAC","sourcesContent":["import { isAbsolute } from \"node:path\";\nimport { validQualifiedId } from \"../value-guards\";\nimport { validInlineFixture } from \"../case-validation\";\nimport {\n  EvaluationConfigurationError,\n  type EvaluationOptions,\n  type HostAdapter,\n  type ResolvedCase,\n} from \"./types\";\nimport {\n  prepareGeneratedFixture,\n  type GeneratedFixture,\n} from \"../fixtures/generated\";\nimport {\n  prepareGitHeadChecks,\n  type GitHeadCheckDeclaration,\n  type PreparedGitHeadCheck,\n} from \"../graders/git-head\";\nimport {\n  isOutputGrader,\n  prepareOutputChecks,\n  type OutputCheckDeclaration,\n  type PreparedOutputCheck,\n} from \"../graders/output\";\nimport {\n  prepareSemanticChecks,\n  type SemanticCheckDeclaration,\n  type PreparedSemanticCheck,\n} from \"../graders/semantic\";\nimport {\n  prepareShellChecks,\n  type ShellCheckDeclaration,\n  type PreparedShellCheck,\n} from \"../graders/shell\";\nimport { hashJson } from \"../identity\";\nimport { fixtureParts } from \"../fixtures/preparation\";\nimport {\n  prepareRepositoryFixture,\n  type RepositoryFixture,\n} from \"../fixtures/repository\";\nimport { resolvedFixture } from \"../resolved-case\";\nimport { isOptionalNonblankString, isRecord } from \"../value-guards\";\nimport type { FixtureSource } from \"../fixtures/create\";\n\ninterface CaseCheckDeclarations {\n  allBuiltinDeclarations: OutputCheckDeclaration[];\n  allShellDeclarations: ShellCheckDeclaration[];\n  allGitHeadDeclarations: GitHeadCheckDeclaration[];\n  allSemanticDeclarations: SemanticCheckDeclaration[];\n  extensionDeclarations: ResolvedCase[\"checks\"];\n  replacedBuiltinGraders: string[];\n  replaced: Set<string>;\n}\n\ninterface ActiveCheckDeclarations {\n  readonly builtinDeclarations: OutputCheckDeclaration[];\n  readonly shellDeclarations: ShellCheckDeclaration[];\n  readonly gitHeadDeclarations: GitHeadCheckDeclaration[];\n  readonly semanticDeclarations: SemanticCheckDeclaration[];\n}\n\ninterface BuiltInCheckPlan {\n  readonly prepared: PreparedOutputCheck[];\n  readonly preparedShell: PreparedShellCheck[];\n  readonly preparedGitHead: PreparedGitHeadCheck[];\n  readonly preparedSemantic: PreparedSemanticCheck[];\n}\n\nexport interface PreparedChecks\n  extends ActiveCheckDeclarations, BuiltInCheckPlan {\n  readonly extensionDeclarations: ResolvedCase[\"checks\"];\n  readonly replacedBuiltinGraders: string[];\n  readonly replaced: ReadonlySet<string>;\n}\n\nexport interface EvaluationCheckPlan {\n  readonly source: FixtureSource<null>;\n  readonly jobs: number;\n  readonly checks: PreparedChecks;\n}\n\nfunction validateRootOptions(options: EvaluationOptions): void {\n  if (\n    !isAbsolute(options.projectRoot) ||\n    !isAbsolute(options.resultsRoot) ||\n    (options.runStateRoot !== undefined && !isAbsolute(options.runStateRoot))\n  )\n    throw new EvaluationConfigurationError(\n      \"project, results, and run-state roots must be absolute\",\n    );\n}\n\nfunction hostIdentityComplete(host: HostAdapter): boolean {\n  return Boolean(host.id && host.model && host.effort);\n}\n\nfunction validateEvaluationIdentity(options: EvaluationOptions): void {\n  if (\n    !options.case.id ||\n    !options.case.prompt ||\n    !hostIdentityComplete(options.host)\n  )\n    throw new EvaluationConfigurationError(\n      \"case and host identities must be nonempty\",\n    );\n}\n\nfunction validateFollowUpPrompt(value: unknown): void {\n  if (!isOptionalNonblankString(value))\n    throw new EvaluationConfigurationError(\"follow-up prompt must be nonempty\");\n}\n\nfunction validateContinuationHost(options: EvaluationOptions): void {\n  if (\n    options.case.followUpPrompt !== undefined &&\n    !options.host.hostCapabilities?.includes(\"sevro.host.continuation\")\n  )\n    throw new EvaluationConfigurationError(\n      \"selected host does not support continuation\",\n    );\n}\n\nfunction validateContinuationNegotiation(options: EvaluationOptions): void {\n  if (\n    options.case.followUpPrompt !== undefined &&\n    options.extension &&\n    !options.extension.session.identity.capabilities.includes(\n      \"sevro.host.continuation\",\n    )\n  )\n    throw new EvaluationConfigurationError(\n      \"continuation capability was not negotiated\",\n    );\n}\n\nfunction validateEvaluationInputs(options: EvaluationOptions): void {\n  validateRootOptions(options);\n  validateEvaluationIdentity(options);\n  validateFollowUpPrompt(options.case.followUpPrompt);\n  validateContinuationHost(options);\n  validateContinuationNegotiation(options);\n}\n\nfunction evaluationFixtureObject(value: unknown): Record<string, unknown> {\n  if (!isRecord(value))\n    throw new EvaluationConfigurationError(\"invalid fixture declaration\");\n  return value;\n}\n\nfunction generatedEvaluationFixture(\n  fixture: Record<string, unknown>,\n): GeneratedFixture {\n  try {\n    return prepareGeneratedFixture(fixture);\n  } catch (cause) {\n    throw new EvaluationConfigurationError(\n      cause instanceof Error ? cause.message : \"invalid generated fixture\",\n      { cause },\n    );\n  }\n}\n\nfunction repositoryEvaluationFixture(\n  fixture: Record<string, unknown>,\n): RepositoryFixture {\n  try {\n    return prepareRepositoryFixture({ kind: \"repository\", ...fixture });\n  } catch (cause) {\n    throw new EvaluationConfigurationError(\n      cause instanceof Error ? cause.message : \"invalid repository fixture\",\n      { cause },\n    );\n  }\n}\n\nfunction validateInlineEvaluationFixture(\n  fixture: Record<string, unknown>,\n): void {\n  if (!validInlineFixture(fixture))\n    throw new EvaluationConfigurationError(\n      \"fixture must declare inline files or a repository source\",\n    );\n}\n\nfunction prepareEvaluationFixture(\n  options: EvaluationOptions,\n): FixtureSource<null> {\n  const fixture = evaluationFixtureObject(options.case.fixture);\n  if (\"kind\" in fixture)\n    return {\n      kind: \"generated\",\n      fixture: generatedEvaluationFixture(fixture),\n    };\n  if (Object.hasOwn(fixture, \"sourceRef\"))\n    return {\n      kind: \"repository\",\n      fixture: repositoryEvaluationFixture(fixture),\n      repository: null,\n    };\n  validateInlineEvaluationFixture(fixture);\n  return { kind: \"inline\", files: options.case.fixture.files };\n}\n\nfunction validateTrialInteger(value: number, message: string): void {\n  if (!Number.isSafeInteger(value) || value < 1)\n    throw new EvaluationConfigurationError(message);\n}\n\nfunction validatePassThreshold(value: number): void {\n  if (!Number.isFinite(value) || value <= 0 || value > 1)\n    throw new EvaluationConfigurationError(\n      \"pass threshold must be greater than zero and at most one\",\n    );\n}\n\nfunction validateRequiredEvidence(ids: string[]): void {\n  if (\n    new Set(ids).size !== ids.length ||\n    ids.some((id) => !validQualifiedId(id))\n  )\n    throw new EvaluationConfigurationError(\"invalid required evidence IDs\");\n}\n\nfunction prepareTrialParameters(options: EvaluationOptions): number {\n  validateTrialInteger(\n    options.trialCount,\n    \"trial count must be a positive integer\",\n  );\n  const jobs = options.jobs ?? 3;\n  validateTrialInteger(jobs, \"jobs must be a positive integer\");\n  validatePassThreshold(options.passThreshold);\n  validateRequiredEvidence(options.case.requiredEvidence);\n  return jobs;\n}\n\nfunction declaredCaseChecks(options: EvaluationOptions): CaseCheckDeclarations {\n  const allBuiltinDeclarations = options.case.checks.filter((check) =>\n    isOutputGrader(check.grader),\n  ) as OutputCheckDeclaration[];\n  const allShellDeclarations = options.case.checks.filter(\n    (check) => check.grader === \"sevro.shell\",\n  ) as ShellCheckDeclaration[];\n  const allGitHeadDeclarations = options.case.checks.filter(\n    (check) => check.grader === \"sevro.git-head\",\n  ) as GitHeadCheckDeclaration[];\n  const allSemanticDeclarations = options.case.checks.filter(\n    (check) => check.grader === \"sevro.semantic\",\n  ) as SemanticCheckDeclaration[];\n  const extensionDeclarations = options.case.checks.filter(\n    (check) =>\n      !isOutputGrader(check.grader) &&\n      check.grader !== \"sevro.shell\" &&\n      check.grader !== \"sevro.git-head\" &&\n      check.grader !== \"sevro.semantic\",\n  );\n  const replacedBuiltinGraders =\n    options.extension?.session.identity.replacedBuiltinGraders ?? [];\n  const replaced = new Set(replacedBuiltinGraders);\n  return {\n    allBuiltinDeclarations,\n    allShellDeclarations,\n    allGitHeadDeclarations,\n    allSemanticDeclarations,\n    extensionDeclarations,\n    replacedBuiltinGraders,\n    replaced,\n  };\n}\n\nfunction validateBuiltInReplacements(context: CaseCheckDeclarations): void {\n  const {\n    allBuiltinDeclarations,\n    allShellDeclarations,\n    allGitHeadDeclarations,\n    allSemanticDeclarations,\n    extensionDeclarations,\n    replacedBuiltinGraders,\n    replaced,\n  } = context;\n  const declaredGraders = new Set<string>(\n    [\n      ...allBuiltinDeclarations,\n      ...allShellDeclarations,\n      ...allGitHeadDeclarations,\n      ...allSemanticDeclarations,\n    ].map((check) => check.grader),\n  );\n  if (\n    replaced.size !== replacedBuiltinGraders.length ||\n    replacedBuiltinGraders.some((id) => !declaredGraders.has(id))\n  )\n    throw new EvaluationConfigurationError(\n      \"unknown or duplicate built-in grader replacement\",\n    );\n  if (replaced.size && !extensionDeclarations.length)\n    throw new EvaluationConfigurationError(\n      \"built-in grader replacement requires an extension check\",\n    );\n}\n\nfunction activeCheckDeclarations(\n  context: CaseCheckDeclarations,\n): ActiveCheckDeclarations {\n  const {\n    allBuiltinDeclarations,\n    allShellDeclarations,\n    allGitHeadDeclarations,\n    allSemanticDeclarations,\n    replaced,\n  } = context;\n  const builtinDeclarations = allBuiltinDeclarations.filter(\n    (check) => !replaced.has(check.grader),\n  );\n  const shellDeclarations = allShellDeclarations.filter(\n    (check) => !replaced.has(check.grader),\n  );\n  const gitHeadDeclarations = allGitHeadDeclarations.filter(\n    (check) => !replaced.has(check.grader),\n  );\n  const semanticDeclarations = allSemanticDeclarations.filter(\n    (check) => !replaced.has(check.grader),\n  );\n  return {\n    builtinDeclarations,\n    shellDeclarations,\n    gitHeadDeclarations,\n    semanticDeclarations,\n  };\n}\n\nfunction validateSemanticRoute(\n  options: EvaluationOptions,\n  checks: SemanticCheckDeclaration[],\n): void {\n  if (checks.length && !options.semanticHost)\n    throw new EvaluationConfigurationError(\n      \"semantic checks require an explicit semantic host\",\n    );\n  if (options.semanticHost && !hostIdentityComplete(options.semanticHost))\n    throw new EvaluationConfigurationError(\n      \"semantic host identity is incomplete\",\n    );\n}\n\nfunction validateAdvisoryGitFixture(source: FixtureSource<null>): void {\n  if (source.kind === \"inline\")\n    throw new EvaluationConfigurationError(\n      \"advisory review requires a Git fixture\",\n    );\n}\n\nfunction validateAdvisoryExclusions(paths: string[]): void {\n  try {\n    for (const path of paths) {\n      fixtureParts(path);\n      if (\n        path.toLowerCase() === \".git\" ||\n        path.toLowerCase().startsWith(\".git/\")\n      )\n        throw new Error(\"repository metadata cannot be an advisory exclusion\");\n    }\n  } catch (cause) {\n    throw new EvaluationConfigurationError(\"invalid advisory exclusion path\", {\n      cause,\n    });\n  }\n}\n\nfunction validateAdvisoryRoute(\n  options: EvaluationOptions,\n  source: FixtureSource<null>,\n): void {\n  const host = options.advisoryHost;\n  if (!host) {\n    if (options.advisoryExcludedPaths?.length)\n      throw new EvaluationConfigurationError(\n        \"advisory exclusions require an advisory host\",\n      );\n    return;\n  }\n  validateAdvisoryGitFixture(source);\n  validateAdvisoryIdentity(host);\n  validateAdvisoryExclusions(options.advisoryExcludedPaths ?? []);\n}\n\nfunction validateAdvisoryIdentity(host: HostAdapter): void {\n  if (!hostIdentityComplete(host))\n    throw new EvaluationConfigurationError(\n      \"advisory host identity is incomplete\",\n    );\n}\n\nfunction validateShellRoute(\n  options: EvaluationOptions,\n  checks: ShellCheckDeclaration[],\n): void {\n  if (checks.length && !options.shellIsolation)\n    throw new EvaluationConfigurationError(\n      \"shell checks require explicit protected source roots\",\n    );\n}\n\nfunction validateGitHeadRoute(\n  source: FixtureSource<null>,\n  checks: GitHeadCheckDeclaration[],\n): void {\n  if (checks.length && source.kind === \"inline\")\n    throw new EvaluationConfigurationError(\n      \"Git HEAD checks require a Git fixture\",\n    );\n}\n\nfunction validateCaseCheckIds(checks: ResolvedCase[\"checks\"]): void {\n  if (new Set(checks.map((check) => check.id)).size !== checks.length)\n    throw new EvaluationConfigurationError(\"case check IDs must be unique\");\n}\n\nfunction validateExtensionGraders(\n  options: EvaluationOptions,\n  declarations: ResolvedCase[\"checks\"],\n): void {\n  if (!declarations.length) return;\n  const extension = options.extension;\n  if (!extension)\n    throw new EvaluationConfigurationError(\n      \"case declares an unavailable extension grader\",\n    );\n  if (\n    declarations.some(\n      (check) => !extension.session.identity.graders.includes(check.grader),\n    )\n  )\n    throw new EvaluationConfigurationError(\n      \"case declares an unavailable extension grader\",\n    );\n}\n\nfunction comparisonCase(caseData: ResolvedCase) {\n  return {\n    id: caseData.id,\n    prompt: caseData.prompt,\n    ...(caseData.followUpPrompt !== undefined\n      ? { followUpPrompt: caseData.followUpPrompt }\n      : {}),\n    fixture: caseData.fixture,\n    checks: caseData.checks,\n    requiredEvidence: caseData.requiredEvidence,\n  };\n}\n\nfunction validateResolvedExtensionCase(options: EvaluationOptions): void {\n  if (!options.extension) return;\n  const resolved = options.extension.resolvedCase;\n  const comparable = comparisonCase({\n    ...resolved,\n    fixture: resolvedFixture(resolved.fixture),\n  });\n  if (hashJson(comparable) !== hashJson(comparisonCase(options.case)))\n    throw new EvaluationConfigurationError(\n      \"resolved extension case does not match the selected case\",\n    );\n}\n\nfunction activePreparedChecks<T>(\n  checks: T[],\n  replaced: Set<string>,\n  grader: string,\n): T[] {\n  return replaced.has(grader) ? [] : checks;\n}\n\nfunction validateInlinePaths(files: ResolvedCase[\"fixture\"][\"files\"]): void {\n  for (const path of Object.keys(files ?? {})) fixtureParts(path);\n}\n\nfunction prepareBuiltInChecks(\n  options: EvaluationOptions,\n  context: CaseCheckDeclarations,\n): BuiltInCheckPlan {\n  const {\n    allBuiltinDeclarations,\n    allShellDeclarations,\n    allGitHeadDeclarations,\n    allSemanticDeclarations,\n    replaced,\n  } = context;\n  try {\n    const prepared = prepareOutputChecks(allBuiltinDeclarations).filter(\n      (check) => !replaced.has(check.grader),\n    );\n    const preparedShell = activePreparedChecks(\n      prepareShellChecks(allShellDeclarations),\n      replaced,\n      \"sevro.shell\",\n    );\n    const preparedGitHead = activePreparedChecks(\n      prepareGitHeadChecks(allGitHeadDeclarations),\n      replaced,\n      \"sevro.git-head\",\n    );\n    const preparedSemantic = activePreparedChecks(\n      prepareSemanticChecks(allSemanticDeclarations),\n      replaced,\n      \"sevro.semantic\",\n    );\n    validateInlinePaths(options.case.fixture.files);\n    return { prepared, preparedShell, preparedGitHead, preparedSemantic };\n  } catch (cause) {\n    throw new EvaluationConfigurationError(\n      cause instanceof Error ? cause.message : \"invalid case configuration\",\n      { cause },\n    );\n  }\n}\n\nfunction prepareCheckPlan(\n  options: EvaluationOptions,\n  source: FixtureSource<null>,\n): PreparedChecks {\n  const declarations = declaredCaseChecks(options);\n  validateBuiltInReplacements(declarations);\n  const active = activeCheckDeclarations(declarations);\n  validateSemanticRoute(options, active.semanticDeclarations);\n  validateAdvisoryRoute(options, source);\n  validateShellRoute(options, active.shellDeclarations);\n  validateGitHeadRoute(source, active.gitHeadDeclarations);\n  validateCaseCheckIds(options.case.checks);\n  validateExtensionGraders(options, declarations.extensionDeclarations);\n  validateResolvedExtensionCase(options);\n  return {\n    ...active,\n    ...prepareBuiltInChecks(options, declarations),\n    extensionDeclarations: declarations.extensionDeclarations,\n    replacedBuiltinGraders: declarations.replacedBuiltinGraders,\n    replaced: declarations.replaced,\n  };\n}\n\nexport function prepareEvaluationChecks(\n  options: EvaluationOptions,\n): EvaluationCheckPlan {\n  validateEvaluationInputs(options);\n  const source = prepareEvaluationFixture(options);\n  const jobs = prepareTrialParameters(options);\n  return { source, jobs, checks: prepareCheckPlan(options, source) };\n}\n"]}

package/dist/src/evaluation/collection.js

@@ -0,0 +1,61 @@
+import { checkpointRunOwner } from "../run-owner.js";
+import { summarizeAssessments } from "../results.js";
+/** Registers retained trials before checkpointing; owns their final ordering. */
+export class ResultCollector {
+    owner;
+    summaries = [];
+    evidence = [];
+    diagnostics = [];
+    constructor(owner) {
+        this.owner = owner;
+    }
+    complete(record) {
+        this.summaries.push(record.summary);
+        this.summaries.sort((left, right) => left.trial - right.trial);
+        this.evidence.push(record.evidence);
+        this.evidence.sort((left, right) => left.trial - right.trial);
+        if (record.diagnostic)
+            this.diagnostics.push({
+                trial: record.summary.trial,
+                diagnostic: record.diagnostic,
+            });
+        this.checkpoint("active");
+    }
+    snapshot(input) {
+        const assessment = !this.summaries.length && input.cancelled
+            ? {
+                execution: { status: "cancelled" },
+                grading: { status: "not_requested" },
+                task: { verdict: "not_assessed" },
+            }
+            : summarizeAssessments(this.summaries, input.passThreshold);
+        const diagnostic = this.diagnostics.sort((left, right) => left.trial - right.trial)[0]?.diagnostic;
+        return {
+            caseSummary: {
+                caseId: input.caseId,
+                ...assessment,
+                trials: this.summaries,
+            },
+            trials: this.evidence,
+            diagnostic,
+        };
+    }
+    checkpoint(status) {
+        checkpointRunOwner(this.owner, this.checkpointTrials(), status);
+    }
+    diagnoseFailure() {
+        try {
+            checkpointRunOwner(this.owner, this.checkpointTrials(), "diagnostic");
+        }
+        catch {
+            /* Preserve the original failure and retained artifacts. */
+        }
+    }
+    checkpointTrials() {
+        return this.summaries.map((trial) => ({
+            trial: trial.trial,
+            artifactPath: trial.artifactPath,
+        }));
+    }
+}
+//# sourceMappingURL=collection.js.map

package/dist/src/evaluation/collection.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"collection.js","sourceRoot":"","sources":["collection.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAiB,wBAAqB;AACjE,OAAO,EAAE,oBAAoB,EAAmB,sBAAmB;AAmBnE,iFAAiF;AACjF,MAAM,OAAO,eAAe;IAMG;IALZ,SAAS,GAAmB,EAAE,CAAC;IAC/B,QAAQ,GAAoB,EAAE,CAAC;IAC/B,WAAW,GAC1B,EAAE,CAAC;IAEL,YAA6B,KAAe;QAAf,UAAK,GAAL,KAAK,CAAU;IAAG,CAAC;IAEhD,QAAQ,CAAC,MAAsB;QAC7B,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACpC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;QAC/D,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QACpC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;QAC9D,IAAI,MAAM,CAAC,UAAU;YACnB,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC;gBACpB,KAAK,EAAE,MAAM,CAAC,OAAO,CAAC,KAAK;gBAC3B,UAAU,EAAE,MAAM,CAAC,UAAU;aAC9B,CAAC,CAAC;QACL,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IAC5B,CAAC;IAED,QAAQ,CAAC,KAIR;QACC,MAAM,UAAU,GACd,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,IAAI,KAAK,CAAC,SAAS;YACvC,CAAC,CAAC;gBACE,SAAS,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE;gBAClC,OAAO,EAAE,EAAE,MAAM,EAAE,eAAe,EAAE;gBACpC,IAAI,EAAE,EAAE,OAAO,EAAE,cAAc,EAAE;aAClC;YACH,CAAC,CAAC,oBAAoB,CAAC,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,aAAa,CAAC,CAAC;QAChE,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CACtC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC,KAAK,CAC1C,CAAC,CAAC,CAAC,EAAE,UAAU,CAAC;QACjB,OAAO;YACL,WAAW,EAAE;gBACX,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,GAAG,UAAU;gBACb,MAAM,EAAE,IAAI,CAAC,SAAS;aACvB;YACD,MAAM,EAAE,IAAI,CAAC,QAAQ;YACrB,UAAU;SACX,CAAC;IACJ,CAAC;IAED,UAAU,CAAC,MAA6C;QACtD,kBAAkB,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,gBAAgB,EAAE,EAAE,MAAM,CAAC,CAAC;IAClE,CAAC;IAED,eAAe;QACb,IAAI,CAAC;YACH,kBAAkB,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,gBAAgB,EAAE,EAAE,YAAY,CAAC,CAAC;QACxE,CAAC;QAAC,MAAM,CAAC;YACP,2DAA2D;QAC7D,CAAC;IACH,CAAC;IAEO,gBAAgB;QACtB,OAAO,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;YACpC,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,YAAY,EAAE,KAAK,CAAC,YAAY;SACjC,CAAC,CAAC,CAAC;IACN,CAAC;CACF","sourcesContent":["import { checkpointRunOwner, type RunOwner } from \"../run-owner\";\nimport { summarizeAssessments, type Assessment } from \"../results\";\nimport type { Diagnostic } from \"../contract-types\";\nimport type { RunEvidenceDataTrialEvidence } from \"../schema-types\";\nimport type { CaseSummary, TrialSummary } from \"./types\";\n\ntype TrialEvidence = RunEvidenceDataTrialEvidence;\n\nexport interface CompletedTrial {\n  readonly summary: TrialSummary;\n  readonly evidence: TrialEvidence;\n  readonly diagnostic?: Diagnostic;\n}\n\nexport interface CollectedResults {\n  readonly caseSummary: CaseSummary;\n  readonly trials: readonly TrialEvidence[];\n  readonly diagnostic: Diagnostic | undefined;\n}\n\n/** Registers retained trials before checkpointing; owns their final ordering. */\nexport class ResultCollector {\n  private readonly summaries: TrialSummary[] = [];\n  private readonly evidence: TrialEvidence[] = [];\n  private readonly diagnostics: { trial: number; diagnostic: Diagnostic }[] =\n    [];\n\n  constructor(private readonly owner: RunOwner) {}\n\n  complete(record: CompletedTrial): void {\n    this.summaries.push(record.summary);\n    this.summaries.sort((left, right) => left.trial - right.trial);\n    this.evidence.push(record.evidence);\n    this.evidence.sort((left, right) => left.trial - right.trial);\n    if (record.diagnostic)\n      this.diagnostics.push({\n        trial: record.summary.trial,\n        diagnostic: record.diagnostic,\n      });\n    this.checkpoint(\"active\");\n  }\n\n  snapshot(input: {\n    caseId: string;\n    passThreshold: number;\n    cancelled: boolean;\n  }): CollectedResults {\n    const assessment: Assessment =\n      !this.summaries.length && input.cancelled\n        ? {\n            execution: { status: \"cancelled\" },\n            grading: { status: \"not_requested\" },\n            task: { verdict: \"not_assessed\" },\n          }\n        : summarizeAssessments(this.summaries, input.passThreshold);\n    const diagnostic = this.diagnostics.sort(\n      (left, right) => left.trial - right.trial,\n    )[0]?.diagnostic;\n    return {\n      caseSummary: {\n        caseId: input.caseId,\n        ...assessment,\n        trials: this.summaries,\n      },\n      trials: this.evidence,\n      diagnostic,\n    };\n  }\n\n  checkpoint(status: \"active\" | \"complete\" | \"interrupted\"): void {\n    checkpointRunOwner(this.owner, this.checkpointTrials(), status);\n  }\n\n  diagnoseFailure(): void {\n    try {\n      checkpointRunOwner(this.owner, this.checkpointTrials(), \"diagnostic\");\n    } catch {\n      /* Preserve the original failure and retained artifacts. */\n    }\n  }\n\n  private checkpointTrials() {\n    return this.summaries.map((trial) => ({\n      trial: trial.trial,\n      artifactPath: trial.artifactPath,\n    }));\n  }\n}\n"]}

package/dist/src/evaluation/context.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=context.js.map

package/dist/src/evaluation/context.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"context.js","sourceRoot":"","sources":["context.ts"],"names":[],"mappings":"","sourcesContent":["import type { ResultCollector } from \"./collection\";\nimport type { PreparedEvaluation } from \"./plan\";\nimport type { PreparationArtifactReference, RunResources } from \"./resources\";\n\nexport interface EvaluationContext {\n  readonly prepared: PreparedEvaluation;\n  readonly resources: RunResources;\n  readonly results: ResultCollector;\n  readonly preparationArtifacts: readonly PreparationArtifactReference[];\n}\n"]}

package/dist/src/evaluation/grading-host.js

@@ -0,0 +1,28 @@
+import { hostArtifacts } from "../evidence/host-results.js";
+import { runtimeSeedSources } from "../runtime/config.js";
+export function scopedHostArtifacts(response, prefix, artifacts, checks, reservedId) {
+    return hostArtifacts({
+        ...response,
+        artifacts: response.artifacts?.map((item) => ({
+            ...item,
+            id: `${prefix}${item.id}`,
+        })),
+    }, new Set([
+        ...artifacts.map((item) => item.id),
+        ...checks.map((item) => item.id),
+        reservedId,
+    ]));
+}
+export function gradingRuntimePolicy(policy, responseOnly = false) {
+    if (!policy)
+        return undefined;
+    return {
+        ...policy,
+        hooks: undefined,
+        seedReadDenyRoots: runtimeSeedSources(policy),
+        seeds: responseOnly
+            ? (policy.seeds ?? []).filter((seed) => seed.roles?.includes("semantic"))
+            : policy.seeds,
+    };
+}
+//# sourceMappingURL=grading-host.js.map

package/dist/src/evaluation/grading-host.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"grading-host.js","sourceRoot":"","sources":["grading-host.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,oCAAiC;AAEzD,OAAO,EAAE,kBAAkB,EAAsB,6BAA0B;AAC3E,MAAM,UAAU,mBAAmB,CACjC,QAAoB,EACpB,MAAc,EACd,SAAoC,EACpC,MAAiC,EACjC,UAAkB;IAElB,OAAO,aAAa,CAClB;QACE,GAAG,QAAQ;QACX,SAAS,EAAE,QAAQ,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YAC5C,GAAG,IAAI;YACP,EAAE,EAAE,GAAG,MAAM,GAAG,IAAI,CAAC,EAAE,EAAE;SAC1B,CAAC,CAAC;KACJ,EACD,IAAI,GAAG,CAAC;QACN,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;QACnC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;QAChC,UAAU;KACX,CAAC,CACH,CAAC;AACJ,CAAC;AACD,MAAM,UAAU,oBAAoB,CAClC,MAAiC,EACjC,YAAY,GAAG,KAAK;IAEpB,IAAI,CAAC,MAAM;QAAE,OAAO,SAAS,CAAC;IAC9B,OAAO;QACL,GAAG,MAAM;QACT,KAAK,EAAE,SAAS;QAChB,iBAAiB,EAAE,kBAAkB,CAAC,MAAM,CAAC;QAC7C,KAAK,EAAE,YAAY;YACjB,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,CAAC,UAAU,CAAC,CAAC;YACzE,CAAC,CAAC,MAAM,CAAC,KAAK;KACjB,CAAC;AACJ,CAAC","sourcesContent":["import { hostArtifacts } from \"../evidence/host-results\";\nimport type { HostResult } from \"./types\";\nimport { runtimeSeedSources, type RuntimePolicy } from \"../runtime/config\";\nexport function scopedHostArtifacts(\n  response: HostResult,\n  prefix: string,\n  artifacts: readonly { id: string }[],\n  checks: readonly { id: string }[],\n  reservedId: string,\n) {\n  return hostArtifacts(\n    {\n      ...response,\n      artifacts: response.artifacts?.map((item) => ({\n        ...item,\n        id: `${prefix}${item.id}`,\n      })),\n    },\n    new Set([\n      ...artifacts.map((item) => item.id),\n      ...checks.map((item) => item.id),\n      reservedId,\n    ]),\n  );\n}\nexport function gradingRuntimePolicy(\n  policy: RuntimePolicy | undefined,\n  responseOnly = false,\n) {\n  if (!policy) return undefined;\n  return {\n    ...policy,\n    hooks: undefined,\n    seedReadDenyRoots: runtimeSeedSources(policy),\n    seeds: responseOnly\n      ? (policy.seeds ?? []).filter((seed) => seed.roles?.includes(\"semantic\"))\n      : policy.seeds,\n  };\n}\n"]}