@velum-labs/routekit-eval-setup 1.3.2 → 1.5.0

package/dist/project-artifacts.js

@@ -3,6 +3,10 @@ import { assertRoutingBasis } from "@velum-labs/routekit-eval-contracts";
 import { writeFileAtomicEffect } from "@velum-labs/routekit-runtime/effect";
 import { Context, Effect, Exit, FileSystem, Layer, Path, Schema } from "effect";
 import { EvalProjectArtifactError } from "./errors.js";
+import { candidateCaseProjection, hiddenOracleProjection } from "./evaluation-structure-policy.js";
+import { EVAL_GRADER_MINIMUM_SCORE, EVAL_GRADER_SYSTEM_PROMPT, renderEvaluationCandidatePromptSource } from "./evaluation-grading-policy.js";
+import { assertEvaluationProposal, evaluationProposalDigest } from "./evaluation-proposal-policy.js";
+import { EvalNamedEventLog as EvalNamedEventLogSchema } from "./eval-event-log.js";
 import { EVAL_PROJECT_VERSION, EvalArtifactApproval as EvalArtifactApprovalSchema, BasisCandidatesArtifactsV3 as BasisCandidatesArtifactsV3Schema, BasisEvaluationProjectionV3 as BasisEvaluationProjectionV3Schema, BasisCandidateCheckpointV3 as BasisCandidateCheckpointV3Schema, BasisGenerationArtifactsV3 as BasisGenerationArtifactsV3Schema, BasisQualificationArtifactsV3 as BasisQualificationArtifactsV3Schema, BasisRepairArtifactsV3 as BasisRepairArtifactsV3Schema, BasisValidationJobCheckpointV3 as BasisValidationJobCheckpointV3Schema, BasisValidationResultsArtifactsV3 as BasisValidationResultsArtifactsV3Schema, BasisValidationArtifactsV3 as BasisValidationArtifactsV3Schema, EvalEvaluationProposal as EvalEvaluationProposalSchema, EvalExecutionPlan as EvalExecutionPlanSchema, EvalRoutingBasisProposal as EvalRoutingBasisProposalSchema, EvalRunReport as EvalRunReportSchema } from "./project-contracts.js";
 const BASIS_PROPOSAL = "routing-basis.proposed.json";
 const BASIS_APPROVAL = "routing-basis.approval.json";
@@ -18,29 +22,38 @@ const BASIS_REPAIR_V3 = "basis/repair.v3.json";
 const BASIS_REPAIR_VALIDATION_V3 = "basis/repair-validation.v3.json";
 const BASIS_REPAIR_VALIDATION_RESULTS_V3 = "basis/repair-validation-results.v3.json";
 const ARTIFACT_ID = /^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$/u;
-const renderCandidatePrompt = () => `const prompt = [
-        testCase.prompt,
-        "",
-        "Response requirements:",
-        "- Answer every part of the request directly and completely.",
-        "- When translating protocols, emit the complete target-protocol envelope and terminal event; do not merely reframe or forward source objects.",
-        "- When asked for a body, bytes, frames, or code, include the concrete output rather than only describing it.",
-        ...(testCase.context === undefined
-          ? []
-          : ["", "Reference material:", "-----", testCase.context, "-----"])
-      ].join("\\n");`;
+const repeatedCase = (byId, selectedCaseId) => {
+    const exact = byId.get(selectedCaseId);
+    if (exact !== undefined)
+        return exact;
+    const repeated = /^(.*)#[1-9][0-9]*$/u.exec(selectedCaseId);
+    const sourceId = repeated?.[1];
+    if (sourceId === undefined) {
+        throw new Error(`execution plan refers to unknown case ${JSON.stringify(selectedCaseId)}`);
+    }
+    const testCase = byId.get(sourceId);
+    if (testCase === undefined) {
+        throw new Error(`execution plan refers to unknown case ${JSON.stringify(selectedCaseId)}`);
+    }
+    return { ...testCase, id: selectedCaseId };
+};
 const renderDimensionSuite = () => `import assert from "node:assert/strict";
 import { test } from "node:test";
 import { setupAgent, setupJudge } from "routekit/eval";
-import cases from "./data/cases.json" with { type: "json" };
+import candidateCases from "./data/candidate-cases.json" with { type: "json" };
+import hiddenOracles from "./data/hidden-oracles.json" with { type: "json" };
 import manifest from "./routekit.eval-manifest.json" with { type: "json" };
 
-assert.equal(cases.length, manifest.caseCount);
-assert.deepEqual(cases.map((testCase) => testCase.id), manifest.caseIds);
+assert.equal(candidateCases.length, manifest.caseCount);
+assert.equal(hiddenOracles.length, manifest.caseCount);
+assert.deepEqual(candidateCases.map((candidateCase) => candidateCase.id), manifest.caseIds);
+assert.deepEqual(hiddenOracles.map((oracle) => oracle.id), manifest.caseIds);
 const candidateReasoningEfforts = manifest.candidateReasoningEfforts ?? {};
+const hiddenOracleById = new Map(hiddenOracles.map((oracle) => [oracle.id, oracle]));
 const judge = setupJudge({
   agent: setupAgent({ model: manifest.judgeModel }),
-  minScore: 0.8
+  minScore: ${String(EVAL_GRADER_MINIMUM_SCORE)},
+  systemPrompt: ${JSON.stringify(EVAL_GRADER_SYSTEM_PROMPT)}
 });
 
 for (const model of manifest.candidateModels) {
@@ -51,10 +64,10 @@ for (const model of manifest.candidateModels) {
       ? {}
       : { parameters: { reasoning: { effort } } })
   });
-  for (const testCase of cases) {
-    test(\`\${model} / \${testCase.id}\`, async () => {
-      ${renderCandidatePrompt()}
-      const run = await candidate.run({ prompt, caseId: testCase.id });
+  for (const candidateCase of candidateCases) {
+    test(\`\${model} / \${candidateCase.id}\`, async () => {
+      ${renderEvaluationCandidatePromptSource()}
+      const run = await candidate.run({ prompt, caseId: candidateCase.id });
       let candidateCompletionError;
       try {
         run.toComplete();
@@ -63,7 +76,9 @@ for (const model of manifest.candidateModels) {
       }
       let judgeError;
       try {
-        await judge.autoEvals({ criteria: testCase.rubric, prompt, run });
+        const oracle = hiddenOracleById.get(candidateCase.id);
+        assert.ok(oracle);
+        await judge.autoEvals({ criteria: oracle.rubric, prompt, run });
       } catch (error) {
         judgeError = error;
       }
@@ -87,96 +102,13 @@ const digest = (value) => createHash("sha256").update(JSON.stringify(value)).dig
 export function routingBasisDigest(dimensions) {
     return digest({ version: 2, dimensions });
 }
-export function evaluationProposalDigest(proposal) {
-    return digest(proposal);
-}
-function assertEvaluationProposal(proposal) {
-    if (proposal.evaluationDigest !==
-        evaluationProposalDigest({
-            version: proposal.version,
-            basisDigest: proposal.basisDigest,
-            candidateModels: proposal.candidateModels,
-            ...(proposal.candidateReasoningEfforts === undefined
-                ? {}
-                : { candidateReasoningEfforts: proposal.candidateReasoningEfforts }),
-            judgeModel: proposal.judgeModel,
-            suites: proposal.suites,
-            decompositionBenchmark: proposal.decompositionBenchmark,
-            compositionSuite: proposal.compositionSuite
-        })) {
-        throw new Error("evaluation proposal digest does not match its contents");
-    }
-    if (proposal.candidateModels.length < 2) {
-        throw new Error("evaluation proposal requires at least two candidate models");
-    }
-    if (new Set(proposal.candidateModels).size !== proposal.candidateModels.length) {
-        throw new Error("evaluation proposal candidate models must be unique");
-    }
-    if (Object.keys(proposal.candidateReasoningEfforts ?? {}).some((model) => !proposal.candidateModels.includes(model))) {
-        throw new Error("evaluation proposal reasoning efforts must belong to candidate models");
-    }
-    if (proposal.judgeModel.trim().length === 0) {
-        throw new Error("evaluation proposal judge must be explicit");
-    }
-    const dimensions = new Set();
-    for (const suite of proposal.suites) {
-        if (dimensions.has(suite.dimensionId)) {
-            throw new Error(`duplicate dimension suite ${JSON.stringify(suite.dimensionId)}`);
-        }
-        dimensions.add(suite.dimensionId);
-        assertDimensionSuite(suite);
-    }
-    if (proposal.decompositionBenchmark.maximumVectorL1Error < 0 ||
-        proposal.decompositionBenchmark.maximumVectorL1Error > 2 ||
-        proposal.decompositionBenchmark.cases.length < 5) {
-        throw new Error("decomposition benchmark must define a reviewed threshold and at least five cases");
-    }
-    if (proposal.compositionSuite.maximumOutputTokens < 1 ||
-        proposal.compositionSuite.minimumWinnerScoreGap < 0 ||
-        proposal.compositionSuite.minimumWinnerScoreGap > 1 ||
-        proposal.compositionSuite.minimumWinnerAgreement < 0 ||
-        proposal.compositionSuite.minimumWinnerAgreement > 1 ||
-        proposal.compositionSuite.cases.length < 5) {
-        throw new Error("composition benchmark must define reviewed thresholds and at least five cases");
-    }
-    for (const [label, cases] of [
-        ["decomposition", proposal.decompositionBenchmark.cases],
-        ["composition", proposal.compositionSuite.cases]
-    ]) {
-        const ids = new Set();
-        for (const testCase of cases) {
-            if (testCase.id.trim().length === 0 || ids.has(testCase.id)) {
-                throw new Error(`${label} benchmark contains an invalid or duplicate case id`);
-            }
-            ids.add(testCase.id);
-        }
-    }
-}
-function assertDimensionSuite(suite) {
-    if (suite.maximumOutputTokens < 1) {
-        throw new Error(`dimension suite ${JSON.stringify(suite.dimensionId)} has no output allowance`);
-    }
-    if (suite.cases.length < 5) {
-        throw new Error(`dimension suite ${JSON.stringify(suite.dimensionId)} must contain at least five cases`);
-    }
-    const ids = new Set();
-    for (const testCase of suite.cases) {
-        if (testCase.id.trim().length === 0 ||
-            testCase.prompt.trim().length === 0 ||
-            testCase.rubric.trim().length === 0) {
-            throw new Error(`dimension suite ${JSON.stringify(suite.dimensionId)} contains an incomplete case`);
-        }
-        if (ids.has(testCase.id)) {
-            throw new Error(`dimension suite ${JSON.stringify(suite.dimensionId)} contains duplicate case ${JSON.stringify(testCase.id)}`);
-        }
-        ids.add(testCase.id);
-    }
-}
+export { evaluationProposalDigest };
 export class EvalProjectArtifacts extends Context.Service()("@velum-labs/routekit-eval-setup/EvalProjectArtifacts") {
 }
 export const makeFileEvalProjectArtifacts = Effect.gen(function* () {
     const fs = yield* FileSystem.FileSystem;
     const paths = yield* Path.Path;
+    const platform = yield* Effect.context();
     const root = (repositoryRoot) => paths.join(paths.resolve(repositoryRoot), ".routekit", "evals");
     const artifactPath = (repositoryRoot, name) => paths.join(root(repositoryRoot), name);
     const candidateCheckpointName = (recipeId) => ARTIFACT_ID.test(recipeId)
@@ -193,7 +125,7 @@ export const makeFileEvalProjectArtifacts = Effect.gen(function* () {
     };
     const writeAtomic = (target, content) => writeFileAtomicEffect(target, content, {
         mode: 0o600
-    }).pipe(Effect.mapError((cause) => artifactFailure("writing", target, cause)), Effect.provideService(FileSystem.FileSystem, fs), Effect.provideService(Path.Path, paths));
+    }).pipe(Effect.mapError((cause) => artifactFailure("writing", target, cause)), Effect.provide(platform));
     const read = (repositoryRoot, name, decode) => Effect.gen(function* () {
         const target = artifactPath(repositoryRoot, name);
         const exists = yield* fs
@@ -292,19 +224,6 @@ export const makeFileEvalProjectArtifacts = Effect.gen(function* () {
             for (const suite of proposal.suites) {
                 const suiteRoot = paths.join("dimensions", suite.dimensionId);
                 yield* write(repositoryRoot, paths.join(suiteRoot, "suite.json"), suite);
-                yield* writeText(repositoryRoot, paths.join(suiteRoot, `${suite.dimensionId}.eval.ts`), renderDimensionSuite());
-                yield* write(repositoryRoot, paths.join(suiteRoot, "data", "cases.json"), suite.cases);
-                yield* write(repositoryRoot, paths.join(suiteRoot, "routekit.eval-manifest.json"), {
-                    version: EVAL_PROJECT_VERSION,
-                    profileId: suite.dimensionId,
-                    candidateModels: proposal.candidateModels,
-                    candidateReasoningEfforts: proposal.candidateReasoningEfforts ?? {},
-                    judgeModel: proposal.judgeModel,
-                    caseCount: suite.cases.length,
-                    caseIds: suite.cases.map((testCase) => testCase.id),
-                    maxOutputTokens: suite.maximumOutputTokens,
-                    expectedCallCount: suite.cases.length * proposal.candidateModels.length * 2
-                });
             }
             yield* write(repositoryRoot, paths.join("benchmarks", "decomposition.json"), proposal.decompositionBenchmark);
             yield* write(repositoryRoot, paths.join("benchmarks", "composition.json"), proposal.compositionSuite);
@@ -324,16 +243,11 @@ export const makeFileEvalProjectArtifacts = Effect.gen(function* () {
                     return yield* artifactFailure("writing", selection.dimensionId, new Error("execution plan refers to an unknown dimension suite"));
                 }
                 const byId = new Map(suite.cases.map((testCase) => [testCase.id, testCase]));
-                const cases = selection.caseIds.map((caseId) => {
-                    const testCase = byId.get(caseId);
-                    if (testCase === undefined) {
-                        throw new Error(`execution plan refers to unknown case ${JSON.stringify(caseId)} in ${JSON.stringify(selection.dimensionId)}`);
-                    }
-                    return testCase;
-                });
+                const cases = selection.caseIds.map((caseId) => repeatedCase(byId, caseId));
                 const suiteRoot = paths.join("plans", plan.planId, "dimensions", selection.dimensionId);
                 yield* writeText(repositoryRoot, paths.join(suiteRoot, `${selection.dimensionId}.eval.ts`), renderDimensionSuite(), mode === "rematerialize");
-                yield* write(repositoryRoot, paths.join(suiteRoot, "data", "cases.json"), cases, mode === "rematerialize");
+                yield* write(repositoryRoot, paths.join(suiteRoot, "data", "candidate-cases.json"), cases.map(candidateCaseProjection), mode === "rematerialize");
+                yield* write(repositoryRoot, paths.join(suiteRoot, "data", "hidden-oracles.json"), cases.map(hiddenOracleProjection), mode === "rematerialize");
                 yield* write(repositoryRoot, paths.join(suiteRoot, "routekit.eval-manifest.json"), {
                     version: EVAL_PROJECT_VERSION,
                     profileId: selection.dimensionId,
@@ -347,16 +261,11 @@ export const makeFileEvalProjectArtifacts = Effect.gen(function* () {
                 }, mode === "rematerialize");
             }
             const compositionById = new Map(proposal.compositionSuite.cases.map((testCase) => [testCase.id, testCase]));
-            const compositionCases = plan.selectedCompositionCaseIds.map((caseId) => {
-                const testCase = compositionById.get(caseId);
-                if (testCase === undefined) {
-                    throw new Error(`execution plan refers to unknown composition case ${JSON.stringify(caseId)}`);
-                }
-                return testCase;
-            });
+            const compositionCases = plan.selectedCompositionCaseIds.map((caseId) => repeatedCase(compositionById, caseId));
             const compositionRoot = paths.join("plans", plan.planId, "composition");
             yield* writeText(repositoryRoot, paths.join(compositionRoot, "composition.eval.ts"), renderDimensionSuite(), mode === "rematerialize");
-            yield* write(repositoryRoot, paths.join(compositionRoot, "data", "cases.json"), compositionCases, mode === "rematerialize");
+            yield* write(repositoryRoot, paths.join(compositionRoot, "data", "candidate-cases.json"), compositionCases.map(candidateCaseProjection), mode === "rematerialize");
+            yield* write(repositoryRoot, paths.join(compositionRoot, "data", "hidden-oracles.json"), compositionCases.map(hiddenOracleProjection), mode === "rematerialize");
             yield* write(repositoryRoot, paths.join(compositionRoot, "routekit.eval-manifest.json"), {
                 version: EVAL_PROJECT_VERSION,
                 profileId: "composition",
@@ -435,7 +344,11 @@ export const makeFileEvalProjectArtifacts = Effect.gen(function* () {
                     reports.push(entry);
             }
             return reports.sort((left, right) => left.localeCompare(right));
-        })
+        }),
+        saveRunEventLog: (repositoryRoot, runId, events) => requireArtifactId("run", runId).pipe(Effect.andThen(Schema.decodeEffect(EvalNamedEventLogSchema)(events).pipe(Effect.mapError((cause) => artifactFailure("writing", artifactPath(repositoryRoot, paths.join("runs", runId, "events.json")), cause)))), Effect.andThen(write(repositoryRoot, paths.join("runs", runId, "events.json"), events))),
+        loadRunEventLog: (repositoryRoot, runId) => requireArtifactId("run", runId).pipe(Effect.andThen(read(repositoryRoot, paths.join("runs", runId, "events.json"), Schema.decodeUnknownEffect(EvalNamedEventLogSchema))), Effect.map((events) => events ?? [])),
+        saveAuthoringEventLog: (repositoryRoot, events) => Schema.decodeEffect(EvalNamedEventLogSchema)(events).pipe(Effect.mapError((cause) => artifactFailure("writing", artifactPath(repositoryRoot, paths.join("authoring", "events.json")), cause)), Effect.andThen(write(repositoryRoot, paths.join("authoring", "events.json"), events))),
+        loadAuthoringEventLog: (repositoryRoot) => read(repositoryRoot, paths.join("authoring", "events.json"), Schema.decodeUnknownEffect(EvalNamedEventLogSchema)).pipe(Effect.map((events) => events ?? []))
     });
 });
 export const EvalProjectArtifactsLive = Layer.effect(EvalProjectArtifacts, makeFileEvalProjectArtifacts);

package/dist/project-authoring.d.ts

@@ -1,11 +1,17 @@
 import { type RoutingBasis } from "@velum-labs/routekit-eval-contracts";
 import { Context, Effect, FileSystem, Layer, Path } from "effect";
 import { EvalProjectAuthoringError } from "./errors.js";
-import { type EvalEvaluationProposal, type EvalProposedDimension, type EvalProjectConfiguration } from "./project-contracts.js";
+import { type EvalEvidenceSource, type EvalEvaluationProposal, type EvalProposedDimension, type EvalProjectConfiguration } from "./project-contracts.js";
 export declare const EVAL_AUTHORING_SOURCE_BYTES = 60000;
 export declare const EVAL_AUTHORING_SOURCE_FILES = 64;
 export declare const EVAL_AUTHORING_CASES_PER_DIMENSION = 20;
 export declare const EVAL_AUTHORING_EVALUATION_OUTPUT_TOKENS = 32768;
+/**
+ * A routing basis admits at most ten dimensions, so cross-dimension preflight
+ * requests at most half as many dossiers as a twenty-case suite with the same
+ * authored-case schema. This is an output allowance, not a quality threshold.
+ */
+export declare const EVAL_AUTHORING_PREFLIGHT_OUTPUT_TOKENS = 16384;
 /**
  * Maximum serialized request body admitted for one authoring call.
  *
@@ -14,37 +20,92 @@ export declare const EVAL_AUTHORING_EVALUATION_OUTPUT_TOKENS = 32768;
  * reserves serialized UTF-8 bytes as a conservative input-token upper bound.
  */
 export declare const EVAL_AUTHORING_REQUEST_BYTES = 512000;
+/**
+ * Complete frozen reviews include both control populations and their replay
+ * history. This separately budgeted allowance is opt-in; archived authoring
+ * envelopes and all non-review roles retain the original 512KB bound.
+ */
+export declare const EVAL_AUTHORING_REVIEW_REQUEST_BYTES = 1000000;
 export type EvalAuthoringSource = {
     readonly path: string;
     readonly content: string;
 };
+export type EvalAuthoringSourceClass = "implementation" | "test" | "protocol" | "documentation" | "operational";
+export type EvalAuthoringSourceManifestEntry = {
+    readonly path: string;
+    readonly sizeBytes: number;
+    readonly sourceClass: EvalAuthoringSourceClass;
+    readonly targetDimensions: readonly string[];
+    readonly inclusionReason: string;
+};
+export type EvalAuthoringSourceManifest = {
+    readonly version: 1;
+    readonly purpose: "dimensions" | "evaluations";
+    readonly workloadDescription: string;
+    readonly basisDigest?: string;
+    readonly packets: readonly {
+        readonly target: string;
+        readonly targetDimensions: readonly string[];
+        readonly sources: readonly EvalAuthoringSourceManifestEntry[];
+    }[];
+};
 export type EvalAuthoringCompletion = {
     readonly operationId: string;
     readonly model: string;
+    readonly foundryRole?: string;
+    readonly reasoningEffort?: string;
+    readonly reviewInputVersion?: 2;
     readonly instructions: string;
     readonly input: string;
     readonly schemaName: string;
     readonly jsonSchema: Readonly<Record<string, unknown>>;
     readonly maximumOutputTokens: number;
 };
+export type EvalAuthoringCompletionResult = {
+    readonly text: string;
+    readonly callId?: string;
+};
 export type EvalAuthoringTransportShape = {
     readonly complete: (input: EvalAuthoringCompletion) => Effect.Effect<string, EvalProjectAuthoringError>;
+    readonly completeDetailed?: (input: EvalAuthoringCompletion) => Effect.Effect<EvalAuthoringCompletionResult, EvalProjectAuthoringError>;
 };
 declare const EvalAuthoringTransport_base: Context.ServiceClass<EvalAuthoringTransport, "@velum-labs/routekit-eval-setup/EvalAuthoringTransport", EvalAuthoringTransportShape>;
 export declare class EvalAuthoringTransport extends EvalAuthoringTransport_base {
 }
-/**
- * Revalidate every selected source at the read boundary. Discovery inventory
- * membership is necessary but not sufficient because the checkout may mutate.
- */
 export declare function readProjectAuthoringSources(input: {
     readonly repositoryRoot: string;
     readonly selectedFiles: readonly string[];
     readonly sourceInventory: readonly string[];
 }): Effect.Effect<readonly EvalAuthoringSource[], EvalProjectAuthoringError, FileSystem.FileSystem | Path.Path>;
+type EvalAuthoringSourcePacket = {
+    readonly sources: readonly EvalAuthoringSourceManifestEntry[];
+};
+type EvalAuthoringEvidencePacket = {
+    readonly packet: EvalAuthoringSourcePacket;
+    readonly evidenceSources: readonly EvalEvidenceSource[];
+};
+type EvalAuthoringDimensionEvidencePacket = EvalAuthoringEvidencePacket & {
+    readonly preflightEvidenceSources: readonly EvalEvidenceSource[];
+};
+export declare const selectProjectAuthoringEvidencePacket: (input: {
+    readonly repositoryRoot: string;
+    readonly sourceInventory: readonly string[];
+    readonly targetDimensions: RoutingBasis["dimensions"];
+}) => Effect.Effect<EvalAuthoringDimensionEvidencePacket, EvalProjectAuthoringError, FileSystem.FileSystem | Path.Path>;
+export declare const selectSharedProjectAuthoringEvidencePacket: (input: {
+    readonly repositoryRoot: string;
+    readonly sourceInventory: readonly string[];
+    readonly targetDimensions: RoutingBasis["dimensions"];
+}) => Effect.Effect<EvalAuthoringEvidencePacket, EvalProjectAuthoringError, FileSystem.FileSystem | Path.Path>;
 export declare function selectProjectAuthoringSourceFiles(input: {
     readonly repositoryRoot: string;
     readonly sourceInventory: readonly string[];
+    readonly workloadDescription?: string;
+    readonly targetDimensions?: RoutingBasis["dimensions"];
+    readonly explicitInclusions?: readonly {
+        readonly path: string;
+        readonly reason: string;
+    }[];
 }): Effect.Effect<readonly string[], EvalProjectAuthoringError, FileSystem.FileSystem | Path.Path>;
 export type EvalProjectAuthorShape = {
     readonly proposeDimensions: (input: {