@velum-labs/routekit-eval-setup 1.3.2 → 1.5.0

package/dist/evaluation-source-retrieval.js

@@ -0,0 +1,513 @@
+export const EVAL_SOURCE_RETRIEVAL_INDEX_FILES = 512;
+export const EVAL_SOURCE_RETRIEVAL_INDEX_BYTES = 8 * 1024 * 1024;
+export const EVAL_SOURCE_RETRIEVAL_FILE_BYTES = 256 * 1024;
+export const EVAL_SOURCE_RETRIEVAL_PACKET_BLOCKS = 20;
+export const EVAL_SOURCE_RETRIEVAL_BLOCKS_PER_SOURCE = 2;
+const STOP_WORDS = new Set([
+    "about",
+    "after",
+    "and",
+    "are",
+    "been",
+    "for",
+    "from",
+    "has",
+    "into",
+    "request",
+    "requests",
+    "that",
+    "the",
+    "this",
+    "with"
+]);
+const canonicalCompare = (left, right) => left < right ? -1 : left > right ? 1 : 0;
+const alphaNumeric = (character) => {
+    const code = character.codePointAt(0);
+    return (code !== undefined &&
+        ((code >= 48 && code <= 57) ||
+            (code >= 65 && code <= 90) ||
+            (code >= 97 && code <= 122)));
+};
+const upper = (character) => {
+    const code = character.codePointAt(0);
+    return code !== undefined && code >= 65 && code <= 90;
+};
+const lowerOrNumber = (character) => {
+    const code = character.codePointAt(0);
+    return (code !== undefined &&
+        ((code >= 48 && code <= 57) || (code >= 97 && code <= 122)));
+};
+const normalizeToken = (value) => {
+    const lower = value.toLowerCase();
+    if (lower.length >= 5 && lower.endsWith("ies")) {
+        return `${lower.slice(0, -3)}y`;
+    }
+    if (lower.length >= 5 &&
+        lower.endsWith("s") &&
+        !lower.endsWith("ss") &&
+        !lower.endsWith("us") &&
+        !lower.endsWith("is")) {
+        return lower.slice(0, -1);
+    }
+    return lower;
+};
+/**
+ * Tokenizes identifiers and prose without asking a model to infer source
+ * relevance. CamelCase boundaries are retained so `providerBackend` can match
+ * the reviewed positive scope `provider backend`.
+ */
+export const evaluationSourceTokens = (value) => {
+    const tokens = [];
+    let token = "";
+    const flush = () => {
+        const normalized = normalizeToken(token);
+        token = "";
+        if (normalized.length >= 3 && !STOP_WORDS.has(normalized)) {
+            tokens.push(normalized);
+        }
+    };
+    const characters = [...value];
+    for (const [index, character] of characters.entries()) {
+        if (!alphaNumeric(character)) {
+            flush();
+            continue;
+        }
+        const previous = token.at(-1);
+        const next = characters[index + 1];
+        if (token.length > 0 &&
+            previous !== undefined &&
+            upper(character) &&
+            (lowerOrNumber(previous) ||
+                (upper(previous) && next !== undefined && lowerOrNumber(next)))) {
+            flush();
+        }
+        token += character;
+    }
+    flush();
+    return tokens;
+};
+const positiveQueryTerms = (input) => {
+    const dimensions = input.targetDimensions ?? [];
+    const values = dimensions.length === 0
+        ? [input.workloadDescription ?? ""]
+        : dimensions.flatMap((dimension) => [
+            dimension.id,
+            dimension.description,
+            ...dimension.includes
+        ]);
+    return [...new Set(values.flatMap(evaluationSourceTokens))].sort(canonicalCompare);
+};
+const positiveQueryFacets = (dimensions) => dimensions.flatMap((dimension) => {
+    const includes = dimension.includes
+        .map((value, index) => ({
+        label: `${dimension.id} include ${String(index + 1)}`,
+        terms: [...new Set(evaluationSourceTokens(value))].sort(canonicalCompare)
+    }))
+        .filter((facet) => facet.terms.length > 0);
+    return includes.length > 0
+        ? includes
+        : [
+            {
+                label: dimension.id,
+                terms: [
+                    ...new Set(evaluationSourceTokens(`${dimension.id} ${dimension.description}`))
+                ].sort(canonicalCompare)
+            }
+        ];
+});
+const ownershipRoot = (path) => {
+    const segments = path.split("/");
+    const first = segments[0] ?? path;
+    if (["apps", "packages", "services", "tooling"].includes(first) &&
+        segments[1] !== undefined) {
+        return `${first}/${segments[1]}`;
+    }
+    return first;
+};
+const retrievalScope = (path) => {
+    const segments = path.split("/");
+    if (["apps", "packages", "services", "tooling"].includes(segments[0] ?? "") &&
+        segments[1] !== undefined &&
+        segments[2] === "src" &&
+        segments[3] !== undefined) {
+        return segments.slice(0, 4).join("/");
+    }
+    return undefined;
+};
+const frequencies = (tokens) => {
+    const values = new Map();
+    for (const token of tokens) {
+        values.set(token, (values.get(token) ?? 0) + 1);
+    }
+    return values;
+};
+const bm25 = (input) => {
+    if (input.termFrequency === 0)
+        return 0;
+    const k1 = 1.2;
+    const b = 0.75;
+    const normalizedLength = input.averageLength <= 0 ? 1 : input.length / input.averageLength;
+    return ((input.termFrequency * (k1 + 1)) /
+        (input.termFrequency + k1 * (1 - b + b * normalizedLength)));
+};
+const pathRanks = (candidates, queryTerms) => {
+    const tokenized = candidates.map((candidate) => ({
+        candidate,
+        tokens: evaluationSourceTokens(candidate.path)
+    }));
+    const averageLength = tokenized.reduce((total, item) => total + item.tokens.length, 0) /
+        Math.max(1, tokenized.length);
+    const documentFrequency = new Map(queryTerms.map((term) => [
+        term,
+        tokenized.filter((item) => item.tokens.includes(term)).length
+    ]));
+    return tokenized
+        .map(({ candidate, tokens }) => {
+        const counts = frequencies(tokens);
+        const matchedTerms = queryTerms.filter((term) => (counts.get(term) ?? 0) > 0);
+        const score = matchedTerms.reduce((total, term) => {
+            const frequency = documentFrequency.get(term) ?? 0;
+            const inverseDocumentFrequency = Math.log(1 +
+                (tokenized.length - frequency + 0.5) /
+                    (frequency + 0.5));
+            return (total +
+                inverseDocumentFrequency *
+                    bm25({
+                        termFrequency: counts.get(term) ?? 0,
+                        length: tokens.length,
+                        averageLength
+                    }));
+        }, 0);
+        return { candidate, score, matchedTerms };
+    })
+        .sort((left, right) => right.score - left.score ||
+        canonicalCompare(left.candidate.path, right.candidate.path));
+};
+/**
+ * Produces a bounded content-index plan. Positive path matches identify likely
+ * ownership roots, then generic filenames inside those roots may compete by
+ * content. This avoids reading an unbounded repository or relying on inventory
+ * order.
+ */
+export function planEvaluationSourceRetrieval(input) {
+    const maximumFiles = input.maximumFiles ?? EVAL_SOURCE_RETRIEVAL_INDEX_FILES;
+    const maximumBytes = input.maximumBytes ?? EVAL_SOURCE_RETRIEVAL_INDEX_BYTES;
+    const queryTerms = positiveQueryTerms(input);
+    const ranked = pathRanks(input.candidates, queryTerms);
+    const eligible = ranked.filter((item) => item.candidate.sizeBytes <= EVAL_SOURCE_RETRIEVAL_FILE_BYTES);
+    const eligibleBytes = eligible.reduce((total, item) => total + item.candidate.sizeBytes, 0);
+    if (eligible.length <= maximumFiles &&
+        eligibleBytes <= maximumBytes) {
+        const scopeScores = new Map();
+        for (const item of eligible) {
+            const scope = retrievalScope(item.candidate.path);
+            if (scope !== undefined) {
+                scopeScores.set(scope, Math.max(scopeScores.get(scope) ?? 0, item.score));
+            }
+        }
+        return eligible
+            .map((item) => {
+            const scope = retrievalScope(item.candidate.path);
+            return {
+                ...item.candidate,
+                pathScore: item.score +
+                    (scope === undefined ? 0 : scopeScores.get(scope) ?? 0)
+            };
+        })
+            .sort((left, right) => canonicalCompare(left.path, right.path));
+    }
+    const positivePathMatches = ranked.filter((item) => item.score > 0);
+    const scopeScores = new Map();
+    for (const item of positivePathMatches) {
+        const scope = retrievalScope(item.candidate.path);
+        if (scope !== undefined) {
+            scopeScores.set(scope, Math.max(scopeScores.get(scope) ?? 0, item.score));
+        }
+    }
+    const explicit = ranked.filter((item) => item.candidate.explicitReason !== undefined);
+    const implicatedRoots = new Set(positivePathMatches
+        .slice(0, 32)
+        .map((item) => ownershipRoot(item.candidate.path)));
+    const retrieved = positivePathMatches.length === 0
+        ? ranked
+        : ranked.filter((item) => item.score > 0 ||
+            implicatedRoots.has(ownershipRoot(item.candidate.path)));
+    const planned = [
+        ...explicit,
+        ...retrieved.filter((item) => item.candidate.explicitReason === undefined)
+    ];
+    const selected = [];
+    let bytes = 0;
+    for (const item of planned) {
+        if (selected.length >= maximumFiles)
+            break;
+        if (item.candidate.sizeBytes > EVAL_SOURCE_RETRIEVAL_FILE_BYTES) {
+            continue;
+        }
+        if (bytes + item.candidate.sizeBytes > maximumBytes)
+            continue;
+        const scope = retrievalScope(item.candidate.path);
+        selected.push({
+            ...item.candidate,
+            pathScore: item.score +
+                (scope === undefined ? 0 : scopeScores.get(scope) ?? 0)
+        });
+        bytes += item.candidate.sizeBytes;
+    }
+    return selected.sort((left, right) => canonicalCompare(left.path, right.path));
+}
+const scoreDocuments = (documents, queryTerms) => {
+    const prepared = documents.map((document) => ({
+        document,
+        pathTokens: evaluationSourceTokens(document.path),
+        contentTokens: evaluationSourceTokens(document.content)
+    }));
+    const averagePathLength = prepared.reduce((total, item) => total + item.pathTokens.length, 0) /
+        Math.max(1, prepared.length);
+    const averageContentLength = prepared.reduce((total, item) => total + item.contentTokens.length, 0) /
+        Math.max(1, prepared.length);
+    const documentFrequency = new Map(queryTerms.map((term) => [
+        term,
+        prepared.filter((item) => item.pathTokens.includes(term) ||
+            item.contentTokens.includes(term)).length
+    ]));
+    return prepared
+        .map(({ document, pathTokens, contentTokens }) => {
+        const pathCounts = frequencies(pathTokens);
+        const contentCounts = frequencies(contentTokens);
+        const pathMatches = queryTerms.filter((term) => (pathCounts.get(term) ?? 0) > 0);
+        const contentMatches = queryTerms.filter((term) => (contentCounts.get(term) ?? 0) > 0);
+        const score = queryTerms.reduce((total, term) => {
+            const frequency = documentFrequency.get(term) ?? 0;
+            const inverseDocumentFrequency = Math.log(1 +
+                (prepared.length - frequency + 0.5) /
+                    (frequency + 0.5));
+            return (total +
+                inverseDocumentFrequency *
+                    (3 *
+                        bm25({
+                            termFrequency: pathCounts.get(term) ?? 0,
+                            length: pathTokens.length,
+                            averageLength: averagePathLength
+                        }) +
+                        bm25({
+                            termFrequency: contentCounts.get(term) ?? 0,
+                            length: contentTokens.length,
+                            averageLength: averageContentLength
+                        })));
+        }, 0);
+        return {
+            document,
+            score: score +
+                (document.explicitReason === undefined ? 0 : 1_000_000),
+            pathMatches,
+            contentMatches
+        };
+    })
+        .sort((left, right) => right.score - left.score ||
+        canonicalCompare(left.document.path, right.document.path));
+};
+export function selectEvaluationSourcePacket(input) {
+    const queryTerms = positiveQueryTerms(input);
+    const ranked = scoreDocuments(input.documents, queryTerms);
+    const hasPositiveMatch = ranked.some((item) => item.pathMatches.length > 0 ||
+        item.contentMatches.length > 0);
+    const candidates = hasPositiveMatch
+        ? ranked.filter((item) => item.document.explicitReason !== undefined ||
+            item.pathMatches.length > 0 ||
+            item.contentMatches.length > 0)
+        : ranked;
+    const selected = [];
+    let bytes = 0;
+    for (const [rankIndex, item] of candidates.entries()) {
+        if (selected.length >= input.maximumFiles)
+            break;
+        if (bytes + item.document.sizeBytes > input.maximumBytes)
+            continue;
+        const terms = [
+            ...new Set([...item.pathMatches, ...item.contentMatches])
+        ].sort(canonicalCompare);
+        selected.push({
+            ...item.document,
+            inclusionReason: item.document.explicitReason === undefined
+                ? terms.length === 0
+                    ? "bounded fallback source; no positive query term matched"
+                    : [
+                        `sparse retrieval rank ${String(rankIndex + 1)} in ${ownershipRoot(item.document.path)}`,
+                        `matched positive terms: ${terms.join(", ")}`,
+                        `fields: ${item.pathMatches.length > 0 &&
+                            item.contentMatches.length > 0
+                            ? "path+content"
+                            : item.pathMatches.length > 0
+                                ? "path"
+                                : "content"}`
+                    ].join("; ")
+                : `explicit inclusion: ${item.document.explicitReason}`
+        });
+        bytes += item.document.sizeBytes;
+    }
+    return selected.sort((left, right) => canonicalCompare(left.path, right.path));
+}
+const rankEvidenceBlocks = (documents, queryTerms) => {
+    const prepared = documents.flatMap((document) => {
+        const pathTokens = evaluationSourceTokens(document.path);
+        return document.evidenceSource.blocks.map((block) => ({
+            document,
+            block,
+            pathTokens,
+            contentTokens: evaluationSourceTokens(block.text)
+        }));
+    });
+    const averagePathLength = prepared.reduce((total, item) => total + item.pathTokens.length, 0) /
+        Math.max(1, prepared.length);
+    const averageContentLength = prepared.reduce((total, item) => total + item.contentTokens.length, 0) /
+        Math.max(1, prepared.length);
+    const documentFrequency = new Map(queryTerms.map((term) => [
+        term,
+        prepared.filter((item) => item.pathTokens.includes(term) ||
+            item.contentTokens.includes(term)).length
+    ]));
+    return prepared
+        .map(({ document, block, pathTokens, contentTokens }) => {
+        const pathCounts = frequencies(pathTokens);
+        const contentCounts = frequencies(contentTokens);
+        const pathMatches = queryTerms.filter((term) => (pathCounts.get(term) ?? 0) > 0);
+        const contentMatches = queryTerms.filter((term) => (contentCounts.get(term) ?? 0) > 0);
+        const score = queryTerms.reduce((total, term) => {
+            const frequency = documentFrequency.get(term) ?? 0;
+            const inverseDocumentFrequency = Math.log(1 +
+                (prepared.length - frequency + 0.5) /
+                    (frequency + 0.5));
+            return (total +
+                inverseDocumentFrequency *
+                    (3 *
+                        bm25({
+                            termFrequency: pathCounts.get(term) ?? 0,
+                            length: pathTokens.length,
+                            averageLength: averagePathLength
+                        }) +
+                        bm25({
+                            termFrequency: contentCounts.get(term) ?? 0,
+                            length: contentTokens.length,
+                            averageLength: averageContentLength
+                        })));
+        }, 0);
+        return {
+            document,
+            block,
+            score: score +
+                (document.pathScore ?? 0) +
+                (document.explicitReason === undefined ? 0 : 1_000_000),
+            matchedTermCount: new Set([
+                ...pathMatches,
+                ...contentMatches
+            ]).size,
+            pathMatches,
+            contentMatches
+        };
+    })
+        .sort((left, right) => Number(right.document.explicitReason !== undefined) -
+        Number(left.document.explicitReason !== undefined) ||
+        right.score - left.score ||
+        right.matchedTermCount - left.matchedTermCount ||
+        canonicalCompare(left.document.path, right.document.path) ||
+        left.block.ordinal - right.block.ordinal);
+};
+/**
+ * Selects compiler-owned evidence blocks rather than whole files. Packet
+ * assembly proceeds in per-source rounds so one large source cannot consume
+ * the complete authoring budget before complementary owners can compete.
+ */
+export function selectEvaluationEvidencePacket(input) {
+    const maximumBlocks = input.maximumBlocks ?? EVAL_SOURCE_RETRIEVAL_PACKET_BLOCKS;
+    const maximumBlocksPerSource = input.maximumBlocksPerSource ??
+        EVAL_SOURCE_RETRIEVAL_BLOCKS_PER_SOURCE;
+    const facets = positiveQueryFacets(input.targetDimensions).map((facet) => {
+        const ranked = rankEvidenceBlocks(input.documents, facet.terms);
+        const positive = ranked.filter((item) => item.document.explicitReason !== undefined ||
+            item.pathMatches.length > 0 ||
+            item.contentMatches.length > 0);
+        return {
+            ...facet,
+            candidates: positive.length > 0 ? positive : ranked
+        };
+    });
+    const selected = [];
+    const selectedIds = new Set();
+    let selectedBytes = 0;
+    const admit = (facet, predicate) => {
+        const item = facet.candidates.find((candidate) => {
+            if (selectedIds.has(candidate.block.id) || !predicate(candidate)) {
+                return false;
+            }
+            const sourceCount = selected.filter((selectedItem) => selectedItem.document.path === candidate.document.path).length;
+            if (sourceCount >= maximumBlocksPerSource)
+                return false;
+            const blockBytes = Buffer.byteLength(candidate.block.text, "utf8");
+            return selectedBytes + blockBytes <= input.maximumBytes;
+        });
+        if (item === undefined)
+            return false;
+        selected.push({ ...item, facetLabel: facet.label });
+        selectedIds.add(item.block.id);
+        selectedBytes += Buffer.byteLength(item.block.text, "utf8");
+        return true;
+    };
+    for (const facet of facets) {
+        if (selected.length >= maximumBlocks)
+            break;
+        admit(facet, (candidate) => (candidate.document.sourceClass === "implementation" ||
+            candidate.document.sourceClass === "protocol") &&
+            !selected.some((selectedItem) => selectedItem.document.path === candidate.document.path));
+    }
+    while (selected.length < maximumBlocks) {
+        let admitted = false;
+        for (const facet of facets) {
+            if (selected.length >= maximumBlocks)
+                break;
+            admitted = admit(facet, () => true) || admitted;
+        }
+        if (!admitted)
+            break;
+    }
+    const grouped = new Map();
+    for (const item of selected) {
+        const values = grouped.get(item.document.path) ?? [];
+        values.push(item);
+        grouped.set(item.document.path, values);
+    }
+    return [...grouped.entries()]
+        .map(([path, items]) => {
+        const first = items[0];
+        const blocks = items
+            .map((item) => item.block)
+            .sort((left, right) => left.ordinal - right.ordinal);
+        const terms = [
+            ...new Set(items.flatMap((item) => [
+                ...item.pathMatches,
+                ...item.contentMatches
+            ]))
+        ].sort(canonicalCompare);
+        const facets = [
+            ...new Set(items.map((item) => item.facetLabel))
+        ].sort(canonicalCompare);
+        return {
+            source: {
+                path,
+                sizeBytes: first.document.sizeBytes,
+                sourceClass: first.document.sourceClass,
+                ...(first.document.explicitReason === undefined
+                    ? {}
+                    : { explicitReason: first.document.explicitReason })
+            },
+            blocks,
+            sizeBytes: blocks.reduce((total, block) => total + Buffer.byteLength(block.text, "utf8"), 0),
+            inclusionReason: first.document.explicitReason === undefined
+                ? terms.length === 0
+                    ? `bounded block fallback in ${ownershipRoot(path)}`
+                    : `compiler-block retrieval in ${ownershipRoot(path)}; facets: ${facets.join(", ")}; matched positive terms: ${terms.join(", ")}`
+                : `explicit inclusion: ${first.document.explicitReason}`
+        };
+    })
+        .sort((left, right) => canonicalCompare(left.source.path, right.source.path));
+}

package/dist/evaluation-structure-policy.d.ts

@@ -0,0 +1,29 @@
+import type { EvalCriterion, EvalDimensionCase, EvalEvaluationProposal, EvalNegativeControl, EvalPositiveControl } from "./project-contracts.js";
+export declare const EVAL_STRUCTURE_POLICY_VERSION: 1;
+export type EvalCandidateCaseProjection = {
+    readonly id: string;
+    readonly prompt: string;
+    readonly context: string;
+};
+export type EvalHiddenOracleProjection = {
+    readonly id: string;
+    readonly rubric: string;
+    readonly criteria: readonly EvalCriterion[];
+    readonly positiveControls: readonly EvalPositiveControl[];
+    readonly negativeControls: readonly EvalNegativeControl[];
+};
+export type EvalStructureReport = {
+    readonly policyVersion: typeof EVAL_STRUCTURE_POLICY_VERSION;
+    readonly evidenceSourceCount: number;
+    readonly evidenceBlockCount: number;
+    readonly caseCount: number;
+    readonly duplicateEvidenceSelectionGroups: readonly (readonly string[])[];
+};
+export declare const renderEvaluationCriteria: (criteria: readonly EvalCriterion[]) => string;
+export declare function candidateCaseProjection(testCase: EvalDimensionCase): EvalCandidateCaseProjection;
+export declare function hiddenOracleProjection(testCase: EvalDimensionCase): EvalHiddenOracleProjection;
+export declare function assertEvaluationStructure(input: {
+    readonly proposal: EvalEvaluationProposal;
+    readonly sourceInventory: readonly string[];
+}): EvalStructureReport;
+export declare function evaluationStructureReportDigest(report: EvalStructureReport): string;

package/dist/evaluation-structure-policy.js

@@ -0,0 +1,138 @@
+import { createHash } from "node:crypto";
+import { assertEvaluationEvidenceSources, renderEvaluationEvidenceContext, resolveEvaluationEvidence } from "./evaluation-evidence.js";
+import { EVAL_AUTHORING_CASE_EVIDENCE_BYTES, EVAL_AUTHORING_EVIDENCE_BLOCK_BYTES } from "./evaluation-authoring-policy.js";
+export const EVAL_STRUCTURE_POLICY_VERSION = 1;
+const sha256 = (value) => createHash("sha256").update(value).digest("hex");
+const uniqueIds = (values, label) => {
+    if (new Set(values.map((value) => value.id)).size !== values.length) {
+        throw new Error(`${label} ids must be unique`);
+    }
+};
+export const renderEvaluationCriteria = (criteria) => criteria
+    .map((criterion) => `${criterion.importance === "critical" ? "Critical" : "Supporting"} criterion ${criterion.id}: ${criterion.description}`)
+    .join("\n");
+const compiledCaseFields = (testCase) => {
+    if (testCase.context === undefined ||
+        testCase.evidenceIds === undefined ||
+        testCase.criteria === undefined ||
+        testCase.positiveControls === undefined ||
+        testCase.negativeControls === undefined) {
+        throw new Error(`evaluation case ${JSON.stringify(testCase.id)} is not compiler-backed`);
+    }
+    return {
+        context: testCase.context,
+        evidenceIds: testCase.evidenceIds,
+        criteria: testCase.criteria,
+        positiveControls: testCase.positiveControls,
+        negativeControls: testCase.negativeControls
+    };
+};
+const assertControls = (caseId, criteria, positiveControls, negativeControls) => {
+    if (criteria.length === 0 || !criteria.some((criterion) => criterion.importance === "critical")) {
+        throw new Error(`evaluation case ${JSON.stringify(caseId)} requires a critical criterion`);
+    }
+    uniqueIds(criteria, `evaluation case ${JSON.stringify(caseId)} criterion`);
+    if (positiveControls.length === 0) {
+        throw new Error(`evaluation case ${JSON.stringify(caseId)} requires a positive control`);
+    }
+    uniqueIds(positiveControls, `evaluation case ${JSON.stringify(caseId)} positive control`);
+    uniqueIds(negativeControls, `evaluation case ${JSON.stringify(caseId)} negative control`);
+    const criterionIds = new Set(criteria.map((criterion) => criterion.id));
+    for (const negative of negativeControls) {
+        if (negative.targetedCriterionIds.length === 0 ||
+            new Set(negative.targetedCriterionIds).size !==
+                negative.targetedCriterionIds.length ||
+            negative.targetedCriterionIds.some((criterionId) => !criterionIds.has(criterionId))) {
+            throw new Error(`evaluation case ${JSON.stringify(caseId)} has an invalid negative-control target`);
+        }
+    }
+    for (const kind of [
+        "missing-criterion",
+        "forbidden-behavior",
+        "non-responsive"
+    ]) {
+        if (!negativeControls.some((control) => control.kind === kind)) {
+            throw new Error(`evaluation case ${JSON.stringify(caseId)} requires a ${kind} negative control`);
+        }
+    }
+};
+export function candidateCaseProjection(testCase) {
+    const fields = compiledCaseFields(testCase);
+    return {
+        id: testCase.id,
+        prompt: testCase.prompt,
+        context: fields.context
+    };
+}
+export function hiddenOracleProjection(testCase) {
+    const fields = compiledCaseFields(testCase);
+    return {
+        id: testCase.id,
+        rubric: testCase.rubric,
+        criteria: fields.criteria,
+        positiveControls: fields.positiveControls,
+        negativeControls: fields.negativeControls
+    };
+}
+export function assertEvaluationStructure(input) {
+    const { proposal } = input;
+    const evidenceSources = proposal.evidenceSources;
+    if (evidenceSources === undefined || evidenceSources.length === 0) {
+        throw new Error("evaluation proposal has no compiler-owned evidence sources");
+    }
+    assertEvaluationEvidenceSources(evidenceSources);
+    const inventory = new Set(input.sourceInventory);
+    for (const source of evidenceSources) {
+        if (!inventory.has(source.path)) {
+            throw new Error(`evaluation evidence source is outside the project inventory: ${JSON.stringify(source.path)}`);
+        }
+        for (const block of source.blocks) {
+            if (Buffer.byteLength(block.text, "utf8") > EVAL_AUTHORING_EVIDENCE_BLOCK_BYTES) {
+                throw new Error(`evaluation evidence block exceeds policy bytes: ${JSON.stringify(block.id)}`);
+            }
+        }
+    }
+    const cases = [
+        ...proposal.suites.flatMap((suite) => suite.cases.map((testCase) => ({
+            identity: `${suite.dimensionId}/${testCase.id}`,
+            testCase
+        }))),
+        ...proposal.compositionSuite.cases.map((testCase) => ({
+            identity: `composition/${testCase.id}`,
+            testCase
+        }))
+    ];
+    const selectionCases = new Map();
+    for (const { identity, testCase } of cases) {
+        const fields = compiledCaseFields(testCase);
+        const selected = resolveEvaluationEvidence({
+            sources: evidenceSources,
+            evidenceIds: fields.evidenceIds,
+            maximumContextBytes: EVAL_AUTHORING_CASE_EVIDENCE_BYTES
+        });
+        if (fields.context !== renderEvaluationEvidenceContext(selected)) {
+            throw new Error(`evaluation case ${JSON.stringify(testCase.id)} context is not compiler-rendered`);
+        }
+        assertControls(testCase.id, fields.criteria, fields.positiveControls, fields.negativeControls);
+        if (testCase.rubric !== renderEvaluationCriteria(fields.criteria)) {
+            throw new Error(`evaluation case ${JSON.stringify(testCase.id)} rubric is not compiler-rendered`);
+        }
+        const fingerprint = fields.evidenceIds.join("\u0000");
+        const matching = selectionCases.get(fingerprint) ?? [];
+        matching.push(identity);
+        selectionCases.set(fingerprint, matching);
+    }
+    return {
+        policyVersion: EVAL_STRUCTURE_POLICY_VERSION,
+        evidenceSourceCount: evidenceSources.length,
+        evidenceBlockCount: evidenceSources.reduce((count, source) => count + source.blocks.length, 0),
+        caseCount: cases.length,
+        duplicateEvidenceSelectionGroups: [...selectionCases.values()]
+            .filter((caseIds) => caseIds.length > 1)
+            .map((caseIds) => [...caseIds].sort((left, right) => left < right ? -1 : left > right ? 1 : 0))
+            .sort((left, right) => left[0] < right[0] ? -1 : left[0] > right[0] ? 1 : 0)
+    };
+}
+export function evaluationStructureReportDigest(report) {
+    return sha256(JSON.stringify(report));
+}