@agent-native/creative-context 0.1.0

package/dist/eval/acceptance.js

@@ -0,0 +1,260 @@
+import { createHash } from "node:crypto";
+import { z } from "zod";
+const DEFAULT_THRESHOLDS = {
+    minimumCases: 4,
+    minimumContextOnWinRate: 0.75,
+    minimumRequiredTermCoverage: 1,
+    minimumMeanEditDistanceReduction: 0,
+};
+const ConditionSchema = z.object({
+    outputHash: z.string().regex(/^[a-f0-9]{64}$/),
+    requiredTermCoverage: z.number().min(0).max(1),
+    forbiddenHits: z.array(z.string()),
+    editDistance: z.number().int().nonnegative(),
+    usedContextKeys: z.array(z.string()),
+    contextPackId: z.string().nullable(),
+    provenanceKeys: z.array(z.string()),
+});
+export const ContextAcceptanceReportSchema = z.object({
+    schemaVersion: z.literal(1),
+    corpusId: z.string().min(1),
+    runId: z.string().regex(/^cc-eval-[a-f0-9]{16}$/),
+    cases: z.array(z.object({
+        id: z.string().min(1),
+        contextOff: ConditionSchema,
+        contextOn: ConditionSchema,
+        winner: z.enum(["context-on", "context-off", "tie"]),
+        contextPackRecorded: z.boolean(),
+        provenanceComplete: z.boolean(),
+        optOutStructurallyClean: z.boolean(),
+    })),
+    summary: z.object({
+        caseCount: z.number().int().nonnegative(),
+        contextOnWins: z.number().int().nonnegative(),
+        contextOffWins: z.number().int().nonnegative(),
+        ties: z.number().int().nonnegative(),
+        contextOnWinRate: z.number().min(0).max(1),
+        contextOnRequiredTermCoverage: z.number().min(0).max(1),
+        meanEditDistanceReduction: z.number(),
+        forbiddenOutputHits: z.number().int().nonnegative(),
+        missingContextPacks: z.number().int().nonnegative(),
+        missingProvenance: z.number().int().nonnegative(),
+        optOutContamination: z.number().int().nonnegative(),
+    }),
+    thresholds: z.object({
+        minimumCases: z.number().int().positive(),
+        minimumContextOnWinRate: z.number().min(0).max(1),
+        minimumRequiredTermCoverage: z.number().min(0).max(1),
+        minimumMeanEditDistanceReduction: z.number(),
+    }),
+    gates: z.object({ passed: z.boolean(), failures: z.array(z.string()) }),
+    manualBlindPreference: z.object({
+        status: z.enum(["pending", "complete"]),
+        scoredTrials: z.number().int().nonnegative(),
+        contextOnPreferenceRate: z.number().min(0).max(1).nullable(),
+    }),
+});
+function normalized(value) {
+    return value.toLocaleLowerCase("en-US").replace(/\s+/g, " ").trim();
+}
+function unique(values) {
+    return [
+        ...new Set(values.map((value) => value.trim()).filter(Boolean)),
+    ].sort();
+}
+function condition(artifact, acceptanceCase) {
+    const output = normalized(artifact.output);
+    const required = unique(acceptanceCase.requiredTerms);
+    const forbidden = unique(acceptanceCase.forbiddenTerms).filter((term) => output.includes(normalized(term)));
+    const matched = required.filter((term) => output.includes(normalized(term)));
+    return {
+        outputHash: createHash("sha256").update(artifact.output).digest("hex"),
+        requiredTermCoverage: required.length
+            ? matched.length / required.length
+            : 1,
+        forbiddenHits: forbidden,
+        editDistance: levenshtein(output, normalized(acceptanceCase.reference)),
+        usedContextKeys: unique(artifact.usedContextKeys),
+        contextPackId: artifact.contextPackId?.trim() || null,
+        provenanceKeys: unique(artifact.provenanceKeys ?? []),
+    };
+}
+function winner(off, on) {
+    if (on.forbiddenHits.length !== off.forbiddenHits.length) {
+        return on.forbiddenHits.length < off.forbiddenHits.length
+            ? "context-on"
+            : "context-off";
+    }
+    if (on.requiredTermCoverage !== off.requiredTermCoverage) {
+        return on.requiredTermCoverage > off.requiredTermCoverage
+            ? "context-on"
+            : "context-off";
+    }
+    if (on.editDistance !== off.editDistance) {
+        return on.editDistance < off.editDistance ? "context-on" : "context-off";
+    }
+    return "tie";
+}
+export function runContextAcceptanceEvaluation(input) {
+    const thresholds = { ...DEFAULT_THRESHOLDS, ...input.thresholds };
+    const cases = [...input.cases]
+        .sort((left, right) => left.id.localeCompare(right.id))
+        .map((acceptanceCase) => {
+        const contextOff = condition(acceptanceCase.contextOff, acceptanceCase);
+        const contextOn = condition(acceptanceCase.contextOn, acceptanceCase);
+        const expected = new Set(unique(acceptanceCase.expectedContextKeys));
+        const contextPackRecorded = Boolean(contextOn.contextPackId);
+        const provenanceComplete = [...expected].every((key) => contextOn.provenanceKeys.includes(key));
+        const optOutStructurallyClean = contextOff.usedContextKeys.length === 0 &&
+            contextOff.contextPackId === null &&
+            contextOff.provenanceKeys.length === 0;
+        return {
+            id: acceptanceCase.id,
+            contextOff,
+            contextOn,
+            winner: winner(contextOff, contextOn),
+            contextPackRecorded,
+            provenanceComplete,
+            optOutStructurallyClean,
+        };
+    });
+    const count = cases.length;
+    const contextOnWins = cases.filter((entry) => entry.winner === "context-on").length;
+    const contextOffWins = cases.filter((entry) => entry.winner === "context-off").length;
+    const ties = count - contextOnWins - contextOffWins;
+    const average = (values) => values.length
+        ? values.reduce((total, value) => total + value, 0) / values.length
+        : 0;
+    const summary = {
+        caseCount: count,
+        contextOnWins,
+        contextOffWins,
+        ties,
+        contextOnWinRate: count ? contextOnWins / count : 0,
+        contextOnRequiredTermCoverage: average(cases.map((entry) => entry.contextOn.requiredTermCoverage)),
+        meanEditDistanceReduction: average(cases.map((entry) => {
+            const baseline = entry.contextOff.editDistance;
+            return baseline > 0
+                ? (baseline - entry.contextOn.editDistance) / baseline
+                : entry.contextOn.editDistance === 0
+                    ? 0
+                    : -1;
+        })),
+        forbiddenOutputHits: cases.reduce((total, entry) => total + entry.contextOn.forbiddenHits.length, 0),
+        missingContextPacks: cases.filter((entry) => !entry.contextPackRecorded)
+            .length,
+        missingProvenance: cases.filter((entry) => !entry.provenanceComplete)
+            .length,
+        optOutContamination: cases.filter((entry) => !entry.optOutStructurallyClean)
+            .length,
+    };
+    const failures = [];
+    if (summary.caseCount < thresholds.minimumCases) {
+        failures.push(`case count ${summary.caseCount} is below ${thresholds.minimumCases}`);
+    }
+    if (summary.contextOnWinRate < thresholds.minimumContextOnWinRate) {
+        failures.push(`context-on win rate ${summary.contextOnWinRate.toFixed(3)} is below ${thresholds.minimumContextOnWinRate.toFixed(3)}`);
+    }
+    if (summary.contextOnRequiredTermCoverage <
+        thresholds.minimumRequiredTermCoverage) {
+        failures.push(`required-term coverage ${summary.contextOnRequiredTermCoverage.toFixed(3)} is below ${thresholds.minimumRequiredTermCoverage.toFixed(3)}`);
+    }
+    if (summary.meanEditDistanceReduction <
+        thresholds.minimumMeanEditDistanceReduction) {
+        failures.push(`edit-distance reduction ${summary.meanEditDistanceReduction.toFixed(3)} is below ${thresholds.minimumMeanEditDistanceReduction.toFixed(3)}`);
+    }
+    if (summary.forbiddenOutputHits)
+        failures.push("forbidden output was emitted");
+    if (summary.missingContextPacks)
+        failures.push("context packs are missing");
+    if (summary.missingProvenance)
+        failures.push("provenance is incomplete");
+    if (summary.optOutContamination)
+        failures.push("context-off contains context evidence");
+    const report = {
+        schemaVersion: 1,
+        corpusId: input.corpusId,
+        runId: `cc-eval-${createHash("sha256")
+            .update(JSON.stringify({
+            corpusId: input.corpusId,
+            cases,
+            thresholds,
+        }))
+            .digest("hex")
+            .slice(0, 16)}`,
+        cases,
+        summary,
+        thresholds,
+        gates: { passed: failures.length === 0, failures },
+        manualBlindPreference: {
+            status: "pending",
+            scoredTrials: 0,
+            contextOnPreferenceRate: null,
+        },
+    };
+    return ContextAcceptanceReportSchema.parse(report);
+}
+export function assertContextAcceptanceGates(report) {
+    const parsed = ContextAcceptanceReportSchema.parse(report);
+    if (!parsed.gates.passed) {
+        throw new Error(`Creative context acceptance gates failed: ${parsed.gates.failures.join("; ")}`);
+    }
+}
+export function createBlindPreferencePacket(cases) {
+    const worksheet = [];
+    const answerKey = [];
+    for (const entry of [...cases].sort((a, b) => a.id.localeCompare(b.id))) {
+        const contextOnCandidate = parseInt(createHash("sha256").update(entry.id).digest("hex").slice(0, 2), 16) % 2
+            ? "A"
+            : "B";
+        worksheet.push({
+            caseId: entry.id,
+            prompt: entry.prompt,
+            candidateA: contextOnCandidate === "A"
+                ? entry.contextOn.output
+                : entry.contextOff.output,
+            candidateB: contextOnCandidate === "B"
+                ? entry.contextOn.output
+                : entry.contextOff.output,
+        });
+        answerKey.push({ caseId: entry.id, contextOnCandidate });
+    }
+    return { worksheet, answerKey };
+}
+export function scoreBlindPreferences(answerKey, scores) {
+    let scoredTrials = 0;
+    let contextOnPreferences = 0;
+    for (const entry of answerKey) {
+        const score = scores[entry.caseId];
+        if (!score)
+            continue;
+        scoredTrials++;
+        if (score === entry.contextOnCandidate)
+            contextOnPreferences++;
+    }
+    return {
+        scoredTrials,
+        contextOnPreferenceRate: scoredTrials
+            ? contextOnPreferences / scoredTrials
+            : 0,
+    };
+}
+function levenshtein(left, right) {
+    if (left === right)
+        return 0;
+    if (!left.length)
+        return right.length;
+    if (!right.length)
+        return left.length;
+    let previous = Array.from({ length: right.length + 1 }, (_, index) => index);
+    for (let leftIndex = 1; leftIndex <= left.length; leftIndex++) {
+        const current = [leftIndex];
+        for (let rightIndex = 1; rightIndex <= right.length; rightIndex++) {
+            current[rightIndex] = Math.min((current[rightIndex - 1] ?? 0) + 1, (previous[rightIndex] ?? 0) + 1, (previous[rightIndex - 1] ?? 0) +
+                (left[leftIndex - 1] === right[rightIndex - 1] ? 0 : 1));
+        }
+        previous = current;
+    }
+    return previous[right.length] ?? 0;
+}
+//# sourceMappingURL=acceptance.js.map

package/dist/eval/acceptance.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"acceptance.js","sourceRoot":"","sources":["../../src/eval/acceptance.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEzC,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AA2BxB,MAAM,kBAAkB,GAAgC;IACtD,YAAY,EAAE,CAAC;IACf,uBAAuB,EAAE,IAAI;IAC7B,2BAA2B,EAAE,CAAC;IAC9B,gCAAgC,EAAE,CAAC;CACpC,CAAC;AAEF,MAAM,eAAe,GAAG,CAAC,CAAC,MAAM,CAAC;IAC/B,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,gBAAgB,CAAC;IAC9C,oBAAoB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC9C,aAAa,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;IAClC,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE;IAC5C,eAAe,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;IACpC,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACpC,cAAc,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;CACpC,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,6BAA6B,GAAG,CAAC,CAAC,MAAM,CAAC;IACpD,aAAa,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IAC3B,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IAC3B,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,wBAAwB,CAAC;IACjD,KAAK,EAAE,CAAC,CAAC,KAAK,CACZ,CAAC,CAAC,MAAM,CAAC;QACP,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;QACrB,UAAU,EAAE,eAAe;QAC3B,SAAS,EAAE,eAAe;QAC1B,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,YAAY,EAAE,aAAa,EAAE,KAAK,CAAC,CAAC;QACpD,mBAAmB,EAAE,CAAC,CAAC,OAAO,EAAE;QAChC,kBAAkB,EAAE,CAAC,CAAC,OAAO,EAAE;QAC/B,uBAAuB,EAAE,CAAC,CAAC,OAAO,EAAE;KACrC,CAAC,CACH;IACD,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC;QAChB,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE;QACzC,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE;QAC7C,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE;QAC9C,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE;QACpC,gBAAgB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC1C,6BAA6B,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACvD,yBAAyB,EAAE,CAAC,CAAC,MAAM,EAAE;QACrC,mBAAmB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE;QACnD,mBAAmB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE;QACnD,iBAAiB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE;QACjD,mBAAmB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE;KACpD,CAAC;IACF,UAAU,EAAE,CAAC,CAAC,MAAM,CAAC;QACnB,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE;QACzC,uBAAuB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACjD,2BAA2B,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACrD,gCAAgC,EAAE,CAAC,CAAC,MAAM,EAAE;KAC7C,CAAC;IACF,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC;IACvE,qBAAqB,EAAE,CAAC,CAAC,MAAM,CAAC;QAC9B,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;QACvC,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE;QAC5C,uBAAuB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;KAC7D,CAAC;CACH,CAAC,CAAC;AAMH,SAAS,UAAU,CAAC,KAAa;IAC/B,OAAO,KAAK,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;AACtE,CAAC;AAED,SAAS,MAAM,CAAC,MAAyB;IACvC,OAAO;QACL,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;KAChE,CAAC,IAAI,EAAE,CAAC;AACX,CAAC;AAED,SAAS,SAAS,CAChB,QAAmC,EACnC,cAAqC;IAErC,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IAC3C,MAAM,QAAQ,GAAG,MAAM,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;IACtD,MAAM,SAAS,GAAG,MAAM,CAAC,cAAc,CAAC,cAAc,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CACtE,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAClC,CAAC;IACF,MAAM,OAAO,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC7E,OAAO;QACL,UAAU,EAAE,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;QACtE,oBAAoB,EAAE,QAAQ,CAAC,MAAM;YACnC,CAAC,CAAC,OAAO,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM;YAClC,CAAC,CAAC,CAAC;QACL,aAAa,EAAE,SAAS;QACxB,YAAY,EAAE,WAAW,CAAC,MAAM,EAAE,UAAU,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC;QACvE,eAAe,EAAE,MAAM,CAAC,QAAQ,CAAC,eAAe,CAAC;QACjD,aAAa,EAAE,QAAQ,CAAC,aAAa,EAAE,IAAI,EAAE,IAAI,IAAI;QACrD,cAAc,EAAE,MAAM,CAAC,QAAQ,CAAC,cAAc,IAAI,EAAE,CAAC;KACtD,CAAC;AACJ,CAAC;AAED,SAAS,MAAM,CACb,GAAiC,EACjC,EAAgC;IAEhC,IAAI,EAAE,CAAC,aAAa,CAAC,MAAM,KAAK,GAAG,CAAC,aAAa,CAAC,MAAM,EAAE,CAAC;QACzD,OAAO,EAAE,CAAC,aAAa,CAAC,MAAM,GAAG,GAAG,CAAC,aAAa,CAAC,MAAM;YACvD,CAAC,CAAC,YAAY;YACd,CAAC,CAAC,aAAa,CAAC;IACpB,CAAC;IACD,IAAI,EAAE,CAAC,oBAAoB,KAAK,GAAG,CAAC,oBAAoB,EAAE,CAAC;QACzD,OAAO,EAAE,CAAC,oBAAoB,GAAG,GAAG,CAAC,oBAAoB;YACvD,CAAC,CAAC,YAAY;YACd,CAAC,CAAC,aAAa,CAAC;IACpB,CAAC;IACD,IAAI,EAAE,CAAC,YAAY,KAAK,GAAG,CAAC,YAAY,EAAE,CAAC;QACzC,OAAO,EAAE,CAAC,YAAY,GAAG,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,aAAa,CAAC;IAC3E,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,8BAA8B,CAAC,KAI9C;IACC,MAAM,UAAU,GAAG,EAAE,GAAG,kBAAkB,EAAE,GAAG,KAAK,CAAC,UAAU,EAAE,CAAC;IAClE,MAAM,KAAK,GAAG,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC;SAC3B,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,aAAa,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;SACtD,GAAG,CAAC,CAAC,cAAc,EAAE,EAAE;QACtB,MAAM,UAAU,GAAG,SAAS,CAAC,cAAc,CAAC,UAAU,EAAE,cAAc,CAAC,CAAC;QACxE,MAAM,SAAS,GAAG,SAAS,CAAC,cAAc,CAAC,SAAS,EAAE,cAAc,CAAC,CAAC;QACtE,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,cAAc,CAAC,mBAAmB,CAAC,CAAC,CAAC;QACrE,MAAM,mBAAmB,GAAG,OAAO,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC;QAC7D,MAAM,kBAAkB,GAAG,CAAC,GAAG,QAAQ,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CACrD,SAAS,CAAC,cAAc,CAAC,QAAQ,CAAC,GAAG,CAAC,CACvC,CAAC;QACF,MAAM,uBAAuB,GAC3B,UAAU,CAAC,eAAe,CAAC,MAAM,KAAK,CAAC;YACvC,UAAU,CAAC,aAAa,KAAK,IAAI;YACjC,UAAU,CAAC,cAAc,CAAC,MAAM,KAAK,CAAC,CAAC;QACzC,OAAO;YACL,EAAE,EAAE,cAAc,CAAC,EAAE;YACrB,UAAU;YACV,SAAS;YACT,MAAM,EAAE,MAAM,CAAC,UAAU,EAAE,SAAS,CAAC;YACrC,mBAAmB;YACnB,kBAAkB;YAClB,uBAAuB;SACxB,CAAC;IACJ,CAAC,CAAC,CAAC;IACL,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC;IAC3B,MAAM,aAAa,GAAG,KAAK,CAAC,MAAM,CAChC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,KAAK,YAAY,CACzC,CAAC,MAAM,CAAC;IACT,MAAM,cAAc,GAAG,KAAK,CAAC,MAAM,CACjC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,KAAK,aAAa,CAC1C,CAAC,MAAM,CAAC;IACT,MAAM,IAAI,GAAG,KAAK,GAAG,aAAa,GAAG,cAAc,CAAC;IACpD,MAAM,OAAO,GAAG,CAAC,MAAyB,EAAE,EAAE,CAC5C,MAAM,CAAC,MAAM;QACX,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,GAAG,KAAK,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM;QACnE,CAAC,CAAC,CAAC,CAAC;IACR,MAAM,OAAO,GAAG;QACd,SAAS,EAAE,KAAK;QAChB,aAAa;QACb,cAAc;QACd,IAAI;QACJ,gBAAgB,EAAE,KAAK,CAAC,CAAC,CAAC,aAAa,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACnD,6BAA6B,EAAE,OAAO,CACpC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,oBAAoB,CAAC,CAC3D;QACD,yBAAyB,EAAE,OAAO,CAChC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;YAClB,MAAM,QAAQ,GAAG,KAAK,CAAC,UAAU,CAAC,YAAY,CAAC;YAC/C,OAAO,QAAQ,GAAG,CAAC;gBACjB,CAAC,CAAC,CAAC,QAAQ,GAAG,KAAK,CAAC,SAAS,CAAC,YAAY,CAAC,GAAG,QAAQ;gBACtD,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,YAAY,KAAK,CAAC;oBAClC,CAAC,CAAC,CAAC;oBACH,CAAC,CAAC,CAAC,CAAC,CAAC;QACX,CAAC,CAAC,CACH;QACD,mBAAmB,EAAE,KAAK,CAAC,MAAM,CAC/B,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,GAAG,KAAK,CAAC,SAAS,CAAC,aAAa,CAAC,MAAM,EAC9D,CAAC,CACF;QACD,mBAAmB,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,mBAAmB,CAAC;aACrE,MAAM;QACT,iBAAiB,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,kBAAkB,CAAC;aAClE,MAAM;QACT,mBAAmB,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,uBAAuB,CAAC;aACzE,MAAM;KACV,CAAC;IACF,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,IAAI,OAAO,CAAC,SAAS,GAAG,UAAU,CAAC,YAAY,EAAE,CAAC;QAChD,QAAQ,CAAC,IAAI,CACX,cAAc,OAAO,CAAC,SAAS,aAAa,UAAU,CAAC,YAAY,EAAE,CACtE,CAAC;IACJ,CAAC;IACD,IAAI,OAAO,CAAC,gBAAgB,GAAG,UAAU,CAAC,uBAAuB,EAAE,CAAC;QAClE,QAAQ,CAAC,IAAI,CACX,uBAAuB,OAAO,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,CAAC,aAAa,UAAU,CAAC,uBAAuB,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CACvH,CAAC;IACJ,CAAC;IACD,IACE,OAAO,CAAC,6BAA6B;QACrC,UAAU,CAAC,2BAA2B,EACtC,CAAC;QACD,QAAQ,CAAC,IAAI,CACX,0BAA0B,OAAO,CAAC,6BAA6B,CAAC,OAAO,CAAC,CAAC,CAAC,aAAa,UAAU,CAAC,2BAA2B,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAC3I,CAAC;IACJ,CAAC;IACD,IACE,OAAO,CAAC,yBAAyB;QACjC,UAAU,CAAC,gCAAgC,EAC3C,CAAC;QACD,QAAQ,CAAC,IAAI,CACX,2BAA2B,OAAO,CAAC,yBAAyB,CAAC,OAAO,CAAC,CAAC,CAAC,aAAa,UAAU,CAAC,gCAAgC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAC7I,CAAC;IACJ,CAAC;IACD,IAAI,OAAO,CAAC,mBAAmB;QAC7B,QAAQ,CAAC,IAAI,CAAC,8BAA8B,CAAC,CAAC;IAChD,IAAI,OAAO,CAAC,mBAAmB;QAAE,QAAQ,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;IAC5E,IAAI,OAAO,CAAC,iBAAiB;QAAE,QAAQ,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;IACzE,IAAI,OAAO,CAAC,mBAAmB;QAC7B,QAAQ,CAAC,IAAI,CAAC,uCAAuC,CAAC,CAAC;IACzD,MAAM,MAAM,GAA4B;QACtC,aAAa,EAAE,CAAC;QAChB,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACxB,KAAK,EAAE,WAAW,UAAU,CAAC,QAAQ,CAAC;aACnC,MAAM,CACL,IAAI,CAAC,SAAS,CAAC;YACb,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,KAAK;YACL,UAAU;SACX,CAAC,CACH;aACA,MAAM,CAAC,KAAK,CAAC;aACb,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE;QACjB,KAAK;QACL,OAAO;QACP,UAAU;QACV,KAAK,EAAE,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,QAAQ,EAAE;QAClD,qBAAqB,EAAE;YACrB,MAAM,EAAE,SAAS;YACjB,YAAY,EAAE,CAAC;YACf,uBAAuB,EAAE,IAAI;SAC9B;KACF,CAAC;IACF,OAAO,6BAA6B,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;AACrD,CAAC;AAED,MAAM,UAAU,4BAA4B,CAC1C,MAA+B;IAE/B,MAAM,MAAM,GAAG,6BAA6B,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IAC3D,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;QACzB,MAAM,IAAI,KAAK,CACb,6CAA6C,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAChF,CAAC;IACJ,CAAC;AACH,CAAC;AAcD,MAAM,UAAU,2BAA2B,CACzC,KAAuC;IAKvC,MAAM,SAAS,GAAkC,EAAE,CAAC;IACpD,MAAM,SAAS,GAA+B,EAAE,CAAC;IACjD,KAAK,MAAM,KAAK,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;QACxE,MAAM,kBAAkB,GACtB,QAAQ,CACN,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAC/D,EAAE,CACH,GAAG,CAAC;YACH,CAAC,CAAC,GAAG;YACL,CAAC,CAAC,GAAG,CAAC;QACV,SAAS,CAAC,IAAI,CAAC;YACb,MAAM,EAAE,KAAK,CAAC,EAAE;YAChB,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,UAAU,EACR,kBAAkB,KAAK,GAAG;gBACxB,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM;gBACxB,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM;YAC7B,UAAU,EACR,kBAAkB,KAAK,GAAG;gBACxB,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM;gBACxB,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM;SAC9B,CAAC,CAAC;QACH,SAAS,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,EAAE,kBAAkB,EAAE,CAAC,CAAC;IAC3D,CAAC;IACD,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC;AAClC,CAAC;AAED,MAAM,UAAU,qBAAqB,CACnC,SAA8C,EAC9C,MAAmD;IAEnD,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,oBAAoB,GAAG,CAAC,CAAC;IAC7B,KAAK,MAAM,KAAK,IAAI,SAAS,EAAE,CAAC;QAC9B,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QACnC,IAAI,CAAC,KAAK;YAAE,SAAS;QACrB,YAAY,EAAE,CAAC;QACf,IAAI,KAAK,KAAK,KAAK,CAAC,kBAAkB;YAAE,oBAAoB,EAAE,CAAC;IACjE,CAAC;IACD,OAAO;QACL,YAAY;QACZ,uBAAuB,EAAE,YAAY;YACnC,CAAC,CAAC,oBAAoB,GAAG,YAAY;YACrC,CAAC,CAAC,CAAC;KACN,CAAC;AACJ,CAAC;AAED,SAAS,WAAW,CAAC,IAAY,EAAE,KAAa;IAC9C,IAAI,IAAI,KAAK,KAAK;QAAE,OAAO,CAAC,CAAC;IAC7B,IAAI,CAAC,IAAI,CAAC,MAAM;QAAE,OAAO,KAAK,CAAC,MAAM,CAAC;IACtC,IAAI,CAAC,KAAK,CAAC,MAAM;QAAE,OAAO,IAAI,CAAC,MAAM,CAAC;IACtC,IAAI,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC;IAC7E,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,IAAI,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,CAAC;QAC9D,MAAM,OAAO,GAAG,CAAC,SAAS,CAAC,CAAC;QAC5B,KAAK,IAAI,UAAU,GAAG,CAAC,EAAE,UAAU,IAAI,KAAK,CAAC,MAAM,EAAE,UAAU,EAAE,EAAE,CAAC;YAClE,OAAO,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC,GAAG,CAC5B,CAAC,OAAO,CAAC,UAAU,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,EAClC,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,EAC/B,CAAC,QAAQ,CAAC,UAAU,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;gBAC7B,CAAC,IAAI,CAAC,SAAS,GAAG,CAAC,CAAC,KAAK,KAAK,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAC1D,CAAC;QACJ,CAAC;QACD,QAAQ,GAAG,OAAO,CAAC;IACrB,CAAC;IACD,OAAO,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;AACrC,CAAC"}

package/dist/eval/fixtures.d.ts

@@ -0,0 +1,16 @@
+import type { RetrievalEvalTask } from "./index.js";
+export interface CreativeContextGoldDocument {
+    key: string;
+    kind: "slide" | "figma-frame" | "notion-section" | "web-page" | "image";
+    title: string;
+    text: string;
+    imageBase64?: string;
+    owner: "personal" | "organization" | "other-organization";
+    status: "active" | "deprecated";
+    revisionOf?: string;
+}
+export declare const CREATIVE_CONTEXT_PURPLE_IMAGE_BASE64 = "iVBORw0KGgoAAAANSUhEUgAAAAgAAAAICAIAAABLbSncAAAAEUlEQVR4nGOI9n+JFTEMLQkAOpFkwUU6BmIAAAAASUVORK5CYII=";
+export declare const CREATIVE_CONTEXT_INK_IMAGE_BASE64 = "iVBORw0KGgoAAAANSUhEUgAAAAgAAAAICAIAAABLbSncAAAAEUlEQVR4nGPg5hbAihiGlgQAtbUJgThTKnUAAAAASUVORK5CYII=";
+export declare const CREATIVE_CONTEXT_GOLD_DOCUMENTS: readonly CreativeContextGoldDocument[];
+export declare const CREATIVE_CONTEXT_GOLD_TASKS: readonly RetrievalEvalTask[];
+//# sourceMappingURL=fixtures.d.ts.map

package/dist/eval/fixtures.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fixtures.d.ts","sourceRoot":"","sources":["../../src/eval/fixtures.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AAEpD,MAAM,WAAW,2BAA2B;IAC1C,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,OAAO,GAAG,aAAa,GAAG,gBAAgB,GAAG,UAAU,GAAG,OAAO,CAAC;IACxE,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,KAAK,EAAE,UAAU,GAAG,cAAc,GAAG,oBAAoB,CAAC;IAC1D,MAAM,EAAE,QAAQ,GAAG,YAAY,CAAC;IAChC,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,eAAO,MAAM,oCAAoC,yGACuD,CAAC;AACzG,eAAO,MAAM,iCAAiC,yGAC0D,CAAC;AAEzG,eAAO,MAAM,+BAA+B,EAAE,SAAS,2BAA2B,EAqEtE,CAAC;AAEb,eAAO,MAAM,2BAA2B,EAAE,SAAS,iBAAiB,EA8C1D,CAAC"}

package/dist/eval/fixtures.js

@@ -0,0 +1,119 @@
+export const CREATIVE_CONTEXT_PURPLE_IMAGE_BASE64 = "iVBORw0KGgoAAAANSUhEUgAAAAgAAAAICAIAAABLbSncAAAAEUlEQVR4nGOI9n+JFTEMLQkAOpFkwUU6BmIAAAAASUVORK5CYII=";
+export const CREATIVE_CONTEXT_INK_IMAGE_BASE64 = "iVBORw0KGgoAAAANSUhEUgAAAAgAAAAICAIAAABLbSncAAAAEUlEQVR4nGPg5hbAihiGlgQAtbUJgThTKnUAAAAASUVORK5CYII=";
+export const CREATIVE_CONTEXT_GOLD_DOCUMENTS = [
+    {
+        key: "slide:metrics-v2",
+        kind: "slide",
+        title: "Weekly metrics scorecard",
+        text: "Dark scorecard with four KPI tiles, a purple trend line, and a concise takeaway.",
+        owner: "organization",
+        status: "active",
+        revisionOf: "slide:metrics-v1",
+    },
+    {
+        key: "slide:metrics-v1",
+        kind: "slide",
+        title: "Weekly metrics scorecard old",
+        text: "Previous scorecard with four KPI tiles.",
+        owner: "organization",
+        status: "deprecated",
+    },
+    {
+        key: "figma:pricing-hero",
+        kind: "figma-frame",
+        title: "Pricing hero",
+        text: "1440 by 820 dark hero, Inter 56 heading, three pricing cards, primary trial button.",
+        imageBase64: CREATIVE_CONTEXT_INK_IMAGE_BASE64,
+        owner: "organization",
+        status: "active",
+    },
+    {
+        key: "notion:dense-guidelines",
+        kind: "notion-section",
+        title: "Dense launch system guidance",
+        text: "For launch narratives, lead with the user outcome, name the concrete workflow, support claims with one metric, keep headings under eight words, use direct active voice, avoid inflated superlatives, preserve the primary purple and ink palette, use compact scorecards for quantitative proof, and close with one explicit trial action. Product screenshots should center the working surface, annotations should identify the changed behavior, and supporting copy should remain subordinate to the artifact. For executive reviews, pair each KPI with a trend and a one-sentence interpretation. For campaign variants, retain the same hierarchy while adapting the evidence and call to action to the audience segment.",
+        owner: "organization",
+        status: "active",
+    },
+    {
+        key: "notion:voice",
+        kind: "notion-section",
+        title: "Launch writing principles",
+        text: "Direct, concise, optimistic product writing. Prefer concrete verbs and short sentences.",
+        owner: "organization",
+        status: "active",
+    },
+    {
+        key: "web:palette",
+        kind: "web-page",
+        title: "Brand home page",
+        text: "Primary #5B4FE9, ink #0B0B10, Inter headings, twelve-pixel radii.",
+        owner: "organization",
+        status: "active",
+    },
+    {
+        key: "image:campaign",
+        kind: "image",
+        title: "Launch campaign hero",
+        text: "Purple gradient product hero with a centered device render and compact white headline.",
+        imageBase64: CREATIVE_CONTEXT_PURPLE_IMAGE_BASE64,
+        owner: "organization",
+        status: "active",
+    },
+    {
+        key: "private:other-org",
+        kind: "slide",
+        title: "Confidential acquisition plan",
+        text: "Restricted content owned by another organization.",
+        owner: "other-organization",
+        status: "active",
+    },
+];
+export const CREATIVE_CONTEXT_GOLD_TASKS = [
+    {
+        id: "exact-metrics-layout",
+        query: { text: "weekly metrics scorecard with four KPI tiles" },
+        relevantKeys: ["slide:metrics-v2"],
+        forbiddenKeys: ["private:other-org"],
+    },
+    {
+        id: "visual-pricing-match",
+        query: { text: "dark pricing hero with three cards" },
+        relevantKeys: ["figma:pricing-hero"],
+        forbiddenKeys: ["private:other-org"],
+    },
+    {
+        id: "voice-match",
+        query: { text: "direct concise optimistic launch copy" },
+        relevantKeys: ["notion:voice"],
+        forbiddenKeys: ["private:other-org"],
+    },
+    {
+        id: "text-to-image-campaign-match",
+        query: { text: "purple campaign hero with centered device" },
+        relevantKeys: ["image:campaign"],
+        forbiddenKeys: ["private:other-org"],
+    },
+    {
+        id: "image-to-image-campaign-match",
+        query: {
+            images: [
+                {
+                    mimeType: "image/png",
+                    base64: CREATIVE_CONTEXT_PURPLE_IMAGE_BASE64,
+                },
+            ],
+        },
+        relevantKeys: ["image:campaign"],
+        forbiddenKeys: ["private:other-org"],
+    },
+    {
+        id: "dense-guidance-match",
+        query: {
+            text: "executive launch KPI trend interpretation active voice purple ink compact scorecard",
+        },
+        relevantKeys: ["notion:dense-guidelines"],
+        forbiddenKeys: ["private:other-org"],
+    },
+];
+//# sourceMappingURL=fixtures.js.map

package/dist/eval/fixtures.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fixtures.js","sourceRoot":"","sources":["../../src/eval/fixtures.ts"],"names":[],"mappings":"AAaA,MAAM,CAAC,MAAM,oCAAoC,GAC/C,sGAAsG,CAAC;AACzG,MAAM,CAAC,MAAM,iCAAiC,GAC5C,sGAAsG,CAAC;AAEzG,MAAM,CAAC,MAAM,+BAA+B,GAC1C;IACE;QACE,GAAG,EAAE,kBAAkB;QACvB,IAAI,EAAE,OAAO;QACb,KAAK,EAAE,0BAA0B;QACjC,IAAI,EAAE,kFAAkF;QACxF,KAAK,EAAE,cAAc;QACrB,MAAM,EAAE,QAAQ;QAChB,UAAU,EAAE,kBAAkB;KAC/B;IACD;QACE,GAAG,EAAE,kBAAkB;QACvB,IAAI,EAAE,OAAO;QACb,KAAK,EAAE,8BAA8B;QACrC,IAAI,EAAE,yCAAyC;QAC/C,KAAK,EAAE,cAAc;QACrB,MAAM,EAAE,YAAY;KACrB;IACD;QACE,GAAG,EAAE,oBAAoB;QACzB,IAAI,EAAE,aAAa;QACnB,KAAK,EAAE,cAAc;QACrB,IAAI,EAAE,qFAAqF;QAC3F,WAAW,EAAE,iCAAiC;QAC9C,KAAK,EAAE,cAAc;QACrB,MAAM,EAAE,QAAQ;KACjB;IACD;QACE,GAAG,EAAE,yBAAyB;QAC9B,IAAI,EAAE,gBAAgB;QACtB,KAAK,EAAE,8BAA8B;QACrC,IAAI,EAAE,msBAAmsB;QACzsB,KAAK,EAAE,cAAc;QACrB,MAAM,EAAE,QAAQ;KACjB;IACD;QACE,GAAG,EAAE,cAAc;QACnB,IAAI,EAAE,gBAAgB;QACtB,KAAK,EAAE,2BAA2B;QAClC,IAAI,EAAE,yFAAyF;QAC/F,KAAK,EAAE,cAAc;QACrB,MAAM,EAAE,QAAQ;KACjB;IACD;QACE,GAAG,EAAE,aAAa;QAClB,IAAI,EAAE,UAAU;QAChB,KAAK,EAAE,iBAAiB;QACxB,IAAI,EAAE,mEAAmE;QACzE,KAAK,EAAE,cAAc;QACrB,MAAM,EAAE,QAAQ;KACjB;IACD;QACE,GAAG,EAAE,gBAAgB;QACrB,IAAI,EAAE,OAAO;QACb,KAAK,EAAE,sBAAsB;QAC7B,IAAI,EAAE,wFAAwF;QAC9F,WAAW,EAAE,oCAAoC;QACjD,KAAK,EAAE,cAAc;QACrB,MAAM,EAAE,QAAQ;KACjB;IACD;QACE,GAAG,EAAE,mBAAmB;QACxB,IAAI,EAAE,OAAO;QACb,KAAK,EAAE,+BAA+B;QACtC,IAAI,EAAE,mDAAmD;QACzD,KAAK,EAAE,oBAAoB;QAC3B,MAAM,EAAE,QAAQ;KACjB;CACO,CAAC;AAEb,MAAM,CAAC,MAAM,2BAA2B,GAAiC;IACvE;QACE,EAAE,EAAE,sBAAsB;QAC1B,KAAK,EAAE,EAAE,IAAI,EAAE,8CAA8C,EAAE;QAC/D,YAAY,EAAE,CAAC,kBAAkB,CAAC;QAClC,aAAa,EAAE,CAAC,mBAAmB,CAAC;KACrC;IACD;QACE,EAAE,EAAE,sBAAsB;QAC1B,KAAK,EAAE,EAAE,IAAI,EAAE,oCAAoC,EAAE;QACrD,YAAY,EAAE,CAAC,oBAAoB,CAAC;QACpC,aAAa,EAAE,CAAC,mBAAmB,CAAC;KACrC;IACD;QACE,EAAE,EAAE,aAAa;QACjB,KAAK,EAAE,EAAE,IAAI,EAAE,uCAAuC,EAAE;QACxD,YAAY,EAAE,CAAC,cAAc,CAAC;QAC9B,aAAa,EAAE,CAAC,mBAAmB,CAAC;KACrC;IACD;QACE,EAAE,EAAE,8BAA8B;QAClC,KAAK,EAAE,EAAE,IAAI,EAAE,2CAA2C,EAAE;QAC5D,YAAY,EAAE,CAAC,gBAAgB,CAAC;QAChC,aAAa,EAAE,CAAC,mBAAmB,CAAC;KACrC;IACD;QACE,EAAE,EAAE,+BAA+B;QACnC,KAAK,EAAE;YACL,MAAM,EAAE;gBACN;oBACE,QAAQ,EAAE,WAAW;oBACrB,MAAM,EAAE,oCAAoC;iBAC7C;aACF;SACF;QACD,YAAY,EAAE,CAAC,gBAAgB,CAAC;QAChC,aAAa,EAAE,CAAC,mBAAmB,CAAC;KACrC;IACD;QACE,EAAE,EAAE,sBAAsB;QAC1B,KAAK,EAAE;YACL,IAAI,EAAE,qFAAqF;SAC5F;QACD,YAAY,EAAE,CAAC,yBAAyB,CAAC;QACzC,aAAa,EAAE,CAAC,mBAAmB,CAAC;KACrC;CACO,CAAC"}

package/dist/eval/index.d.ts

@@ -0,0 +1,63 @@
+import type { EmbeddingFamily, MultimodalEmbeddingInput } from "../embeddings/types.js";
+export { assertContextAcceptanceGates, ContextAcceptanceReportSchema, createBlindPreferencePacket, runContextAcceptanceEvaluation, scoreBlindPreferences, type BlindPreferenceAnswerKey, type BlindPreferenceWorksheetRow, type ContextAcceptanceArtifact, type ContextAcceptanceCase, type ContextAcceptanceReport, type ContextAcceptanceThresholds, } from "./acceptance.js";
+export { CREATIVE_CONTEXT_ACCEPTANCE_CASES } from "./acceptance-fixtures.js";
+export { CREATIVE_CONTEXT_INK_IMAGE_BASE64, CREATIVE_CONTEXT_GOLD_DOCUMENTS, CREATIVE_CONTEXT_GOLD_TASKS, CREATIVE_CONTEXT_PURPLE_IMAGE_BASE64, type CreativeContextGoldDocument, } from "./fixtures.js";
+export interface RetrievalEvalTask {
+    id: string;
+    query: MultimodalEmbeddingInput;
+    relevantKeys: readonly string[];
+    forbiddenKeys?: readonly string[];
+}
+export interface RetrievalEvalMetrics {
+    taskCount: number;
+    top5Recall: number;
+    meanReciprocalRank: number;
+    permissionLeaks: number;
+}
+export interface ContextQualityTrial {
+    id: string;
+    preferred: "context-on" | "context-off" | "tie";
+    baselineEditDistance: number;
+    contextEditDistance: number;
+}
+export interface ContextQualityMetrics {
+    trialCount: number;
+    contextPreferenceRate: number;
+    meanEditDistanceReduction: number;
+}
+export interface CreativeContextCorrectnessEvidence {
+    permissionLeaks: number;
+    generationsMissingPacks: number;
+    generationsMissingProvenance: number;
+    importsResumable: boolean;
+    importsIdempotent: boolean;
+    importsDeterministic: boolean;
+    revocationRemovesAccess: boolean;
+    lexicalLaneRebuilds: boolean;
+    vectorLaneRebuilds: boolean;
+    connectorsPassing: readonly string[];
+    consumersUsingReuseLadder: readonly string[];
+    nativeCodeRetrievalPassing: readonly string[];
+    nativeCloneFidelityPassing: readonly string[];
+    nativeCloneVersionsPinned: boolean;
+    supportedNativeElementsEditable: boolean;
+    runtimeExcludesFullResolutionReferenceRenders: boolean;
+    contextOptOutExcludesAllContext: boolean;
+    contextOnBeatsContextOff: boolean;
+}
+export declare function evaluateRankings(tasks: readonly RetrievalEvalTask[], rankings: Readonly<Record<string, readonly string[]>>): RetrievalEvalMetrics;
+export declare function evaluateContextQuality(trials: readonly ContextQualityTrial[]): ContextQualityMetrics;
+export declare function creativeContextCompletionFailures(evidence: CreativeContextCorrectnessEvidence): string[];
+export declare function assertCreativeContextCompletionGates(evidence: CreativeContextCorrectnessEvidence): void;
+export declare function bakeOffEmbeddingFamilies(input: {
+    families: readonly EmbeddingFamily[];
+    documents: Readonly<Record<string, MultimodalEmbeddingInput>>;
+    tasks: readonly RetrievalEvalTask[];
+    allowedDocumentKeys?: readonly string[];
+    topK?: number;
+}): Promise<{
+    family: EmbeddingFamily;
+    metrics: RetrievalEvalMetrics;
+}[]>;
+export declare function embeddingBakeoffPasses(metrics: RetrievalEvalMetrics): boolean;
+//# sourceMappingURL=index.d.ts.map

package/dist/eval/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/eval/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,eAAe,EACf,wBAAwB,EACzB,MAAM,wBAAwB,CAAC;AAEhC,OAAO,EACL,4BAA4B,EAC5B,6BAA6B,EAC7B,2BAA2B,EAC3B,8BAA8B,EAC9B,qBAAqB,EACrB,KAAK,wBAAwB,EAC7B,KAAK,2BAA2B,EAChC,KAAK,yBAAyB,EAC9B,KAAK,qBAAqB,EAC1B,KAAK,uBAAuB,EAC5B,KAAK,2BAA2B,GACjC,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,iCAAiC,EAAE,MAAM,0BAA0B,CAAC;AAE7E,OAAO,EACL,iCAAiC,EACjC,+BAA+B,EAC/B,2BAA2B,EAC3B,oCAAoC,EACpC,KAAK,2BAA2B,GACjC,MAAM,eAAe,CAAC;AAEvB,MAAM,WAAW,iBAAiB;IAChC,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,wBAAwB,CAAC;IAChC,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;IAChC,aAAa,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;CACnC;AAED,MAAM,WAAW,oBAAoB;IACnC,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,eAAe,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,mBAAmB;IAClC,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,YAAY,GAAG,aAAa,GAAG,KAAK,CAAC;IAChD,oBAAoB,EAAE,MAAM,CAAC;IAC7B,mBAAmB,EAAE,MAAM,CAAC;CAC7B;AAED,MAAM,WAAW,qBAAqB;IACpC,UAAU,EAAE,MAAM,CAAC;IACnB,qBAAqB,EAAE,MAAM,CAAC;IAC9B,yBAAyB,EAAE,MAAM,CAAC;CACnC;AAED,MAAM,WAAW,kCAAkC;IACjD,eAAe,EAAE,MAAM,CAAC;IACxB,uBAAuB,EAAE,MAAM,CAAC;IAChC,4BAA4B,EAAE,MAAM,CAAC;IACrC,gBAAgB,EAAE,OAAO,CAAC;IAC1B,iBAAiB,EAAE,OAAO,CAAC;IAC3B,oBAAoB,EAAE,OAAO,CAAC;IAC9B,uBAAuB,EAAE,OAAO,CAAC;IACjC,mBAAmB,EAAE,OAAO,CAAC;IAC7B,kBAAkB,EAAE,OAAO,CAAC;IAC5B,iBAAiB,EAAE,SAAS,MAAM,EAAE,CAAC;IACrC,yBAAyB,EAAE,SAAS,MAAM,EAAE,CAAC;IAC7C,0BAA0B,EAAE,SAAS,MAAM,EAAE,CAAC;IAC9C,0BAA0B,EAAE,SAAS,MAAM,EAAE,CAAC;IAC9C,yBAAyB,EAAE,OAAO,CAAC;IACnC,+BAA+B,EAAE,OAAO,CAAC;IACzC,6CAA6C,EAAE,OAAO,CAAC;IACvD,+BAA+B,EAAE,OAAO,CAAC;IACzC,wBAAwB,EAAE,OAAO,CAAC;CACnC;AAYD,wBAAgB,gBAAgB,CAC9B,KAAK,EAAE,SAAS,iBAAiB,EAAE,EACnC,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC,CAAC,GACpD,oBAAoB,CA2BtB;AAED,wBAAgB,sBAAsB,CACpC,MAAM,EAAE,SAAS,mBAAmB,EAAE,GACrC,qBAAqB,CAwBvB;AAED,wBAAgB,iCAAiC,CAC/C,QAAQ,EAAE,kCAAkC,GAC3C,MAAM,EAAE,CA6DV;AAED,wBAAgB,oCAAoC,CAClD,QAAQ,EAAE,kCAAkC,GAC3C,IAAI,CAON;AAkBD,wBAAsB,wBAAwB,CAAC,KAAK,EAAE;IACpD,QAAQ,EAAE,SAAS,eAAe,EAAE,CAAC;IACrC,SAAS,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,wBAAwB,CAAC,CAAC,CAAC;IAC9D,KAAK,EAAE,SAAS,iBAAiB,EAAE,CAAC;IACpC,mBAAmB,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACxC,IAAI,CAAC,EAAE,MAAM,CAAC;CACf,GAAG,OAAO,CAAC;IAAE,MAAM,EAAE,eAAe,CAAC;IAAC,OAAO,EAAE,oBAAoB,CAAA;CAAE,EAAE,CAAC,CAqCxE;AAED,wBAAgB,sBAAsB,CAAC,OAAO,EAAE,oBAAoB,GAAG,OAAO,CAO7E"}