@tenphi/akno-core 0.1.0

package/dist/bench/mixed-retrieval.js

@@ -0,0 +1,355 @@
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { performance } from 'node:perf_hooks';
+import { open } from "../open.js";
+const K = 3;
+const PAGE_K = 2;
+const FIXTURES = {
+    'people/ada-marlow.md': `# Ada Marlow
+
+The northern lantern archive uses the albatross key for cataloguing.
+The albatross conduit starts here. Continue through [[organisations/vulpine-mutual]].
+`,
+    'organisations/vulpine-mutual.md': `# Vulpine Mutual
+
+The southern lantern archive lists Vulpine Mutual under the foxglove index.
+The conduit continues to [[products/zephyr-qx-100]].
+`,
+    'products/zephyr-qx-100.md': `# Zephyr QX-100
+
+Calibration notes for the Zephyr QX-100 use invented benchmark markers.
+`,
+    'records/vulpine-policy.md': `# Vulpine Mutual policy
+
+The attached record is the canonical coverage evidence.
+
+![[vulpine-policy.txt]]
+`,
+    'records/vulpine-policy.txt': 'Silverpine coverage marker 1111 confirms the invented Vulpine Mutual policy record.\n',
+    'inbox/zephyr-service.txt': 'Amberlark service code belongs to the invented Zephyr QX-100 maintenance record.\n',
+    'inbox/blackwater-route.txt': 'Copperfin route marker points to the invented archive at Blackwater Bay.\n',
+    'inbox/vulpine-renewal.txt': 'Willowstamp renewal marker belongs to the invented Vulpine Mutual notice.\n',
+    'inbox/northern-lantern-source.txt': 'The northern lantern archive source uses the invented kestrel register.\n',
+    'inbox/northern-lantern-catalogue.txt': 'The northern lantern archive catalogue uses the invented osprey register.\n',
+    'inbox/southern-lantern-source.txt': 'The southern lantern archive source uses the invented heron register.\n',
+    'inbox/southern-lantern-catalogue.txt': 'The southern lantern archive catalogue uses the invented tern register.\n',
+};
+const ORPHAN_CASES = [
+    { query: 'amberlark service code', path: 'inbox/zephyr-service.txt' },
+    { query: 'copperfin route marker', path: 'inbox/blackwater-route.txt' },
+    { query: 'willowstamp renewal marker', path: 'inbox/vulpine-renewal.txt' },
+];
+const PAGE_CASES = [
+    { query: 'northern lantern archive', slug: 'people/ada-marlow' },
+    { query: 'southern lantern archive', slug: 'organisations/vulpine-mutual' },
+];
+const OWNED_CASE = {
+    query: 'silverpine',
+    slug: 'records/vulpine-policy',
+    path: 'records/vulpine-policy.txt',
+};
+const GRAPH_CASE = {
+    query: 'albatross conduit destination',
+    slug: 'products/zephyr-qx-100',
+};
+/**
+ * A fixed, invented corpus measures retrieval quality independently of the user's
+ * knowledge base. The model stack is intentionally absent: optional capabilities
+ * must report degradation while lexical mixed recall stays correct.
+ */
+export async function runMixedRetrievalBench(options = {}) {
+    const iterations = normalizedIterations(options.iterations ?? 12);
+    const root = fs.mkdtempSync(path.join(os.tmpdir(), 'akno-mixed-bench-kb-'));
+    const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), 'akno-mixed-bench-state-'));
+    const corpus = {
+        pages: Object.keys(FIXTURES).filter((file) => file.endsWith('.md')).length,
+        ownedDocuments: 1,
+        orphanDocuments: 7,
+        queries: ORPHAN_CASES.length + PAGE_CASES.length + 2,
+        graphQueries: 1,
+        orphanK: K,
+        pageK: PAGE_K,
+    };
+    let mem = null;
+    try {
+        writeFixtures(root);
+        mem = await open({
+            aknoPath: root,
+            stateDir,
+            isolated: true,
+            overrides: {
+                akno_path: root,
+                state_dir: stateDir,
+                providers: {},
+                models: {
+                    embedding: { id: null, dimensions: 8 },
+                    reranker: { id: null, enabled: false },
+                    derive: { id: null },
+                    expansion: { id: null },
+                    vision: { id: null, enabled: false },
+                },
+            },
+        });
+        await mem.index({});
+        const orphanRuns = [];
+        let orphanHits = 0;
+        let honestDegradedHits = 0;
+        for (const benchCase of ORPHAN_CASES) {
+            const result = await mem.recall({
+                query: benchCase.query,
+                mode: 'lookup',
+                limit: K,
+                budget: 1200,
+            });
+            orphanRuns.push(result);
+            const found = result.results.some((candidate) => candidate.type === 'document' && candidate.path === benchCase.path);
+            if (found)
+                orphanHits++;
+            if (found &&
+                result.status === 'degraded' &&
+                result.degraded?.includes('no_embedding_model') &&
+                result.degraded.includes('no_expansion_model')) {
+                honestDegradedHits++;
+            }
+        }
+        const owned = await mem.recall({
+            query: OWNED_CASE.query,
+            mode: 'lookup',
+            limit: K,
+            budget: 1200,
+        });
+        const duplicate = duplicateRate(owned, OWNED_CASE.slug, OWNED_CASE.path);
+        let pageOnlyHits = 0;
+        let mixedPageHits = 0;
+        let mixedDocumentCases = 0;
+        let graphAnchorHits = 0;
+        for (const benchCase of PAGE_CASES) {
+            const pageOnly = await mem.recall({
+                query: benchCase.query,
+                mode: 'lookup',
+                limit: PAGE_K,
+                budget: 1200,
+                graph: false,
+                filter: { source: 'page' },
+            });
+            const graphAssisted = await mem.recall({
+                query: benchCase.query,
+                mode: 'lookup',
+                limit: PAGE_K,
+                budget: 1200,
+                graph: true,
+                filter: { source: 'page' },
+            });
+            const mixed = await mem.recall({
+                query: benchCase.query,
+                mode: 'lookup',
+                limit: PAGE_K,
+                budget: 1200,
+            });
+            if (hasPage(pageOnly, benchCase.slug))
+                pageOnlyHits++;
+            if (graphAssisted.results[0]?.type === 'page' && graphAssisted.results[0].slug === benchCase.slug) {
+                graphAnchorHits++;
+            }
+            if (hasPage(mixed, benchCase.slug))
+                mixedPageHits++;
+            if (mixed.results.some((candidate) => candidate.type === 'document'))
+                mixedDocumentCases++;
+        }
+        const pageOnlyRecall = pageOnlyHits / PAGE_CASES.length;
+        const mixedPageRecall = mixedPageHits / PAGE_CASES.length;
+        const allCasesMixed = mixedDocumentCases === PAGE_CASES.length;
+        const graphBaseline = await mem.recall({
+            query: GRAPH_CASE.query,
+            mode: 'lookup',
+            expand: false,
+            graph: false,
+            limit: 5,
+            budget: 1200,
+        });
+        const graphAssisted = await mem.recall({
+            query: GRAPH_CASE.query,
+            mode: 'lookup',
+            expand: false,
+            graph: true,
+            limit: 5,
+            budget: 1200,
+        });
+        const graphTarget = graphAssisted.results.find((candidate) => candidate.type === 'page' && candidate.slug === GRAPH_CASE.slug);
+        const graphOnlyDiscovery = !hasPage(graphBaseline, GRAPH_CASE.slug) && Boolean(graphTarget);
+        const completeGraphProvenance = graphTarget?.type === 'page' &&
+            graphTarget.matched_by?.includes('graph') === true &&
+            graphTarget.graph_paths?.some((entry) => entry.hops === 2 &&
+                entry.nodes.length === 3 &&
+                entry.relations.length === 2 &&
+                entry.evidence.length === 2 &&
+                entry.evidence.every((locator) => Boolean(locator.slug ?? locator.document))) === true;
+        const latency = await mixedLatency(mem, iterations);
+        const results = [
+            metric({
+                name: `orphan recall@${K}`,
+                value: orphanHits / ORPHAN_CASES.length,
+                target: 1,
+                comparison: 'at_least',
+                unit: 'ratio',
+                detail: `${orphanHits}/${ORPHAN_CASES.length} orphan queries returned the expected document`,
+            }),
+            metric({
+                name: 'duplicate document result rate',
+                value: duplicate.rate,
+                target: 0,
+                comparison: 'at_most',
+                unit: 'ratio',
+                detail: duplicate.detail,
+            }),
+            metric({
+                name: `page recall@${PAGE_K}, page-only baseline`,
+                value: pageOnlyRecall,
+                target: 1,
+                comparison: 'at_least',
+                unit: 'ratio',
+                detail: `${pageOnlyHits}/${PAGE_CASES.length} expected pages returned`,
+            }),
+            metric({
+                name: 'page recall change after mixed assembly',
+                value: allCasesMixed ? mixedPageRecall - pageOnlyRecall : -1,
+                target: 0,
+                comparison: 'at_least',
+                unit: 'ratio',
+                detail: `page-only ${percent(pageOnlyRecall)}, mixed ${percent(mixedPageRecall)}; ` +
+                    `${mixedDocumentCases}/${PAGE_CASES.length} cases also returned document evidence`,
+            }),
+            metric({
+                name: 'graph multi-hop discovery@5',
+                value: graphOnlyDiscovery ? 1 : 0,
+                target: 1,
+                comparison: 'at_least',
+                unit: 'ratio',
+                detail: graphOnlyDiscovery
+                    ? 'the two-hop target appeared only with graph assistance'
+                    : 'the target was already lexical or graph assistance did not return it',
+            }),
+            metric({
+                name: 'graph path provenance completeness',
+                value: completeGraphProvenance ? 1 : 0,
+                target: 1,
+                comparison: 'at_least',
+                unit: 'ratio',
+                detail: completeGraphProvenance
+                    ? 'the returned target carried all nodes, relations, and evidence locators'
+                    : 'the returned target lacked a complete bounded path explanation',
+            }),
+            metric({
+                name: 'graph direct-query top-1 preservation',
+                value: graphAnchorHits / PAGE_CASES.length,
+                target: 1,
+                comparison: 'at_least',
+                unit: 'ratio',
+                detail: `${graphAnchorHits}/${PAGE_CASES.length} direct page queries kept their expected top result`,
+            }),
+            metric({
+                name: 'lexical recall with model degradation',
+                value: honestDegradedHits / ORPHAN_CASES.length,
+                target: 1,
+                comparison: 'at_least',
+                unit: 'ratio',
+                detail: `${honestDegradedHits}/${ORPHAN_CASES.length} hits remained visible with typed model degradation`,
+            }),
+            metric({
+                name: 'mixed assembly + budget fit, p50',
+                value: latency,
+                target: options.latencyBudgetMs ?? 20,
+                comparison: 'at_most',
+                unit: 'milliseconds',
+                detail: `${iterations} samples after one warm-up`,
+            }),
+        ];
+        return { results, passed: results.every((result) => result.passed), corpus };
+    }
+    catch (error) {
+        const results = [
+            metric({
+                name: 'mixed retrieval benchmark execution',
+                value: 1,
+                target: 0,
+                comparison: 'at_most',
+                unit: 'count',
+                detail: error instanceof Error ? error.message : String(error),
+            }),
+        ];
+        return { results, passed: false, corpus };
+    }
+    finally {
+        if (mem)
+            await mem.close().catch(() => undefined);
+        fs.rmSync(root, { recursive: true, force: true });
+        fs.rmSync(stateDir, { recursive: true, force: true });
+    }
+}
+function writeFixtures(root) {
+    for (const [relPath, content] of Object.entries(FIXTURES)) {
+        const target = path.join(root, relPath);
+        fs.mkdirSync(path.dirname(target), { recursive: true });
+        fs.writeFileSync(target, content, 'utf8');
+    }
+}
+function duplicateRate(result, expectedSlug, expectedPath) {
+    const ownedPaths = new Set(result.results.flatMap((candidate) => candidate.type === 'page'
+        ? (candidate.documents ?? []).flatMap((document) => (document.rel_path ? [document.rel_path] : []))
+        : []));
+    const standalonePaths = new Set(result.results.flatMap((candidate) => (candidate.type === 'document' ? [candidate.path] : [])));
+    const expectedPage = result.results.some((candidate) => candidate.type === 'page' &&
+        candidate.slug === expectedSlug &&
+        candidate.documents?.some((document) => document.rel_path === expectedPath));
+    const duplicates = [...standalonePaths].filter((documentPath) => ownedPaths.has(documentPath));
+    const identities = new Set([...ownedPaths, ...standalonePaths]);
+    const rate = expectedPage ? duplicates.length / Math.max(1, identities.size) : 1;
+    return {
+        rate,
+        detail: expectedPage
+            ? `${duplicates.length}/${identities.size} document identities appeared both owned and standalone`
+            : 'the expected owned-document page was not returned',
+    };
+}
+function hasPage(result, expectedSlug) {
+    return result.results.some((candidate) => candidate.type === 'page' && candidate.slug === expectedSlug);
+}
+async function mixedLatency(mem, iterations) {
+    const count = normalizedIterations(iterations);
+    const run = () => mem.recall({
+        query: PAGE_CASES[0].query,
+        mode: 'lookup',
+        expand: false,
+        limit: PAGE_K,
+        budget: 400,
+    });
+    const warmup = await run();
+    if (!warmup.results.some((candidate) => candidate.type === 'page') ||
+        !warmup.results.some((candidate) => candidate.type === 'document') ||
+        warmup.budget_used > 400) {
+        throw new Error('the latency case did not assemble both result types inside its shared budget');
+    }
+    const samples = [];
+    for (let index = 0; index < count; index++) {
+        const started = performance.now();
+        await run();
+        samples.push(performance.now() - started);
+    }
+    samples.sort((left, right) => left - right);
+    return round(samples[Math.min(samples.length - 1, Math.floor(samples.length * 0.5))]);
+}
+function normalizedIterations(value) {
+    return Number.isFinite(value) && value > 0 ? Math.floor(value) : 12;
+}
+function metric(input) {
+    const passed = input.comparison === 'at_least' ? input.value >= input.target : input.value <= input.target;
+    return { ...input, value: round(input.value), passed };
+}
+function percent(value) {
+    return `${Math.round(value * 100)}%`;
+}
+function round(value) {
+    return Math.round(value * 100) / 100;
+}
+//# sourceMappingURL=mixed-retrieval.js.map

package/dist/bench/mixed-retrieval.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"mixed-retrieval.js","sourceRoot":"","sources":["../../src/bench/mixed-retrieval.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAE9C,OAAO,EAAE,IAAI,EAAa,MAAM,YAAY,CAAC;AAkC7C,MAAM,CAAC,GAAG,CAAC,CAAC;AACZ,MAAM,MAAM,GAAG,CAAC,CAAC;AACjB,MAAM,QAAQ,GAA2B;IACvC,sBAAsB,EAAE;;;;CAIzB;IACC,iCAAiC,EAAE;;;;CAIpC;IACC,2BAA2B,EAAE;;;CAG9B;IACC,2BAA2B,EAAE;;;;;CAK9B;IACC,4BAA4B,EAC1B,uFAAuF;IACzF,0BAA0B,EACxB,oFAAoF;IACtF,4BAA4B,EAAE,4EAA4E;IAC1G,2BAA2B,EAAE,6EAA6E;IAC1G,mCAAmC,EACjC,2EAA2E;IAC7E,sCAAsC,EACpC,6EAA6E;IAC/E,mCAAmC,EACjC,yEAAyE;IAC3E,sCAAsC,EACpC,2EAA2E;CAC9E,CAAC;AAEF,MAAM,YAAY,GAAG;IACnB,EAAE,KAAK,EAAE,wBAAwB,EAAE,IAAI,EAAE,0BAA0B,EAAE;IACrE,EAAE,KAAK,EAAE,wBAAwB,EAAE,IAAI,EAAE,4BAA4B,EAAE;IACvE,EAAE,KAAK,EAAE,4BAA4B,EAAE,IAAI,EAAE,2BAA2B,EAAE;CAC3E,CAAC;AACF,MAAM,UAAU,GAAG;IACjB,EAAE,KAAK,EAAE,0BAA0B,EAAE,IAAI,EAAE,mBAAmB,EAAE;IAChE,EAAE,KAAK,EAAE,0BAA0B,EAAE,IAAI,EAAE,8BAA8B,EAAE;CAC5E,CAAC;AACF,MAAM,UAAU,GAAG;IACjB,KAAK,EAAE,YAAY;IACnB,IAAI,EAAE,wBAAwB;IAC9B,IAAI,EAAE,4BAA4B;CACnC,CAAC;AACF,MAAM,UAAU,GAAG;IACjB,KAAK,EAAE,+BAA+B;IACtC,IAAI,EAAE,wBAAwB;CAC/B,CAAC;AAEF;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAC1C,UAAsC,EAAE;IAExC,MAAM,UAAU,GAAG,oBAAoB,CAAC,OAAO,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC;IAClE,MAAM,IAAI,GAAG,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,sBAAsB,CAAC,CAAC,CAAC;IAC5E,MAAM,QAAQ,GAAG,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,yBAAyB,CAAC,CAAC,CAAC;IACnF,MAAM,MAAM,GAAG;QACb,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM;QAC1E,cAAc,EAAE,CAAC;QACjB,eAAe,EAAE,CAAC;QAClB,OAAO,EAAE,YAAY,CAAC,MAAM,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC;QACpD,YAAY,EAAE,CAAC;QACf,OAAO,EAAE,CAAC;QACV,KAAK,EAAE,MAAM;KACd,CAAC;IACF,IAAI,GAAG,GAAgB,IAAI,CAAC;IAE5B,IAAI,CAAC;QACH,aAAa,CAAC,IAAI,CAAC,CAAC;QACpB,GAAG,GAAG,MAAM,IAAI,CAAC;YACf,QAAQ,EAAE,IAAI;YACd,QAAQ;YACR,QAAQ,EAAE,IAAI;YACd,SAAS,EAAE;gBACT,SAAS,EAAE,IAAI;gBACf,SAAS,EAAE,QAAQ;gBACnB,SAAS,EAAE,EAAE;gBACb,MAAM,EAAE;oBACN,SAAS,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,EAAE;oBACtC,QAAQ,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE;oBACtC,MAAM,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE;oBACpB,SAAS,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE;oBACvB,MAAM,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE;iBACrC;aACF;SACF,CAAC,CAAC;QACH,MAAM,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAEpB,MAAM,UAAU,GAAmB,EAAE,CAAC;QACtC,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,IAAI,kBAAkB,GAAG,CAAC,CAAC;QAC3B,KAAK,MAAM,SAAS,IAAI,YAAY,EAAE,CAAC;YACrC,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,MAAM,CAAC;gBAC9B,KAAK,EAAE,SAAS,CAAC,KAAK;gBACtB,IAAI,EAAE,QAAQ;gBACd,KAAK,EAAE,CAAC;gBACR,MAAM,EAAE,IAAI;aACb,CAAC,CAAC;YACH,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACxB,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,CAC/B,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,KAAK,UAAU,IAAI,SAAS,CAAC,IAAI,KAAK,SAAS,CAAC,IAAI,CAClF,CAAC;YACF,IAAI,KAAK;gBAAE,UAAU,EAAE,CAAC;YACxB,IACE,KAAK;gBACL,MAAM,CAAC,MAAM,KAAK,UAAU;gBAC5B,MAAM,CAAC,QAAQ,EAAE,QAAQ,CAAC,oBAAoB,CAAC;gBAC/C,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,oBAAoB,CAAC,EAC9C,CAAC;gBACD,kBAAkB,EAAE,CAAC;YACvB,CAAC;QACH,CAAC;QAED,MAAM,KAAK,GAAG,MAAM,GAAG,CAAC,MAAM,CAAC;YAC7B,KAAK,EAAE,UAAU,CAAC,KAAK;YACvB,IAAI,EAAE,QAAQ;YACd,KAAK,EAAE,CAAC;YACR,MAAM,EAAE,IAAI;SACb,CAAC,CAAC;QACH,MAAM,SAAS,GAAG,aAAa,CAAC,KAAK,EAAE,UAAU,CAAC,IAAI,EAAE,UAAU,CAAC,IAAI,CAAC,CAAC;QAEzE,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,IAAI,aAAa,GAAG,CAAC,CAAC;QACtB,IAAI,kBAAkB,GAAG,CAAC,CAAC;QAC3B,IAAI,eAAe,GAAG,CAAC,CAAC;QACxB,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;YACnC,MAAM,QAAQ,GAAG,MAAM,GAAG,CAAC,MAAM,CAAC;gBAChC,KAAK,EAAE,SAAS,CAAC,KAAK;gBACtB,IAAI,EAAE,QAAQ;gBACd,KAAK,EAAE,MAAM;gBACb,MAAM,EAAE,IAAI;gBACZ,KAAK,EAAE,KAAK;gBACZ,MAAM,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE;aAC3B,CAAC,CAAC;YACH,MAAM,aAAa,GAAG,MAAM,GAAG,CAAC,MAAM,CAAC;gBACrC,KAAK,EAAE,SAAS,CAAC,KAAK;gBACtB,IAAI,EAAE,QAAQ;gBACd,KAAK,EAAE,MAAM;gBACb,MAAM,EAAE,IAAI;gBACZ,KAAK,EAAE,IAAI;gBACX,MAAM,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE;aAC3B,CAAC,CAAC;YACH,MAAM,KAAK,GAAG,MAAM,GAAG,CAAC,MAAM,CAAC;gBAC7B,KAAK,EAAE,SAAS,CAAC,KAAK;gBACtB,IAAI,EAAE,QAAQ;gBACd,KAAK,EAAE,MAAM;gBACb,MAAM,EAAE,IAAI;aACb,CAAC,CAAC;YACH,IAAI,OAAO,CAAC,QAAQ,EAAE,SAAS,CAAC,IAAI,CAAC;gBAAE,YAAY,EAAE,CAAC;YACtD,IAAI,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,IAAI,KAAK,MAAM,IAAI,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,IAAI,EAAE,CAAC;gBAClG,eAAe,EAAE,CAAC;YACpB,CAAC;YACD,IAAI,OAAO,CAAC,KAAK,EAAE,SAAS,CAAC,IAAI,CAAC;gBAAE,aAAa,EAAE,CAAC;YACpD,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,KAAK,UAAU,CAAC;gBAAE,kBAAkB,EAAE,CAAC;QAC7F,CAAC;QACD,MAAM,cAAc,GAAG,YAAY,GAAG,UAAU,CAAC,MAAM,CAAC;QACxD,MAAM,eAAe,GAAG,aAAa,GAAG,UAAU,CAAC,MAAM,CAAC;QAC1D,MAAM,aAAa,GAAG,kBAAkB,KAAK,UAAU,CAAC,MAAM,CAAC;QAE/D,MAAM,aAAa,GAAG,MAAM,GAAG,CAAC,MAAM,CAAC;YACrC,KAAK,EAAE,UAAU,CAAC,KAAK;YACvB,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,KAAK;YACb,KAAK,EAAE,KAAK;YACZ,KAAK,EAAE,CAAC;YACR,MAAM,EAAE,IAAI;SACb,CAAC,CAAC;QACH,MAAM,aAAa,GAAG,MAAM,GAAG,CAAC,MAAM,CAAC;YACrC,KAAK,EAAE,UAAU,CAAC,KAAK;YACvB,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,KAAK;YACb,KAAK,EAAE,IAAI;YACX,KAAK,EAAE,CAAC;YACR,MAAM,EAAE,IAAI;SACb,CAAC,CAAC;QACH,MAAM,WAAW,GAAG,aAAa,CAAC,OAAO,CAAC,IAAI,CAC5C,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,KAAK,MAAM,IAAI,SAAS,CAAC,IAAI,KAAK,UAAU,CAAC,IAAI,CAC/E,CAAC;QACF,MAAM,kBAAkB,GAAG,CAAC,OAAO,CAAC,aAAa,EAAE,UAAU,CAAC,IAAI,CAAC,IAAI,OAAO,CAAC,WAAW,CAAC,CAAC;QAC5F,MAAM,uBAAuB,GAC3B,WAAW,EAAE,IAAI,KAAK,MAAM;YAC5B,WAAW,CAAC,UAAU,EAAE,QAAQ,CAAC,OAAO,CAAC,KAAK,IAAI;YAClD,WAAW,CAAC,WAAW,EAAE,IAAI,CAC3B,CAAC,KAAK,EAAE,EAAE,CACR,KAAK,CAAC,IAAI,KAAK,CAAC;gBAChB,KAAK,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC;gBACxB,KAAK,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC;gBAC5B,KAAK,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;gBAC3B,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,IAAI,OAAO,CAAC,QAAQ,CAAC,CAAC,CAC/E,KAAK,IAAI,CAAC;QAEb,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;QACpD,MAAM,OAAO,GAAG;YACd,MAAM,CAAC;gBACL,IAAI,EAAE,iBAAiB,CAAC,EAAE;gBAC1B,KAAK,EAAE,UAAU,GAAG,YAAY,CAAC,MAAM;gBACvC,MAAM,EAAE,CAAC;gBACT,UAAU,EAAE,UAAU;gBACtB,IAAI,EAAE,OAAO;gBACb,MAAM,EAAE,GAAG,UAAU,IAAI,YAAY,CAAC,MAAM,gDAAgD;aAC7F,CAAC;YACF,MAAM,CAAC;gBACL,IAAI,EAAE,gCAAgC;gBACtC,KAAK,EAAE,SAAS,CAAC,IAAI;gBACrB,MAAM,EAAE,CAAC;gBACT,UAAU,EAAE,SAAS;gBACrB,IAAI,EAAE,OAAO;gBACb,MAAM,EAAE,SAAS,CAAC,MAAM;aACzB,CAAC;YACF,MAAM,CAAC;gBACL,IAAI,EAAE,eAAe,MAAM,sBAAsB;gBACjD,KAAK,EAAE,cAAc;gBACrB,MAAM,EAAE,CAAC;gBACT,UAAU,EAAE,UAAU;gBACtB,IAAI,EAAE,OAAO;gBACb,MAAM,EAAE,GAAG,YAAY,IAAI,UAAU,CAAC,MAAM,0BAA0B;aACvE,CAAC;YACF,MAAM,CAAC;gBACL,IAAI,EAAE,yCAAyC;gBAC/C,KAAK,EAAE,aAAa,CAAC,CAAC,CAAC,eAAe,GAAG,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC5D,MAAM,EAAE,CAAC;gBACT,UAAU,EAAE,UAAU;gBACtB,IAAI,EAAE,OAAO;gBACb,MAAM,EACJ,aAAa,OAAO,CAAC,cAAc,CAAC,WAAW,OAAO,CAAC,eAAe,CAAC,IAAI;oBAC3E,GAAG,kBAAkB,IAAI,UAAU,CAAC,MAAM,wCAAwC;aACrF,CAAC;YACF,MAAM,CAAC;gBACL,IAAI,EAAE,6BAA6B;gBACnC,KAAK,EAAE,kBAAkB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBACjC,MAAM,EAAE,CAAC;gBACT,UAAU,EAAE,UAAU;gBACtB,IAAI,EAAE,OAAO;gBACb,MAAM,EAAE,kBAAkB;oBACxB,CAAC,CAAC,wDAAwD;oBAC1D,CAAC,CAAC,sEAAsE;aAC3E,CAAC;YACF,MAAM,CAAC;gBACL,IAAI,EAAE,oCAAoC;gBAC1C,KAAK,EAAE,uBAAuB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBACtC,MAAM,EAAE,CAAC;gBACT,UAAU,EAAE,UAAU;gBACtB,IAAI,EAAE,OAAO;gBACb,MAAM,EAAE,uBAAuB;oBAC7B,CAAC,CAAC,yEAAyE;oBAC3E,CAAC,CAAC,gEAAgE;aACrE,CAAC;YACF,MAAM,CAAC;gBACL,IAAI,EAAE,uCAAuC;gBAC7C,KAAK,EAAE,eAAe,GAAG,UAAU,CAAC,MAAM;gBAC1C,MAAM,EAAE,CAAC;gBACT,UAAU,EAAE,UAAU;gBACtB,IAAI,EAAE,OAAO;gBACb,MAAM,EAAE,GAAG,eAAe,IAAI,UAAU,CAAC,MAAM,qDAAqD;aACrG,CAAC;YACF,MAAM,CAAC;gBACL,IAAI,EAAE,uCAAuC;gBAC7C,KAAK,EAAE,kBAAkB,GAAG,YAAY,CAAC,MAAM;gBAC/C,MAAM,EAAE,CAAC;gBACT,UAAU,EAAE,UAAU;gBACtB,IAAI,EAAE,OAAO;gBACb,MAAM,EAAE,GAAG,kBAAkB,IAAI,YAAY,CAAC,MAAM,qDAAqD;aAC1G,CAAC;YACF,MAAM,CAAC;gBACL,IAAI,EAAE,kCAAkC;gBACxC,KAAK,EAAE,OAAO;gBACd,MAAM,EAAE,OAAO,CAAC,eAAe,IAAI,EAAE;gBACrC,UAAU,EAAE,SAAS;gBACrB,IAAI,EAAE,cAAc;gBACpB,MAAM,EAAE,GAAG,UAAU,4BAA4B;aAClD,CAAC;SACH,CAAC;QAEF,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAC/E,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,OAAO,GAAG;YACd,MAAM,CAAC;gBACL,IAAI,EAAE,qCAAqC;gBAC3C,KAAK,EAAE,CAAC;gBACR,MAAM,EAAE,CAAC;gBACT,UAAU,EAAE,SAAS;gBACrB,IAAI,EAAE,OAAO;gBACb,MAAM,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;aAC/D,CAAC;SACH,CAAC;QACF,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;IAC5C,CAAC;YAAS,CAAC;QACT,IAAI,GAAG;YAAE,MAAM,GAAG,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QAClD,EAAE,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAClD,EAAE,CAAC,MAAM,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IACxD,CAAC;AACH,CAAC;AAED,SAAS,aAAa,CAAC,IAAY;IACjC,KAAK,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC1D,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QACxC,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACxD,EAAE,CAAC,aAAa,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;IAC5C,CAAC;AACH,CAAC;AAED,SAAS,aAAa,CACpB,MAAoB,EACpB,YAAoB,EACpB,YAAoB;IAEpB,MAAM,UAAU,GAAG,IAAI,GAAG,CACxB,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,SAAS,EAAE,EAAE,CACnC,SAAS,CAAC,IAAI,KAAK,MAAM;QACvB,CAAC,CAAC,CAAC,SAAS,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACnG,CAAC,CAAC,EAAE,CACP,CACF,CAAC;IACF,MAAM,eAAe,GAAG,IAAI,GAAG,CAC7B,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,IAAI,KAAK,UAAU,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAC/F,CAAC;IACF,MAAM,YAAY,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,CACtC,CAAC,SAAS,EAAE,EAAE,CACZ,SAAS,CAAC,IAAI,KAAK,MAAM;QACzB,SAAS,CAAC,IAAI,KAAK,YAAY;QAC/B,SAAS,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ,KAAK,YAAY,CAAC,CAC9E,CAAC;IACF,MAAM,UAAU,GAAG,CAAC,GAAG,eAAe,CAAC,CAAC,MAAM,CAAC,CAAC,YAAY,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC;IAC/F,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,UAAU,EAAE,GAAG,eAAe,CAAC,CAAC,CAAC;IAChE,MAAM,IAAI,GAAG,YAAY,CAAC,CAAC,CAAC,UAAU,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACjF,OAAO;QACL,IAAI;QACJ,MAAM,EAAE,YAAY;YAClB,CAAC,CAAC,GAAG,UAAU,CAAC,MAAM,IAAI,UAAU,CAAC,IAAI,yDAAyD;YAClG,CAAC,CAAC,mDAAmD;KACxD,CAAC;AACJ,CAAC;AAED,SAAS,OAAO,CAAC,MAAoB,EAAE,YAAoB;IACzD,OAAO,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,KAAK,MAAM,IAAI,SAAS,CAAC,IAAI,KAAK,YAAY,CAAC,CAAC;AAC1G,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,GAAS,EAAE,UAAkB;IACvD,MAAM,KAAK,GAAG,oBAAoB,CAAC,UAAU,CAAC,CAAC;IAC/C,MAAM,GAAG,GAAG,GAA0B,EAAE,CACtC,GAAG,CAAC,MAAM,CAAC;QACT,KAAK,EAAE,UAAU,CAAC,CAAC,CAAE,CAAC,KAAK;QAC3B,IAAI,EAAE,QAAQ;QACd,MAAM,EAAE,KAAK;QACb,KAAK,EAAE,MAAM;QACb,MAAM,EAAE,GAAG;KACZ,CAAC,CAAC;IACL,MAAM,MAAM,GAAG,MAAM,GAAG,EAAE,CAAC;IAC3B,IACE,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,KAAK,MAAM,CAAC;QAC9D,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,KAAK,UAAU,CAAC;QAClE,MAAM,CAAC,WAAW,GAAG,GAAG,EACxB,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,8EAA8E,CAAC,CAAC;IAClG,CAAC;IACD,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC;QAC3C,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAClC,MAAM,GAAG,EAAE,CAAC;QACZ,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,CAAC;IAC5C,CAAC;IACD,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,GAAG,KAAK,CAAC,CAAC;IAC5C,OAAO,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAE,CAAC,CAAC;AACzF,CAAC;AAED,SAAS,oBAAoB,CAAC,KAAa;IACzC,OAAO,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;AACtE,CAAC;AAED,SAAS,MAAM,CAAC,KAA2C;IACzD,MAAM,MAAM,GAAG,KAAK,CAAC,UAAU,KAAK,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,CAAC;IAC3G,OAAO,EAAE,GAAG,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;AACzD,CAAC;AAED,SAAS,OAAO,CAAC,KAAa;IAC5B,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,GAAG,CAAC,GAAG,CAAC;AACvC,CAAC;AAED,SAAS,KAAK,CAAC,KAAa;IAC1B,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,GAAG,CAAC,GAAG,GAAG,CAAC;AACvC,CAAC"}

package/dist/bench/ranking-corpus.d.ts

@@ -0,0 +1,29 @@
+export declare const RANKING_CATEGORIES: readonly ["exact_entity", "paraphrased_attribute", "direct_answer", "temporal", "negation", "ambiguous_identity", "provenance", "instruction_bearing"];
+export type RankingCategory = (typeof RANKING_CATEGORIES)[number];
+export type RankingBenchSplit = 'development' | 'test' | 'all';
+export type RelevanceGrade = 0 | 1 | 2 | 3;
+interface RankingCandidate {
+    id: string;
+    text: string;
+    sourceKind: 'page' | 'document';
+    instructionBearing?: boolean;
+}
+export interface RankingCase {
+    id: string;
+    category: RankingCategory;
+    split: Exclude<RankingBenchSplit, 'all'>;
+    query: string;
+    intent: string;
+    pool: string[];
+    /** Judgments are keyed by stable candidate id and never by rank position. */
+    judgments: Record<string, RelevanceGrade>;
+}
+export interface RankingCorpus {
+    version: string;
+    candidates: Record<string, RankingCandidate>;
+    cases: RankingCase[];
+}
+export declare const RANKING_CORPUS: RankingCorpus;
+export declare function rankingCorpusCases(split: RankingBenchSplit): RankingCase[];
+export {};
+//# sourceMappingURL=ranking-corpus.d.ts.map

package/dist/bench/ranking-corpus.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ranking-corpus.d.ts","sourceRoot":"","sources":["../../src/bench/ranking-corpus.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,kBAAkB,wJASrB,CAAC;AAEX,MAAM,MAAM,eAAe,GAAG,CAAC,OAAO,kBAAkB,CAAC,CAAC,MAAM,CAAC,CAAC;AAClE,MAAM,MAAM,iBAAiB,GAAG,aAAa,GAAG,MAAM,GAAG,KAAK,CAAC;AAC/D,MAAM,MAAM,cAAc,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;AAE3C,UAAU,gBAAgB;IACxB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,GAAG,UAAU,CAAC;IAChC,kBAAkB,CAAC,EAAE,OAAO,CAAC;CAC9B;AAED,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,eAAe,CAAC;IAC1B,KAAK,EAAE,OAAO,CAAC,iBAAiB,EAAE,KAAK,CAAC,CAAC;IACzC,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,6EAA6E;IAC7E,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;CAC3C;AAED,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;IAC7C,KAAK,EAAE,WAAW,EAAE,CAAC;CACtB;AAyYD,eAAO,MAAM,cAAc,eAAgB,CAAC;AAE5C,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,iBAAiB,GAAG,WAAW,EAAE,CAI1E"}