fastrmemory 0.5.0 → 0.6.0

package/dist/src/bench/public-harness.d.ts

@@ -14,6 +14,8 @@ export interface PublicEngine {
     retrieval: RetrievalConfig;
     requiresEmbeddings?: boolean;
     embeddingModel?: 'hash' | 'minilm';
+    /** Result depth the engine returns (default 10). */
+    limit?: number;
 }
 export declare const PUBLIC_ENGINES: PublicEngine[];
 export interface PublicCaseResult {
@@ -27,11 +29,16 @@ export interface PublicCaseResult {
     hitAt10: number;
     recallAt5: number;
     fullRecallAt5: number;
+    /** Strict@10: every gold id is present in the top 10. */
+    fullRecallAt10: number;
     goldCount: number;
     retrieved: string[];
     goldIds: string[];
     tokens: number;
     latencyMs: number;
+    /** Parsed temporal window applied to this case, when the engine used one. */
+    temporalWindow?: string;
+    temporalFallback?: boolean;
 }
 export interface PublicMetrics {
     count: number;
@@ -41,6 +48,8 @@ export interface PublicMetrics {
     mrr: number;
     evidenceRecallAt5: number;
     fullRecallAt5: number;
+    /** Strict@10 = all gold ids in the top 10, averaged over cases. */
+    fullRecallAt10: number;
     avgTokens: number;
     p50LatencyMs: number;
     p95LatencyMs: number;
@@ -48,11 +57,16 @@ export interface PublicMetrics {
 export interface PublicCategoryMetrics extends PublicMetrics {
     category: string;
 }
+/** Metrics restricted to cases with an exact gold count (the aggregate-question slice). */
+export interface PublicGoldBucket extends PublicMetrics {
+    goldCount: number;
+}
 export interface PublicEngineEval {
     name: string;
     config: RetrievalConfig;
     overall: PublicMetrics;
     byCategory: PublicCategoryMetrics[];
+    byGoldCount: PublicGoldBucket[];
     cases: PublicCaseResult[];
 }
 export interface PublicHarnessOptions {

package/dist/src/bench/public-harness.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"public-harness.d.ts","sourceRoot":"","sources":["../../../src/bench/public-harness.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3C,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAEtD,OAAO,EAAkB,KAAK,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAC/E,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAEtE,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,eAAe,CAAC;IAC3B,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,cAAc,CAAC,EAAE,MAAM,GAAG,QAAQ,CAAC;CACpC;AAED,eAAO,MAAM,cAAc,EAAE,YAAY,EAiCxC,CAAC;AAEF,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;IACb,cAAc,EAAE,MAAM,CAAC;IACvB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,GAAG,EAAE,MAAM,CAAC;IACZ,iBAAiB,EAAE,MAAM,CAAC;IAC1B,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,qBAAsB,SAAQ,aAAa;IAC1D,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,eAAe,CAAC;IACxB,OAAO,EAAE,aAAa,CAAC;IACvB,UAAU,EAAE,qBAAqB,EAAE,CAAC;IACpC,KAAK,EAAE,gBAAgB,EAAE,CAAC;CAC3B;AAED,MAAM,WAAW,oBAAoB;IACnC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED,wBAAgB,iBAAiB,CAAC,OAAO,EAAE,aAAa,GAAG;IAAE,EAAE,EAAE,YAAY,CAAC;IAAC,IAAI,EAAE,eAAe,CAAA;CAAE,CAmCrG;AAED,wBAAsB,iBAAiB,CACrC,EAAE,EAAE,YAAY,EAChB,IAAI,EAAE,eAAe,EACrB,OAAO,EAAE,aAAa,EACtB,MAAM,EAAE,YAAY,GACnB,OAAO,CAAC,MAAM,CAAC,CAQjB;AAaD,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,gBAAgB,EAAE,GAAG,aAAa,CA6BzE;AAED,wBAAgB,UAAU,CACxB,IAAI,EAAE,UAAU,EAChB,SAAS,EAAE,MAAM,EAAE,EACnB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,GAChB,gBAAgB,CAwBlB;AAED,wBAAsB,oBAAoB,CACxC,EAAE,EAAE,YAAY,EAChB,OAAO,EAAE,aAAa,EACtB,MAAM,EAAE,YAAY,EACpB,OAAO,GAAE,oBAAyB,GACjC,OAAO,CAAC,gBAAgB,CAAC,CAqC3B"}
+{"version":3,"file":"public-harness.d.ts","sourceRoot":"","sources":["../../../src/bench/public-harness.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3C,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAGtD,OAAO,EAAkB,KAAK,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAC/E,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAEtE,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,eAAe,CAAC;IAC3B,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,cAAc,CAAC,EAAE,MAAM,GAAG,QAAQ,CAAC;IACnC,oDAAoD;IACpD,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,eAAO,MAAM,cAAc,EAAE,YAAY,EAyFxC,CAAC;AAEF,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;IACb,cAAc,EAAE,MAAM,CAAC;IACvB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,EAAE,MAAM,CAAC;IACtB,yDAAyD;IACzD,cAAc,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,6EAA6E;IAC7E,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,GAAG,EAAE,MAAM,CAAC;IACZ,iBAAiB,EAAE,MAAM,CAAC;IAC1B,aAAa,EAAE,MAAM,CAAC;IACtB,mEAAmE;IACnE,cAAc,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,qBAAsB,SAAQ,aAAa;IAC1D,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,2FAA2F;AAC3F,MAAM,WAAW,gBAAiB,SAAQ,aAAa;IACrD,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,eAAe,CAAC;IACxB,OAAO,EAAE,aAAa,CAAC;IACvB,UAAU,EAAE,qBAAqB,EAAE,CAAC;IACpC,WAAW,EAAE,gBAAgB,EAAE,CAAC;IAChC,KAAK,EAAE,gBAAgB,EAAE,CAAC;CAC3B;AAED,MAAM,WAAW,oBAAoB;IACnC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED,wBAAgB,iBAAiB,CAAC,OAAO,EAAE,aAAa,GAAG;IAAE,EAAE,EAAE,YAAY,CAAC;IAAC,IAAI,EAAE,eAAe,CAAA;CAAE,CAsCrG;AAED,wBAAsB,iBAAiB,CACrC,EAAE,EAAE,YAAY,EAChB,IAAI,EAAE,eAAe,EACrB,OAAO,EAAE,aAAa,EACtB,MAAM,EAAE,YAAY,GACnB,OAAO,CAAC,MAAM,CAAC,CAQjB;AAaD,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,gBAAgB,EAAE,GAAG,aAAa,CA+BzE;AAED,wBAAgB,UAAU,CACxB,IAAI,EAAE,UAAU,EAChB,SAAS,EAAE,MAAM,EAAE,EACnB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,GAChB,gBAAgB,CA2BlB;AAED,wBAAsB,oBAAoB,CACxC,EAAE,EAAE,YAAY,EAChB,OAAO,EAAE,aAAa,EACtB,MAAM,EAAE,YAAY,EACpB,OAAO,GAAE,oBAAyB,GACjC,OAAO,CAAC,gBAAgB,CAAC,CA8C3B"}

package/dist/src/bench/public-harness.js

@@ -9,22 +9,78 @@ import { DatabaseSync } from 'node:sqlite';
 import { applySchema } from '../db/schema.js';
 import { FastrRepository } from '../db/repository.js';
 import { backfillEmbeddings } from '../search/backfill.js';
+import { drainFtsQueue } from '../db/fts-queue.js';
 import { searchMemories } from '../search/hybrid-rrf.js';
 export const PUBLIC_ENGINES = [
     {
         name: 'bm25_only',
-        retrieval: { enableBm25: true, enableVector: false, enableGraph: false, trapBoost: 1, verifiedBoost: 1 }
+        retrieval: {
+            enableBm25: true,
+            enableVector: false,
+            enableGraph: false,
+            trapBoost: 1,
+            verifiedBoost: 1,
+            aggregateCoverage: false
+        }
+    },
+    {
+        // Same first stage, plus the parsed date window as a pre-ranking filter.
+        name: 'bm25_temporal',
+        retrieval: {
+            enableBm25: true,
+            enableVector: false,
+            enableGraph: false,
+            trapBoost: 1,
+            verifiedBoost: 1,
+            temporalFilter: true,
+            aggregateCoverage: false
+        }
+    },
+    {
+        // Counting questions search with their content terms only and return the full set.
+        name: 'bm25_aggregate',
+        retrieval: {
+            enableBm25: true,
+            enableVector: false,
+            enableGraph: false,
+            trapBoost: 1,
+            verifiedBoost: 1,
+            aggregateCoverage: true,
+            aggregateLimit: 20
+        }
+    },
+    {
+        // Both stages: the window removes out-of-range rows, the aggregate pass removes
+        // scaffolding from the lexical query.
+        name: 'bm25_temporal_aggregate',
+        retrieval: {
+            enableBm25: true,
+            enableVector: false,
+            enableGraph: false,
+            trapBoost: 1,
+            verifiedBoost: 1,
+            temporalFilter: true,
+            aggregateCoverage: true,
+            aggregateLimit: 20
+        }
+    },
+    {
+        // The shipped default after P0: full pipeline, vector stream off unless a semantic
+        // provider is loaded. This is what a user gets with no configuration at all.
+        name: 'default_config',
+        retrieval: {}
     },
     {
         name: 'full_hash',
         retrieval: {
             enableBm25: true,
-            enableVector: true,
+            vectorMode: 'hash',
             enableGraph: true,
             graphSeedFromCandidates: false,
             trapBoost: 2.0,
             trapBoostMode: 'conditional',
-            verifiedBoost: 1.25
+            verifiedBoost: 1.25,
+            aggregateCoverage: false
         },
         requiresEmbeddings: true,
         embeddingModel: 'hash'
@@ -33,12 +89,13 @@ export const PUBLIC_ENGINES = [
         name: 'full_minilm',
         retrieval: {
             enableBm25: true,
-            enableVector: true,
+            vectorMode: 'semantic',
             enableGraph: true,
             graphSeedFromCandidates: false,
             trapBoost: 2.0,
             trapBoostMode: 'conditional',
-            verifiedBoost: 1.25
+            verifiedBoost: 1.25,
+            aggregateCoverage: false
         },
         requiresEmbeddings: true,
         embeddingModel: 'minilm'
@@ -65,6 +122,8 @@ export function loadPublicDataset(dataset) {
         db.exec('ROLLBACK');
         throw error;
     }
+    // Seeding is not timed; index everything so the first measured query is a warm query.
+    drainFtsQueue(db, { limit: 1_000_000, schema: 'main' });
     return { db, repo };
 }
 export async function prepareEmbeddings(db, repo, dataset, engine) {
@@ -98,6 +157,7 @@ export function metricsFromCases(cases) {
             mrr: 0,
             evidenceRecallAt5: 0,
             fullRecallAt5: 0,
+            fullRecallAt10: 0,
             avgTokens: 0,
             p50LatencyMs: 0,
             p95LatencyMs: 0
@@ -112,6 +172,7 @@ export function metricsFromCases(cases) {
         mrr: round(cases.reduce((sum, item) => sum + item.reciprocalRank, 0) / count, 3),
         evidenceRecallAt5: round(cases.reduce((sum, item) => sum + item.recallAt5, 0) / count, 3),
         fullRecallAt5: round((cases.reduce((sum, item) => sum + item.fullRecallAt5, 0) / count) * 100, 1),
+        fullRecallAt10: round((cases.reduce((sum, item) => sum + item.fullRecallAt10, 0) / count) * 100, 1),
         avgTokens: Math.round(cases.reduce((sum, item) => sum + item.tokens, 0) / count),
         p50LatencyMs: round(percentile(latencies, 50), 2),
         p95LatencyMs: round(percentile(latencies, 95), 2)
@@ -124,6 +185,8 @@ export function caseResult(item, retrieved, tokens, latencyMs) {
     const top5 = retrieved.slice(0, 5);
     const foundAt5 = top5.filter((id) => gold.has(id)).length;
     const recallAt5 = gold.size === 0 ? 0 : foundAt5 / gold.size;
+    const foundAt10 = retrieved.slice(0, 10).filter((id) => gold.has(id)).length;
+    const fullRecallAt10 = gold.size > 0 && foundAt10 === gold.size ? 1 : 0;
     return {
         caseId: item.id,
         category: item.category,
@@ -135,6 +198,7 @@ export function caseResult(item, retrieved, tokens, latencyMs) {
         hitAt10: rank > 0 && rank <= 10 ? 1 : 0,
         recallAt5: round(recallAt5, 3),
         fullRecallAt5: recallAt5 === 1 ? 1 : 0,
+        fullRecallAt10,
         goldCount: item.goldIds.length,
         retrieved,
         goldIds: item.goldIds,
@@ -145,6 +209,7 @@ export function caseResult(item, retrieved, tokens, latencyMs) {
 export async function evaluatePublicEngine(db, dataset, engine, options = {}) {
     const selected = options.limit ? dataset.cases.slice(0, options.limit) : dataset.cases;
     const budgetTokens = options.budgetTokens ?? 1_000_000;
+    const resultLimit = engine.limit ?? 10;
     const results = [];
     for (const item of selected) {
         const start = performance.now();
@@ -152,14 +217,20 @@ export async function evaluatePublicEngine(db, dataset, engine, options = {}) {
             projectId: item.projectId,
             query: item.query,
             budgetTokens,
-            limit: 10,
+            limit: resultLimit,
             backfill: false,
-            retrieval: engine.retrieval
+            retrieval: { ...engine.retrieval, questionDate: item.questionDate }
         });
         const latencyMs = performance.now() - start;
-        results.push(caseResult(item, response.memories.map((memory) => memory.memory.id), response.totalTokens, latencyMs));
+        const caseRow = caseResult(item, response.memories.map((memory) => memory.memory.id), response.totalTokens, latencyMs);
+        if (response.temporalWindow) {
+            caseRow.temporalWindow = response.temporalWindow.label;
+            caseRow.temporalFallback = response.temporalFallback === true;
+        }
+        results.push(caseRow);
     }
     const categories = Array.from(new Set(results.map((item) => item.category))).sort();
+    const goldCounts = Array.from(new Set(results.map((item) => item.goldCount))).sort((a, b) => a - b);
     return {
         name: engine.name,
         config: engine.retrieval,
@@ -168,6 +239,10 @@ export async function evaluatePublicEngine(db, dataset, engine, options = {}) {
             category,
             ...metricsFromCases(results.filter((item) => item.category === category))
         })),
+        byGoldCount: goldCounts.map((goldCount) => ({
+            goldCount,
+            ...metricsFromCases(results.filter((item) => item.goldCount === goldCount))
+        })),
         cases: results
     };
 }

package/dist/src/bench/public-harness.js.map

@@ -1 +1 @@
-{"version":3,"file":"public-harness.js","sourceRoot":"","sources":["../../../src/bench/public-harness.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAC9C,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,EAAE,kBAAkB,EAAE,MAAM,uBAAuB,CAAC;AAC3D,OAAO,EAAE,cAAc,EAAwB,MAAM,yBAAyB,CAAC;AAU/E,MAAM,CAAC,MAAM,cAAc,GAAmB;IAC5C;QACE,IAAI,EAAE,WAAW;QACjB,SAAS,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,YAAY,EAAE,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE;KACzG;IACD;QACE,IAAI,EAAE,WAAW;QACjB,SAAS,EAAE;YACT,UAAU,EAAE,IAAI;YAChB,YAAY,EAAE,IAAI;YAClB,WAAW,EAAE,IAAI;YACjB,uBAAuB,EAAE,KAAK;YAC9B,SAAS,EAAE,GAAG;YACd,aAAa,EAAE,aAAa;YAC5B,aAAa,EAAE,IAAI;SACpB;QACD,kBAAkB,EAAE,IAAI;QACxB,cAAc,EAAE,MAAM;KACvB;IACD;QACE,IAAI,EAAE,aAAa;QACnB,SAAS,EAAE;YACT,UAAU,EAAE,IAAI;YAChB,YAAY,EAAE,IAAI;YAClB,WAAW,EAAE,IAAI;YACjB,uBAAuB,EAAE,KAAK;YAC9B,SAAS,EAAE,GAAG;YACd,aAAa,EAAE,aAAa;YAC5B,aAAa,EAAE,IAAI;SACpB;QACD,kBAAkB,EAAE,IAAI;QACxB,cAAc,EAAE,QAAQ;KACzB;CACF,CAAC;AAmDF,MAAM,UAAU,iBAAiB,CAAC,OAAsB;IACtD,MAAM,EAAE,GAAG,IAAI,YAAY,CAAC,UAAU,CAAC,CAAC;IACxC,WAAW,CAAC,EAAE,CAAC,CAAC;IAChB,MAAM,IAAI,GAAG,IAAI,eAAe,CAAC,EAAE,CAAC,CAAC;IACrC,MAAM,MAAM,GAAG,EAAE,CAAC,OAAO,CAAC;;;;;GAKzB,CAAC,CAAC;IAEH,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACjB,IAAI,CAAC;QACH,KAAK,MAAM,MAAM,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;YACtC,MAAM,CAAC,GAAG,CACR,MAAM,CAAC,EAAE,EACT,MAAM,CAAC,SAAS,EAChB,cAAc,EACd,UAAU,EACV,MAAM,CAAC,KAAK,EACZ,MAAM,CAAC,OAAO,EACd,UAAU,EACV,GAAG,EACH,MAAM,CAAC,SAAS,EAChB,IAAI,EACJ,MAAM,CAAC,SAAS,CACjB,CAAC;QACJ,CAAC;QACD,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACpB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACpB,MAAM,KAAK,CAAC;IACd,CAAC;IAED,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC;AACtB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,EAAgB,EAChB,IAAqB,EACrB,OAAsB,EACtB,MAAoB;IAEpB,IAAI,CAAC,MAAM,CAAC,kBAAkB;QAAE,OAAO,CAAC,CAAC;IACzC,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;IACpF,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACnC,QAAQ,IAAI,MAAM,kBAAkB,CAAC,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,CAAC;IAClF,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,SAAS,UAAU,CAAC,MAAgB,EAAE,CAAS;IAC7C,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAClC,MAAM,MAAM,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACjD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IACjG,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;AACvB,CAAC;AAED,SAAS,KAAK,CAAC,KAAa,EAAE,MAAc;IAC1C,OAAO,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;AACvC,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,KAAyB;IACxD,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC;IAC3B,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;QAChB,OAAO;YACL,KAAK,EAAE,CAAC;YACR,MAAM,EAAE,CAAC;YACT,MAAM,EAAE,CAAC;YACT,OAAO,EAAE,CAAC;YACV,GAAG,EAAE,CAAC;YACN,iBAAiB,EAAE,CAAC;YACpB,aAAa,EAAE,CAAC;YAChB,SAAS,EAAE,CAAC;YACZ,YAAY,EAAE,CAAC;YACf,YAAY,EAAE,CAAC;SAChB,CAAC;IACJ,CAAC;IACD,MAAM,SAAS,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACtD,OAAO;QACL,KAAK;QACL,MAAM,EAAE,KAAK,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,GAAG,GAAG,EAAE,CAAC,CAAC;QACnF,MAAM,EAAE,KAAK,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,GAAG,GAAG,EAAE,CAAC,CAAC;QACnF,OAAO,EAAE,KAAK,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,GAAG,GAAG,EAAE,CAAC,CAAC;QACrF,GAAG,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC,CAAC,GAAG,KAAK,EAAE,CAAC,CAAC;QAChF,iBAAiB,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC,GAAG,KAAK,EAAE,CAAC,CAAC;QACzF,aAAa,EAAE,KAAK,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,GAAG,GAAG,EAAE,CAAC,CAAC;QACjG,SAAS,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC;QAChF,YAAY,EAAE,KAAK,CAAC,UAAU,CAAC,SAAS,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC;QACjD,YAAY,EAAE,KAAK,CAAC,UAAU,CAAC,SAAS,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC;KAClD,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,UAAU,CACxB,IAAgB,EAChB,SAAmB,EACnB,MAAc,EACd,SAAiB;IAEjB,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACnC,MAAM,SAAS,GAAG,SAAS,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;IAC5D,MAAM,IAAI,GAAG,SAAS,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAChD,MAAM,IAAI,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACnC,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;IAC1D,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC;IAC7D,OAAO;QACL,MAAM,EAAE,IAAI,CAAC,EAAE;QACf,QAAQ,EAAE,IAAI,CAAC,QAAQ;QACvB,SAAS,EAAE,IAAI,CAAC,SAAS;QACzB,IAAI;QACJ,cAAc,EAAE,IAAI,GAAG,CAAC,IAAI,IAAI,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QACrD,MAAM,EAAE,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC1B,MAAM,EAAE,IAAI,GAAG,CAAC,IAAI,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACrC,OAAO,EAAE,IAAI,GAAG,CAAC,IAAI,IAAI,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACvC,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC;QAC9B,aAAa,EAAE,SAAS,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACtC,SAAS,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM;QAC9B,SAAS;QACT,OAAO,EAAE,IAAI,CAAC,OAAO;QACrB,MAAM;QACN,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC;KAC/B,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,EAAgB,EAChB,OAAsB,EACtB,MAAoB,EACpB,UAAgC,EAAE;IAElC,MAAM,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC;IACvF,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,SAAS,CAAC;IACvD,MAAM,OAAO,GAAuB,EAAE,CAAC;IAEvC,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;QAC5B,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAChC,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,EAAE,EAAE;YACxC,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,YAAY;YACZ,KAAK,EAAE,EAAE;YACT,QAAQ,EAAE,KAAK;YACf,SAAS,EAAE,MAAM,CAAC,SAAS;SAC5B,CAAC,CAAC;QACH,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;QAC5C,OAAO,CAAC,IAAI,CACV,UAAU,CACR,IAAI,EACJ,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,EACnD,QAAQ,CAAC,WAAW,EACpB,SAAS,CACV,CACF,CAAC;IACJ,CAAC;IAED,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACpF,OAAO;QACL,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,MAAM,EAAE,MAAM,CAAC,SAAS;QACxB,OAAO,EAAE,gBAAgB,CAAC,OAAO,CAAC;QAClC,UAAU,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;YACxC,QAAQ;YACR,GAAG,gBAAgB,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC;SAC1E,CAAC,CAAC;QACH,KAAK,EAAE,OAAO;KACf,CAAC;AACJ,CAAC"}
+{"version":3,"file":"public-harness.js","sourceRoot":"","sources":["../../../src/bench/public-harness.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAC9C,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,EAAE,kBAAkB,EAAE,MAAM,uBAAuB,CAAC;AAC3D,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,cAAc,EAAwB,MAAM,yBAAyB,CAAC;AAY/E,MAAM,CAAC,MAAM,cAAc,GAAmB;IAC5C;QACE,IAAI,EAAE,WAAW;QACjB,SAAS,EAAE;YACT,UAAU,EAAE,IAAI;YAChB,YAAY,EAAE,KAAK;YACnB,WAAW,EAAE,KAAK;YAClB,SAAS,EAAE,CAAC;YACZ,aAAa,EAAE,CAAC;YAChB,iBAAiB,EAAE,KAAK;SACzB;KACF;IACD;QACE,yEAAyE;QACzE,IAAI,EAAE,eAAe;QACrB,SAAS,EAAE;YACT,UAAU,EAAE,IAAI;YAChB,YAAY,EAAE,KAAK;YACnB,WAAW,EAAE,KAAK;YAClB,SAAS,EAAE,CAAC;YACZ,aAAa,EAAE,CAAC;YAChB,cAAc,EAAE,IAAI;YACpB,iBAAiB,EAAE,KAAK;SACzB;KACF;IACD;QACE,mFAAmF;QACnF,IAAI,EAAE,gBAAgB;QACtB,SAAS,EAAE;YACT,UAAU,EAAE,IAAI;YAChB,YAAY,EAAE,KAAK;YACnB,WAAW,EAAE,KAAK;YAClB,SAAS,EAAE,CAAC;YACZ,aAAa,EAAE,CAAC;YAChB,iBAAiB,EAAE,IAAI;YACvB,cAAc,EAAE,EAAE;SACnB;KACF;IACD;QACE,gFAAgF;QAChF,sCAAsC;QACtC,IAAI,EAAE,yBAAyB;QAC/B,SAAS,EAAE;YACT,UAAU,EAAE,IAAI;YAChB,YAAY,EAAE,KAAK;YACnB,WAAW,EAAE,KAAK;YAClB,SAAS,EAAE,CAAC;YACZ,aAAa,EAAE,CAAC;YAChB,cAAc,EAAE,IAAI;YACpB,iBAAiB,EAAE,IAAI;YACvB,cAAc,EAAE,EAAE;SACnB;KACF;IACD;QACE,mFAAmF;QACnF,6EAA6E;QAC7E,IAAI,EAAE,gBAAgB;QACtB,SAAS,EAAE,EAAE;KACd;IACD;QACE,IAAI,EAAE,WAAW;QACjB,SAAS,EAAE;YACT,UAAU,EAAE,IAAI;YAChB,UAAU,EAAE,MAAM;YAClB,WAAW,EAAE,IAAI;YACjB,uBAAuB,EAAE,KAAK;YAC9B,SAAS,EAAE,GAAG;YACd,aAAa,EAAE,aAAa;YAC5B,aAAa,EAAE,IAAI;YACnB,iBAAiB,EAAE,KAAK;SACzB;QACD,kBAAkB,EAAE,IAAI;QACxB,cAAc,EAAE,MAAM;KACvB;IACD;QACE,IAAI,EAAE,aAAa;QACnB,SAAS,EAAE;YACT,UAAU,EAAE,IAAI;YAChB,UAAU,EAAE,UAAU;YACtB,WAAW,EAAE,IAAI;YACjB,uBAAuB,EAAE,KAAK;YAC9B,SAAS,EAAE,GAAG;YACd,aAAa,EAAE,aAAa;YAC5B,aAAa,EAAE,IAAI;YACnB,iBAAiB,EAAE,KAAK;SACzB;QACD,kBAAkB,EAAE,IAAI;QACxB,cAAc,EAAE,QAAQ;KACzB;CACF,CAAC;AAgEF,MAAM,UAAU,iBAAiB,CAAC,OAAsB;IACtD,MAAM,EAAE,GAAG,IAAI,YAAY,CAAC,UAAU,CAAC,CAAC;IACxC,WAAW,CAAC,EAAE,CAAC,CAAC;IAChB,MAAM,IAAI,GAAG,IAAI,eAAe,CAAC,EAAE,CAAC,CAAC;IACrC,MAAM,MAAM,GAAG,EAAE,CAAC,OAAO,CAAC;;;;;GAKzB,CAAC,CAAC;IAEH,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACjB,IAAI,CAAC;QACH,KAAK,MAAM,MAAM,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;YACtC,MAAM,CAAC,GAAG,CACR,MAAM,CAAC,EAAE,EACT,MAAM,CAAC,SAAS,EAChB,cAAc,EACd,UAAU,EACV,MAAM,CAAC,KAAK,EACZ,MAAM,CAAC,OAAO,EACd,UAAU,EACV,GAAG,EACH,MAAM,CAAC,SAAS,EAChB,IAAI,EACJ,MAAM,CAAC,SAAS,CACjB,CAAC;QACJ,CAAC;QACD,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACpB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACpB,MAAM,KAAK,CAAC;IACd,CAAC;IAED,sFAAsF;IACtF,aAAa,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;IAExD,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC;AACtB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,EAAgB,EAChB,IAAqB,EACrB,OAAsB,EACtB,MAAoB;IAEpB,IAAI,CAAC,MAAM,CAAC,kBAAkB;QAAE,OAAO,CAAC,CAAC;IACzC,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;IACpF,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACnC,QAAQ,IAAI,MAAM,kBAAkB,CAAC,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,CAAC;IAClF,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,SAAS,UAAU,CAAC,MAAgB,EAAE,CAAS;IAC7C,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAClC,MAAM,MAAM,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACjD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IACjG,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;AACvB,CAAC;AAED,SAAS,KAAK,CAAC,KAAa,EAAE,MAAc;IAC1C,OAAO,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;AACvC,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,KAAyB;IACxD,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC;IAC3B,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;QAChB,OAAO;YACL,KAAK,EAAE,CAAC;YACR,MAAM,EAAE,CAAC;YACT,MAAM,EAAE,CAAC;YACT,OAAO,EAAE,CAAC;YACV,GAAG,EAAE,CAAC;YACN,iBAAiB,EAAE,CAAC;YACpB,aAAa,EAAE,CAAC;YAChB,cAAc,EAAE,CAAC;YACjB,SAAS,EAAE,CAAC;YACZ,YAAY,EAAE,CAAC;YACf,YAAY,EAAE,CAAC;SAChB,CAAC;IACJ,CAAC;IACD,MAAM,SAAS,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACtD,OAAO;QACL,KAAK;QACL,MAAM,EAAE,KAAK,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,GAAG,GAAG,EAAE,CAAC,CAAC;QACnF,MAAM,EAAE,KAAK,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,GAAG,GAAG,EAAE,CAAC,CAAC;QACnF,OAAO,EAAE,KAAK,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,GAAG,GAAG,EAAE,CAAC,CAAC;QACrF,GAAG,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC,CAAC,GAAG,KAAK,EAAE,CAAC,CAAC;QAChF,iBAAiB,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC,GAAG,KAAK,EAAE,CAAC,CAAC;QACzF,aAAa,EAAE,KAAK,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,GAAG,GAAG,EAAE,CAAC,CAAC;QACjG,cAAc,EAAE,KAAK,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,GAAG,GAAG,EAAE,CAAC,CAAC;QACnG,SAAS,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC;QAChF,YAAY,EAAE,KAAK,CAAC,UAAU,CAAC,SAAS,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC;QACjD,YAAY,EAAE,KAAK,CAAC,UAAU,CAAC,SAAS,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC;KAClD,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,UAAU,CACxB,IAAgB,EAChB,SAAmB,EACnB,MAAc,EACd,SAAiB;IAEjB,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACnC,MAAM,SAAS,GAAG,SAAS,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;IAC5D,MAAM,IAAI,GAAG,SAAS,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAChD,MAAM,IAAI,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACnC,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;IAC1D,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC;IAC7D,MAAM,SAAS,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;IAC7E,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,GAAG,CAAC,IAAI,SAAS,KAAK,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACxE,OAAO;QACL,MAAM,EAAE,IAAI,CAAC,EAAE;QACf,QAAQ,EAAE,IAAI,CAAC,QAAQ;QACvB,SAAS,EAAE,IAAI,CAAC,SAAS;QACzB,IAAI;QACJ,cAAc,EAAE,IAAI,GAAG,CAAC,IAAI,IAAI,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QACrD,MAAM,EAAE,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC1B,MAAM,EAAE,IAAI,GAAG,CAAC,IAAI,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACrC,OAAO,EAAE,IAAI,GAAG,CAAC,IAAI,IAAI,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACvC,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC;QAC9B,aAAa,EAAE,SAAS,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACtC,cAAc;QACd,SAAS,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM;QAC9B,SAAS;QACT,OAAO,EAAE,IAAI,CAAC,OAAO;QACrB,MAAM;QACN,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC;KAC/B,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,EAAgB,EAChB,OAAsB,EACtB,MAAoB,EACpB,UAAgC,EAAE;IAElC,MAAM,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC;IACvF,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,SAAS,CAAC;IACvD,MAAM,WAAW,GAAG,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC;IACvC,MAAM,OAAO,GAAuB,EAAE,CAAC;IAEvC,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;QAC5B,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAChC,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,EAAE,EAAE;YACxC,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,YAAY;YACZ,KAAK,EAAE,WAAW;YAClB,QAAQ,EAAE,KAAK;YACf,SAAS,EAAE,EAAE,GAAG,MAAM,CAAC,SAAS,EAAE,YAAY,EAAE,IAAI,CAAC,YAAY,EAAE;SACpE,CAAC,CAAC;QACH,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;QAC5C,MAAM,OAAO,GAAG,UAAU,CACxB,IAAI,EACJ,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,EACnD,QAAQ,CAAC,WAAW,EACpB,SAAS,CACV,CAAC;QACF,IAAI,QAAQ,CAAC,cAAc,EAAE,CAAC;YAC5B,OAAO,CAAC,cAAc,GAAG,QAAQ,CAAC,cAAc,CAAC,KAAK,CAAC;YACvD,OAAO,CAAC,gBAAgB,GAAG,QAAQ,CAAC,gBAAgB,KAAK,IAAI,CAAC;QAChE,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACxB,CAAC;IAED,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACpF,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACpG,OAAO;QACL,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,MAAM,EAAE,MAAM,CAAC,SAAS;QACxB,OAAO,EAAE,gBAAgB,CAAC,OAAO,CAAC;QAClC,UAAU,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;YACxC,QAAQ;YACR,GAAG,gBAAgB,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC;SAC1E,CAAC,CAAC;QACH,WAAW,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;YAC1C,SAAS;YACT,GAAG,gBAAgB,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC;SAC5E,CAAC,CAAC;QACH,KAAK,EAAE,OAAO;KACf,CAAC;AACJ,CAAC"}

package/dist/src/bench/public-pins.js

@@ -8,11 +8,11 @@ import * as crypto from 'node:crypto';
 import * as fs from 'node:fs';
 export const PUBLIC_DATASET_PINS = {
     longmemeval: {
-        file: 'longmemeval_s.json',
-        revision: '2ec2a557f339b6c0369619b1ed5793734cc87533',
-        sha256: '08D8DAD4BE43EE2049A22FF5674EB86725D0CE5FF434CDE2627E5E8E7E117894',
-        url: 'https://huggingface.co/datasets/xiaowu0162/longmemeval',
-        source: 'huggingface:xiaowu0162/longmemeval'
+        file: 'longmemeval_s_cleaned.json',
+        revision: '98d7416c24c778c2fee6e6f3006e7a073259d48f',
+        sha256: 'D6F21EA9D60A0D56F34A05B609C79C88A451D2AE03597821EA3D5A9678C3A442',
+        url: 'https://huggingface.co/datasets/xiaowu0162/longmemeval-cleaned',
+        source: 'huggingface:xiaowu0162/longmemeval-cleaned'
     },
     locomo: {
         file: 'locomo10.json',

package/dist/src/bench/public-pins.js.map

@@ -1 +1 @@
-{"version":3,"file":"public-pins.js","sourceRoot":"","sources":["../../../src/bench/public-pins.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,MAAM,MAAM,aAAa,CAAC;AACtC,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAY9B,MAAM,CAAC,MAAM,mBAAmB,GAA0C;IACxE,WAAW,EAAE;QACX,IAAI,EAAE,oBAAoB;QAC1B,QAAQ,EAAE,0CAA0C;QACpD,MAAM,EAAE,kEAAkE;QAC1E,GAAG,EAAE,wDAAwD;QAC7D,MAAM,EAAE,oCAAoC;KAC7C;IACD,MAAM,EAAE;QACN,IAAI,EAAE,eAAe;QACrB,QAAQ,EAAE,0CAA0C;QACpD,MAAM,EAAE,kEAAkE;QAC1E,GAAG,EAAE,yCAAyC;QAC9C,MAAM,EAAE,6BAA6B;KACtC;CACF,CAAC;AAEF,MAAM,UAAU,UAAU,CAAC,IAAY;IACrC,OAAO,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC;AAC/F,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,GAAe,EAAE,IAAY,EAAE,aAAsB;IACpF,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;QACzB,MAAM,IAAI,KAAK,CACb,yBAAyB,IAAI,sBAAsB,GAAG,CAAC,GAAG,gBAAgB,GAAG,CAAC,QAAQ,oBAAoB,IAAI,EAAE,CACjH,CAAC;IACJ,CAAC;IACD,MAAM,MAAM,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;IAChC,IAAI,MAAM,KAAK,GAAG,CAAC,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;QAC5C,MAAM,IAAI,KAAK,CACb,6BAA6B,IAAI,cAAc,GAAG,CAAC,MAAM,cAAc,MAAM,+DAA+D,CAC7I,CAAC;IACJ,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,IAAY;IACnC,OAAO,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;AACnD,CAAC"}
+{"version":3,"file":"public-pins.js","sourceRoot":"","sources":["../../../src/bench/public-pins.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,MAAM,MAAM,aAAa,CAAC;AACtC,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAY9B,MAAM,CAAC,MAAM,mBAAmB,GAA0C;IACxE,WAAW,EAAE;QACX,IAAI,EAAE,4BAA4B;QAClC,QAAQ,EAAE,0CAA0C;QACpD,MAAM,EAAE,kEAAkE;QAC1E,GAAG,EAAE,gEAAgE;QACrE,MAAM,EAAE,4CAA4C;KACrD;IACD,MAAM,EAAE;QACN,IAAI,EAAE,eAAe;QACrB,QAAQ,EAAE,0CAA0C;QACpD,MAAM,EAAE,kEAAkE;QAC1E,GAAG,EAAE,yCAAyC;QAC9C,MAAM,EAAE,6BAA6B;KACtC;CACF,CAAC;AAEF,MAAM,UAAU,UAAU,CAAC,IAAY;IACrC,OAAO,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC;AAC/F,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,GAAe,EAAE,IAAY,EAAE,aAAsB;IACpF,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;QACzB,MAAM,IAAI,KAAK,CACb,yBAAyB,IAAI,sBAAsB,GAAG,CAAC,GAAG,gBAAgB,GAAG,CAAC,QAAQ,oBAAoB,IAAI,EAAE,CACjH,CAAC;IACJ,CAAC;IACD,MAAM,MAAM,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;IAChC,IAAI,MAAM,KAAK,GAAG,CAAC,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;QAC5C,MAAM,IAAI,KAAK,CACb,6BAA6B,IAAI,cAAc,GAAG,CAAC,MAAM,cAAc,MAAM,+DAA+D,CAC7I,CAAC;IACJ,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,IAAY;IACnC,OAAO,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;AACnD,CAAC"}

package/dist/src/bench/split.d.ts

@@ -0,0 +1,33 @@
+/**
+ * Deterministic evaluation split.
+ *
+ * The split is derived from a SHA-256 of the case id and never from the case content,
+ * question type, or result. Sorting by that hash and cutting at the midpoint produces an
+ * exactly balanced 60/60 split for a 120-case slice and is stable across machines, Node
+ * versions, and runs.
+ *
+ * Tuning happens on `dev` only. The `test` half is reported but never used to choose a
+ * default, a threshold, or a fusion weight.
+ */
+export type BenchSplitName = 'dev' | 'test';
+export interface SplitCase {
+    id: string;
+}
+export declare function splitKey(id: string): string;
+/**
+ * Partitions cases into dev/test halves. The ordering of the input never matters: both
+ * halves are returned in the input order, but membership is decided by the hash rank.
+ */
+export declare function splitCases<T extends SplitCase>(cases: T[], devFraction?: number): {
+    dev: T[];
+    test: T[];
+};
+export declare function selectSplit<T extends SplitCase>(cases: T[], split: BenchSplitName | 'all'): T[];
+/**
+ * Takes the first `perType` cases of every category, in dataset order. This is the
+ * documented slice rule: first 20 answerable questions per question_type x 6 types.
+ */
+export declare function slicePerType<T extends {
+    category: string;
+}>(cases: T[], perType: number): T[];
+//# sourceMappingURL=split.d.ts.map

package/dist/src/bench/split.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"split.d.ts","sourceRoot":"","sources":["../../../src/bench/split.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAIH,MAAM,MAAM,cAAc,GAAG,KAAK,GAAG,MAAM,CAAC;AAE5C,MAAM,WAAW,SAAS;IACxB,EAAE,EAAE,MAAM,CAAC;CACZ;AAED,wBAAgB,QAAQ,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,CAE3C;AAED;;;GAGG;AACH,wBAAgB,UAAU,CAAC,CAAC,SAAS,SAAS,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE,WAAW,SAAM,GAAG;IAC9E,GAAG,EAAE,CAAC,EAAE,CAAC;IACT,IAAI,EAAE,CAAC,EAAE,CAAC;CACX,CAaA;AAED,wBAAgB,WAAW,CAAC,CAAC,SAAS,SAAS,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE,KAAK,EAAE,cAAc,GAAG,KAAK,GAAG,CAAC,EAAE,CAG/F;AAED;;;GAGG;AACH,wBAAgB,YAAY,CAAC,CAAC,SAAS;IAAE,QAAQ,EAAE,MAAM,CAAA;CAAE,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE,OAAO,EAAE,MAAM,GAAG,CAAC,EAAE,CAU7F"}

package/dist/src/bench/split.js

@@ -0,0 +1,56 @@
+/**
+ * Deterministic evaluation split.
+ *
+ * The split is derived from a SHA-256 of the case id and never from the case content,
+ * question type, or result. Sorting by that hash and cutting at the midpoint produces an
+ * exactly balanced 60/60 split for a 120-case slice and is stable across machines, Node
+ * versions, and runs.
+ *
+ * Tuning happens on `dev` only. The `test` half is reported but never used to choose a
+ * default, a threshold, or a fusion weight.
+ */
+import { createHash } from 'node:crypto';
+export function splitKey(id) {
+    return createHash('sha256').update(id, 'utf8').digest('hex');
+}
+/**
+ * Partitions cases into dev/test halves. The ordering of the input never matters: both
+ * halves are returned in the input order, but membership is decided by the hash rank.
+ */
+export function splitCases(cases, devFraction = 0.5) {
+    if (cases.length === 0)
+        return { dev: [], test: [] };
+    const ranked = [...cases].sort((a, b) => {
+        const ka = splitKey(a.id);
+        const kb = splitKey(b.id);
+        return ka < kb ? -1 : ka > kb ? 1 : 0;
+    });
+    const devCount = Math.round(ranked.length * devFraction);
+    const devIds = new Set(ranked.slice(0, devCount).map((item) => item.id));
+    return {
+        dev: cases.filter((item) => devIds.has(item.id)),
+        test: cases.filter((item) => !devIds.has(item.id))
+    };
+}
+export function selectSplit(cases, split) {
+    if (split === 'all')
+        return cases;
+    return splitCases(cases)[split];
+}
+/**
+ * Takes the first `perType` cases of every category, in dataset order. This is the
+ * documented slice rule: first 20 answerable questions per question_type x 6 types.
+ */
+export function slicePerType(cases, perType) {
+    const counts = new Map();
+    const selected = [];
+    for (const item of cases) {
+        const current = counts.get(item.category) ?? 0;
+        if (current >= perType)
+            continue;
+        counts.set(item.category, current + 1);
+        selected.push(item);
+    }
+    return selected;
+}
+//# sourceMappingURL=split.js.map

package/dist/src/bench/split.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"split.js","sourceRoot":"","sources":["../../../src/bench/split.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAQzC,MAAM,UAAU,QAAQ,CAAC,EAAU;IACjC,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC/D,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,UAAU,CAAsB,KAAU,EAAE,WAAW,GAAG,GAAG;IAI3E,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,GAAG,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC;IACrD,MAAM,MAAM,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACtC,MAAM,EAAE,GAAG,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAC1B,MAAM,EAAE,GAAG,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAC1B,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACxC,CAAC,CAAC,CAAC;IACH,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,GAAG,WAAW,CAAC,CAAC;IACzD,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;IACzE,OAAO;QACL,GAAG,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAChD,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;KACnD,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,WAAW,CAAsB,KAAU,EAAE,KAA6B;IACxF,IAAI,KAAK,KAAK,KAAK;QAAE,OAAO,KAAK,CAAC;IAClC,OAAO,UAAU,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC;AAClC,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,YAAY,CAAiC,KAAU,EAAE,OAAe;IACtF,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IACzC,MAAM,QAAQ,GAAQ,EAAE,CAAC;IACzB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,OAAO,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC/C,IAAI,OAAO,IAAI,OAAO;YAAE,SAAS;QACjC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,GAAG,CAAC,CAAC,CAAC;QACvC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACtB,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC"}

package/dist/src/bench/stats.d.ts

@@ -0,0 +1,40 @@
+/**
+ * Paired statistics for retrieval benchmarks.
+ *
+ * Every published delta in this repo is reported with a confidence interval, and every
+ * strict-recall comparison uses an exact McNemar test, because the two engines are run on
+ * the same cases and the correct comparison is the discordant pairs, not two independent
+ * proportions.
+ */
+/** Wilson score interval for a binomial proportion, in percentage points. */
+export declare function wilson(successes: number, n: number, z?: number): [number, number];
+/**
+ * Two-sided exact McNemar test on discordant pairs.
+ * `b` = baseline correct / candidate wrong, `c` = baseline wrong / candidate correct.
+ * Returns the exact binomial p-value; when b + c is 0 the result is p = 1.
+ */
+export declare function mcnemarExact(b: number, c: number): number;
+/**
+ * Paired bootstrap over per-case differences.
+ * Resamples case indices with replacement, so the interval reflects case-level variance
+ * rather than the variance of two independent samples.
+ */
+export declare function pairedBootstrap(diffs: number[], options?: {
+    iterations?: number;
+    seed?: number;
+}): {
+    mean: number;
+    low: number;
+    high: number;
+    iterations: number;
+};
+/** Convenience wrapper: paired comparison of two per-case 0/1 vectors keyed by case id. */
+export declare function pairedCompare(baseline: Map<string, number>, candidate: Map<string, number>): {
+    n: number;
+    fixed: number;
+    broken: number;
+    unchanged: number;
+    mcnemarP: number;
+    bootstrap: ReturnType<typeof pairedBootstrap>;
+};
+//# sourceMappingURL=stats.d.ts.map

package/dist/src/bench/stats.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"stats.d.ts","sourceRoot":"","sources":["../../../src/bench/stats.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,6EAA6E;AAC7E,wBAAgB,MAAM,CAAC,SAAS,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,SAAO,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAU/E;AAQD;;;;GAIG;AACH,wBAAgB,YAAY,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM,CAOzD;AAED;;;;GAIG;AACH,wBAAgB,eAAe,CAC7B,KAAK,EAAE,MAAM,EAAE,EACf,OAAO,GAAE;IAAE,UAAU,CAAC,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAA;CAAO,GACnD;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,CAwBjE;AAED,2FAA2F;AAC3F,wBAAgB,aAAa,CAC3B,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,EAC7B,SAAS,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,GAC7B;IAAE,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,UAAU,CAAC,OAAO,eAAe,CAAC,CAAA;CAAE,CAoBlI"}

package/dist/src/bench/stats.js

@@ -0,0 +1,99 @@
+/**
+ * Paired statistics for retrieval benchmarks.
+ *
+ * Every published delta in this repo is reported with a confidence interval, and every
+ * strict-recall comparison uses an exact McNemar test, because the two engines are run on
+ * the same cases and the correct comparison is the discordant pairs, not two independent
+ * proportions.
+ */
+/** Wilson score interval for a binomial proportion, in percentage points. */
+export function wilson(successes, n, z = 1.96) {
+    if (n === 0)
+        return [0, 0];
+    const p = successes / n;
+    const denom = 1 + (z * z) / n;
+    const center = (p + (z * z) / (2 * n)) / denom;
+    const spread = (z * Math.sqrt((p * (1 - p)) / n + (z * z) / (4 * n * n))) / denom;
+    return [
+        Math.round(Math.max(0, center - spread) * 1000) / 10,
+        Math.round(Math.min(1, center + spread) * 1000) / 10
+    ];
+}
+function logChoose(n, k) {
+    let sum = 0;
+    for (let i = 1; i <= k; i++)
+        sum += Math.log(n - k + i) - Math.log(i);
+    return sum;
+}
+/**
+ * Two-sided exact McNemar test on discordant pairs.
+ * `b` = baseline correct / candidate wrong, `c` = baseline wrong / candidate correct.
+ * Returns the exact binomial p-value; when b + c is 0 the result is p = 1.
+ */
+export function mcnemarExact(b, c) {
+    const n = b + c;
+    if (n === 0)
+        return 1;
+    const k = Math.min(b, c);
+    let tail = 0;
+    for (let i = 0; i <= k; i++)
+        tail += Math.exp(logChoose(n, i) - n * Math.LN2);
+    return Math.min(1, Math.round(tail * 2 * 1e6) / 1e6);
+}
+/**
+ * Paired bootstrap over per-case differences.
+ * Resamples case indices with replacement, so the interval reflects case-level variance
+ * rather than the variance of two independent samples.
+ */
+export function pairedBootstrap(diffs, options = {}) {
+    const iterations = options.iterations ?? 10_000;
+    if (diffs.length === 0)
+        return { mean: 0, low: 0, high: 0, iterations };
+    const mean = diffs.reduce((sum, value) => sum + value, 0) / diffs.length;
+    // Deterministic LCG so a published interval is reproducible without a seed file.
+    let state = options.seed ?? 0x9e3779b9;
+    const next = () => {
+        state = (state * 1664525 + 1013904223) >>> 0;
+        return state / 0x100000000;
+    };
+    const means = [];
+    for (let i = 0; i < iterations; i++) {
+        let sum = 0;
+        for (let j = 0; j < diffs.length; j++)
+            sum += diffs[Math.floor(next() * diffs.length)];
+        means.push(sum / diffs.length);
+    }
+    means.sort((a, b) => a - b);
+    const at = (q) => means[Math.min(means.length - 1, Math.max(0, Math.floor(q * means.length)))];
+    return {
+        mean: Math.round(mean * 1000) / 1000,
+        low: Math.round(at(0.025) * 1000) / 1000,
+        high: Math.round(at(0.975) * 1000) / 1000,
+        iterations
+    };
+}
+/** Convenience wrapper: paired comparison of two per-case 0/1 vectors keyed by case id. */
+export function pairedCompare(baseline, candidate) {
+    const ids = [...baseline.keys()].filter((id) => candidate.has(id));
+    let fixed = 0;
+    let broken = 0;
+    const diffs = [];
+    for (const id of ids) {
+        const a = baseline.get(id);
+        const b = candidate.get(id);
+        if (a === 0 && b === 1)
+            fixed++;
+        else if (a === 1 && b === 0)
+            broken++;
+        diffs.push(b - a);
+    }
+    return {
+        n: ids.length,
+        fixed,
+        broken,
+        unchanged: ids.length - fixed - broken,
+        mcnemarP: mcnemarExact(broken, fixed),
+        bootstrap: pairedBootstrap(diffs)
+    };
+}
+//# sourceMappingURL=stats.js.map

package/dist/src/bench/stats.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"stats.js","sourceRoot":"","sources":["../../../src/bench/stats.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,6EAA6E;AAC7E,MAAM,UAAU,MAAM,CAAC,SAAiB,EAAE,CAAS,EAAE,CAAC,GAAG,IAAI;IAC3D,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC3B,MAAM,CAAC,GAAG,SAAS,GAAG,CAAC,CAAC;IACxB,MAAM,KAAK,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;IAC9B,MAAM,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC;IAC/C,MAAM,MAAM,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC;IAClF,OAAO;QACL,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE;QACpD,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE;KACrD,CAAC;AACJ,CAAC;AAED,SAAS,SAAS,CAAC,CAAS,EAAE,CAAS;IACrC,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE;QAAE,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IACtE,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,YAAY,CAAC,CAAS,EAAE,CAAS;IAC/C,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAChB,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACtB,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACzB,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE;QAAE,IAAI,IAAI,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC;IAC9E,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,GAAG,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC;AACvD,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,eAAe,CAC7B,KAAe,EACf,UAAkD,EAAE;IAEpD,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,MAAM,CAAC;IAChD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,IAAI,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC;IACxE,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC;IACzE,iFAAiF;IACjF,IAAI,KAAK,GAAG,OAAO,CAAC,IAAI,IAAI,UAAU,CAAC;IACvC,MAAM,IAAI,GAAG,GAAW,EAAE;QACxB,KAAK,GAAG,CAAC,KAAK,GAAG,OAAO,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;QAC7C,OAAO,KAAK,GAAG,WAAW,CAAC;IAC7B,CAAC,CAAC;IACF,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;QACpC,IAAI,GAAG,GAAG,CAAC,CAAC;QACZ,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE;YAAE,GAAG,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;QACvF,KAAK,CAAC,IAAI,CAAC,GAAG,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;IACjC,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC5B,MAAM,EAAE,GAAG,CAAC,CAAS,EAAU,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/G,OAAO;QACL,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,GAAG,IAAI;QACpC,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,GAAG,IAAI;QACxC,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,GAAG,IAAI;QACzC,UAAU;KACX,CAAC;AACJ,CAAC;AAED,2FAA2F;AAC3F,MAAM,UAAU,aAAa,CAC3B,QAA6B,EAC7B,SAA8B;IAE9B,MAAM,GAAG,GAAG,CAAC,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;IACnE,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,EAAE,IAAI,GAAG,EAAE,CAAC;QACrB,MAAM,CAAC,GAAG,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAE,CAAC;QAC5B,MAAM,CAAC,GAAG,SAAS,CAAC,GAAG,CAAC,EAAE,CAAE,CAAC;QAC7B,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC;YAAE,KAAK,EAAE,CAAC;aAC3B,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC;YAAE,MAAM,EAAE,CAAC;QACtC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACpB,CAAC;IACD,OAAO;QACL,CAAC,EAAE,GAAG,CAAC,MAAM;QACb,KAAK;QACL,MAAM;QACN,SAAS,EAAE,GAAG,CAAC,MAAM,GAAG,KAAK,GAAG,MAAM;QACtC,QAAQ,EAAE,YAAY,CAAC,MAAM,EAAE,KAAK,CAAC;QACrC,SAAS,EAAE,eAAe,CAAC,KAAK,CAAC;KAClC,CAAC;AACJ,CAAC"}

package/dist/src/bench/vs-engines.d.ts

@@ -0,0 +1,51 @@
+/**
+ * Engine descriptors for the cross-engine comparison.
+ *
+ * Each descriptor pins a package version, states how to start it over MCP stdio, how to
+ * isolate its on-disk state, and how to map one marker document and one query onto its
+ * tools. Nothing engine-specific lives outside this file.
+ *
+ * Isolation was verified by probing, not assumed:
+ *   - fastrmemory honours `FASTR_HOME` for the global database and `--project` for the
+ *     project database.
+ *   - agentmemory ignores `AGENTMEMORY_DATA_DIR` and resolves `homedir()/.agentmemory`,
+ *     so `USERPROFILE` and `HOME` both point at a fresh directory per case.
+ *   - hippo's MCP server ignores `HIPPO_HOME` and writes to `~/.hippo` unless the working
+ *     directory contains a `.hippo` store, so each case runs `hippo init` in its own
+ *     project directory first and also gets a fresh home.
+ */
+import type { MarkerDoc } from './marker-corpus.js';
+import type { McpServerSpec } from './mcp-stdio.js';
+export interface CaseDirs {
+    /** Root of the throwaway directory tree for this engine run. */
+    root: string;
+    /** Project directory handed to the engine. */
+    project: string;
+    /** Fresh home directory, so nothing escapes into the real user profile. */
+    home: string;
+}
+export interface VsEngine {
+    id: string;
+    label: string;
+    /** Pinned version, reported in the artifact. */
+    version: string;
+    /** Absolute path of the JavaScript entry point that is launched with `node`. */
+    entry: string;
+    bootstrap: (dirs: CaseDirs) => Promise<void>;
+    spec: (dirs: CaseDirs) => McpServerSpec;
+    rememberTool: string;
+    rememberArgs: (doc: MarkerDoc) => Record<string, unknown>;
+    recallTool: string;
+    recallArgs: (query: string, budgetTokens: number) => Record<string, unknown>;
+    /** True when the engine needs a project directory that already exists. */
+    needsProjectDir: boolean;
+}
+export interface EngineEnvironment {
+    repoRoot: string;
+    /** Directory holding the pinned third-party installs. */
+    externalDir: string;
+}
+export declare function createVsEngines(environment: EngineEnvironment): VsEngine[];
+/** Fails early with an actionable message instead of timing out inside the MCP handshake. */
+export declare function assertEngineAvailable(engine: VsEngine, externalDir: string): void;
+//# sourceMappingURL=vs-engines.d.ts.map

package/dist/src/bench/vs-engines.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"vs-engines.d.ts","sourceRoot":"","sources":["../../../src/bench/vs-engines.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAKH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AACpD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAEpD,MAAM,WAAW,QAAQ;IACvB,gEAAgE;IAChE,IAAI,EAAE,MAAM,CAAC;IACb,8CAA8C;IAC9C,OAAO,EAAE,MAAM,CAAC;IAChB,2EAA2E;IAC3E,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,QAAQ;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,gDAAgD;IAChD,OAAO,EAAE,MAAM,CAAC;IAChB,gFAAgF;IAChF,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,CAAC,IAAI,EAAE,QAAQ,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAC7C,IAAI,EAAE,CAAC,IAAI,EAAE,QAAQ,KAAK,aAAa,CAAC;IACxC,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,CAAC,GAAG,EAAE,SAAS,KAAK,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC1D,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,KAAK,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC7E,0EAA0E;IAC1E,eAAe,EAAE,OAAO,CAAC;CAC1B;AAED,MAAM,WAAW,iBAAiB;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,yDAAyD;IACzD,WAAW,EAAE,MAAM,CAAC;CACrB;AAOD,wBAAgB,eAAe,CAAC,WAAW,EAAE,iBAAiB,GAAG,QAAQ,EAAE,CAyF1E;AAED,6FAA6F;AAC7F,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,QAAQ,EAAE,WAAW,EAAE,MAAM,GAAG,IAAI,CAQjF"}