cognitive-core 0.2.4 → 0.3.1

package/dist/eval/memory-qa/loaders.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"loaders.js","sourceRoot":"","sources":["../../../src/eval/memory-qa/loaders.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,UAAU,EAAE,eAAe,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAE/F,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAG7D,OAAO,EAAE,UAAU,EAAE,eAAe,EAAE,YAAY,EAAE,mBAAmB,EAAE,CAAC;AAE1E;;;;GAIG;AACH,MAAM,UAAU,eAAe,CAAC,QAAuB;IACrD,OAAO,mBAAmB,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAC/C,iBAAiB,CAAC;QAChB,EAAE,EAAE,GAAG,CAAC,EAAE;QACV,KAAK,EAAE,GAAG,CAAC,MAAM;QACjB,IAAI,EAAE,GAAG,CAAC,IAAI;QACd,MAAM,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC;QACxB,QAAQ,EAAE,EAAE;QACZ,IAAI,EAAE,CAAC,GAAG,CAAC,SAAS,CAAC;QACrB,UAAU,EAAE,GAAG;QACf,MAAM,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE;KAC/B,CAAC,CACH,CAAC;AACJ,CAAC"}

package/dist/eval/memory-qa/metrics.d.ts

@@ -0,0 +1,3 @@
+export { normalizeAnswer, f1Score, bleu1, rougeL, overlapScores } from 'swarmkit-eval';
+export type { OverlapScores } from 'swarmkit-eval';
+//# sourceMappingURL=metrics.d.ts.map

package/dist/eval/memory-qa/metrics.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../../../src/eval/memory-qa/metrics.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AACvF,YAAY,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC"}

package/dist/eval/memory-qa/metrics.js

@@ -0,0 +1,2 @@
+export { normalizeAnswer, f1Score, bleu1, rougeL, overlapScores } from 'swarmkit-eval';
+//# sourceMappingURL=metrics.js.map

package/dist/eval/memory-qa/metrics.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"metrics.js","sourceRoot":"","sources":["../../../src/eval/memory-qa/metrics.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC"}

package/dist/eval/memory-qa/qa.d.ts

@@ -0,0 +1,186 @@
+import type { SearchProvider } from '../../memory/search-provider.js';
+import type { MemQAInstance, MemQuestion } from './types.js';
+import { type IterativeConfig, type CompleteFn } from './iterative.js';
+import { type MemoryQAServingGate, type MemoryQAServingQuestionDecision } from './serving.js';
+/**
+ * Phase B — end-to-end QA over retrieved memory (LLM read + judge).
+ *
+ * For each answerable question: retrieve top-k notes with a {@link SearchProvider},
+ * have the model READ them and answer, then JUDGE the answer against gold. Run
+ * with two retrievers (e.g. lexical vs hybrid) to test whether the Phase-A
+ * recall gain *converts* to answer-accuracy — the rung-3 of the proxy→outcome
+ * ladder. Costs agent budget (≈1 read + ≤1 judge call per question per arm), so
+ * it's gated. See docs/DESIGN-memory-qa-benchmarks.md §4 (Phase B).
+ *
+ * Budget savers: abstention is detected by a `NO_ANSWER` sentinel (no judge
+ * call); a correct answer that literally contains the gold short-circuits the
+ * judge (high-precision positive). Only ambiguous cases hit the LLM judge.
+ */
+/** Minimal LLM hook: prompt in, completion text out. */
+export type { CompleteFn } from './iterative.js';
+export interface QARecord {
+    id: string;
+    category: string;
+    /** Question text — kept so a report can be re-judged post-hoc (J metric). */
+    question: string;
+    answer: string;
+    /** Gold reference answer (for post-hoc word-overlap scoring). */
+    gold: string;
+    correct: boolean;
+    judgedBy: 'contains' | 'llm' | 'abstain-sentinel' | 'overlap' | 'mem0-judge' | 'error';
+    /** Retrieval rounds run (iterative mode only) — for cost accounting. */
+    iterations?: number;
+}
+export interface QAReport {
+    provider: string;
+    k: number;
+    overall: {
+        accuracy: number;
+        n: number;
+    };
+    byCategory: Record<string, {
+        accuracy: number;
+        n: number;
+    }>;
+    records: QARecord[];
+}
+/**
+ * True when an answer is a refusal / "no information" response. For LoCoMo
+ * category 5 (adversarial) the `adversarial_answer` is a planted misattribution
+ * distractor (e.g. asks what *Caroline* realized when the transcript shows
+ * *Melanie* said it), so the CORRECT behavior is to refuse — and refusing scores
+ * 0 under both token-F1 and the J judge (both match the distractor as "gold").
+ * Refusal accuracy is therefore the meaningful cat-5 metric: high = robust to the
+ * bait. See docs/DESIGN-memory-qa-benchmarks.md §4h.
+ */
+export declare function isRefusal(answer: string): boolean;
+/** Fraction of a category's answers that are refusals (= correct on adversarial). */
+export declare function refusalAccuracy(report: QAReport, category: string): {
+    accuracy: number;
+    n: number;
+};
+/** Run the mem0 J-judge for one (question, gold, answer); true = CORRECT. */
+export declare function judgeCorrect(complete: CompleteFn, question: string, gold: string, answer: string): Promise<boolean>;
+/** A question paired with the instance whose haystack it queries. */
+export interface SampledQuestion {
+    instance: MemQAInstance;
+    question: MemQuestion;
+}
+/** Deterministic strided down-sample so both A/B arms score the identical set. */
+export declare function sampleQuestions(instances: MemQAInstance[], maxQuestions?: number): SampledQuestion[];
+/**
+ * Category-stratified sample: up to `perCategory` answerable questions per
+ * `question.category`, strided within each category for spread. Gives
+ * representative coverage across question types (the first-N-instances sampler
+ * over type-grouped data collapses to one category).
+ */
+export declare function sampleStratified(instances: MemQAInstance[], perCategory: number, opts?: {
+    includeAbstain?: boolean;
+    categoryOffset?: number;
+}): SampledQuestion[];
+/** Run end-to-end QA for one retriever over a (sampled) question set. */
+export declare function runMemoryQA(instances: MemQAInstance[], retriever: SearchProvider, complete: CompleteFn, opts?: {
+    k?: number;
+    maxQuestions?: number;
+    /** Explicit pre-sampled question set — pass the SAME list to both arms for a paired A/B. */
+    questions?: SampledQuestion[];
+    /**
+     * 'judge' (default): LLM-judge correctness (conversion A/B).
+     * 'overlap': natural answer + record gold, NO judge call — for word-overlap
+     * (F1/BLEU-1) scoring vs A-MEM. Halves the call count (read only).
+     */
+    scoreMode?: 'judge' | 'overlap';
+    /**
+     * Custom answer-generation prompt (e.g. to match A-MEM's category-specific
+     * F1-optimized prompts). When set, overrides the default read prompt — the
+     * fair way to compare memory/retrieval without answer-format confounds.
+     */
+    buildPrompt?: (question: MemQuestion, excerpts: string[]) => string;
+    /**
+     * A-MEM's `retrieve_memory_llm` re-rank: after retrieval, an LLM distills the
+     * top-k excerpts to "the relevant parts" before the answer read (+1 call/q).
+     */
+    rerank?: boolean;
+    /**
+     * In `overlap` mode, also score each answer with the mem0 LLM-judge ("J")
+     * via this LLM (so a report carries BOTH word-overlap F1/BLEU and J accuracy).
+     * Pass a stronger model than the reader to reduce self-judging bias.
+     */
+    judge?: CompleteFn;
+    /**
+     * Iterative reflective-retrieval (MemR3-style closed loop) instead of
+     * single-shot top-k. Same retriever + answer prompt, so an A/B vs the
+     * default isolates the loop. Takes precedence over `rerank`. See §4i.
+     */
+    iterative?: IterativeConfig;
+    /**
+     * Process up to N questions concurrently (default 1 = sequential). Safe for
+     * HTTP backends (Azure OpenAI) that handle parallel requests; keep at 1 for
+     * the local `claude` CLI. Per-question work (and the iterative loop within a
+     * question) stays sequential — only DIFFERENT questions run in parallel.
+     */
+    concurrency?: number;
+    /**
+     * Optional eval-only post-retrieval serving gate. The default path is
+     * unchanged: top-k hits are all served to the reader. When set, the gate
+     * decides which retrieved candidates are actually shown.
+     */
+    servingGate?: MemoryQAServingGate;
+    /** Capture per-question served/withheld diagnostics for gate experiments. */
+    onServingDecision?: (decision: MemoryQAServingQuestionDecision) => void;
+    onProgress?: (done: number, total: number) => void;
+}): Promise<QAReport>;
+/**
+ * Re-derive a report over a subset of question ids. Used to drop questions that
+ * ERRORED (transient LLM failure, `judgedBy: 'error'`) from BOTH arms of a paired
+ * A/B, so the comparison stays on a common, fully-scored question set.
+ */
+export declare function filterReport(report: QAReport, keep: Set<string>): QAReport;
+export interface QACompare {
+    baseline: string;
+    treatment: string;
+    baselineAccuracy: number;
+    treatmentAccuracy: number;
+    delta: number;
+    n: number;
+}
+/** Accuracy A/B between two QA reports (same question set). */
+export declare function compareQA(baseline: QAReport, treatment: QAReport): QACompare;
+export interface PairedAccuracy {
+    /** Treatment fixed a baseline miss (baseline wrong → treatment right). */
+    b: number;
+    /** Treatment broke a baseline hit (baseline right → treatment wrong). */
+    c: number;
+    delta: number;
+    /** McNemar χ² (Yates continuity-corrected), 1 df. */
+    chi2: number;
+    /** χ² > 3.841 ⇒ paired difference significant at p<0.05. */
+    significant: boolean;
+    n: number;
+}
+/**
+ * McNemar's paired test on per-question correctness (records matched by id).
+ * The correct test for paired binary outcomes — only the *discordant* pairs
+ * (one arm right, the other wrong) carry signal.
+ */
+export declare function pairedAccuracy(baseline: QAReport, treatment: QAReport): PairedAccuracy;
+export declare function formatQA(r: QAReport): string;
+export declare function formatQACompare(c: QACompare): string;
+export interface OverlapCategoryScore {
+    f1: number;
+    bleu1: number;
+    rougeL: number;
+    n: number;
+}
+export interface OverlapReport {
+    byCategory: Record<string, OverlapCategoryScore>;
+    overall: OverlapCategoryScore;
+}
+/**
+ * Post-hoc word-overlap scores from a QA report's (answer, gold) pairs, grouped
+ * by category — the A-MEM metric family. Deterministic, no extra LLM calls.
+ */
+export declare function overlapByCategory(report: QAReport): OverlapReport;
+/** F1×100 table by category (A-MEM reports F1/BLEU-1 on a 0–100 scale). */
+export declare function formatOverlap(r: OverlapReport, categoryNames?: Record<string, string>): string;
+//# sourceMappingURL=qa.d.ts.map

package/dist/eval/memory-qa/qa.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"qa.d.ts","sourceRoot":"","sources":["../../../src/eval/memory-qa/qa.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,iCAAiC,CAAC;AAEtE,OAAO,KAAK,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAE7D,OAAO,EAAyB,KAAK,eAAe,EAAE,KAAK,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAC9F,OAAO,EAIL,KAAK,mBAAmB,EACxB,KAAK,+BAA+B,EACrC,MAAM,cAAc,CAAC;AAEtB;;;;;;;;;;;;;GAaG;AAEH,wDAAwD;AACxD,YAAY,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAIjD,MAAM,WAAW,QAAQ;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,6EAA6E;IAC7E,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,iEAAiE;IACjE,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,OAAO,CAAC;IACjB,QAAQ,EAAE,UAAU,GAAG,KAAK,GAAG,kBAAkB,GAAG,SAAS,GAAG,YAAY,GAAG,OAAO,CAAC;IACvF,wEAAwE;IACxE,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,QAAQ;IACvB,QAAQ,EAAE,MAAM,CAAC;IACjB,CAAC,EAAE,MAAM,CAAC;IACV,OAAO,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IACzC,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC5D,OAAO,EAAE,QAAQ,EAAE,CAAC;CACrB;AA6ED;;;;;;;;GAQG;AACH,wBAAgB,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAIjD;AAED,qFAAqF;AACrF,wBAAgB,eAAe,CAC7B,MAAM,EAAE,QAAQ,EAChB,QAAQ,EAAE,MAAM,GACf;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,CAAC,EAAE,MAAM,CAAA;CAAE,CAIjC;AAED,6EAA6E;AAC7E,wBAAsB,YAAY,CAChC,QAAQ,EAAE,UAAU,EACpB,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,OAAO,CAAC,CASlB;AAED,qEAAqE;AACrE,MAAM,WAAW,eAAe;IAC9B,QAAQ,EAAE,aAAa,CAAC;IACxB,QAAQ,EAAE,WAAW,CAAC;CACvB;AAiBD,kFAAkF;AAClF,wBAAgB,eAAe,CAC7B,SAAS,EAAE,aAAa,EAAE,EAC1B,YAAY,CAAC,EAAE,MAAM,GACpB,eAAe,EAAE,CAOnB;AAED;;;;;GAKG;AACH,wBAAgB,gBAAgB,CAC9B,SAAS,EAAE,aAAa,EAAE,EAC1B,WAAW,EAAE,MAAM,EACnB,IAAI,GAAE;IAAE,cAAc,CAAC,EAAE,OAAO,CAAC;IAAC,cAAc,CAAC,EAAE,MAAM,CAAA;CAAO,GAC/D,eAAe,EAAE,CAyBnB;AAED,yEAAyE;AACzE,wBAAsB,WAAW,CAC/B,SAAS,EAAE,aAAa,EAAE,EAC1B,SAAS,EAAE,cAAc,EACzB,QAAQ,EAAE,UAAU,EACpB,IAAI,GAAE;IACJ,CAAC,CAAC,EAAE,MAAM,CAAC;IACX,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,4FAA4F;IAC5F,SAAS,CAAC,EAAE,eAAe,EAAE,CAAC;IAC9B;;;;OAIG;IACH,SAAS,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAChC;;;;OAIG;IACH,WAAW,CAAC,EAAE,CAAC,QAAQ,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,MAAM,CAAC;IACpE;;;OAGG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB;;;;OAIG;IACH,KAAK,CAAC,EAAE,UAAU,CAAC;IACnB;;;;OAIG;IACH,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B;;;;;OAKG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;OAIG;IACH,WAAW,CAAC,EAAE,mBAAmB,CAAC;IAClC,6EAA6E;IAC7E,iBAAiB,CAAC,EAAE,CAAC,QAAQ,EAAE,+BAA+B,KAAK,IAAI,CAAC;IACxE,UAAU,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;CAC/C,GACL,OAAO,CAAC,QAAQ,CAAC,CA0InB;AAuBD;;;;GAIG;AACH,wBAAgB,YAAY,CAAC,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,QAAQ,CAM1E;AAED,MAAM,WAAW,SAAS;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,CAAC;IACzB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,KAAK,EAAE,MAAM,CAAC;IACd,CAAC,EAAE,MAAM,CAAC;CACX;AAED,+DAA+D;AAC/D,wBAAgB,SAAS,CAAC,QAAQ,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,GAAG,SAAS,CAS5E;AAED,MAAM,WAAW,cAAc;IAC7B,0EAA0E;IAC1E,CAAC,EAAE,MAAM,CAAC;IACV,yEAAyE;IACzE,CAAC,EAAE,MAAM,CAAC;IACV,KAAK,EAAE,MAAM,CAAC;IACd,qDAAqD;IACrD,IAAI,EAAE,MAAM,CAAC;IACb,4DAA4D;IAC5D,WAAW,EAAE,OAAO,CAAC;IACrB,CAAC,EAAE,MAAM,CAAC;CACX;AAED;;;;GAIG;AACH,wBAAgB,cAAc,CAAC,QAAQ,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,GAAG,cAAc,CAsBtF;AAID,wBAAgB,QAAQ,CAAC,CAAC,EAAE,QAAQ,GAAG,MAAM,CAU5C;AAED,wBAAgB,eAAe,CAAC,CAAC,EAAE,SAAS,GAAG,MAAM,CAMpD;AAMD,MAAM,WAAW,oBAAoB;IACnC,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,CAAC,EAAE,MAAM,CAAC;CACX;AAED,MAAM,WAAW,aAAa;IAC5B,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,oBAAoB,CAAC,CAAC;IACjD,OAAO,EAAE,oBAAoB,CAAC;CAC/B;AAED;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,QAAQ,GAAG,aAAa,CA0BjE;AAED,2EAA2E;AAC3E,wBAAgB,aAAa,CAAC,CAAC,EAAE,aAAa,EAAE,aAAa,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,MAAM,CAkB9F"}

package/dist/eval/memory-qa/qa.js

@@ -0,0 +1,424 @@
+import { instanceToNotes } from './loaders.js';
+import { overlapScores } from './metrics.js';
+import { runIterativeRetrieval } from './iterative.js';
+import { searchHitsToGateCandidates, serveAllMemoryQAGate, toServingDecision, } from './serving.js';
+const NO_ANSWER = 'NO_ANSWER';
+function normalize(s) {
+    return String(s ?? '')
+        .toLowerCase()
+        .replace(/[^a-z0-9 ]+/g, ' ')
+        .replace(/\s+/g, ' ')
+        .trim();
+}
+function buildReadPrompt(question, excerpts, natural = false) {
+    const numbered = excerpts.map((e, i) => `${i + 1}. ${e}`).join('\n');
+    // `natural` (parity mode): no NO_ANSWER sentinel — answer as naturally as the
+    // A-MEM QA setup, so word-overlap (F1/BLEU-1) is scored on a real answer.
+    const instruction = natural
+        ? 'Answer the question in one short phrase, based on the conversation memory above.'
+        : `Answer in one short phrase. If the excerpts do not contain the answer, reply exactly: ${NO_ANSWER}`;
+    return [
+        'You are answering a question using ONLY the memory excerpts below (from a past conversation).',
+        '',
+        'Memory excerpts:',
+        numbered,
+        '',
+        `Question: ${question}`,
+        '',
+        instruction,
+    ].join('\n');
+}
+/** A-MEM's retrieve_memory_llm re-rank: distill retrieved memories to the relevant parts. */
+function buildRerankPrompt(question, excerpts) {
+    const numbered = excerpts.map((e, i) => `${i + 1}. ${e}`).join('\n');
+    return [
+        'Given the following conversation memories and a question, select the most relevant',
+        'parts of the conversation that would help answer the question. Include the date/time',
+        'if available.',
+        '',
+        `Memories:\n${numbered}`,
+        '',
+        `Question: ${question}`,
+        '',
+        'Return only the relevant parts of the conversation that would help answer this question.',
+    ].join('\n');
+}
+function buildJudgePrompt(question, gold, candidate) {
+    return [
+        'Judge whether the candidate answer is correct given the reference answer.',
+        'Minor wording or formatting differences are fine — judge semantic correctness.',
+        '',
+        `Question: ${question}`,
+        `Reference answer: ${gold}`,
+        `Candidate answer: ${candidate}`,
+        '',
+        'Reply with exactly one word: YES or NO.',
+    ].join('\n');
+}
+/**
+ * mem0's LLM-judge ("J") prompt verbatim — the LoCoMo-leaderboard metric used by
+ * Mem0 / MemR3. Deliberately GENEROUS (topic match counts; format/length/relative
+ * dates forgiven), so J ≫ token-F1. Lets our numbers line up with both the F1
+ * papers (A-MEM, MGRetrieval) and the J-leaderboard. See docs §4h.
+ */
+function buildMem0JudgePrompt(question, gold, candidate) {
+    return `Your task is to label an answer to a question as 'CORRECT' or 'WRONG'. You will be given:
+(1) a question, (2) a 'gold' (ground truth) answer, (3) a generated answer to score as CORRECT/WRONG.
+
+The gold answer is usually concise. The generated answer may be much longer — be GENEROUS: as long as it touches on the same topic / refers to the same thing as the gold answer, count it CORRECT. For time questions, differing formats or relative references that denote the same date/period are CORRECT (e.g. "May 7th" vs "7 May").
+
+Question: ${question}
+Gold answer: ${gold}
+Generated answer: ${candidate}
+
+Reply with exactly one word: CORRECT or WRONG.`;
+}
+/**
+ * True when an answer is a refusal / "no information" response. For LoCoMo
+ * category 5 (adversarial) the `adversarial_answer` is a planted misattribution
+ * distractor (e.g. asks what *Caroline* realized when the transcript shows
+ * *Melanie* said it), so the CORRECT behavior is to refuse — and refusing scores
+ * 0 under both token-F1 and the J judge (both match the distractor as "gold").
+ * Refusal accuracy is therefore the meaningful cat-5 metric: high = robust to the
+ * bait. See docs/DESIGN-memory-qa-benchmarks.md §4h.
+ */
+export function isRefusal(answer) {
+    return /not mentioned|no information|not in the conversation|cannot|can'?t|do(es)?n'?t (know|have|mention|appear|say|specify)|isn'?t (mentioned|available|specified)|unable to|not (specified|stated|provided|discussed|available|mentioned)/i.test(answer ?? '');
+}
+/** Fraction of a category's answers that are refusals (= correct on adversarial). */
+export function refusalAccuracy(report, category) {
+    const rs = report.records.filter((r) => r.category === category);
+    const refused = rs.filter((r) => isRefusal(r.answer)).length;
+    return { accuracy: rs.length ? refused / rs.length : 0, n: rs.length };
+}
+/** Run the mem0 J-judge for one (question, gold, answer); true = CORRECT. */
+export async function judgeCorrect(complete, question, gold, answer) {
+    try {
+        const out = (await complete(buildMem0JudgePrompt(question, gold, answer))).trim().toLowerCase();
+        // "wrong" checked first (a CORRECT/WRONG reply won't contain both per the prompt).
+        if (/\bwrong\b/.test(out))
+            return false;
+        return /\bcorrect\b/.test(out);
+    }
+    catch {
+        return false;
+    }
+}
+/** Answerable questions only (those with evidence) — abstention is a separate axis. */
+function answerable(q) {
+    return !q.abstain && (q.evidenceTurnIds.length > 0 || q.evidenceSessionIds.length > 0);
+}
+function allAnswerable(instances) {
+    const all = [];
+    for (const instance of instances) {
+        for (const question of instance.questions) {
+            if (answerable(question))
+                all.push({ instance, question });
+        }
+    }
+    return all;
+}
+/** Deterministic strided down-sample so both A/B arms score the identical set. */
+export function sampleQuestions(instances, maxQuestions) {
+    const all = allAnswerable(instances);
+    if (!maxQuestions || all.length <= maxQuestions)
+        return all;
+    const stride = all.length / maxQuestions;
+    const out = [];
+    for (let i = 0; i < maxQuestions; i++)
+        out.push(all[Math.floor(i * stride)]);
+    return out;
+}
+/**
+ * Category-stratified sample: up to `perCategory` answerable questions per
+ * `question.category`, strided within each category for spread. Gives
+ * representative coverage across question types (the first-N-instances sampler
+ * over type-grouped data collapses to one category).
+ */
+export function sampleStratified(instances, perCategory, opts = {}) {
+    const source = opts.includeAbstain
+        ? instances.flatMap((instance) => instance.questions.map((question) => ({ instance, question })))
+        : allAnswerable(instances);
+    const byCat = new Map();
+    for (const sq of source) {
+        const arr = byCat.get(sq.question.category) ?? [];
+        arr.push(sq);
+        byCat.set(sq.question.category, arr);
+    }
+    const out = [];
+    for (const arr of byCat.values()) {
+        const take = Math.min(perCategory, arr.length);
+        if (take <= 0)
+            continue;
+        if (opts.categoryOffset !== undefined) {
+            const offset = Math.max(0, Math.floor(opts.categoryOffset));
+            out.push(...arr.slice(offset, offset + take));
+            continue;
+        }
+        const stride = arr.length / take;
+        for (let i = 0; i < take; i++)
+            out.push(arr[Math.floor(i * stride)]);
+    }
+    return out;
+}
+/** Run end-to-end QA for one retriever over a (sampled) question set. */
+export async function runMemoryQA(instances, retriever, complete, opts = {}) {
+    const k = opts.k ?? 5;
+    const overlapMode = opts.scoreMode === 'overlap';
+    const sample = opts.questions ?? sampleQuestions(instances, opts.maxQuestions);
+    const concurrency = Math.max(1, Math.floor(opts.concurrency ?? 1));
+    if (opts.iterative && (opts.servingGate || opts.onServingDecision)) {
+        throw new Error('memory-QA serving gates are only supported for single-shot retrieval; disable iterative mode for gate experiments.');
+    }
+    // Pre-build notes (+ id index) per instance once — avoids redundant builds and
+    // races when questions run concurrently. instanceToNotes is pure, cheap.
+    const notesByInstance = new Map();
+    const noteByIdByInstance = new Map();
+    for (const { instance } of sample) {
+        if (notesByInstance.has(instance.id))
+            continue;
+        const notes = instanceToNotes(instance);
+        notesByInstance.set(instance.id, notes);
+        noteByIdByInstance.set(instance.id, new Map(notes.map((n) => [n.frontmatter.id, n])));
+    }
+    const buildAnswerPrompt = (q, ex) => opts.buildPrompt ? opts.buildPrompt(q, ex) : buildReadPrompt(q.question, ex, overlapMode);
+    // One question, end-to-end (retrieve → [iterative loop] → answer → score).
+    // Always resolves to a QARecord (errors → 'error' sentinel) so one failure
+    // never aborts the run; paired callers drop error ids via filterReport.
+    async function processOne(sq) {
+        const { instance, question } = sq;
+        const notes = notesByInstance.get(instance.id);
+        const noteById = noteByIdByInstance.get(instance.id);
+        try {
+            let answer;
+            let iterations;
+            if (opts.iterative) {
+                const res = await runIterativeRetrieval(question, notes, noteById, retriever, complete, buildAnswerPrompt, opts.iterative);
+                answer = res.answer;
+                iterations = res.iterations;
+            }
+            else {
+                const hits = await retriever.search(question.question, notes, { maxResults: k });
+                let servedHits = hits;
+                if (opts.servingGate || opts.onServingDecision) {
+                    const candidates = searchHitsToGateCandidates(hits, noteById);
+                    const context = { instance, question, provider: retriever.name, candidates, k };
+                    const gateDecision = opts.servingGate
+                        ? await opts.servingGate(context)
+                        : serveAllMemoryQAGate(context);
+                    opts.onServingDecision?.(toServingDecision(context, gateDecision));
+                    const servedIds = new Set(gateDecision.servedNoteIds);
+                    servedHits = hits.filter((hit) => servedIds.has(hit.noteId));
+                }
+                let excerpts = servedHits.map((h) => noteById.get(h.noteId)?.body ?? '').filter(Boolean);
+                if (opts.rerank && excerpts.length > 0) {
+                    try {
+                        const distilled = (await complete(buildRerankPrompt(question.question, excerpts))).trim();
+                        if (distilled)
+                            excerpts = [distilled];
+                    }
+                    catch {
+                        // Keep the raw excerpts on rerank failure.
+                    }
+                }
+                answer = (await complete(buildAnswerPrompt(question, excerpts))).trim();
+            }
+            let correct = false;
+            let judgedBy;
+            if (overlapMode) {
+                if (opts.judge) {
+                    correct = await judgeCorrect(opts.judge, question.question, question.answer, answer);
+                    judgedBy = 'mem0-judge';
+                }
+                else {
+                    judgedBy = 'overlap';
+                }
+            }
+            else if (answer.toUpperCase().includes(NO_ANSWER)) {
+                judgedBy = 'abstain-sentinel';
+            }
+            else if (question.answer && normalize(answer).includes(normalize(question.answer))) {
+                correct = true; // high-precision positive — skip the judge call
+                judgedBy = 'contains';
+            }
+            else {
+                const verdict = await complete(buildJudgePrompt(question.question, question.answer, answer));
+                correct = /^\s*yes/i.test(verdict.trim());
+                judgedBy = 'llm';
+            }
+            return {
+                id: question.id,
+                category: question.category,
+                question: question.question,
+                answer,
+                gold: question.answer,
+                correct,
+                judgedBy,
+                iterations,
+            };
+        }
+        catch {
+            return {
+                id: question.id,
+                category: question.category,
+                question: question.question,
+                answer: '',
+                gold: question.answer,
+                correct: false,
+                judgedBy: 'error',
+            };
+        }
+    }
+    // Bounded worker pool: at most `concurrency` questions in flight. Records are
+    // index-assigned to preserve sample order (deterministic artifact); done-count
+    // drives progress regardless of completion order.
+    const records = new Array(sample.length);
+    let nextIdx = 0;
+    let done = 0;
+    const runners = Array.from({ length: Math.min(concurrency, sample.length) }, async () => {
+        for (;;) {
+            const i = nextIdx++;
+            if (i >= sample.length)
+                return;
+            records[i] = await processOne(sample[i]);
+            opts.onProgress?.(++done, sample.length);
+        }
+    });
+    await Promise.all(runners);
+    return buildReport(retriever.name, k, records);
+}
+/** Aggregate per-question records into a QAReport (overall + per-category accuracy). */
+function buildReport(provider, k, records) {
+    const byCatAcc = {};
+    const catAgg = new Map();
+    for (const r of records) {
+        const a = catAgg.get(r.category) ?? { hits: 0, n: 0 };
+        a.n++;
+        if (r.correct)
+            a.hits++;
+        catAgg.set(r.category, a);
+    }
+    for (const [cat, a] of catAgg)
+        byCatAcc[cat] = { accuracy: a.n ? a.hits / a.n : 0, n: a.n };
+    const hits = records.filter((r) => r.correct).length;
+    return {
+        provider,
+        k,
+        overall: { accuracy: records.length ? hits / records.length : 0, n: records.length },
+        byCategory: byCatAcc,
+        records,
+    };
+}
+/**
+ * Re-derive a report over a subset of question ids. Used to drop questions that
+ * ERRORED (transient LLM failure, `judgedBy: 'error'`) from BOTH arms of a paired
+ * A/B, so the comparison stays on a common, fully-scored question set.
+ */
+export function filterReport(report, keep) {
+    return buildReport(report.provider, report.k, report.records.filter((r) => keep.has(r.id)));
+}
+/** Accuracy A/B between two QA reports (same question set). */
+export function compareQA(baseline, treatment) {
+    return {
+        baseline: baseline.provider,
+        treatment: treatment.provider,
+        baselineAccuracy: baseline.overall.accuracy,
+        treatmentAccuracy: treatment.overall.accuracy,
+        delta: treatment.overall.accuracy - baseline.overall.accuracy,
+        n: treatment.overall.n,
+    };
+}
+/**
+ * McNemar's paired test on per-question correctness (records matched by id).
+ * The correct test for paired binary outcomes — only the *discordant* pairs
+ * (one arm right, the other wrong) carry signal.
+ */
+export function pairedAccuracy(baseline, treatment) {
+    const base = new Map(baseline.records.map((r) => [r.id, r.correct]));
+    let b = 0;
+    let c = 0;
+    let n = 0;
+    for (const r of treatment.records) {
+        if (!base.has(r.id))
+            continue;
+        n++;
+        const lex = base.get(r.id);
+        if (!lex && r.correct)
+            b++;
+        else if (lex && !r.correct)
+            c++;
+    }
+    const disc = b + c;
+    const chi2 = disc > 0 ? (Math.abs(b - c) - 1) ** 2 / disc : 0;
+    return {
+        b,
+        c,
+        delta: treatment.overall.accuracy - baseline.overall.accuracy,
+        chi2,
+        significant: chi2 > 3.841,
+        n,
+    };
+}
+const pct = (x) => `${(x * 100).toFixed(0)}%`;
+export function formatQA(r) {
+    const cats = Object.keys(r.byCategory).sort();
+    const lines = [
+        `QA accuracy (${r.provider}, k=${r.k}, n=${r.overall.n}): ${pct(r.overall.accuracy)}`,
+    ];
+    for (const c of cats)
+        lines.push(`  ${c.padEnd(26)} ${pct(r.byCategory[c].accuracy).padStart(5)}  (n=${r.byCategory[c].n})`);
+    return lines.join('\n');
+}
+export function formatQACompare(c) {
+    return (`QA A/B: ${c.treatment} vs ${c.baseline} | ` +
+        `${pct(c.treatmentAccuracy)} vs ${pct(c.baselineAccuracy)} | ` +
+        `Δ ${c.delta >= 0 ? '+' : ''}${pct(c.delta)} (n=${c.n})`);
+}
+/**
+ * Post-hoc word-overlap scores from a QA report's (answer, gold) pairs, grouped
+ * by category — the A-MEM metric family. Deterministic, no extra LLM calls.
+ */
+export function overlapByCategory(report) {
+    const agg = new Map();
+    const overall = { f1: 0, bleu1: 0, rougeL: 0, n: 0 };
+    for (const r of report.records) {
+        const s = overlapScores(r.answer, r.gold);
+        const a = agg.get(r.category) ?? { f1: 0, bleu1: 0, rougeL: 0, n: 0 };
+        a.f1 += s.f1;
+        a.bleu1 += s.bleu1;
+        a.rougeL += s.rougeL;
+        a.n++;
+        agg.set(r.category, a);
+        overall.f1 += s.f1;
+        overall.bleu1 += s.bleu1;
+        overall.rougeL += s.rougeL;
+        overall.n++;
+    }
+    const mean = (a) => ({
+        f1: a.n ? a.f1 / a.n : 0,
+        bleu1: a.n ? a.bleu1 / a.n : 0,
+        rougeL: a.n ? a.rougeL / a.n : 0,
+        n: a.n,
+    });
+    const byCategory = {};
+    for (const [c, a] of agg)
+        byCategory[c] = mean(a);
+    return { byCategory, overall: mean(overall) };
+}
+/** F1×100 table by category (A-MEM reports F1/BLEU-1 on a 0–100 scale). */
+export function formatOverlap(r, categoryNames) {
+    const x100 = (x) => (x * 100).toFixed(1);
+    const cats = Object.keys(r.byCategory).sort();
+    const lines = [
+        `word-overlap (×100) by category:`,
+        `  ${'category'.padEnd(16)} ${'F1'.padStart(6)} ${'BLEU1'.padStart(6)} ${'ROUGE-L'.padStart(7)}   n`,
+    ];
+    for (const c of cats) {
+        const s = r.byCategory[c];
+        const name = categoryNames?.[c] ?? c;
+        lines.push(`  ${name.padEnd(16)} ${x100(s.f1).padStart(6)} ${x100(s.bleu1).padStart(6)} ${x100(s.rougeL).padStart(7)}   ${s.n}`);
+    }
+    lines.push(`  ${'OVERALL'.padEnd(16)} ${x100(r.overall.f1).padStart(6)} ${x100(r.overall.bleu1).padStart(6)} ${x100(r.overall.rougeL).padStart(7)}   ${r.overall.n}`);
+    return lines.join('\n');
+}
+//# sourceMappingURL=qa.js.map