@eidentic/bench 0.1.0 → 0.1.1

package/dist/index.d.cts

@@ -179,10 +179,263 @@ declare function loadLongMemEval(jsonPath: string, opts?: {
  * @param opts.maxBytes - Maximum allowed file size in bytes (default 256 MiB).
  *   Increase this only if you are loading a vetted, trusted dataset file.
  */
-declare function loadLoCoMo(jsonPath: string, opts?: {
+declare function loadLoCoMo$1(jsonPath: string, opts?: {
     maxBytes?: number;
 }): Promise<BenchDataset>;
 
+/** One turn parsed from a session. */
+interface LocomoTurn {
+    speaker: string;
+    diaId: string;
+    text: string;
+}
+/** One session parsed from the conversation object. */
+interface LocomoSession {
+    /** 1-based session index from the dataset key. */
+    index: number;
+    /** Human-readable date-time string from the dataset, e.g. "1:56 pm on 8 May, 2023". */
+    dateTime: string;
+    /** Epoch milliseconds parsed from dateTime (0 when parse fails). */
+    dateTimeMs: number;
+    turns: LocomoTurn[];
+}
+/**
+ * Category definitions:
+ *  1 = multi-hop (282 questions)
+ *  2 = temporal  (321 questions)
+ *  3 = open-domain (96 questions)
+ *  4 = single-hop (841 questions)
+ *  5 = adversarial/unanswerable (446 questions)
+ *
+ * NOTE: the numeric → semantic mapping above comes from the actual file counts.
+ * Primary score = categories 1–4 (denominator 1540). Category 5 is scored separately as refusal rate.
+ */
+type LocomoCategory = 1 | 2 | 3 | 4 | 5;
+/** One QA pair with typed evidence references. */
+interface LocomoQA {
+    question: string;
+    /** Gold answer (string or number). Absent for category-5 (adversarial) questions. */
+    answer?: string;
+    category: LocomoCategory;
+    /** Dia-id references into the conversation sessions, e.g. ["D1:9", "D3:2"]. */
+    evidence: string[];
+    /** The trap answer for category 5 — a plausible-sounding but wrong answer. */
+    adversarialAnswer?: string;
+}
+/** One LoCoMo sample with parsed sessions and QA. */
+interface LocomoSample {
+    sampleId: string;
+    speakerA: string;
+    speakerB: string;
+    sessions: LocomoSession[];
+    qa: LocomoQA[];
+}
+/** Typed dataset returned by loadLoCoMo. */
+interface LocomoDataset {
+    samples: LocomoSample[];
+}
+/**
+ * Resolve evidence dia-ids to their actual turn texts.
+ *
+ * @param sample  - The LoCoMo sample.
+ * @param diaIds  - Array of dia-id strings, e.g. ["D1:9", "D3:2"].
+ * @returns       - Array of matched turn texts (unmatched ids are silently skipped).
+ */
+declare function resolveEvidence(sample: LocomoSample, diaIds: string[]): string[];
+
+/**
+ * LoCoMo dataset loader — real schema.
+ *
+ * The real locomo10.json root is a bare JSON array of samples. Sessions are stored as dynamic
+ * keys session_N / session_N_date_time in the conversation object (not a nested array).
+ *
+ * Dataset source: https://github.com/snap-research/locomo
+ * License: CC BY-NC 4.0 — raw data must NOT be committed; results are publishable.
+ *
+ * LOCOMO_SOURCE_SHA below records the upstream commit that produced the locomo10.json file
+ * used in this implementation, for provenance.
+ */
+
+/** Upstream commit SHA of snap-research/locomo at the time the loader was written. */
+declare const LOCOMO_SOURCE_SHA = "3eb6f2c585f5e1699204e3c3bdf7adc5c28cb376";
+/**
+ * Load a LoCoMo JSON file (real schema — bare array of samples with dynamic session keys).
+ *
+ * @param jsonPath  - Absolute or relative path to the locomo10.json file.
+ *   **Security note:** callers must validate untrusted paths before passing them here.
+ * @param opts.maxBytes  - Maximum allowed file size (default 256 MiB).
+ * @returns          - Typed LocomoDataset with parsed sessions and QA.
+ */
+declare function loadLoCoMo(jsonPath: string, opts?: {
+    maxBytes?: number;
+}): Promise<LocomoDataset>;
+
+/**
+ * Fair-run LoCoMo benchmark harness.
+ *
+ * Key fair-run rules (non-negotiable, documented for methodology transparency):
+ *
+ * 1. BOTH speakers are humans — never map to user/assistant roles.
+ *    Turns are ingested as "[SpeakerName]: <text>" into one Memory scope per conversation.
+ *
+ * 2. Timestamps structurally — session headers are prepended ("Session N — <date>") so
+ *    temporal questions are answerable; ingestedAt metadata carries the epoch-ms.
+ *
+ * 3. Memory-mode answer step — retrieve topK <= ~10 snippets (never inflate topK to
+ *    bypass retrieval quality), build a prompt from retrieved snippets, answer concisely.
+ *
+ * 4. Full-context mode — the MANDATORY baseline. Entire conversation fed as context.
+ *
+ * 5. Judging — strict LLM judge: correct only when the model answer contains the gold
+ *    answer's specific information (paraphrase ok; vague/topical-only = wrong).
+ *    For category 5: correct = model declined; adversarial_answer match = wrong.
+ *
+ * 6. Metrics — per-category accuracy, overall J(1-4) with exact denominator 1540,
+ *    category-5 refusal rate separately, token/cost accounting, wall-clock.
+ *
+ * 7. Determinism — seed recorded; seeded shuffle used when sampleLimit/questionLimit set.
+ *
+ * 8. Resilience — per-question try/catch; checkpoint-resume via JSONL file.
+ */
+
+/** Per-question scored row (also used as checkpoint entry). */
+interface LocomoQuestionResult {
+    sampleId: string;
+    questionIndex: number;
+    question: string;
+    goldAnswer?: string;
+    category: 1 | 2 | 3 | 4 | 5;
+    modelAnswer: string;
+    correct: boolean;
+    /** True when category=5 and the model gave the adversarial trap answer. */
+    trapTriggered?: boolean;
+    error?: string;
+    /** Tokens used in the answer step for this question. */
+    answerInputTokens: number;
+    answerOutputTokens: number;
+    /** Tokens used in the judge step for this question. */
+    judgeInputTokens: number;
+    judgeOutputTokens: number;
+}
+/** Token / cost summary (no vendor pricing hard-coded — caller provides rates). */
+interface TokenSummary {
+    ingestInputTokens: number;
+    ingestOutputTokens: number;
+    answerInputTokens: number;
+    answerOutputTokens: number;
+    judgeInputTokens: number;
+    judgeOutputTokens: number;
+    totalInputTokens: number;
+    totalOutputTokens: number;
+}
+/** Per-category accuracy. */
+interface CategoryStats {
+    correct: number;
+    total: number;
+    accuracy: number;
+}
+/** The full benchmark report. */
+interface LocomoReport {
+    /** Run configuration (included in every published result for transparency). */
+    config: {
+        mode: "memory" | "full-context";
+        topK: number;
+        answerModelId: string;
+        judgeModelId: string;
+        datasetSha: string;
+        seed: number;
+        categories: number[];
+        samplesRun: number;
+        questionsRun: number;
+    };
+    /** Overall J(1–4): accuracy on categories 1–4 only; denominator = questions actually run in 1–4. */
+    overallJ14: CategoryStats;
+    /** Per-category breakdown (keys "1" through "5"). */
+    byCategory: Record<string, CategoryStats>;
+    /** Category-5 refusal rate (separate from J14). */
+    cat5RefusalRate?: {
+        correct: number;
+        total: number;
+        rate: number;
+    };
+    /** Token usage accounting. */
+    tokens: TokenSummary;
+    /** Wall-clock duration of the run in milliseconds. */
+    wallClockMs: number;
+    /** Individual question results. */
+    questions: LocomoQuestionResult[];
+    /** Count of questions that threw errors (counted as wrong, not skipped). */
+    errorCount: number;
+}
+/** Factory for a fresh Memory instance, called once per sample. */
+type MemoryFactory = (sampleId: string) => Memory | Promise<Memory>;
+/** Options for runLocomoBench. */
+interface LocomoBenchOptions {
+    /** Path to locomo10.json. */
+    dataPath: string;
+    /** Factory for a fresh Memory per conversation (used only when mode="memory"). */
+    memoryFactory?: MemoryFactory;
+    /** Model used to generate answers. */
+    answerModel: ModelPort;
+    /** Model used to judge correctness. */
+    judgeModel: ModelPort;
+    /** "memory" requires memoryFactory; "full-context" feeds the full conversation as context. */
+    mode: "memory" | "full-context";
+    /** Categories to include (default: [1,2,3,4,5]). */
+    categories?: number[];
+    /** Cap on samples to process (for quick pilot runs). */
+    sampleLimit?: number;
+    /** Cap on questions per sample (applied after seeded shuffle when set). */
+    questionLimit?: number;
+    /** Random seed for shuffle reproducibility. Default 42. */
+    seed?: number;
+    /** Max snippets retrieved per question in memory mode. MUST be <= 10 to avoid trivialising retrieval. */
+    topK?: number;
+    /** Concurrency (questions in flight simultaneously). Default 1. */
+    concurrency?: number;
+    /** Progress callback: (questionsCompleted, questionsTotal) */
+    onProgress?: (done: number, total: number) => void;
+    /** Path to a JSONL checkpoint file. Existing rows are skipped on resume. */
+    checkpointPath?: string;
+    /** Pre-loaded dataset (avoids re-reading the file if already loaded). */
+    dataset?: LocomoDataset;
+}
+/**
+ * Run the LoCoMo benchmark with the given options.
+ *
+ * @param opts - Configuration (see LocomoBenchOptions).
+ * @returns    - Full LocomoReport with metrics, token accounting, and per-question details.
+ */
+declare function runLocomoBench(opts: LocomoBenchOptions): Promise<LocomoReport>;
+
+/**
+ * Markdown results renderer for LoCoMo benchmark reports.
+ *
+ * Produces a defensible, methodology-transparent table suitable for publication.
+ * Per the mandatory fair-run rules, results MUST include:
+ *   - Model ids and judge model id
+ *   - topK value
+ *   - Dataset SHA (provenance)
+ *   - Mode (memory | full-context)
+ *   - Seed and n-questions
+ */
+
+/** Optional price table for cost estimates (per 1M tokens, input/output). */
+interface PriceTable {
+    /** Per-million input tokens in USD. */
+    inputPer1M: number;
+    /** Per-million output tokens in USD. */
+    outputPer1M: number;
+}
+/**
+ * Render one or more LoCoMo benchmark reports as a Markdown table with mandatory methodology notes.
+ *
+ * @param reports  - Array of LocomoReport objects to compare.
+ * @param prices   - Optional price table for cost-per-run estimates (per 1M tokens).
+ * @returns        - Markdown string ready to write to a .md file.
+ */
+declare function renderLocomoReportMarkdown(reports: LocomoReport[], prices?: PriceTable): string;
+
 /**
  * Write-quality benchmark for the Eidentic memory harness.
  *
@@ -493,4 +746,4 @@ interface TemporalBenchOptions {
  */
 declare function runTemporalBench(memory: Memory, dataset: SyntheticTemporalDataset, opts?: TemporalBenchOptions): Promise<TemporalBenchReport>;
 
-export { type BenchCase, type BenchDataset, type BenchOptions, type BenchQuestion, type BenchReport, type BenchTurn, CONTRADICTION_FIXTURES, type CaseResult, type ContradictionFixture, JUNK_STREAM_FIXTURES, type JunkItem, type QuestionResult, type StateTransition, type SyntheticTemporalDataset, type TemporalBenchOptions, type TemporalBenchReport, type TemporalEntity, type TemporalQuestion, type TemporalQuestionResult, type WriteQualityDetail, type WriteQualityOptions, type WriteQualityReport, factRecall, loadLoCoMo, loadLongMemEval, normalizeText, normalizedIncludes, recallAtK, runMemoryBench, runTemporalBench, runWriteQualityBench, syntheticDataset, syntheticTemporalDataset };
+export { type BenchCase, type BenchDataset, type BenchOptions, type BenchQuestion, type BenchReport, type BenchTurn, CONTRADICTION_FIXTURES, type CaseResult, type CategoryStats, type ContradictionFixture, JUNK_STREAM_FIXTURES, type JunkItem, LOCOMO_SOURCE_SHA, type LocomoBenchOptions, type LocomoCategory, type LocomoDataset, type LocomoQA, type LocomoQuestionResult, type LocomoReport, type LocomoSample, type LocomoSession, type LocomoTurn, type MemoryFactory, type PriceTable, type QuestionResult, type StateTransition, type SyntheticTemporalDataset, type TemporalBenchOptions, type TemporalBenchReport, type TemporalEntity, type TemporalQuestion, type TemporalQuestionResult, type TokenSummary, type WriteQualityDetail, type WriteQualityOptions, type WriteQualityReport, factRecall, loadLoCoMo, loadLoCoMo$1 as loadLoCoMoLegacy, loadLongMemEval, normalizeText, normalizedIncludes, recallAtK, renderLocomoReportMarkdown, resolveEvidence, runLocomoBench, runMemoryBench, runTemporalBench, runWriteQualityBench, syntheticDataset, syntheticTemporalDataset };

package/dist/index.d.ts

@@ -179,10 +179,263 @@ declare function loadLongMemEval(jsonPath: string, opts?: {
  * @param opts.maxBytes - Maximum allowed file size in bytes (default 256 MiB).
  *   Increase this only if you are loading a vetted, trusted dataset file.
  */
-declare function loadLoCoMo(jsonPath: string, opts?: {
+declare function loadLoCoMo$1(jsonPath: string, opts?: {
     maxBytes?: number;
 }): Promise<BenchDataset>;
 
+/** One turn parsed from a session. */
+interface LocomoTurn {
+    speaker: string;
+    diaId: string;
+    text: string;
+}
+/** One session parsed from the conversation object. */
+interface LocomoSession {
+    /** 1-based session index from the dataset key. */
+    index: number;
+    /** Human-readable date-time string from the dataset, e.g. "1:56 pm on 8 May, 2023". */
+    dateTime: string;
+    /** Epoch milliseconds parsed from dateTime (0 when parse fails). */
+    dateTimeMs: number;
+    turns: LocomoTurn[];
+}
+/**
+ * Category definitions:
+ *  1 = multi-hop (282 questions)
+ *  2 = temporal  (321 questions)
+ *  3 = open-domain (96 questions)
+ *  4 = single-hop (841 questions)
+ *  5 = adversarial/unanswerable (446 questions)
+ *
+ * NOTE: the numeric → semantic mapping above comes from the actual file counts.
+ * Primary score = categories 1–4 (denominator 1540). Category 5 is scored separately as refusal rate.
+ */
+type LocomoCategory = 1 | 2 | 3 | 4 | 5;
+/** One QA pair with typed evidence references. */
+interface LocomoQA {
+    question: string;
+    /** Gold answer (string or number). Absent for category-5 (adversarial) questions. */
+    answer?: string;
+    category: LocomoCategory;
+    /** Dia-id references into the conversation sessions, e.g. ["D1:9", "D3:2"]. */
+    evidence: string[];
+    /** The trap answer for category 5 — a plausible-sounding but wrong answer. */
+    adversarialAnswer?: string;
+}
+/** One LoCoMo sample with parsed sessions and QA. */
+interface LocomoSample {
+    sampleId: string;
+    speakerA: string;
+    speakerB: string;
+    sessions: LocomoSession[];
+    qa: LocomoQA[];
+}
+/** Typed dataset returned by loadLoCoMo. */
+interface LocomoDataset {
+    samples: LocomoSample[];
+}
+/**
+ * Resolve evidence dia-ids to their actual turn texts.
+ *
+ * @param sample  - The LoCoMo sample.
+ * @param diaIds  - Array of dia-id strings, e.g. ["D1:9", "D3:2"].
+ * @returns       - Array of matched turn texts (unmatched ids are silently skipped).
+ */
+declare function resolveEvidence(sample: LocomoSample, diaIds: string[]): string[];
+
+/**
+ * LoCoMo dataset loader — real schema.
+ *
+ * The real locomo10.json root is a bare JSON array of samples. Sessions are stored as dynamic
+ * keys session_N / session_N_date_time in the conversation object (not a nested array).
+ *
+ * Dataset source: https://github.com/snap-research/locomo
+ * License: CC BY-NC 4.0 — raw data must NOT be committed; results are publishable.
+ *
+ * LOCOMO_SOURCE_SHA below records the upstream commit that produced the locomo10.json file
+ * used in this implementation, for provenance.
+ */
+
+/** Upstream commit SHA of snap-research/locomo at the time the loader was written. */
+declare const LOCOMO_SOURCE_SHA = "3eb6f2c585f5e1699204e3c3bdf7adc5c28cb376";
+/**
+ * Load a LoCoMo JSON file (real schema — bare array of samples with dynamic session keys).
+ *
+ * @param jsonPath  - Absolute or relative path to the locomo10.json file.
+ *   **Security note:** callers must validate untrusted paths before passing them here.
+ * @param opts.maxBytes  - Maximum allowed file size (default 256 MiB).
+ * @returns          - Typed LocomoDataset with parsed sessions and QA.
+ */
+declare function loadLoCoMo(jsonPath: string, opts?: {
+    maxBytes?: number;
+}): Promise<LocomoDataset>;
+
+/**
+ * Fair-run LoCoMo benchmark harness.
+ *
+ * Key fair-run rules (non-negotiable, documented for methodology transparency):
+ *
+ * 1. BOTH speakers are humans — never map to user/assistant roles.
+ *    Turns are ingested as "[SpeakerName]: <text>" into one Memory scope per conversation.
+ *
+ * 2. Timestamps structurally — session headers are prepended ("Session N — <date>") so
+ *    temporal questions are answerable; ingestedAt metadata carries the epoch-ms.
+ *
+ * 3. Memory-mode answer step — retrieve topK <= ~10 snippets (never inflate topK to
+ *    bypass retrieval quality), build a prompt from retrieved snippets, answer concisely.
+ *
+ * 4. Full-context mode — the MANDATORY baseline. Entire conversation fed as context.
+ *
+ * 5. Judging — strict LLM judge: correct only when the model answer contains the gold
+ *    answer's specific information (paraphrase ok; vague/topical-only = wrong).
+ *    For category 5: correct = model declined; adversarial_answer match = wrong.
+ *
+ * 6. Metrics — per-category accuracy, overall J(1-4) with exact denominator 1540,
+ *    category-5 refusal rate separately, token/cost accounting, wall-clock.
+ *
+ * 7. Determinism — seed recorded; seeded shuffle used when sampleLimit/questionLimit set.
+ *
+ * 8. Resilience — per-question try/catch; checkpoint-resume via JSONL file.
+ */
+
+/** Per-question scored row (also used as checkpoint entry). */
+interface LocomoQuestionResult {
+    sampleId: string;
+    questionIndex: number;
+    question: string;
+    goldAnswer?: string;
+    category: 1 | 2 | 3 | 4 | 5;
+    modelAnswer: string;
+    correct: boolean;
+    /** True when category=5 and the model gave the adversarial trap answer. */
+    trapTriggered?: boolean;
+    error?: string;
+    /** Tokens used in the answer step for this question. */
+    answerInputTokens: number;
+    answerOutputTokens: number;
+    /** Tokens used in the judge step for this question. */
+    judgeInputTokens: number;
+    judgeOutputTokens: number;
+}
+/** Token / cost summary (no vendor pricing hard-coded — caller provides rates). */
+interface TokenSummary {
+    ingestInputTokens: number;
+    ingestOutputTokens: number;
+    answerInputTokens: number;
+    answerOutputTokens: number;
+    judgeInputTokens: number;
+    judgeOutputTokens: number;
+    totalInputTokens: number;
+    totalOutputTokens: number;
+}
+/** Per-category accuracy. */
+interface CategoryStats {
+    correct: number;
+    total: number;
+    accuracy: number;
+}
+/** The full benchmark report. */
+interface LocomoReport {
+    /** Run configuration (included in every published result for transparency). */
+    config: {
+        mode: "memory" | "full-context";
+        topK: number;
+        answerModelId: string;
+        judgeModelId: string;
+        datasetSha: string;
+        seed: number;
+        categories: number[];
+        samplesRun: number;
+        questionsRun: number;
+    };
+    /** Overall J(1–4): accuracy on categories 1–4 only; denominator = questions actually run in 1–4. */
+    overallJ14: CategoryStats;
+    /** Per-category breakdown (keys "1" through "5"). */
+    byCategory: Record<string, CategoryStats>;
+    /** Category-5 refusal rate (separate from J14). */
+    cat5RefusalRate?: {
+        correct: number;
+        total: number;
+        rate: number;
+    };
+    /** Token usage accounting. */
+    tokens: TokenSummary;
+    /** Wall-clock duration of the run in milliseconds. */
+    wallClockMs: number;
+    /** Individual question results. */
+    questions: LocomoQuestionResult[];
+    /** Count of questions that threw errors (counted as wrong, not skipped). */
+    errorCount: number;
+}
+/** Factory for a fresh Memory instance, called once per sample. */
+type MemoryFactory = (sampleId: string) => Memory | Promise<Memory>;
+/** Options for runLocomoBench. */
+interface LocomoBenchOptions {
+    /** Path to locomo10.json. */
+    dataPath: string;
+    /** Factory for a fresh Memory per conversation (used only when mode="memory"). */
+    memoryFactory?: MemoryFactory;
+    /** Model used to generate answers. */
+    answerModel: ModelPort;
+    /** Model used to judge correctness. */
+    judgeModel: ModelPort;
+    /** "memory" requires memoryFactory; "full-context" feeds the full conversation as context. */
+    mode: "memory" | "full-context";
+    /** Categories to include (default: [1,2,3,4,5]). */
+    categories?: number[];
+    /** Cap on samples to process (for quick pilot runs). */
+    sampleLimit?: number;
+    /** Cap on questions per sample (applied after seeded shuffle when set). */
+    questionLimit?: number;
+    /** Random seed for shuffle reproducibility. Default 42. */
+    seed?: number;
+    /** Max snippets retrieved per question in memory mode. MUST be <= 10 to avoid trivialising retrieval. */
+    topK?: number;
+    /** Concurrency (questions in flight simultaneously). Default 1. */
+    concurrency?: number;
+    /** Progress callback: (questionsCompleted, questionsTotal) */
+    onProgress?: (done: number, total: number) => void;
+    /** Path to a JSONL checkpoint file. Existing rows are skipped on resume. */
+    checkpointPath?: string;
+    /** Pre-loaded dataset (avoids re-reading the file if already loaded). */
+    dataset?: LocomoDataset;
+}
+/**
+ * Run the LoCoMo benchmark with the given options.
+ *
+ * @param opts - Configuration (see LocomoBenchOptions).
+ * @returns    - Full LocomoReport with metrics, token accounting, and per-question details.
+ */
+declare function runLocomoBench(opts: LocomoBenchOptions): Promise<LocomoReport>;
+
+/**
+ * Markdown results renderer for LoCoMo benchmark reports.
+ *
+ * Produces a defensible, methodology-transparent table suitable for publication.
+ * Per the mandatory fair-run rules, results MUST include:
+ *   - Model ids and judge model id
+ *   - topK value
+ *   - Dataset SHA (provenance)
+ *   - Mode (memory | full-context)
+ *   - Seed and n-questions
+ */
+
+/** Optional price table for cost estimates (per 1M tokens, input/output). */
+interface PriceTable {
+    /** Per-million input tokens in USD. */
+    inputPer1M: number;
+    /** Per-million output tokens in USD. */
+    outputPer1M: number;
+}
+/**
+ * Render one or more LoCoMo benchmark reports as a Markdown table with mandatory methodology notes.
+ *
+ * @param reports  - Array of LocomoReport objects to compare.
+ * @param prices   - Optional price table for cost-per-run estimates (per 1M tokens).
+ * @returns        - Markdown string ready to write to a .md file.
+ */
+declare function renderLocomoReportMarkdown(reports: LocomoReport[], prices?: PriceTable): string;
+
 /**
  * Write-quality benchmark for the Eidentic memory harness.
  *
@@ -493,4 +746,4 @@ interface TemporalBenchOptions {
  */
 declare function runTemporalBench(memory: Memory, dataset: SyntheticTemporalDataset, opts?: TemporalBenchOptions): Promise<TemporalBenchReport>;
 
-export { type BenchCase, type BenchDataset, type BenchOptions, type BenchQuestion, type BenchReport, type BenchTurn, CONTRADICTION_FIXTURES, type CaseResult, type ContradictionFixture, JUNK_STREAM_FIXTURES, type JunkItem, type QuestionResult, type StateTransition, type SyntheticTemporalDataset, type TemporalBenchOptions, type TemporalBenchReport, type TemporalEntity, type TemporalQuestion, type TemporalQuestionResult, type WriteQualityDetail, type WriteQualityOptions, type WriteQualityReport, factRecall, loadLoCoMo, loadLongMemEval, normalizeText, normalizedIncludes, recallAtK, runMemoryBench, runTemporalBench, runWriteQualityBench, syntheticDataset, syntheticTemporalDataset };
+export { type BenchCase, type BenchDataset, type BenchOptions, type BenchQuestion, type BenchReport, type BenchTurn, CONTRADICTION_FIXTURES, type CaseResult, type CategoryStats, type ContradictionFixture, JUNK_STREAM_FIXTURES, type JunkItem, LOCOMO_SOURCE_SHA, type LocomoBenchOptions, type LocomoCategory, type LocomoDataset, type LocomoQA, type LocomoQuestionResult, type LocomoReport, type LocomoSample, type LocomoSession, type LocomoTurn, type MemoryFactory, type PriceTable, type QuestionResult, type StateTransition, type SyntheticTemporalDataset, type TemporalBenchOptions, type TemporalBenchReport, type TemporalEntity, type TemporalQuestion, type TemporalQuestionResult, type TokenSummary, type WriteQualityDetail, type WriteQualityOptions, type WriteQualityReport, factRecall, loadLoCoMo, loadLoCoMo$1 as loadLoCoMoLegacy, loadLongMemEval, normalizeText, normalizedIncludes, recallAtK, renderLocomoReportMarkdown, resolveEvidence, runLocomoBench, runMemoryBench, runTemporalBench, runWriteQualityBench, syntheticDataset, syntheticTemporalDataset };