@eidentic/bench 0.1.0 → 0.1.2

package/dist/index.d.ts

@@ -167,7 +167,7 @@ declare const syntheticDataset: BenchDataset;
  * @param opts.maxBytes - Maximum allowed file size in bytes (default 256 MiB).
  *   Increase this only if you are loading a vetted, trusted dataset file.
  */
-declare function loadLongMemEval(jsonPath: string, opts?: {
+declare function loadLongMemEval$1(jsonPath: string, opts?: {
     maxBytes?: number;
 }): Promise<BenchDataset>;
 /**
@@ -179,10 +179,542 @@ declare function loadLongMemEval(jsonPath: string, opts?: {
  * @param opts.maxBytes - Maximum allowed file size in bytes (default 256 MiB).
  *   Increase this only if you are loading a vetted, trusted dataset file.
  */
-declare function loadLoCoMo(jsonPath: string, opts?: {
+declare function loadLoCoMo$1(jsonPath: string, opts?: {
     maxBytes?: number;
 }): Promise<BenchDataset>;
 
+/** One turn parsed from a session. */
+interface LocomoTurn {
+    speaker: string;
+    diaId: string;
+    text: string;
+}
+/** One session parsed from the conversation object. */
+interface LocomoSession {
+    /** 1-based session index from the dataset key. */
+    index: number;
+    /** Human-readable date-time string from the dataset, e.g. "1:56 pm on 8 May, 2023". */
+    dateTime: string;
+    /** Epoch milliseconds parsed from dateTime (0 when parse fails). */
+    dateTimeMs: number;
+    turns: LocomoTurn[];
+}
+/**
+ * Category definitions:
+ *  1 = multi-hop (282 questions)
+ *  2 = temporal  (321 questions)
+ *  3 = open-domain (96 questions)
+ *  4 = single-hop (841 questions)
+ *  5 = adversarial/unanswerable (446 questions)
+ *
+ * NOTE: the numeric → semantic mapping above comes from the actual file counts.
+ * Primary score = categories 1–4 (denominator 1540). Category 5 is scored separately as refusal rate.
+ */
+type LocomoCategory = 1 | 2 | 3 | 4 | 5;
+/** One QA pair with typed evidence references. */
+interface LocomoQA {
+    question: string;
+    /** Gold answer (string or number). Absent for category-5 (adversarial) questions. */
+    answer?: string;
+    category: LocomoCategory;
+    /** Dia-id references into the conversation sessions, e.g. ["D1:9", "D3:2"]. */
+    evidence: string[];
+    /** The trap answer for category 5 — a plausible-sounding but wrong answer. */
+    adversarialAnswer?: string;
+}
+/** One LoCoMo sample with parsed sessions and QA. */
+interface LocomoSample {
+    sampleId: string;
+    speakerA: string;
+    speakerB: string;
+    sessions: LocomoSession[];
+    qa: LocomoQA[];
+}
+/** Typed dataset returned by loadLoCoMo. */
+interface LocomoDataset {
+    samples: LocomoSample[];
+}
+/**
+ * Resolve evidence dia-ids to their actual turn texts.
+ *
+ * @param sample  - The LoCoMo sample.
+ * @param diaIds  - Array of dia-id strings, e.g. ["D1:9", "D3:2"].
+ * @returns       - Array of matched turn texts (unmatched ids are silently skipped).
+ */
+declare function resolveEvidence(sample: LocomoSample, diaIds: string[]): string[];
+
+/**
+ * LoCoMo dataset loader — real schema.
+ *
+ * The real locomo10.json root is a bare JSON array of samples. Sessions are stored as dynamic
+ * keys session_N / session_N_date_time in the conversation object (not a nested array).
+ *
+ * Dataset source: https://github.com/snap-research/locomo
+ * License: CC BY-NC 4.0 — raw data must NOT be committed; results are publishable.
+ *
+ * LOCOMO_SOURCE_SHA below records the upstream commit that produced the locomo10.json file
+ * used in this implementation, for provenance.
+ */
+
+/** Upstream commit SHA of snap-research/locomo at the time the loader was written. */
+declare const LOCOMO_SOURCE_SHA = "3eb6f2c585f5e1699204e3c3bdf7adc5c28cb376";
+/**
+ * Load a LoCoMo JSON file (real schema — bare array of samples with dynamic session keys).
+ *
+ * @param jsonPath  - Absolute or relative path to the locomo10.json file.
+ *   **Security note:** callers must validate untrusted paths before passing them here.
+ * @param opts.maxBytes  - Maximum allowed file size (default 256 MiB).
+ * @returns          - Typed LocomoDataset with parsed sessions and QA.
+ */
+declare function loadLoCoMo(jsonPath: string, opts?: {
+    maxBytes?: number;
+}): Promise<LocomoDataset>;
+
+/**
+ * Fair-run LoCoMo benchmark harness.
+ *
+ * Key fair-run rules (non-negotiable, documented for methodology transparency):
+ *
+ * 1. BOTH speakers are humans — never map to user/assistant roles.
+ *    Turns are ingested as "[SpeakerName]: <text>" into one Memory scope per conversation.
+ *
+ * 2. Timestamps structurally — session headers are prepended ("Session N — <date>") so
+ *    temporal questions are answerable; ingestedAt metadata carries the epoch-ms.
+ *
+ * 3. Memory-mode answer step — retrieve topK <= ~10 snippets (never inflate topK to
+ *    bypass retrieval quality), build a prompt from retrieved snippets, answer concisely.
+ *
+ * 4. Full-context mode — the MANDATORY baseline. Entire conversation fed as context.
+ *
+ * 5. Judging — strict LLM judge: correct only when the model answer contains the gold
+ *    answer's specific information (paraphrase ok; vague/topical-only = wrong).
+ *    For category 5: correct = model declined; adversarial_answer match = wrong.
+ *
+ * 6. Metrics — per-category accuracy, overall J(1-4) with exact denominator 1540,
+ *    category-5 refusal rate separately, token/cost accounting, wall-clock.
+ *
+ * 7. Determinism — seed recorded; seeded shuffle used when sampleLimit/questionLimit set.
+ *
+ * 8. Resilience — per-question try/catch; checkpoint-resume via JSONL file.
+ */
+
+/** Per-question scored row (also used as checkpoint entry). */
+interface LocomoQuestionResult {
+    sampleId: string;
+    questionIndex: number;
+    question: string;
+    goldAnswer?: string;
+    category: 1 | 2 | 3 | 4 | 5;
+    modelAnswer: string;
+    correct: boolean;
+    /** True when category=5 and the model gave the adversarial trap answer. */
+    trapTriggered?: boolean;
+    error?: string;
+    /** Tokens used in the answer step for this question. */
+    answerInputTokens: number;
+    answerOutputTokens: number;
+    /** Tokens used in the judge step for this question. */
+    judgeInputTokens: number;
+    judgeOutputTokens: number;
+}
+/** Token / cost summary (no vendor pricing hard-coded — caller provides rates). */
+interface TokenSummary {
+    ingestInputTokens: number;
+    ingestOutputTokens: number;
+    answerInputTokens: number;
+    answerOutputTokens: number;
+    judgeInputTokens: number;
+    judgeOutputTokens: number;
+    totalInputTokens: number;
+    totalOutputTokens: number;
+}
+/** Per-category accuracy. */
+interface CategoryStats {
+    correct: number;
+    total: number;
+    accuracy: number;
+}
+/** The full benchmark report. */
+interface LocomoReport {
+    /** Run configuration (included in every published result for transparency). */
+    config: {
+        mode: "memory" | "full-context";
+        topK: number;
+        answerModelId: string;
+        judgeModelId: string;
+        datasetSha: string;
+        seed: number;
+        categories: number[];
+        samplesRun: number;
+        questionsRun: number;
+    };
+    /** Overall J(1–4): accuracy on categories 1–4 only; denominator = questions actually run in 1–4. */
+    overallJ14: CategoryStats;
+    /** Per-category breakdown (keys "1" through "5"). */
+    byCategory: Record<string, CategoryStats>;
+    /** Category-5 refusal rate (separate from J14). */
+    cat5RefusalRate?: {
+        correct: number;
+        total: number;
+        rate: number;
+    };
+    /** Token usage accounting. */
+    tokens: TokenSummary;
+    /** Wall-clock duration of the run in milliseconds. */
+    wallClockMs: number;
+    /** Individual question results. */
+    questions: LocomoQuestionResult[];
+    /** Count of questions that threw errors (counted as wrong, not skipped). */
+    errorCount: number;
+}
+/** Factory for a fresh Memory instance, called once per sample. */
+type MemoryFactory = (sampleId: string) => Memory | Promise<Memory>;
+/** Options for runLocomoBench. */
+interface LocomoBenchOptions {
+    /** Path to locomo10.json. */
+    dataPath: string;
+    /** Factory for a fresh Memory per conversation (used only when mode="memory"). */
+    memoryFactory?: MemoryFactory;
+    /** Model used to generate answers. */
+    answerModel: ModelPort;
+    /** Model used to judge correctness. */
+    judgeModel: ModelPort;
+    /** "memory" requires memoryFactory; "full-context" feeds the full conversation as context. */
+    mode: "memory" | "full-context";
+    /** Categories to include (default: [1,2,3,4,5]). */
+    categories?: number[];
+    /** Cap on samples to process (for quick pilot runs). */
+    sampleLimit?: number;
+    /** Cap on questions per sample (applied after seeded shuffle when set). */
+    questionLimit?: number;
+    /** Random seed for shuffle reproducibility. Default 42. */
+    seed?: number;
+    /** Max snippets retrieved per question in memory mode. MUST be <= 10 to avoid trivialising retrieval. */
+    topK?: number;
+    /** Concurrency (questions in flight simultaneously). Default 1. */
+    concurrency?: number;
+    /** Progress callback: (questionsCompleted, questionsTotal) */
+    onProgress?: (done: number, total: number) => void;
+    /** Path to a JSONL checkpoint file. Existing rows are skipped on resume. */
+    checkpointPath?: string;
+    /** Pre-loaded dataset (avoids re-reading the file if already loaded). */
+    dataset?: LocomoDataset;
+}
+/**
+ * Run the LoCoMo benchmark with the given options.
+ *
+ * @param opts - Configuration (see LocomoBenchOptions).
+ * @returns    - Full LocomoReport with metrics, token accounting, and per-question details.
+ */
+declare function runLocomoBench(opts: LocomoBenchOptions): Promise<LocomoReport>;
+
+/**
+ * Markdown results renderer for LoCoMo benchmark reports.
+ *
+ * Produces a defensible, methodology-transparent table suitable for publication.
+ * Per the mandatory fair-run rules, results MUST include:
+ *   - Model ids and judge model id
+ *   - topK value
+ *   - Dataset SHA (provenance)
+ *   - Mode (memory | full-context)
+ *   - Seed and n-questions
+ */
+
+/** Optional price table for cost estimates (per 1M tokens, input/output). */
+interface PriceTable {
+    /** Per-million input tokens in USD. */
+    inputPer1M: number;
+    /** Per-million output tokens in USD. */
+    outputPer1M: number;
+}
+/**
+ * Render one or more LoCoMo benchmark reports as a Markdown table with mandatory methodology notes.
+ *
+ * @param reports  - Array of LocomoReport objects to compare.
+ * @param prices   - Optional price table for cost-per-run estimates (per 1M tokens).
+ * @returns        - Markdown string ready to write to a .md file.
+ */
+declare function renderLocomoReportMarkdown(reports: LocomoReport[], prices?: PriceTable): string;
+
+/** One turn inside a parsed session. */
+interface LmeTurn {
+    role: "user" | "assistant";
+    content: string;
+    /** True when the turn contains the gold answer (may be absent in all turns). */
+    hasAnswer: boolean;
+}
+/** One session inside a question's haystack. */
+interface LmeSession {
+    /** The session's original id from the dataset. */
+    id: string;
+    /** Human-readable date-time string from the dataset, e.g. "2023/05/20 (Sat) 02:36". */
+    dateTime: string;
+    /** Epoch milliseconds parsed from dateTime (0 when parse fails). */
+    dateTimeMs: number;
+    turns: LmeTurn[];
+}
+/**
+ * LongMemEval question types (7 base types, plus `*_abs` abstention variants).
+ *
+ * Base types:
+ *   single-session-user           — fact stated by the user in one session
+ *   single-session-assistant      — fact stated by the assistant in one session
+ *   single-session-preference     — user preference expressed in one session
+ *   multi-session                 — evidence spans multiple sessions
+ *   temporal-reasoning            — requires reasoning about time/dates
+ *   knowledge-update              — the fact was updated in a later session
+ *
+ * Abstention variants (append "_abs"):
+ *   The correct answer is to recognize the information is not present / premise
+ *   is flawed and respond with "no information" rather than fabricating an answer.
+ */
+type LmeBaseType = "single-session-user" | "single-session-assistant" | "single-session-preference" | "multi-session" | "temporal-reasoning" | "knowledge-update";
+type LmeQuestionType = LmeBaseType | `${LmeBaseType}_abs` | (string & Record<never, never>);
+/** One parsed LongMemEval question with its haystack. */
+interface LmeQuestion {
+    /** Original question_id from the dataset, e.g. "e47becba" or "gpt4_xxxx". */
+    id: string;
+    /** Full question_type string (may include "_abs" suffix). */
+    type: LmeQuestionType;
+    /** Base type without any "_abs" suffix. */
+    baseType: LmeBaseType | string;
+    /** Whether this is an abstention variant (type ends with "_abs"). */
+    isAbstention: boolean;
+    question: string;
+    /** Gold answer string. For abstention questions, the correct response is to abstain. */
+    answer: string;
+    /** Human-readable question date string from the dataset. */
+    questionDate: string;
+    /** Epoch ms for the question date (0 when parse fails). */
+    questionDateMs: number;
+    /** Haystack sessions in date order. */
+    sessions: LmeSession[];
+    /** Session ids that contain the answer evidence. */
+    answerSessionIds: string[];
+}
+/** Typed dataset returned by loadLongMemEval (new real-schema loader). */
+interface LmeDataset {
+    questions: LmeQuestion[];
+}
+
+/**
+ * LongMemEval dataset loader — real schema.
+ *
+ * Parses the real longmemeval_s.json (or _m / _oracle) format into typed LmeDataset.
+ *
+ * Dataset source:   https://github.com/xiaowu0162/LongMemEval
+ * HuggingFace:      https://huggingface.co/datasets/xiaowu0162/longmemeval
+ * License:          MIT — results are publishable; raw data must NOT be committed.
+ *
+ * LONGMEMEVAL_SOURCE below records the upstream HuggingFace dataset repo + snapshot sha
+ * used when this loader was written, for provenance.
+ */
+
+/** Provenance constant for the LongMemEval dataset. */
+declare const LONGMEMEVAL_SOURCE: {
+    readonly url: "https://huggingface.co/datasets/xiaowu0162/longmemeval";
+    readonly snapshotSha: "2ec2a557f339b6c0369619b1ed5793734cc87533";
+    readonly file: "longmemeval_s";
+    readonly license: "MIT";
+};
+/**
+ * Parse a LongMemEval date-time string to epoch milliseconds.
+ *
+ * Observed format: "2023/05/20 (Sat) 02:36"
+ * Normalise: strip the "(Weekday)" token, replace "/" separators.
+ *
+ * Returns 0 when parsing fails (the date is optional / informational).
+ */
+declare function parseLmeDateTimeString(raw: string): number;
+/**
+ * Load a LongMemEval JSON file (real schema — JSON array of questions).
+ *
+ * @param jsonPath  - Absolute or relative path to the longmemeval_s.json file.
+ *   **Security note:** callers must validate untrusted paths before passing them here.
+ * @param opts.maxBytes  - Maximum allowed file size (default 512 MiB).
+ * @returns          - Typed LmeDataset with parsed sessions and questions.
+ */
+declare function loadLongMemEval(jsonPath: string, opts?: {
+    maxBytes?: number;
+}): Promise<LmeDataset>;
+
+/**
+ * Fair-run LongMemEval benchmark harness.
+ *
+ * Key fair-run rules (non-negotiable, documented for methodology transparency):
+ *
+ * 1. PER-QUESTION memory scope — each question gets its own fresh haystack ingested into
+ *    a fresh Memory instance. (Unlike LoCoMo which is per-conversation, LongMemEval
+ *    is per-question: each question has its own haystack of ~50 sessions.)
+ *
+ * 2. Dual-granularity ingest — per-turn entries carry the session date in text so
+ *    retrieved snippets are temporally anchored; one per-session chunk entry preserves
+ *    multi-turn context for questions whose evidence spans adjacent turns.
+ *
+ * 3. Current date context — the question_date is passed to the answer prompt so temporal
+ *    questions can reason about recency (e.g. "last week").
+ *
+ * 4. Memory-mode answer step — retrieve topK <= 10 snippets (never inflate topK to
+ *    bypass retrieval quality), build a prompt from retrieved snippets + question + current date.
+ *
+ * 5. Full-context mode — the MANDATORY baseline. Sessions are concatenated in date order
+ *    with session headers; if a haystack exceeds the context cap (120k chars ≈ ~90k tokens
+ *    for gpt-4o-mini's 128k context), oldest sessions are truncated first and recorded.
+ *
+ * 6. Judging — strict LLM judge: correct only when model answer contains the gold answer's
+ *    specific info (paraphrase ok; temporal: equivalent date expressions ok).
+ *    For abstention questions: correct = model declined / said no-info / identified the
+ *    flawed premise; fabricating a concrete answer = wrong.
+ *
+ * 7. Metrics — overall accuracy + per-question-type accuracy + abstention accuracy reported
+ *    separately + token/cost accounting per phase + wall-clock. Full config disclosure in report.
+ *
+ * 8. Determinism — seed recorded; seeded shuffle used when questionLimit is set.
+ *
+ * 9. Resilience — per-question try/catch; checkpoint-resume via JSONL file.
+ */
+
+/** Per-question scored row (also used as checkpoint entry). */
+interface LmeQuestionResult {
+    questionId: string;
+    questionType: string;
+    isAbstention: boolean;
+    question: string;
+    goldAnswer: string;
+    modelAnswer: string;
+    correct: boolean;
+    /** True if the model appeared to abstain (no-info / declined). */
+    appearedToAbstain: boolean;
+    /** True if context was truncated in full-context mode. */
+    contextTruncated?: boolean;
+    error?: string;
+    /** Tokens used in the answer step. */
+    answerInputTokens: number;
+    answerOutputTokens: number;
+    /** Tokens used in the judge step. */
+    judgeInputTokens: number;
+    judgeOutputTokens: number;
+}
+/** Token / cost summary. */
+interface LmeTokenSummary {
+    ingestEmbedTokens: number;
+    answerInputTokens: number;
+    answerOutputTokens: number;
+    judgeInputTokens: number;
+    judgeOutputTokens: number;
+    totalInputTokens: number;
+    totalOutputTokens: number;
+}
+/** Accuracy stats for a question type or overall. */
+interface LmeTypeStats {
+    correct: number;
+    total: number;
+    accuracy: number;
+}
+/** Full benchmark report. */
+interface LmeReport {
+    /** Run configuration (included in every published result for transparency). */
+    config: {
+        mode: "memory" | "full-context";
+        topK: number;
+        answerModelId: string;
+        judgeModelId: string;
+        datasetSource: typeof LONGMEMEVAL_SOURCE;
+        seed: number;
+        types: string[];
+        questionsRun: number;
+    };
+    /** Overall accuracy on all non-abstention questions. */
+    overall: LmeTypeStats;
+    /** Per-question-type accuracy (keys = type strings without "_abs" suffix). */
+    byType: Record<string, LmeTypeStats>;
+    /** Abstention accuracy: correct = model declined; wrong = fabricated concrete answer. */
+    abstentionAccuracy?: LmeTypeStats;
+    /** Token usage accounting. */
+    tokens: LmeTokenSummary;
+    /** Wall-clock duration in milliseconds. */
+    wallClockMs: number;
+    /** Individual question results. */
+    questions: LmeQuestionResult[];
+    /** Count of questions that threw errors (counted as wrong, not skipped). */
+    errorCount: number;
+}
+/** Factory for a fresh Memory instance, called once per question. */
+type LmeMemoryFactory = (questionId: string) => Memory | Promise<Memory>;
+/** Options for runLongMemEvalBench. */
+interface LmeBenchOptions {
+    /** Path to longmemeval_s.json (required unless dataset is provided). */
+    dataPath?: string;
+    /** Pre-loaded dataset (avoids re-reading the file if already loaded). */
+    dataset?: LmeDataset;
+    /** Factory for a fresh Memory per question (required when mode="memory"). */
+    memoryFactory?: LmeMemoryFactory;
+    /** Model used to generate answers. */
+    answerModel: ModelPort;
+    /** Model used to judge correctness. */
+    judgeModel: ModelPort;
+    /** "memory" requires memoryFactory; "full-context" feeds the full haystack as context. */
+    mode: "memory" | "full-context";
+    /** Question types to include (default: all). */
+    types?: string[];
+    /** Cap on questions to process (for quick pilot runs). */
+    questionLimit?: number;
+    /** Random seed for shuffle reproducibility. Default 42. */
+    seed?: number;
+    /** Max snippets retrieved per question in memory mode. MUST be <= 10. Default 10. */
+    topK?: number;
+    /** Concurrency (questions in flight simultaneously). Default 1. */
+    concurrency?: number;
+    /** Progress callback: (questionsCompleted, questionsTotal) */
+    onProgress?: (done: number, total: number) => void;
+    /** Path to a JSONL checkpoint file. Existing rows are skipped on resume. */
+    checkpointPath?: string;
+    /**
+     * Max characters for full-context haystack (to avoid exceeding model context).
+     * Default 480000 chars (~120k tokens at ~4 chars/token, fitting gpt-4o-mini 128k).
+     * When exceeded, oldest sessions are dropped first and contextTruncated is recorded.
+     */
+    fullContextMaxChars?: number;
+}
+/**
+ * Run the LongMemEval benchmark with the given options.
+ *
+ * @param opts - Configuration (see LmeBenchOptions).
+ * @returns    - Full LmeReport with metrics, token accounting, and per-question details.
+ */
+declare function runLongMemEvalBench(opts: LmeBenchOptions): Promise<LmeReport>;
+
+/**
+ * Markdown results renderer for LongMemEval benchmark reports.
+ *
+ * Produces a defensible, methodology-transparent table suitable for publication.
+ * Per the mandatory fair-run rules, results MUST include:
+ *   - Model ids and judge model id
+ *   - topK value (memory mode)
+ *   - Dataset provenance (source URL + snapshot sha)
+ *   - Mode (memory | full-context)
+ *   - Seed and n-questions
+ *   - Per-type accuracy breakdown
+ *   - Abstention accuracy reported separately
+ */
+
+/** Optional price table for cost estimates (per 1M tokens, input/output). */
+interface LmePriceTable {
+    /** Per-million input tokens in USD. */
+    inputPer1M: number;
+    /** Per-million output tokens in USD. */
+    outputPer1M: number;
+}
+/**
+ * Render one or more LongMemEval benchmark reports as a Markdown table
+ * with mandatory methodology notes.
+ *
+ * @param reports  - Array of LmeReport objects to compare.
+ * @param prices   - Optional price table for cost-per-run estimates (per 1M tokens).
+ * @returns        - Markdown string ready to write to a .md file.
+ */
+declare function renderLongMemEvalReportMarkdown(reports: LmeReport[], prices?: LmePriceTable): string;
+
 /**
  * Write-quality benchmark for the Eidentic memory harness.
  *
@@ -493,4 +1025,4 @@ interface TemporalBenchOptions {
  */
 declare function runTemporalBench(memory: Memory, dataset: SyntheticTemporalDataset, opts?: TemporalBenchOptions): Promise<TemporalBenchReport>;
 
-export { type BenchCase, type BenchDataset, type BenchOptions, type BenchQuestion, type BenchReport, type BenchTurn, CONTRADICTION_FIXTURES, type CaseResult, type ContradictionFixture, JUNK_STREAM_FIXTURES, type JunkItem, type QuestionResult, type StateTransition, type SyntheticTemporalDataset, type TemporalBenchOptions, type TemporalBenchReport, type TemporalEntity, type TemporalQuestion, type TemporalQuestionResult, type WriteQualityDetail, type WriteQualityOptions, type WriteQualityReport, factRecall, loadLoCoMo, loadLongMemEval, normalizeText, normalizedIncludes, recallAtK, runMemoryBench, runTemporalBench, runWriteQualityBench, syntheticDataset, syntheticTemporalDataset };
+export { type BenchCase, type BenchDataset, type BenchOptions, type BenchQuestion, type BenchReport, type BenchTurn, CONTRADICTION_FIXTURES, type CaseResult, type CategoryStats, type ContradictionFixture, JUNK_STREAM_FIXTURES, type JunkItem, LOCOMO_SOURCE_SHA, LONGMEMEVAL_SOURCE, type LmeBaseType, type LmeBenchOptions, type LmeDataset, type LmeMemoryFactory, type LmePriceTable, type LmeQuestion, type LmeQuestionResult, type LmeQuestionType, type LmeReport, type LmeSession, type LmeTokenSummary, type LmeTurn, type LmeTypeStats, type LocomoBenchOptions, type LocomoCategory, type LocomoDataset, type LocomoQA, type LocomoQuestionResult, type LocomoReport, type LocomoSample, type LocomoSession, type LocomoTurn, type MemoryFactory, type PriceTable, type QuestionResult, type StateTransition, type SyntheticTemporalDataset, type TemporalBenchOptions, type TemporalBenchReport, type TemporalEntity, type TemporalQuestion, type TemporalQuestionResult, type TokenSummary, type WriteQualityDetail, type WriteQualityOptions, type WriteQualityReport, factRecall, loadLoCoMo, loadLoCoMo$1 as loadLoCoMoLegacy, loadLongMemEval, loadLongMemEval$1 as loadLongMemEvalLegacy, normalizeText, normalizedIncludes, parseLmeDateTimeString, recallAtK, renderLocomoReportMarkdown, renderLongMemEvalReportMarkdown, resolveEvidence, runLocomoBench, runLongMemEvalBench, runMemoryBench, runTemporalBench, runWriteQualityBench, syntheticDataset, syntheticTemporalDataset };