@eidentic/bench 0.1.0

package/dist/index.d.cts

@@ -0,0 +1,496 @@
+import { Memory } from '@eidentic/memory';
+import { ModelPort, Scope, AssertFactInput } from '@eidentic/types';
+
+/**
+ * Core dataset shapes for the Eidentic memory benchmark harness (§6.10).
+ *
+ * A BenchDataset is a named collection of BenchCase objects. Each case represents a
+ * multi-session conversation (turns) followed by a set of retrieval questions. Each question
+ * specifies one or more gold facts — substrings that MUST appear in the retrieved context for
+ * the answer to be considered recalled.
+ */
+/** A single conversation turn ingested into Memory before recall is evaluated. */
+interface BenchTurn {
+    role: "user" | "assistant";
+    text: string;
+    /** Optional session identifier — used to group turns into multi-session conversations. */
+    sessionId?: string;
+    /** Optional ISO timestamp — used for temporal ordering and knowledge-update cases. */
+    ts?: string;
+}
+/** A single retrieval question with gold evidence. */
+interface BenchQuestion {
+    question: string;
+    /**
+     * Substrings that must appear (normalized) in the top-K retrieved context for this question to
+     * be considered correctly recalled. Recall@K = fraction of goldFacts that are present.
+     */
+    goldFacts: string[];
+    /** Optional ground-truth answer string (used by the optional LLM judge for answer-correctness). */
+    answer?: string;
+    /** Category for per-category breakdown in BenchReport. */
+    category?: "single-session" | "multi-session" | "temporal" | "knowledge-update";
+}
+/** A single benchmark case: ingested turns + evaluation questions. */
+interface BenchCase {
+    id: string;
+    turns: BenchTurn[];
+    questions: BenchQuestion[];
+}
+/** A named benchmark dataset containing one or more cases. */
+interface BenchDataset {
+    name: string;
+    cases: BenchCase[];
+}
+/** Per-question result recorded in BenchReport. */
+interface QuestionResult {
+    caseId: string;
+    question: string;
+    category?: BenchQuestion["category"];
+    recallAtK: number;
+    /** Snippets retrieved (text only, for inspection). */
+    retrieved: string[];
+    /** Gold facts found vs expected. */
+    foundFacts: number;
+    totalFacts: number;
+}
+/** Per-case summary in BenchReport. */
+interface CaseResult {
+    caseId: string;
+    recallAtK: {
+        mean: number;
+        n: number;
+    };
+    questions: QuestionResult[];
+}
+/** The complete report produced by runMemoryBench. */
+interface BenchReport {
+    dataset: string;
+    recallAtK: {
+        mean: number;
+        n: number;
+    };
+    byCategory: Record<string, {
+        mean: number;
+        n: number;
+    }>;
+    perCase: CaseResult[];
+}
+
+/**
+ * Deterministic recall metric for the memory benchmark harness (§6.10).
+ *
+ * recallAtK: fraction of gold facts whose text appears (normalized substring match) in the top-K
+ * retrieved context snippets. No model required — fully deterministic.
+ */
+/**
+ * Normalize a string for comparison: lowercase, collapse whitespace, strip leading/trailing
+ * punctuation from each word so that "Alice's" and "Alice" both match. This is intentionally
+ * lenient — the point is to detect whether the semantic content was retrieved, not to penalize
+ * minor formatting differences.
+ */
+declare function normalizeText(text: string): string;
+/**
+ * Check whether `needle` appears in `haystack` after normalization.
+ * Uses substring matching so partial matches count (e.g. "TypeScript" found in a longer sentence).
+ */
+declare function normalizedIncludes(haystack: string, needle: string): boolean;
+/**
+ * Compute recall@K: the fraction of gold facts that appear as substrings in the retrieved context.
+ *
+ * @param retrievedSnippets - The text of the top-K retrieved memory snippets.
+ * @param goldFacts - The gold evidence substrings that must be found in the retrieved context.
+ * @returns A value in [0, 1]. Returns 1.0 when goldFacts is empty (vacuously true).
+ */
+declare function recallAtK(retrievedSnippets: string[], goldFacts: string[]): number;
+/**
+ * Compute fact recall against an asserted knowledge graph (KG).
+ *
+ * Given a set of expected (subject, predicate, object) triples as plain text strings (e.g.
+ * "Alice lives in London"), checks how many appear in the provided fact texts. Useful when the
+ * benchmark asserts facts into the temporal KG and queries them back.
+ *
+ * @param factTexts - Texts of facts retrieved from the KG (e.g. "subject predicate object" strings).
+ * @param goldFacts - Expected fact substrings.
+ * @returns Fraction found in [0, 1].
+ */
+declare function factRecall(factTexts: string[], goldFacts: string[]): number;
+
+/**
+ * Core benchmark runner for the Eidentic memory benchmark harness (§6.10).
+ *
+ * runMemoryBench: ingests a multi-session conversation into a fresh Memory instance, then for each
+ * question retrieves and checks whether the gold evidence was recalled. Aggregates into BenchReport.
+ */
+
+/** Options for runMemoryBench. */
+interface BenchOptions {
+    /** Number of top snippets to retrieve per question. Default 8. */
+    topK?: number;
+    /**
+     * Optional LLM judge (a ModelPort). When provided AND a question has an `answer` field,
+     * also scores answer-correctness via eval's llmJudge pattern (gated/optional).
+     * No judge means deterministic recall-only mode — safe for CI.
+     */
+    judge?: ModelPort;
+}
+/**
+ * Run a BenchDataset through a fresh Memory instance per case and produce a BenchReport.
+ *
+ * @param makeMemory - Factory called once per case to produce a fresh, empty Memory.
+ * @param dataset - The benchmark dataset to evaluate.
+ * @param opts - Optional: topK, judge.
+ */
+declare function runMemoryBench(makeMemory: () => Promise<Memory> | Memory, dataset: BenchDataset, opts?: BenchOptions): Promise<BenchReport>;
+
+/**
+ * Bundled synthetic benchmark dataset for CI (§6.10).
+ *
+ * Covers all four categories: single-session, multi-session, temporal, knowledge-update.
+ * Deterministic, hand-written, no real models or large files needed.
+ *
+ * Design notes for FakeEmbedder compatibility:
+ *   FakeEmbedder hashes word tokens into a fixed-dim bag-of-words vector. Recall is highest
+ *   when the gold fact shares tokens with the question OR the ingested turn. Gold facts are
+ *   therefore written as verbatim substrings of ingested turns so that lexical recall works,
+ *   and questions use overlapping vocabulary so semantic (vector) recall also fires.
+ */
+
+declare const syntheticDataset: BenchDataset;
+
+/**
+ * Load a LongMemEval JSON file and convert it to BenchDataset.
+ *
+ * @param jsonPath - Absolute or relative path to the LongMemEval JSON file.
+ *   **Security note:** callers must validate untrusted paths before passing
+ *   them here; this function does not perform path-traversal checks.
+ * @param opts.maxBytes - Maximum allowed file size in bytes (default 256 MiB).
+ *   Increase this only if you are loading a vetted, trusted dataset file.
+ */
+declare function loadLongMemEval(jsonPath: string, opts?: {
+    maxBytes?: number;
+}): Promise<BenchDataset>;
+/**
+ * Load a LoCoMo JSON file and convert it to BenchDataset.
+ *
+ * @param jsonPath - Absolute or relative path to the LoCoMo JSON file.
+ *   **Security note:** callers must validate untrusted paths before passing
+ *   them here; this function does not perform path-traversal checks.
+ * @param opts.maxBytes - Maximum allowed file size in bytes (default 256 MiB).
+ *   Increase this only if you are loading a vetted, trusted dataset file.
+ */
+declare function loadLoCoMo(jsonPath: string, opts?: {
+    maxBytes?: number;
+}): Promise<BenchDataset>;
+
+/**
+ * Write-quality benchmark for the Eidentic memory harness.
+ *
+ * Measures three orthogonal write-side properties that retrieval-only benchmarks miss:
+ *
+ *   1. Contradiction suppression — when a newer fact contradicts an older one, the
+ *      stale fact should be suppressed (invalidated) and the current fact should win.
+ *
+ *   2. Junk resistance — the system should NOT store system-prompt dumps, tool outputs,
+ *      transient task chatter, or other non-durable content. Measured as:
+ *        junkRate = junk items stored / junk items fed  (want: 0)
+ *        factRecall = real facts stored / real facts fed  (want: 1)
+ *
+ *   3. Duplicate resistance — re-ingesting the same events (across simulated sessions,
+ *      including recalled-echo patterns) should not inflate the store.
+ *        duplicateRate = extra copies written / total re-ingest events  (want: 0)
+ *
+ * All three metrics are paired with cost-transparency fields: llmCallsPerWrite and
+ * tokensUsedIfAny. With MockModel these are deterministic counts.
+ *
+ * None of these metrics require a real LLM or external infrastructure.
+ */
+
+/** A single contradiction fixture: subject + predicate, initial value, then an updated value. */
+interface ContradictionFixture {
+    subject: string;
+    predicate: string;
+    /** The stale (earlier) value. Should be invalidated after the update is asserted. */
+    staleObject: string;
+    /** The current (later) value. Should win after assertion. */
+    currentObject: string;
+    /** ISO timestamps establishing temporal order. staleFrom < currentFrom. */
+    staleFrom: string;
+    currentFrom: string;
+}
+/** A single junk-resistance item in the mixed-stream fixture. */
+interface JunkItem {
+    /**
+     * "real"  — a genuine durable user fact that SHOULD be recorded.
+     * "junk"  — content that SHOULD NOT be stored (system prompt, tool output, chatter).
+     */
+    kind: "real" | "junk";
+    /** Sub-classification for reporting (not used in scoring). */
+    junkKind?: "system-prompt" | "tool-output" | "transient-state" | "agent-scratchpad";
+    text: string;
+    /** For "real" items: the expected fact triple that proves storage. */
+    expectedFact?: {
+        subject: string;
+        predicate: string;
+        object: string;
+    };
+}
+/** Per-item detail entry in WriteQualityReport. */
+interface WriteQualityDetail {
+    kind: "contradiction" | "junk" | "duplicate";
+    label: string;
+    passed: boolean;
+    note?: string;
+}
+/** Full report produced by runWriteQualityBench. */
+interface WriteQualityReport {
+    /**
+     * Fraction of contradiction pairs where the CURRENT fact wins and the stale fact is
+     * suppressed / invalidated. Range [0, 1]. Higher is better.
+     */
+    contradictionAccuracy: number;
+    /**
+     * Fraction of junk items that were stored (lower is better — want 0).
+     * junkRate = junk items stored / junk items fed.
+     */
+    junkRate: number;
+    /**
+     * Fraction of real facts that were stored (higher is better — want 1).
+     * factRecall = real facts stored / real facts fed.
+     */
+    factRecall: number;
+    /**
+     * Fraction of re-ingested duplicate events that resulted in an additional write
+     * (lower is better — want 0).
+     * duplicateRate = extra copies written / total re-ingest events.
+     */
+    duplicateRate: number;
+    /**
+     * LLM calls issued per ingest write (averaged across all writes in the bench run).
+     * With MockModel this is a deterministic count.
+     */
+    llmCallsPerWrite: number;
+    /**
+     * Total tokens consumed across all LLM calls in the bench run (input + output).
+     * With MockModel this is a deterministic count.
+     */
+    tokensUsedIfAny: number;
+    /** Per-item details for inspection and debugging. */
+    details: WriteQualityDetail[];
+}
+/**
+ * Built-in contradiction fixtures.
+ * Each pair represents a fact that changes over time: employer, city, job title.
+ */
+declare const CONTRADICTION_FIXTURES: ContradictionFixture[];
+/**
+ * Built-in mixed-stream fixture set for junk-resistance testing.
+ *
+ * Designed to exercise the Consolidator / passive-extraction rejection gates:
+ *   - System-prompt content → REJECT (config, not user fact)
+ *   - Tool output / API response → REJECT (derived data, not stated fact)
+ *   - Transient in-progress state → REJECT (ephemeral)
+ *   - Agent scratchpad / reasoning → REJECT (internal)
+ *   - Real user facts → KEEP
+ */
+declare const JUNK_STREAM_FIXTURES: JunkItem[];
+interface WriteQualityOptions {
+    /**
+     * Custom contradiction fixtures. Defaults to CONTRADICTION_FIXTURES.
+     */
+    contradictionFixtures?: ContradictionFixture[];
+    /**
+     * Custom junk-stream fixtures. Defaults to JUNK_STREAM_FIXTURES.
+     */
+    junkStreamFixtures?: JunkItem[];
+    /**
+     * Number of simulated sessions for duplicate-resistance measurement.
+     * Each session re-ingests the same real facts. Default: 3.
+     */
+    duplicateSessions?: number;
+    /**
+     * Scope to use for all bench operations. Defaults to a stable bench scope.
+     */
+    scope?: Scope;
+}
+/**
+ * Run the write-quality benchmark against a Memory instance.
+ *
+ * The Memory instance must have a graph configured (to use assertFact / queryFacts).
+ * For junk-resistance, the test exercises the passive-extraction path via
+ * assertFact directly using a MockModel-driven Consolidator pattern — or, when no
+ * model is configured, falls back to direct assertFact to verify the KG behavior.
+ *
+ * Cost transparency: every LLM call and token count is tracked and reported.
+ *
+ * @param memory - A fresh Memory instance with a graph backend.
+ * @param opts   - Optional customization of fixtures and parameters.
+ */
+declare function runWriteQualityBench(memory: Memory, opts?: WriteQualityOptions): Promise<WriteQualityReport>;
+
+/**
+ * Synthetic temporal benchmark dataset generator.
+ *
+ * Generates K entities with state histories (employer / city / preference changing
+ * over time at known dates). Produces:
+ *   - A sequence of AssertFactInput calls that establish the temporal record.
+ *   - A question set "what was X's <property> at <date>?" with gold answers.
+ *     Covers dates:
+ *       • BETWEEN two changes (mid-interval)
+ *       • AT the exact change boundary
+ *       • BEFORE first known fact (expect: no answer)
+ *       • AT the current/latest state
+ *
+ * This benchmark is ONLY passable by systems with timestamped fact validity (validAt).
+ * Pure retrieval systems that do not record fact-level temporal intervals cannot pass it —
+ * this distinction is documented plainly in BASELINES.md.
+ *
+ * The generator is seeded for determinism (seed parameter controls entity names +
+ * timestamps so the same seed always produces the same question/answer pairs).
+ */
+
+/** A single state transition for one entity property. */
+interface StateTransition {
+    /** ISO timestamp at which this state becomes valid. */
+    validFrom: string;
+    /** The value of the property at this point in time. */
+    value: string;
+}
+/** One entity with one or more tracked properties and their state history. */
+interface TemporalEntity {
+    /** Stable entity name (e.g. "entity_0"). */
+    name: string;
+    /** Map from predicate name (e.g. "employer") to ordered history of transitions. */
+    history: Record<string, StateTransition[]>;
+}
+/** A single point-in-time question with gold answer. */
+interface TemporalQuestion {
+    subject: string;
+    predicate: string;
+    /** The ISO date at which the question is asked. */
+    askedAt: string;
+    /**
+     * Gold answer: the value of the property at askedAt, or null if no fact was known
+     * before askedAt (query before first known state).
+     */
+    goldAnswer: string | null;
+    /** Human-readable description of why this answer is correct. */
+    rationale: string;
+}
+/** The complete synthetic temporal dataset. */
+interface SyntheticTemporalDataset {
+    /** Human-readable name for display. */
+    name: string;
+    /** Seed used to generate this dataset (for reproducibility). */
+    seed: number;
+    /** Generated entities with full state histories. */
+    entities: TemporalEntity[];
+    /**
+     * AssertFactInput objects ready to be passed to memory.assertFact, in ascending
+     * validFrom order. Call with the scope of your choice.
+     */
+    asserts: AssertFactInput[];
+    /** Point-in-time QA pairs for evaluation. */
+    questions: TemporalQuestion[];
+}
+/**
+ * Generate a synthetic temporal dataset.
+ *
+ * @param opts.entityCount  Number of entities (default 4).
+ * @param opts.seed         Seed for determinism (default 42).
+ * @param opts.changesPerProperty  Number of state changes per property per entity (default 3).
+ */
+declare function syntheticTemporalDataset(opts?: {
+    entityCount?: number;
+    seed?: number;
+    changesPerProperty?: number;
+}): SyntheticTemporalDataset;
+
+/**
+ * Temporal point-in-time QA benchmark (`runTemporalBench`).
+ *
+ * Evaluates whether a Memory instance with a graph backend can answer point-in-time
+ * questions: "what was X's <property> at <date>?" using the `validAt` parameter of
+ * queryFacts. No LLM is required — answers are compared directly to gold.
+ *
+ * Two metrics:
+ *   - pointInTimeAccuracy: fraction of non-current-state questions answered correctly.
+ *   - currentStateAccuracy: fraction of "latest state" questions answered correctly.
+ *
+ * Cost transparency: llmCallsPerWrite and tokensUsedIfAny are always 0 for this
+ * deterministic benchmark (no model calls). Reported for API consistency.
+ *
+ * This benchmark is ONLY passable by systems with timestamped fact validity. A system
+ * that stores facts without validFrom/validUntil intervals cannot correctly answer
+ * "what was X at <past date>?" — it will always return the current state.
+ */
+
+/** Per-question result in TemporalBenchReport. */
+interface TemporalQuestionResult {
+    subject: string;
+    predicate: string;
+    askedAt: string;
+    goldAnswer: string | null;
+    systemAnswer: string | null;
+    correct: boolean;
+    /** "before-first-fact" | "at-boundary" | "mid-interval" | "current-state" */
+    questionType: string;
+    rationale: string;
+}
+/** Full report produced by runTemporalBench. */
+interface TemporalBenchReport {
+    datasetName: string;
+    /**
+     * Accuracy on questions whose askedAt is BEFORE or BETWEEN known states
+     * (excludes current-state questions). Range [0, 1]. Higher is better.
+     */
+    pointInTimeAccuracy: number;
+    /**
+     * Accuracy on questions asking for the latest / current state.
+     * Range [0, 1]. Higher is better.
+     */
+    currentStateAccuracy: number;
+    /**
+     * Accuracy on questions asked BEFORE the first known fact (correct answer: null/no result).
+     * Range [0, 1]. A system that returns stale facts here fails this metric.
+     */
+    beforeFirstFactAccuracy: number;
+    /** Total questions evaluated. */
+    totalQuestions: number;
+    /**
+     * LLM calls per write. Always 0 for this deterministic benchmark.
+     * Present for cost-transparency API consistency.
+     */
+    llmCallsPerWrite: number;
+    /**
+     * Tokens used. Always 0 for this deterministic benchmark.
+     * Present for cost-transparency API consistency.
+     */
+    tokensUsedIfAny: number;
+    /** Per-question results for inspection. */
+    results: TemporalQuestionResult[];
+}
+interface TemporalBenchOptions {
+    /**
+     * Scope to use for assertFact and queryFacts calls.
+     * Default: { kind: "agent", agentId: "bench:temporal" }
+     */
+    scope?: Scope;
+}
+/**
+ * Run the temporal point-in-time QA benchmark.
+ *
+ * Workflow:
+ *   1. Assert all facts from dataset.asserts into the memory graph, in ascending
+ *      validFrom order (required for temporal-order correctness).
+ *   2. For each question in dataset.questions, call queryFacts({ validAt: q.askedAt }).
+ *   3. Compare the returned fact's object to q.goldAnswer.
+ *   4. Aggregate into TemporalBenchReport.
+ *
+ * @param memory  A Memory instance with a configured graph backend (required).
+ * @param dataset The synthetic temporal dataset produced by syntheticTemporalDataset().
+ * @param opts    Optional scope configuration.
+ */
+declare function runTemporalBench(memory: Memory, dataset: SyntheticTemporalDataset, opts?: TemporalBenchOptions): Promise<TemporalBenchReport>;
+
+export { type BenchCase, type BenchDataset, type BenchOptions, type BenchQuestion, type BenchReport, type BenchTurn, CONTRADICTION_FIXTURES, type CaseResult, type ContradictionFixture, JUNK_STREAM_FIXTURES, type JunkItem, type QuestionResult, type StateTransition, type SyntheticTemporalDataset, type TemporalBenchOptions, type TemporalBenchReport, type TemporalEntity, type TemporalQuestion, type TemporalQuestionResult, type WriteQualityDetail, type WriteQualityOptions, type WriteQualityReport, factRecall, loadLoCoMo, loadLongMemEval, normalizeText, normalizedIncludes, recallAtK, runMemoryBench, runTemporalBench, runWriteQualityBench, syntheticDataset, syntheticTemporalDataset };