hippo-memory 1.69.0 → 1.71.0

package/dist/{eval-stats.d.ts → eval/eval-stats.d.ts}

@@ -1,16 +1,5 @@
-/**
- * Statistics and cost accounting for the token-efficiency evals.
- *
- * - Cost: price provider usage over four buckets (uncached input, cache
- *   write, cache read, output). Raw token counts overstate savings when most
- *   input is already cache reads, so every dollar claim goes through here.
- * - Uncertainty: paired bootstrap over tasks, cluster bootstrap (tasks that
- *   share a repository are not independent), and a paired ratio bootstrap for
- *   dollars per resolved task.
- *
- * Deterministic: every resampling function takes a seed, so a published
- * result can be reproduced exactly.
- */
+/** Statistics and cost accounting for the token-efficiency evals: cost is priced over four buckets (uncached input, cache write, cache read, output).
+ *  Uncertainty uses paired, cluster (same-repository tasks are not independent) and paired-ratio bootstraps; every resampler takes a seed. */
 /** Token usage for one model call or one whole task, split by how it is billed. */
 export interface Usage {
     /** Input tokens billed at the base input price (not read from or written to a cache). */
@@ -22,11 +11,7 @@ export interface Usage {
     /** Output tokens, including any reasoning tokens the provider bills as output. */
     outputTokens: number;
 }
-/**
- * Prices in dollars per million tokens. Take them from the provider's
- * current price page for the exact model; this module has no built-in
- * prices because they change.
- */
+/** Prices in dollars per million tokens, taken from the provider's current price page for the exact model; this module has no built-in prices. */
 export interface Prices {
     inputPerMTok: number;
     cacheWritePerMTok: number;
@@ -37,12 +22,8 @@ export interface Prices {
 export declare function addUsage(...usages: Usage[]): Usage;
 /** Dollar cost of `usage` at `prices`. */
 export declare function priceUsage(usage: Usage, prices: Prices): number;
-/**
- * Relative prices of the cache buckets against the base input price, for
- * cost in "uncached-equivalent tokens" when no dollar prices are given.
- * Defaults follow Anthropic's published ratios (5-minute cache write 1.25x,
- * cache read 0.1x); pass the ratios for another provider when needed.
- */
+/** Relative cache-bucket prices against the base input price, for cost in "uncached-equivalent tokens" when no dollar prices are given.
+ *  Defaults follow Anthropic's ratios (5-minute cache write 1.25x, cache read 0.1x); pass another provider's ratios when needed. */
 export interface CacheRatios {
     write: number;
     read: number;
@@ -51,10 +32,8 @@ export interface CacheRatios {
 export declare const DEFAULT_CACHE_RATIOS: Readonly<CacheRatios>;
 /** Input cost of `usage` in uncached-equivalent tokens (output excluded). */
 export declare function uncachedEquivalentInput(usage: Usage, ratios?: CacheRatios): number;
-/**
- * Mulberry32: a small seeded PRNG returning floats in [0, 1). Same seed,
- * same stream, on every platform.
- */
+/** Mulberry32: a small seeded PRNG returning floats in [0, 1). Same seed,
+ *  same stream, on every platform. */
 export declare function seededRandom(seed: number): () => number;
 /** A point estimate with a percentile bootstrap confidence interval. */
 export interface Estimate {
@@ -73,17 +52,9 @@ export interface BootstrapOpts {
     /** PRNG seed. Default 1. */
     seed?: number;
 }
-/**
- * Paired bootstrap for the mean of per-task differences (treatment minus
- * control on the same task). An interval that excludes zero is the bar for
- * calling a difference real.
- */
+/** Paired bootstrap for the mean of per-task differences (treatment minus control, same task); an interval excluding zero is the bar for a real difference. */
 export declare function pairedBootstrap(diffs: number[], opts?: BootstrapOpts): Estimate;
-/**
- * Cluster bootstrap for the mean of per-task differences: resamples whole
- * clusters (for example all tasks from one repository), because tasks in a
- * cluster share causes and are not independent draws.
- */
+/** Cluster bootstrap for the mean of per-task differences: resamples whole clusters (e.g. one repository), because tasks in a cluster are not independent. */
 export declare function clusteredPairedBootstrap(diffsByCluster: ReadonlyMap<string, number[]>, opts?: BootstrapOpts): Estimate;
 /** One task's outcome in one arm, for {@link costPerResolvedDelta}. */
 export interface ArmOutcome {
@@ -101,23 +72,14 @@ export interface CostPerResolvedDelta {
     /** (treatment minus control) / control, with its interval. Negative is a saving. */
     relative: Estimate;
 }
-/**
- * Cost per resolved task in two arms run on the same tasks, with a paired
- * bootstrap over tasks (a task is resampled with both of its arm outcomes).
- * `control[i]` and `treatment[i]` must be the same task. Resamples in which
- * an arm resolves nothing are dropped; `iterations` reports how many were
- * kept.
- */
+/** Cost per resolved task in two arms on the same tasks, with a paired bootstrap (`control[i]` and `treatment[i]` must be the same task).
+ *  Resamples in which an arm resolves nothing are dropped; `iterations` reports how many were kept. */
 export declare function costPerResolvedDelta(control: ArmOutcome[], treatment: ArmOutcome[], opts?: BootstrapOpts): CostPerResolvedDelta;
-/**
- * pass@k: share of tasks with at least one success in their first k runs.
- * NaN when no task has k runs (not measured, which is not the same as 0).
- */
+/** pass@k: share of tasks with at least one success in their first k runs.
+ *  NaN when no task has k runs (not measured, which is not the same as 0). */
 export declare function passAtK(runsByTask: boolean[][], k: number): number;
-/**
- * pass^k: share of tasks whose first k runs all succeed (consistency).
- * NaN when no task has k runs.
- */
+/** pass^k: share of tasks whose first k runs all succeed (consistency).
+ *  NaN when no task has k runs. */
 export declare function passHatK(runsByTask: boolean[][], k: number): number;
 /** All units of one family (tasks by seeds), resampled as one block so seeds stay together. */
 export type Family<T> = readonly T[];

package/dist/{eval-stats.js → eval/eval-stats.js}

@@ -1,16 +1,8 @@
-/**
- * Statistics and cost accounting for the token-efficiency evals.
- *
- * - Cost: price provider usage over four buckets (uncached input, cache
- *   write, cache read, output). Raw token counts overstate savings when most
- *   input is already cache reads, so every dollar claim goes through here.
- * - Uncertainty: paired bootstrap over tasks, cluster bootstrap (tasks that
- *   share a repository are not independent), and a paired ratio bootstrap for
- *   dollars per resolved task.
- *
- * Deterministic: every resampling function takes a seed, so a published
- * result can be reproduced exactly.
- */
+/** Statistics and cost accounting for the token-efficiency evals: cost is priced over four buckets (uncached input, cache write, cache read, output).
+ *  Uncertainty uses paired, cluster (same-repository tasks are not independent) and paired-ratio bootstraps; every resampler takes a seed. */
+const DEFAULT_ITERATIONS = 5000;
+const DEFAULT_TWO_LEVEL_ITERATIONS = 10_000;
+const DEFAULT_ALPHA = 0.05;
 /** Sum several usages bucket by bucket. */
 export function addUsage(...usages) {
     const total = { inputTokens: 0, cacheWriteTokens: 0, cacheReadTokens: 0, outputTokens: 0 };
@@ -35,10 +27,8 @@ export const DEFAULT_CACHE_RATIOS = { write: 1.25, read: 0.1 };
 export function uncachedEquivalentInput(usage, ratios = DEFAULT_CACHE_RATIOS) {
     return usage.inputTokens + usage.cacheWriteTokens * ratios.write + usage.cacheReadTokens * ratios.read;
 }
-/**
- * Mulberry32: a small seeded PRNG returning floats in [0, 1). Same seed,
- * same stream, on every platform.
- */
+/** Mulberry32: a small seeded PRNG returning floats in [0, 1). Same seed,
+ *  same stream, on every platform. */
 export function seededRandom(seed) {
     let a = seed >>> 0;
     return () => {
@@ -58,14 +48,10 @@ function percentileInterval(samples, alpha) {
 function mean(xs) {
     return xs.length === 0 ? 0 : xs.reduce((s, x) => s + x, 0) / xs.length;
 }
-/**
- * Paired bootstrap for the mean of per-task differences (treatment minus
- * control on the same task). An interval that excludes zero is the bar for
- * calling a difference real.
- */
+/** Paired bootstrap for the mean of per-task differences (treatment minus control, same task); an interval excluding zero is the bar for a real difference. */
 export function pairedBootstrap(diffs, opts = {}) {
-    const iterations = opts.iterations ?? 5000;
-    const alpha = opts.alpha ?? 0.05;
+    const iterations = opts.iterations ?? DEFAULT_ITERATIONS;
+    const alpha = opts.alpha ?? DEFAULT_ALPHA;
     if (diffs.length === 0)
         return { estimate: 0, low: 0, high: 0, iterations: 0 };
     const rand = seededRandom(opts.seed ?? 1);
@@ -79,14 +65,10 @@ export function pairedBootstrap(diffs, opts = {}) {
     }
     return { estimate: mean(diffs), ...percentileInterval(samples, alpha), iterations };
 }
-/**
- * Cluster bootstrap for the mean of per-task differences: resamples whole
- * clusters (for example all tasks from one repository), because tasks in a
- * cluster share causes and are not independent draws.
- */
+/** Cluster bootstrap for the mean of per-task differences: resamples whole clusters (e.g. one repository), because tasks in a cluster are not independent. */
 export function clusteredPairedBootstrap(diffsByCluster, opts = {}) {
-    const iterations = opts.iterations ?? 5000;
-    const alpha = opts.alpha ?? 0.05;
+    const iterations = opts.iterations ?? DEFAULT_ITERATIONS;
+    const alpha = opts.alpha ?? DEFAULT_ALPHA;
     const clusters = [...diffsByCluster.values()].filter((c) => c.length > 0);
     const all = clusters.flat();
     if (all.length === 0)
@@ -112,19 +94,14 @@ function costPerResolved(outcomes) {
     const cost = outcomes.reduce((s, o) => s + o.cost, 0);
     return resolved === 0 ? Number.POSITIVE_INFINITY : cost / resolved;
 }
-/**
- * Cost per resolved task in two arms run on the same tasks, with a paired
- * bootstrap over tasks (a task is resampled with both of its arm outcomes).
- * `control[i]` and `treatment[i]` must be the same task. Resamples in which
- * an arm resolves nothing are dropped; `iterations` reports how many were
- * kept.
- */
+/** Cost per resolved task in two arms on the same tasks, with a paired bootstrap (`control[i]` and `treatment[i]` must be the same task).
+ *  Resamples in which an arm resolves nothing are dropped; `iterations` reports how many were kept. */
 export function costPerResolvedDelta(control, treatment, opts = {}) {
     if (control.length !== treatment.length) {
         throw new Error('control and treatment must list the same tasks in the same order');
     }
-    const iterations = opts.iterations ?? 5000;
-    const alpha = opts.alpha ?? 0.05;
+    const iterations = opts.iterations ?? DEFAULT_ITERATIONS;
+    const alpha = opts.alpha ?? DEFAULT_ALPHA;
     const c = costPerResolved(control);
     const t = costPerResolved(treatment);
     const rand = seededRandom(opts.seed ?? 1);
@@ -163,20 +140,16 @@ export function costPerResolvedDelta(control, treatment, opts = {}) {
         },
     };
 }
-/**
- * pass@k: share of tasks with at least one success in their first k runs.
- * NaN when no task has k runs (not measured, which is not the same as 0).
- */
+/** pass@k: share of tasks with at least one success in their first k runs.
+ *  NaN when no task has k runs (not measured, which is not the same as 0). */
 export function passAtK(runsByTask, k) {
     const eligible = runsByTask.filter((r) => r.length >= k);
     if (eligible.length === 0)
         return Number.NaN;
     return eligible.filter((r) => r.slice(0, k).some(Boolean)).length / eligible.length;
 }
-/**
- * pass^k: share of tasks whose first k runs all succeed (consistency).
- * NaN when no task has k runs.
- */
+/** pass^k: share of tasks whose first k runs all succeed (consistency).
+ *  NaN when no task has k runs. */
 export function passHatK(runsByTask, k) {
     const eligible = runsByTask.filter((r) => r.length >= k);
     if (eligible.length === 0)
@@ -213,7 +186,7 @@ function twoSidedP(samples, nullValue) {
 /** Resamples repositories, then families inside each; only the repository draw carries a shared-store fault.
  * The statistic never uses the PRNG, so a second call on one seed draws the same resamples. */
 export function twoLevelBootstrap(repos, statistic, opts = {}) {
-    const requested = opts.iterations ?? 10_000;
+    const requested = opts.iterations ?? DEFAULT_TWO_LEVEL_ITERATIONS;
     if (!Number.isInteger(requested) || requested <= 0) {
         throw new RangeError(`iterations must be a positive integer, got ${requested}`);
     }
@@ -233,7 +206,7 @@ export function twoLevelBootstrap(repos, statistic, opts = {}) {
         return notANumber(nullValue, dropped);
     const estimate = statistic(kept.flatMap((r) => r.flat()));
     const p = kept.length < 2 ? Number.NaN : twoSidedP(samples, nullValue);
-    const interval = percentileInterval(samples, opts.alpha ?? 0.05);
+    const interval = percentileInterval(samples, opts.alpha ?? DEFAULT_ALPHA);
     return { estimate, ...interval, p, iterations: samples.length, dropped, nullValue };
 }
 /** Holm step-down in input order; the family size is `ps.length`, as preregistered.
@@ -263,7 +236,7 @@ export function verdict(e, adjustedP, spec) {
         return inconclusive;
     const nullValue = e.nullValue;
     const lowerIsHelpful = spec.helpful === 'lower';
-    if (adjustedP < (spec.alpha ?? 0.05)) {
+    if (adjustedP < (spec.alpha ?? DEFAULT_ALPHA)) {
         const helpfulSide = lowerIsHelpful ? e.estimate < nullValue : e.estimate > nullValue;
         const harmfulSide = lowerIsHelpful ? e.estimate > nullValue : e.estimate < nullValue;
         if (harmfulSide)

package/dist/{eval-suite.d.ts → eval/eval-suite.d.ts}

@@ -1,17 +1,6 @@
-/**
- * Self-contained feature evaluation suite for hippo.
- *
- * Creates a synthetic memory corpus with known ground truth, runs searches
- * per feature category, and reports per-feature metrics with regression
- * detection against a saved baseline.
- *
- * Design goals:
- *  - Zero API calls (no LLM judge, no embeddings)
- *  - Deterministic (fixed timestamps, content, IDs)
- *  - Fast (<60s for full suite)
- *  - Per-feature breakdown so you see exactly what a change helped/hurt
- */
-import { type MemoryEntry } from './memory.js';
+/** Self-contained feature eval suite: a synthetic corpus with known ground truth, per-feature metrics and regression detection against a saved baseline.
+ *  Zero API calls (no LLM judge, no embeddings), deterministic (fixed timestamps, content, IDs) and fast. */
+import { type MemoryEntry } from '../core/memory.js';
 export type FeatureCategory = 'direct-recall' | 'extraction-preference' | 'dag-drilldown' | 'temporal' | 'noise-resistance' | 'multi-hop';
 export interface FeatureTestCase {
     id: string;

package/dist/{eval-suite.js → eval/eval-suite.js}

@@ -1,26 +1,13 @@
-/**
- * Self-contained feature evaluation suite for hippo.
- *
- * Creates a synthetic memory corpus with known ground truth, runs searches
- * per feature category, and reports per-feature metrics with regression
- * detection against a saved baseline.
- *
- * Design goals:
- *  - Zero API calls (no LLM judge, no embeddings)
- *  - Deterministic (fixed timestamps, content, IDs)
- *  - Fast (<60s for full suite)
- *  - Per-feature breakdown so you see exactly what a change helped/hurt
- */
-import { createMemory, Layer } from './memory.js';
-import { search } from './search/bm25-search.js';
-import { multihopSearch } from './multihop.js';
+/** Self-contained feature eval suite: a synthetic corpus with known ground truth, per-feature metrics and regression detection against a saved baseline.
+ *  Zero API calls (no LLM judge, no embeddings), deterministic (fixed timestamps, content, IDs) and fast. */
+import { createMemory, Layer } from '../core/memory.js';
+import { search } from '../search/bm25-search.js';
+import { multihopSearch } from '../search/multihop.js';
 import { mrr, recallAtK, ndcgAtK } from './eval.js';
-// ---------------------------------------------------------------------------
-// Synthetic corpus — deterministic, no API calls
-// ---------------------------------------------------------------------------
+import { DAY_MS } from '../util/time.js';
 const BASE_DATE = new Date('2026-01-15T10:00:00Z');
 function dateOffset(days) {
-    const d = new Date(BASE_DATE.getTime() + days * 86400000);
+    const d = new Date(BASE_DATE.getTime() + days * DAY_MS);
     return d.toISOString();
 }
 function mem(id, content, opts = {}) {
@@ -39,8 +26,28 @@ function mem(id, content, opts = {}) {
 }
 /** Direct recall: basic keyword matching. */
 function addDirectRecall(entries, cases) {
-    entries.push(mem('dr-1', 'The PostgreSQL database migration failed because the users table had a NOT NULL constraint on the email column', { created: dateOffset(1), tags: ['topic:database'] }), mem('dr-2', 'React component rendering performance improved by 40% after memoizing the expensive computation in useMemo', { created: dateOffset(2), tags: ['topic:frontend'] }), mem('dr-3', 'The API rate limiter should use a sliding window algorithm instead of fixed window to prevent burst traffic', { created: dateOffset(3), tags: ['topic:api'] }), mem('dr-4', 'Docker container memory limits need to be set to 512MB for the worker service to prevent OOM kills', { created: dateOffset(4), tags: ['topic:devops'] }), mem('dr-5', 'The JWT token expiration was set to 24 hours but should be reduced to 1 hour for security compliance', { created: dateOffset(5), tags: ['topic:security'] }), mem('dr-6', 'Webpack bundle size grew to 2.3MB because lodash was imported as a whole instead of cherry-picking', { created: dateOffset(6), tags: ['topic:frontend'] }), mem('dr-7', 'The Redis cache TTL for user sessions should match the JWT expiration to prevent stale sessions', { created: dateOffset(7), tags: ['topic:caching'] }), mem('dr-8', 'GraphQL resolver for nested comments has N+1 query problem solved by DataLoader batching', { created: dateOffset(8), tags: ['topic:api'] }));
-    cases.push({ id: 'dr-q1', category: 'direct-recall', query: 'PostgreSQL migration NOT NULL constraint', expectedIds: ['dr-1'], description: 'exact keyword match on DB migration' }, { id: 'dr-q2', category: 'direct-recall', query: 'React useMemo performance memoizing', expectedIds: ['dr-2'], description: 'React performance optimization' }, { id: 'dr-q3', category: 'direct-recall', query: 'sliding window rate limiter API', expectedIds: ['dr-3'], description: 'API rate limiting approach' }, { id: 'dr-q4', category: 'direct-recall', query: 'Docker OOM memory limits worker', expectedIds: ['dr-4'], description: 'Docker memory config' }, { id: 'dr-q5', category: 'direct-recall', query: 'JWT token expiration security', expectedIds: ['dr-5'], description: 'JWT security setting' }, { id: 'dr-q6', category: 'direct-recall', query: 'webpack bundle size lodash', expectedIds: ['dr-6'], description: 'bundle size issue' }, { id: 'dr-q7', category: 'direct-recall', query: 'Redis cache TTL session expiration', expectedIds: ['dr-7'], description: 'cache TTL config' }, { id: 'dr-q8', category: 'direct-recall', query: 'GraphQL N+1 DataLoader batching', expectedIds: ['dr-8'], description: 'GraphQL N+1 fix' });
+    entries.push(mem('dr-1', 'The PostgreSQL database migration failed because the users table had a NOT NULL constraint on the email column', {
+        created: dateOffset(1), tags: ['topic:database']
+    }), mem('dr-2', 'React component rendering performance improved by 40% after memoizing the expensive computation in useMemo', {
+        created: dateOffset(2), tags: ['topic:frontend']
+    }), mem('dr-3', 'The API rate limiter should use a sliding window algorithm instead of fixed window to prevent burst traffic', {
+        created: dateOffset(3), tags: ['topic:api']
+    }), mem('dr-4', 'Docker container memory limits need to be set to 512MB for the worker service to prevent OOM kills', {
+        created: dateOffset(4), tags: ['topic:devops']
+    }), mem('dr-5', 'The JWT token expiration was set to 24 hours but should be reduced to 1 hour for security compliance', {
+        created: dateOffset(5), tags: ['topic:security']
+    }), mem('dr-6', 'Webpack bundle size grew to 2.3MB because lodash was imported as a whole instead of cherry-picking', {
+        created: dateOffset(6), tags: ['topic:frontend']
+    }), mem('dr-7', 'The Redis cache TTL for user sessions should match the JWT expiration to prevent stale sessions', {
+        created: dateOffset(7), tags: ['topic:caching']
+    }), mem('dr-8', 'GraphQL resolver for nested comments has N+1 query problem solved by DataLoader batching', { created: dateOffset(8), tags: ['topic:api'] }));
+    cases.push({
+        id: 'dr-q1', category: 'direct-recall', query: 'PostgreSQL migration NOT NULL constraint', expectedIds: ['dr-1'],
+        description: 'exact keyword match on DB migration'
+    }, {
+        id: 'dr-q2', category: 'direct-recall', query: 'React useMemo performance memoizing', expectedIds: ['dr-2'],
+        description: 'React performance optimization'
+    }, { id: 'dr-q3', category: 'direct-recall', query: 'sliding window rate limiter API', expectedIds: ['dr-3'], description: 'API rate limiting approach' }, { id: 'dr-q4', category: 'direct-recall', query: 'Docker OOM memory limits worker', expectedIds: ['dr-4'], description: 'Docker memory config' }, { id: 'dr-q5', category: 'direct-recall', query: 'JWT token expiration security', expectedIds: ['dr-5'], description: 'JWT security setting' }, { id: 'dr-q6', category: 'direct-recall', query: 'webpack bundle size lodash', expectedIds: ['dr-6'], description: 'bundle size issue' }, { id: 'dr-q7', category: 'direct-recall', query: 'Redis cache TTL session expiration', expectedIds: ['dr-7'], description: 'cache TTL config' }, { id: 'dr-q8', category: 'direct-recall', query: 'GraphQL N+1 DataLoader batching', expectedIds: ['dr-8'], description: 'GraphQL N+1 fix' });
 }
 /** Extraction preference: extracted facts should rank above raw source. */
 function addExtractionPreference(entries, cases) {
@@ -60,7 +67,16 @@ function addExtractionPreference(entries, cases) {
         created: dateOffset(12), layer: Layer.Semantic, tags: ['speaker:Carol', 'topic:auth', 'extracted'],
         extracted_from: 'ep-src-3',
     }));
-    cases.push({ id: 'ep-q1', category: 'extraction-preference', query: 'Kubernetes upgrade staging environment', expectedIds: ['ep-ext-1'], description: 'extracted fact about K8s upgrade should rank above raw utterance' }, { id: 'ep-q2', category: 'extraction-preference', query: 'Datadog Grafana alerting migration', expectedIds: ['ep-ext-2'], description: 'extracted fact about monitoring switch' }, { id: 'ep-q3', category: 'extraction-preference', query: 'OAuth2 PKCE login page redesign', expectedIds: ['ep-ext-3'], description: 'extracted fact about auth work' });
+    cases.push({
+        id: 'ep-q1', category: 'extraction-preference', query: 'Kubernetes upgrade staging environment', expectedIds: ['ep-ext-1'],
+        description: 'extracted fact about K8s upgrade should rank above raw utterance'
+    }, {
+        id: 'ep-q2', category: 'extraction-preference', query: 'Datadog Grafana alerting migration', expectedIds: ['ep-ext-2'],
+        description: 'extracted fact about monitoring switch'
+    }, {
+        id: 'ep-q3', category: 'extraction-preference', query: 'OAuth2 PKCE login page redesign', expectedIds: ['ep-ext-3'],
+        description: 'extracted fact about auth work'
+    });
 }
 /** Dag drill-down: summary nodes should surface children. */
 function addDagDrilldown(entries, cases) {
@@ -81,12 +97,30 @@ function addDagDrilldown(entries, cases) {
     for (const child of [entries.find(e => e.id === 'dag-child-1'), entries.find(e => e.id === 'dag-child-2'), entries.find(e => e.id === 'dag-child-3')]) {
         child.dag_parent_id = 'dag-summary-1';
     }
-    cases.push({ id: 'dag-q1', category: 'dag-drilldown', query: 'API performance problems latency', expectedIds: ['dag-child-1', 'dag-child-2', 'dag-child-3', 'dag-summary-1'], description: 'summary should drill down to all children' }, { id: 'dag-q2', category: 'dag-drilldown', query: 'endpoint response time degradation', expectedIds: ['dag-child-1', 'dag-child-2', 'dag-summary-1'], description: 'query matching summary should surface relevant children' });
+    cases.push({
+        id: 'dag-q1', category: 'dag-drilldown', query: 'API performance problems latency',
+        expectedIds: ['dag-child-1', 'dag-child-2', 'dag-child-3', 'dag-summary-1'], description: 'summary should drill down to all children'
+    }, {
+        id: 'dag-q2', category: 'dag-drilldown', query: 'endpoint response time degradation',
+        expectedIds: ['dag-child-1', 'dag-child-2', 'dag-summary-1'], description: 'query matching summary should surface relevant children'
+    });
 }
 /** Temporal: recency/oldest cues should affect ranking. */
 function addTemporal(entries, cases) {
     entries.push(mem('tmp-1', 'The team decided to use TypeScript for the new service', { created: dateOffset(-30), tags: ['topic:architecture'] }), mem('tmp-2', 'The team evaluated Rust as an alternative language for the service', { created: dateOffset(-20), tags: ['topic:architecture'] }), mem('tmp-3', 'The team added Go as a candidate language for the service rewrite', { created: dateOffset(-10), tags: ['topic:architecture'] }), mem('tmp-4', 'The team finalized the language choice as Go for the service rewrite', { created: dateOffset(-1), tags: ['topic:architecture'] }));
-    cases.push({ id: 'tmp-q1', category: 'temporal', query: 'what did the team recently decide about the service language', expectedIds: ['tmp-4'], description: 'recent cue should boost newest entry' }, { id: 'tmp-q2', category: 'temporal', query: 'what was the first language choice for the service', expectedIds: ['tmp-1'], description: 'oldest cue should boost earliest entry' }, { id: 'tmp-q3', category: 'temporal', query: 'latest update on the service rewrite language', expectedIds: ['tmp-4'], description: 'latest should boost most recent' }, { id: 'tmp-q4', category: 'temporal', query: 'original architecture decision for the service', expectedIds: ['tmp-1'], description: 'original should boost earliest' });
+    cases.push({
+        id: 'tmp-q1', category: 'temporal', query: 'what did the team recently decide about the service language', expectedIds: ['tmp-4'],
+        description: 'recent cue should boost newest entry'
+    }, {
+        id: 'tmp-q2', category: 'temporal', query: 'what was the first language choice for the service', expectedIds: ['tmp-1'],
+        description: 'oldest cue should boost earliest entry'
+    }, {
+        id: 'tmp-q3', category: 'temporal', query: 'latest update on the service rewrite language', expectedIds: ['tmp-4'],
+        description: 'latest should boost most recent'
+    }, {
+        id: 'tmp-q4', category: 'temporal', query: 'original architecture decision for the service', expectedIds: ['tmp-1'],
+        description: 'original should boost earliest'
+    });
 }
 /** Noise resistance: relevant memories found despite noise. */
 function addNoiseResistance(entries, cases) {
@@ -116,7 +150,18 @@ function addNoiseResistance(entries, cases) {
     }), mem('nr-2', 'The S3 bucket policy was misconfigured allowing public read access to customer data uploads', {
         created: dateOffset(21), tags: ['topic:security', 'topic:s3'],
     }));
-    cases.push({ id: 'nr-q1', category: 'noise-resistance', query: 'database connection pool exhausted max connections', expectedIds: ['nr-1'], description: 'find DB issue despite 30 noise entries' }, { id: 'nr-q2', category: 'noise-resistance', query: 'S3 bucket public access security misconfiguration', expectedIds: ['nr-2'], description: 'find S3 issue despite noise' }, { id: 'nr-q3', category: 'noise-resistance', query: 'PostgreSQL migration constraint', expectedIds: ['dr-1'], description: 'find earlier DB entry despite noise' }, { id: 'nr-q4', category: 'noise-resistance', query: 'JWT token security expiration', expectedIds: ['dr-5'], description: 'find security entry despite noise' });
+    cases.push({
+        id: 'nr-q1', category: 'noise-resistance', query: 'database connection pool exhausted max connections', expectedIds: ['nr-1'],
+        description: 'find DB issue despite 30 noise entries'
+    }, {
+        id: 'nr-q2', category: 'noise-resistance', query: 'S3 bucket public access security misconfiguration', expectedIds: ['nr-2'],
+        description: 'find S3 issue despite noise'
+    }, {
+        id: 'nr-q3', category: 'noise-resistance', query: 'PostgreSQL migration constraint', expectedIds: ['dr-1'],
+        description: 'find earlier DB entry despite noise'
+    }, {
+        id: 'nr-q4', category: 'noise-resistance', query: 'JWT token security expiration', expectedIds: ['dr-5'], description: 'find security entry despite noise'
+    });
 }
 /** Multi-hop: entity chaining across sessions. */
 function addMultiHop(entries, cases) {
@@ -129,7 +174,13 @@ function addMultiHop(entries, cases) {
     }), mem('mh-4', 'speaker:Carol reported that the billing dashboard shows incorrect revenue numbers because it reads from the payments_raw table instead of payments_reconciled.', {
         created: dateOffset(28), tags: ['speaker:Carol', 'topic:billing'],
     }));
-    cases.push({ id: 'mh-q1', category: 'multi-hop', query: 'Who fixed the Stripe webhook bug and what was the root cause?', expectedIds: ['mh-1', 'mh-2', 'mh-3'], description: 'chain Alice -> Stripe -> Bob -> fix' }, { id: 'mh-q2', category: 'multi-hop', query: 'What are all the payment-related issues the team discussed?', expectedIds: ['mh-1', 'mh-2', 'mh-3', 'mh-4'], description: 'find all payment topics across speakers' });
+    cases.push({
+        id: 'mh-q1', category: 'multi-hop', query: 'Who fixed the Stripe webhook bug and what was the root cause?',
+        expectedIds: ['mh-1', 'mh-2', 'mh-3'], description: 'chain Alice -> Stripe -> Bob -> fix'
+    }, {
+        id: 'mh-q2', category: 'multi-hop', query: 'What are all the payment-related issues the team discussed?',
+        expectedIds: ['mh-1', 'mh-2', 'mh-3', 'mh-4'], description: 'find all payment topics across speakers'
+    });
 }
 export function buildSyntheticCorpus() {
     const entries = [];
@@ -142,9 +193,25 @@ export function buildSyntheticCorpus() {
     addMultiHop(entries, cases);
     return { entries, cases };
 }
-// ---------------------------------------------------------------------------
-// Runner — evaluates each case against the synthetic corpus
-// ---------------------------------------------------------------------------
+/** Each category's mean scores, in the order the corpus first names the category. */
+function featureAverages(cases, caseResults) {
+    const categories = [...new Set(cases.map(c => c.category))];
+    return categories.map(cat => {
+        const catCases = caseResults.filter(r => r.case.category === cat);
+        const n = catCases.length;
+        const avgMrr = catCases.reduce((s, r) => s + r.mrrVal, 0) / n;
+        const avgR5 = catCases.reduce((s, r) => s + r.r5, 0) / n;
+        const avgNdcg5 = catCases.reduce((s, r) => s + r.ndcg5, 0) / n;
+        return {
+            category: cat,
+            cases: n,
+            mrr: avgMrr,
+            recallAt5: avgR5,
+            ndcgAt5: avgNdcg5,
+            passed: true,
+        };
+    });
+}
 export async function runFeatureEval(version) {
     const start = Date.now();
     const { entries, cases } = buildSyntheticCorpus();
@@ -166,22 +233,7 @@ export async function runFeatureEval(version) {
             ndcg5: ndcgAtK(returnedIds, c.expectedIds, 5),
         });
     }
-    const categories = [...new Set(cases.map(c => c.category))];
-    const features = categories.map(cat => {
-        const catCases = caseResults.filter(r => r.case.category === cat);
-        const n = catCases.length;
-        const avgMrr = catCases.reduce((s, r) => s + r.mrrVal, 0) / n;
-        const avgR5 = catCases.reduce((s, r) => s + r.r5, 0) / n;
-        const avgNdcg5 = catCases.reduce((s, r) => s + r.ndcg5, 0) / n;
-        return {
-            category: cat,
-            cases: n,
-            mrr: avgMrr,
-            recallAt5: avgR5,
-            ndcgAt5: avgNdcg5,
-            passed: true,
-        };
-    });
+    const features = featureAverages(cases, caseResults);
     const totalCases = caseResults.length;
     const overallMrr = caseResults.reduce((s, r) => s + r.mrrVal, 0) / totalCases;
     const overallR5 = caseResults.reduce((s, r) => s + r.r5, 0) / totalCases;
@@ -195,9 +247,6 @@ export async function runFeatureEval(version) {
         durationMs: Date.now() - start,
     };
 }
-// ---------------------------------------------------------------------------
-// Regression detection
-// ---------------------------------------------------------------------------
 const REGRESSION_THRESHOLD = 0.05;
 export function detectRegressions(baseline, current) {
     const regressions = [];
@@ -225,11 +274,7 @@ export function detectRegressions(baseline, current) {
     };
 }
 export function resultToBaseline(result) {
-    // SAFETY: every FeatureCategory key is populated by the loop immediately
-    // below (one iteration per entry in result.features), before this object
-    // is read anywhere; result.features is built from every category in
-    // buildSyntheticCorpus's cases (runFeatureEval derives `categories` the
-    // same way), so no FeatureCategory key is left unset.
+    // SAFETY: the loop below sets every FeatureCategory key from result.features, which holds every category in buildSyntheticCorpus's cases.
     const features = {};
     for (const f of result.features) {
         features[f.category] = { mrr: f.mrr, recallAt5: f.recallAt5, ndcgAt5: f.ndcgAt5 };
@@ -241,9 +286,6 @@ export function resultToBaseline(result) {
         overall: result.overall,
     };
 }
-// ---------------------------------------------------------------------------
-// Formatters
-// ---------------------------------------------------------------------------
 function pct(n) {
     return (n * 100).toFixed(1) + '%';
 }

package/dist/{eval.d.ts → eval/eval.d.ts}

@@ -1,14 +1,6 @@
-/**
- * Recall eval harness.
- *
- * Given a corpus of (query, expected_memory_ids) cases, run recall against
- * the store and report ranking-quality metrics: MRR, Recall@K, NDCG@K.
- *
- * The goal is to make recall quality measurable so MMR lambda, embedding
- * weights, and future scoring tweaks can be tuned against evidence instead
- * of intuition.
- */
-import type { MemoryEntry } from './memory.js';
+/** Recall eval harness: runs recall over (query, expected_memory_ids) cases and reports MRR, Recall@K and NDCG@K.
+ *  It exists so MMR lambda, embedding weights and scoring tweaks are tuned against evidence instead of intuition. */
+import type { MemoryEntry } from '../core/memory.js';
 export interface EvalCase {
     /** Free-form ID for humans to reference the case. */
     id: string;
@@ -58,10 +50,8 @@ export interface RunEvalOptions {
 export declare function mrr(returned: string[], expected: string[]): number;
 /** Recall@K — fraction of expected items found in the top-K. */
 export declare function recallAtK(returned: string[], expected: string[], k: number): number;
-/**
- * Normalized Discounted Cumulative Gain at K with binary relevance.
- * gain_i = 1 if returned[i] ∈ expected else 0. discount = log2(i + 2).
- */
+/** Normalized Discounted Cumulative Gain at K with binary relevance.
+ *  gain_i = 1 if returned[i] ∈ expected else 0. discount = log2(i + 2). */
 export declare function ndcgAtK(returned: string[], expected: string[], k: number): number;
 export declare function runEval(cases: EvalCase[], entries: MemoryEntry[], options?: RunEvalOptions): Promise<EvalSummary>;
 export interface EvalDelta {
@@ -88,16 +78,8 @@ export interface EvalComparison {
     onlyInBaseline: string[];
     onlyInCurrent: string[];
 }
-/**
- * Compute pairwise deltas between a baseline and a current eval summary.
- * Cases are matched by EvalCase.id; any mismatch is surfaced in the onlyIn
- * arrays so the caller knows the corpora diverged.
- */
+/** Compute pairwise deltas between a baseline and a current eval summary, matching cases by EvalCase.id; mismatches land in the onlyIn arrays. */
 export declare function compareSummaries(baseline: EvalSummary, current: EvalSummary): EvalComparison;
-/**
- * For each memory, take its first 8 content words as a trivial query and
- * expect that memory back. Useful as a smoke test: if recall can't find a
- * memory by its own opening words, something is broken.
- */
+/** For each memory, use its first 8 content words as a trivial query and expect that memory back: a smoke test for recall. */
 export declare function bootstrapCorpus(entries: MemoryEntry[], maxCases?: number): EvalCase[];
 //# sourceMappingURL=eval.d.ts.map