claude-memory-layer 1.0.30 → 1.0.32

package/tests/core/replay-evaluator.test.ts

@@ -1,181 +0,0 @@
-import { describe, expect, it } from 'vitest';
-
-import {
-  evaluateReplayFixture,
-  formatReplayEvaluationMarkdown,
-  type ReplayRetrievalRunner
-} from '../../src/core/replay-evaluator.js';
-
-const fixture = {
-  name: 'private-real-session-qrels',
-  description: 'contains raw real session text that reports must not leak',
-  ks: [1, 3],
-  queries: [
-    {
-      queryId: 'q-secret-1',
-      query: 'SECRET vector search recall regression',
-      expectedIds: ['m-secret-1'],
-      expectedRelevance: { 'm-secret-1': 2 }
-    }
-  ],
-  memories: [
-    {
-      id: 'm-secret-1',
-      content: 'SECRET vector search recall regression fix uses retriever pipeline replay'
-    },
-    {
-      id: 'm-noise',
-      content: 'unrelated dashboard layout memory'
-    }
-  ]
-};
-
-describe('replay fixture evaluator', () => {
-  it('evaluates through the retriever pipeline runner and returns a sanitized report', async () => {
-    const calls: Array<{ query: string; queryId: string; topK: number }> = [];
-    const retrievalRunner: ReplayRetrievalRunner = async (query, input) => {
-      calls.push({ query, queryId: input.query.queryId, topK: input.topK });
-      return {
-        retrievedIds: ['m-secret-1', 'm-noise'],
-        candidateIds: ['m-secret-1', 'm-noise'],
-        confidence: 'high',
-        fallbackTrace: ['stage:primary:fast']
-      };
-    };
-
-    const report = await evaluateReplayFixture(fixture, {
-      generatedAt: '2026-05-05T00:00:00.000Z',
-      retrievalRunner
-    });
-    const serialized = JSON.stringify(report);
-
-    expect(calls).toEqual([
-      { query: 'SECRET vector search recall regression', queryId: 'q-secret-1', topK: 3 }
-    ]);
-    expect(report).toMatchObject({
-      name: 'private-real-session-qrels',
-      evaluator: 'retriever-pipeline-v1',
-      generatedAt: '2026-05-05T00:00:00.000Z',
-      fixtureStats: {
-        queryCount: 1,
-        memoryCount: 2,
-        ks: [1, 3]
-      },
-      summary: {
-        queryCount: 1,
-        precisionAtK: { 1: 1, 3: 1 / 3 },
-        recallAtK: { 1: 1, 3: 1 },
-        ndcgAtK: { 1: 1, 3: 1 },
-        hitAtK: { 1: 1, 3: 1 },
-        mrr: 1,
-        failedQueryCount: 0
-      }
-    });
-    expect(report.perQuery).toEqual([
-      {
-        queryId: 'q-secret-1',
-        retrievedIds: ['m-secret-1', 'm-noise'],
-        candidateIds: ['m-secret-1', 'm-noise'],
-        confidence: 'high',
-        fallbackTrace: ['stage:primary:fast'],
-        reciprocalRank: 1,
-        at: {
-          1: { precision: 1, recall: 1, hits: 1, ndcg: 1 },
-          3: { precision: 1 / 3, recall: 1, hits: 1, ndcg: 1 }
-        }
-      }
-    ]);
-    expect(serialized).not.toContain('SECRET');
-    expect(serialized).not.toContain('vector search recall regression');
-  });
-
-  it('uses the real in-memory Retriever/RetrievalOrchestrator pipeline by default', async () => {
-    const report = await evaluateReplayFixture(fixture, {
-      generatedAt: '2026-05-05T00:00:00.000Z',
-      retrievalOptions: { strategy: 'fast' }
-    });
-
-    expect(report.evaluator).toBe('retriever-pipeline-v1');
-    expect(report.perQuery[0]).toMatchObject({
-      queryId: 'q-secret-1',
-      retrievedIds: expect.arrayContaining(['m-secret-1']),
-      candidateIds: expect.arrayContaining(['m-secret-1']),
-      fallbackTrace: expect.arrayContaining(['stage:primary:fast'])
-    });
-    expect(report.summary.hitAtK[1]).toBe(1);
-  });
-
-  it('counts no-match qrels separately from positive retrieval misses', async () => {
-    const report = await evaluateReplayFixture({
-      name: 'negative-qrels-fixture',
-      ks: [1, 3],
-      queries: [
-        {
-          queryId: 'q-positive',
-          query: 'retriever pipeline replay answer',
-          expectation: 'match',
-          expectedIds: ['m-positive'],
-          expectedRelevance: { 'm-positive': 2 },
-          knownAnswer: 'Retriever pipeline replay answer should be found.'
-        },
-        {
-          queryId: 'q-command-artifact-no-match',
-          query: 'local-command-stdout command-name opus',
-          expectation: 'no_match',
-          expectedIds: [],
-          expectedRelevance: {},
-          forbiddenIds: ['m-positive']
-        }
-      ],
-      memories: [
-        {
-          id: 'm-positive',
-          content: 'Retriever pipeline replay answer should be found.'
-        }
-      ]
-    }, {
-      generatedAt: '2026-05-05T00:00:00.000Z',
-      retrievalOptions: { strategy: 'auto' }
-    });
-
-    expect(report.summary).toMatchObject({
-      queryCount: 2,
-      positiveQueryCount: 1,
-      noMatchQueryCount: 1,
-      noMatchCorrect: 1,
-      noMatchAccuracy: 1,
-      failedQueryCount: 0,
-      precisionAtK: { 1: 1, 3: 1 / 3 },
-      recallAtK: { 1: 1, 3: 1 },
-      hitAtK: { 1: 1, 3: 1 }
-    });
-    expect(report.perQuery[1]).toMatchObject({
-      queryId: 'q-command-artifact-no-match',
-      expectation: 'no_match',
-      retrievedIds: [],
-      forbiddenHitIds: [],
-      noMatchSatisfied: true,
-      confidence: 'none'
-    });
-  });
-
-  it('formats markdown reports without raw query or memory content', async () => {
-    const report = await evaluateReplayFixture(fixture, {
-      generatedAt: '2026-05-05T00:00:00.000Z',
-      includePerQuery: false
-    });
-
-    const markdown = formatReplayEvaluationMarkdown(report, {
-      qrelsPath: '.claude-memory/benchmarks/real-session-qrels.json'
-    });
-
-    expect(markdown).toContain('# Retrieval Replay Benchmark Report');
-    expect(markdown).toContain('private-real-session-qrels');
-    expect(markdown).toContain('nDCG@1');
-    expect(markdown).toContain('Hit@1');
-    expect(markdown).toContain('MRR');
-    expect(markdown).toContain('.claude-memory/benchmarks/real-session-qrels.json');
-    expect(markdown).not.toContain('SECRET');
-    expect(markdown).not.toContain('vector search recall regression');
-  });
-});

package/tests/core/retrieval-analytics-service.test.ts

@@ -1,210 +0,0 @@
-import { mkdtempSync, rmSync } from 'node:fs';
-import { tmpdir } from 'node:os';
-import { join } from 'node:path';
-import { afterEach, describe, expect, it } from 'vitest';
-import { RetrievalAnalyticsService } from '../../src/core/engine/retrieval-analytics-service.js';
-import type {
-  RetrievalAnalyticsStore,
-  RetrievalTrace
-} from '../../src/core/engine/retrieval-analytics-service.js';
-import type { MemoryEvent } from '../../src/core/types.js';
-
-const tempDirs: string[] = [];
-
-afterEach(() => {
-  while (tempDirs.length > 0) {
-    const dir = tempDirs.pop();
-    if (dir) {
-      rmSync(dir, { recursive: true, force: true });
-    }
-  }
-});
-
-function tempStoragePath(): string {
-  const dir = mkdtempSync(join(tmpdir(), 'retrieval-analytics-service-'));
-  tempDirs.push(dir);
-  return dir;
-}
-
-function event(
-  id: string,
-  content: string,
-  timestamp = new Date('2026-02-24T00:00:00.000Z')
-): MemoryEvent & { access_count?: number; last_accessed_at?: string } {
-  return {
-    id,
-    sessionId: 's1',
-    eventType: 'user_prompt',
-    content,
-    canonicalKey: `test/${id}`,
-    dedupeKey: `s1:${id}`,
-    timestamp,
-    metadata: {}
-  };
-}
-
-function baseStore(overrides: Partial<RetrievalAnalyticsStore> = {}): RetrievalAnalyticsStore {
-  return {
-    getRetrievalTraceStats: async () => ({
-      totalQueries: 0,
-      avgCandidateCount: 0,
-      avgSelectedCount: 0,
-      selectionRate: 0
-    }),
-    getRecentRetrievalTraces: async (_limit = 50) => [],
-    getMostAccessed: async (_limit = 10) => [],
-    evaluateSessionHelpfulness: async (_sessionId: string) => {},
-    getUnevaluatedSessions: async (_currentSessionId: string, _limit = 5) => [],
-    getHelpfulMemories: async (_limit = 10) => [],
-    getHelpfulnessStats: async () => ({
-      avgScore: 0,
-      totalEvaluated: 0,
-      totalRetrievals: 0,
-      helpful: 0,
-      neutral: 0,
-      unhelpful: 0
-    }),
-    ...overrides
-  };
-}
-
-describe('RetrievalAnalyticsService', () => {
-  it('maps most-accessed events to dashboard memories and extracts topics from content', async () => {
-    let initialized = 0;
-    let requestedLimit: number | undefined;
-    const accessed = event(
-      'e1',
-      `${'x'.repeat(205)}\n## Thin Core Architecture\nUse **Retrieval Analytics** to keep MemoryService small.`
-    );
-    accessed.access_count = 3;
-    accessed.last_accessed_at = '2026-02-25T00:00:00.000Z';
-    const untouched = event('e2', 'plain content without explicit access metadata');
-
-    const service = new RetrievalAnalyticsService({
-      initialize: async () => { initialized += 1; },
-      retrievalStore: baseStore({
-        getMostAccessed: async (limit = 10) => {
-          requestedLimit = limit;
-          return [accessed, untouched];
-        }
-      })
-    });
-
-    const memories = await service.getMostAccessedMemories(7);
-
-    expect(initialized).toBe(0);
-    expect(requestedLimit).toBe(7);
-    expect(memories).toHaveLength(2);
-    expect(memories[0]).toMatchObject({
-      memoryId: 'e1',
-      summary: `${'x'.repeat(200)}...`,
-      topics: ['Thin Core Architecture', 'Retrieval Analytics'],
-      accessCount: 3,
-      lastAccessed: '2026-02-25T00:00:00.000Z',
-      confidence: 1.0,
-      createdAt: accessed.timestamp
-    });
-    expect(memories[1]).toMatchObject({
-      memoryId: 'e2',
-      accessCount: 0,
-      lastAccessed: null,
-      confidence: 1.0,
-      createdAt: untouched.timestamp
-    });
-  });
-
-  it('evaluates pending sessions best-effort and ignores individual failures', async () => {
-    let initialized = 0;
-    const evaluated: string[] = [];
-
-    const service = new RetrievalAnalyticsService({
-      initialize: async () => { initialized += 1; },
-      retrievalStore: baseStore({
-        getUnevaluatedSessions: async (currentSessionId: string, limit = 5) => {
-          expect(currentSessionId).toBe('current-session');
-          expect(limit).toBe(5);
-          return ['ok-1', 'fails', 'ok-2'];
-        },
-        evaluateSessionHelpfulness: async (sessionId: string) => {
-          if (sessionId === 'fails') {
-            throw new Error('transient evaluation failure');
-          }
-          evaluated.push(sessionId);
-        }
-      })
-    });
-
-    await service.evaluatePendingSessions('current-session');
-
-    expect(initialized).toBe(1);
-    expect(evaluated).toEqual(['ok-1', 'ok-2']);
-  });
-
-  it('delegates trace and helpfulness read-model methods after initialization', async () => {
-    let initialized = 0;
-    const traceStats = {
-      totalQueries: 12,
-      avgCandidateCount: 4,
-      avgSelectedCount: 2,
-      selectionRate: 0.5
-    };
-    const traceRows = [{
-      traceId: 't1',
-      sessionId: 's1',
-      projectHash: 'project-hash',
-      queryText: 'thin core',
-      strategy: 'auto',
-      candidateEventIds: ['e1', 'e2'],
-      selectedEventIds: ['e1'],
-      candidateDetails: [{ eventId: 'e1', score: 0.9 }],
-      selectedDetails: [{ eventId: 'e1', score: 0.9 }],
-      candidateCount: 2,
-      selectedCount: 1,
-      confidence: 'high',
-      fallbackTrace: ['stage:primary:deep'],
-      createdAt: new Date('2026-02-24T01:00:00.000Z')
-    }] satisfies RetrievalTrace[];
-    const helpfulMemories = [{
-      eventId: 'e1',
-      summary: 'helpful memory',
-      helpfulnessScore: 0.8,
-      accessCount: 4,
-      evaluationCount: 2
-    }];
-    const helpfulnessStats = {
-      avgScore: 0.75,
-      totalEvaluated: 8,
-      totalRetrievals: 10,
-      helpful: 6,
-      neutral: 1,
-      unhelpful: 1
-    };
-    const evaluated: string[] = [];
-
-    const service = new RetrievalAnalyticsService({
-      initialize: async () => { initialized += 1; },
-      retrievalStore: baseStore({
-        getRetrievalTraceStats: async () => traceStats,
-        getRecentRetrievalTraces: async (limit = 50) => {
-          expect(limit).toBe(3);
-          return traceRows;
-        },
-        evaluateSessionHelpfulness: async (sessionId: string) => { evaluated.push(sessionId); },
-        getHelpfulMemories: async (limit = 10) => {
-          expect(limit).toBe(2);
-          return helpfulMemories;
-        },
-        getHelpfulnessStats: async () => helpfulnessStats
-      })
-    });
-
-    await expect(service.getRetrievalTraceStats()).resolves.toEqual(traceStats);
-    await expect(service.getRecentRetrievalTraces(3)).resolves.toEqual(traceRows);
-    await service.evaluateSessionHelpfulness('s1');
-    await expect(service.getHelpfulMemories(2)).resolves.toEqual(helpfulMemories);
-    await expect(service.getHelpfulnessStats()).resolves.toEqual(helpfulnessStats);
-
-    expect(evaluated).toEqual(['s1']);
-    expect(initialized).toBe(5);
-  });
-});

package/tests/core/retrieval-benchmark.test.ts

@@ -1,93 +0,0 @@
-import { describe, expect, it } from 'vitest';
-
-import { computePrecisionRecallAtK, summarizeReplayMetrics } from '../../src/core/retrieval-benchmark.js';
-
-describe('retrieval replay benchmark metrics', () => {
-  it('computes Precision@k and Recall@k for replay queries', () => {
-    const queryMetrics = computePrecisionRecallAtK(
-      [
-        { queryId: 'q1', expectedIds: ['a', 'b'], retrievedIds: ['a', 'x', 'b'] },
-        { queryId: 'q2', expectedIds: ['c'], retrievedIds: ['x', 'y', 'z'] }
-      ],
-      [1, 3]
-    );
-
-    expect(queryMetrics[0].at[1]).toMatchObject({ precision: 1, recall: 0.5, hits: 1 });
-    expect(queryMetrics[0].at[3]).toMatchObject({ precision: 2 / 3, recall: 1, hits: 2 });
-    expect(queryMetrics[0].at[1].ndcg).toBe(1);
-    expect(queryMetrics[0].at[3].ndcg).toBeCloseTo(0.91972, 4);
-    expect(queryMetrics[1].at[1]).toMatchObject({ precision: 0, recall: 0, hits: 0 });
-    expect(queryMetrics[1].at[3]).toMatchObject({ precision: 0, recall: 0, hits: 0 });
-
-    const summary = summarizeReplayMetrics(queryMetrics, [1, 3]);
-    expect(summary).toMatchObject({
-      queryCount: 2,
-      precisionAtK: { 1: 0.5, 3: 1 / 3 },
-      recallAtK: { 1: 0.25, 3: 0.5 }
-    });
-    expect(summary.ndcgAtK[1]).toBe(0.5);
-    expect(summary.ndcgAtK[3]).toBeCloseTo(0.45986, 4);
-  });
-
-  it('computes graded nDCG@k from qrels relevance labels', () => {
-    const [queryMetrics] = computePrecisionRecallAtK(
-      [
-        {
-          queryId: 'q-graded',
-          expectedIds: ['a', 'b'],
-          expectedRelevance: { a: 3, b: 1 },
-          retrievedIds: ['b', 'a', 'noise']
-        }
-      ],
-      [2]
-    );
-
-    expect(queryMetrics.at[2]).toMatchObject({ precision: 1, recall: 1, hits: 2 });
-    expect(queryMetrics.at[2].ndcg).toBeCloseTo(0.70981, 4);
-  });
-
-  it('deduplicates retrieved ids so replay metrics cannot over-count repeated hits', () => {
-    const queryMetrics = computePrecisionRecallAtK(
-      [
-        { queryId: 'q-duplicate', expectedIds: ['a'], retrievedIds: ['a', 'a', 'a', 'x'] }
-      ],
-      [1, 3]
-    );
-
-    expect(queryMetrics).toEqual([
-      {
-        queryId: 'q-duplicate',
-        at: {
-          1: { precision: 1, recall: 1, hits: 1, ndcg: 1 },
-          3: { precision: 1 / 3, recall: 1, hits: 1, ndcg: 1 }
-        }
-      }
-    ]);
-    expect(queryMetrics[0].at[3].ndcg).toBe(1);
-  });
-
-  it('normalizes k values without losing zero-result replay rows', () => {
-    const queryMetrics = computePrecisionRecallAtK(
-      [{ queryId: 'q-empty', expectedIds: ['a'], retrievedIds: [] }],
-      [3.9, 1, 1, -2]
-    );
-
-    expect(queryMetrics).toEqual([
-      {
-        queryId: 'q-empty',
-        at: {
-          0: { precision: 0, recall: 0, hits: 0, ndcg: 0 },
-          1: { precision: 0, recall: 0, hits: 0, ndcg: 0 },
-          3: { precision: 0, recall: 0, hits: 0, ndcg: 0 }
-        }
-      }
-    ]);
-
-    expect(summarizeReplayMetrics(queryMetrics, [3.9, 1, 1, -2])).toEqual({
-      queryCount: 1,
-      precisionAtK: { 0: 0, 1: 0, 3: 0 },
-      recallAtK: { 0: 0, 1: 0, 3: 0 },
-      ndcgAtK: { 0: 0, 1: 0, 3: 0 }
-    });
-  });
-});