claude-memory-layer 1.0.26 → 1.0.28

package/tests/core/product-validation-matrix.test.ts

@@ -0,0 +1,61 @@
+import { describe, expect, it } from 'vitest';
+
+import {
+  getProductValidationMatrixSummary,
+  productValidationMatrix,
+  renderProductValidationMatrixMarkdown
+} from '../../src/core/product-validation-matrix.js';
+
+const requiredSurfaces = [
+  'claude.adapter.import',
+  'claude.adapter.search',
+  'claude.adapter.disclosure',
+  'codex.adapter.scan',
+  'codex.adapter.import',
+  'codex.adapter.replay',
+  'hermes.adapter.scan',
+  'hermes.adapter.import',
+  'hermes.adapter.replay',
+  'mcp.context.pack',
+  'mcp.project.timeline',
+  'mcp.source.ref',
+  'cli.api.reporting',
+  'safety.dryRun'
+];
+
+describe('product validation matrix', () => {
+  it('covers the product-level validation surfaces with requirements and evidence', () => {
+    const surfaceIds = new Set(productValidationMatrix.map((surface) => surface.id));
+
+    for (const id of requiredSurfaces) {
+      expect(surfaceIds.has(id), `missing surface ${id}`).toBe(true);
+    }
+
+    for (const surface of productValidationMatrix) {
+      expect(surface.title).toBeTruthy();
+      expect(surface.requirements.length, `${surface.id} requirements`).toBeGreaterThan(0);
+      expect(surface.evidence.length, `${surface.id} evidence`).toBeGreaterThan(0);
+      expect(['ready', 'covered', 'partial', 'planned']).toContain(surface.status);
+    }
+  });
+
+  it('summarizes and renders a stable reporting-friendly matrix', () => {
+    const summary = getProductValidationMatrixSummary(productValidationMatrix);
+    expect(summary.totalSurfaces).toBeGreaterThanOrEqual(requiredSurfaces.length);
+    expect(summary.surfacesByArea.codex).toBeGreaterThanOrEqual(3);
+    expect(summary.surfacesByArea.hermes).toBeGreaterThanOrEqual(3);
+    expect(summary.surfacesByArea.mcp).toBeGreaterThanOrEqual(3);
+    expect(summary.surfacesByArea.claude).toBeGreaterThanOrEqual(3);
+    expect(summary.evidenceCount).toBeGreaterThanOrEqual(requiredSurfaces.length);
+
+    const markdown = renderProductValidationMatrixMarkdown(productValidationMatrix);
+    expect(markdown).toContain('# Product Validation Matrix');
+    expect(markdown).toContain('Codex adapter replay');
+    expect(markdown).toContain('Hermes adapter replay');
+    expect(markdown).toContain('MCP context pack');
+    expect(markdown).toContain('MCP source reference');
+    expect(markdown).toContain('Safety / dry-run');
+    expect(markdown).toContain('tests/core/codex-session-history-importer-validation.test.ts');
+    expect(markdown).toContain('tests/core/hermes-session-history-importer-validation.test.ts');
+  });
+});

package/tests/core/project-registry.test.ts

@@ -0,0 +1,78 @@
+import { afterEach, describe, expect, it, vi } from 'vitest';
+import * as fs from 'node:fs/promises';
+import * as os from 'node:os';
+import * as path from 'node:path';
+import {
+  getProjectStoragePath,
+  hashProjectPath,
+  normalizeProjectPath,
+  resolveProjectStoragePath
+} from '../../src/core/registry/project-path.js';
+
+describe('project-path registry utilities', () => {
+  it('normalizes paths and generates stable hashes', async () => {
+    const root = await fs.mkdtemp(path.join(os.tmpdir(), 'cml-project-path-'));
+    const projectDir = path.join(root, 'project');
+    await fs.mkdir(projectDir, { recursive: true });
+
+    const hashA = hashProjectPath(projectDir);
+    const hashB = hashProjectPath(projectDir + '/');
+
+    const normalized = normalizeProjectPath(projectDir + '/');
+
+    expect(normalized.endsWith('/project')).toBe(true);
+    expect(hashA).toBe(hashB);
+    expect(hashA).toMatch(/^[a-f0-9]{8}$/);
+  });
+
+  it('resolves storage paths for both project paths and explicit hashes', async () => {
+    const root = await fs.mkdtemp(path.join(os.tmpdir(), 'cml-storage-path-'));
+    const projectDir = path.join(root, 'project');
+    await fs.mkdir(projectDir, { recursive: true });
+
+    const projectHash = hashProjectPath(projectDir);
+    const storageFromPath = getProjectStoragePath(projectDir);
+    const storageFromResolverPath = resolveProjectStoragePath(projectDir);
+    const storageFromResolverHash = resolveProjectStoragePath(projectHash);
+
+    expect(storageFromPath).toBe(storageFromResolverPath);
+    expect(storageFromResolverHash).toContain(path.join('.claude-code', 'memory', 'projects', projectHash));
+  });
+});
+
+describe('session registry utilities', () => {
+  afterEach(() => {
+    vi.resetModules();
+    vi.doUnmock('os');
+  });
+
+  it('registers and loads project mapping from the isolated home directory', async () => {
+    const tempHome = await fs.mkdtemp(path.join(os.tmpdir(), 'cml-home-'));
+    const projectDir = path.join(tempHome, 'workspace', 'project');
+    await fs.mkdir(projectDir, { recursive: true });
+
+    vi.doMock('os', async () => {
+      const actual = await vi.importActual<typeof import('os')>('os');
+      return {
+        ...actual,
+        homedir: () => tempHome
+      };
+    });
+
+    const registryModule = await import('../../src/core/registry/session-registry.js');
+
+    registryModule.registerSession('session-123', projectDir);
+    const projectInfo = registryModule.getSessionProject('session-123');
+
+    expect(projectInfo).not.toBeNull();
+    expect(projectInfo?.projectPath.endsWith('/workspace/project')).toBe(true);
+    expect(projectInfo?.projectHash).toBe(hashProjectPath(projectDir));
+
+    const registryPath = path.join(tempHome, '.claude-code', 'memory', 'session-registry.json');
+    const saved = JSON.parse(await fs.readFile(registryPath, 'utf8')) as {
+      sessions: Record<string, { projectPath: string }>;
+    };
+
+    expect(saved.sessions['session-123']?.projectPath.endsWith('/workspace/project')).toBe(true);
+  });
+});

package/tests/core/replay-evaluator.test.ts

@@ -0,0 +1,181 @@
+import { describe, expect, it } from 'vitest';
+
+import {
+  evaluateReplayFixture,
+  formatReplayEvaluationMarkdown,
+  type ReplayRetrievalRunner
+} from '../../src/core/replay-evaluator.js';
+
+const fixture = {
+  name: 'private-real-session-qrels',
+  description: 'contains raw real session text that reports must not leak',
+  ks: [1, 3],
+  queries: [
+    {
+      queryId: 'q-secret-1',
+      query: 'SECRET vector search recall regression',
+      expectedIds: ['m-secret-1'],
+      expectedRelevance: { 'm-secret-1': 2 }
+    }
+  ],
+  memories: [
+    {
+      id: 'm-secret-1',
+      content: 'SECRET vector search recall regression fix uses retriever pipeline replay'
+    },
+    {
+      id: 'm-noise',
+      content: 'unrelated dashboard layout memory'
+    }
+  ]
+};
+
+describe('replay fixture evaluator', () => {
+  it('evaluates through the retriever pipeline runner and returns a sanitized report', async () => {
+    const calls: Array<{ query: string; queryId: string; topK: number }> = [];
+    const retrievalRunner: ReplayRetrievalRunner = async (query, input) => {
+      calls.push({ query, queryId: input.query.queryId, topK: input.topK });
+      return {
+        retrievedIds: ['m-secret-1', 'm-noise'],
+        candidateIds: ['m-secret-1', 'm-noise'],
+        confidence: 'high',
+        fallbackTrace: ['stage:primary:fast']
+      };
+    };
+
+    const report = await evaluateReplayFixture(fixture, {
+      generatedAt: '2026-05-05T00:00:00.000Z',
+      retrievalRunner
+    });
+    const serialized = JSON.stringify(report);
+
+    expect(calls).toEqual([
+      { query: 'SECRET vector search recall regression', queryId: 'q-secret-1', topK: 3 }
+    ]);
+    expect(report).toMatchObject({
+      name: 'private-real-session-qrels',
+      evaluator: 'retriever-pipeline-v1',
+      generatedAt: '2026-05-05T00:00:00.000Z',
+      fixtureStats: {
+        queryCount: 1,
+        memoryCount: 2,
+        ks: [1, 3]
+      },
+      summary: {
+        queryCount: 1,
+        precisionAtK: { 1: 1, 3: 1 / 3 },
+        recallAtK: { 1: 1, 3: 1 },
+        ndcgAtK: { 1: 1, 3: 1 },
+        hitAtK: { 1: 1, 3: 1 },
+        mrr: 1,
+        failedQueryCount: 0
+      }
+    });
+    expect(report.perQuery).toEqual([
+      {
+        queryId: 'q-secret-1',
+        retrievedIds: ['m-secret-1', 'm-noise'],
+        candidateIds: ['m-secret-1', 'm-noise'],
+        confidence: 'high',
+        fallbackTrace: ['stage:primary:fast'],
+        reciprocalRank: 1,
+        at: {
+          1: { precision: 1, recall: 1, hits: 1, ndcg: 1 },
+          3: { precision: 1 / 3, recall: 1, hits: 1, ndcg: 1 }
+        }
+      }
+    ]);
+    expect(serialized).not.toContain('SECRET');
+    expect(serialized).not.toContain('vector search recall regression');
+  });
+
+  it('uses the real in-memory Retriever/RetrievalOrchestrator pipeline by default', async () => {
+    const report = await evaluateReplayFixture(fixture, {
+      generatedAt: '2026-05-05T00:00:00.000Z',
+      retrievalOptions: { strategy: 'fast' }
+    });
+
+    expect(report.evaluator).toBe('retriever-pipeline-v1');
+    expect(report.perQuery[0]).toMatchObject({
+      queryId: 'q-secret-1',
+      retrievedIds: expect.arrayContaining(['m-secret-1']),
+      candidateIds: expect.arrayContaining(['m-secret-1']),
+      fallbackTrace: expect.arrayContaining(['stage:primary:fast'])
+    });
+    expect(report.summary.hitAtK[1]).toBe(1);
+  });
+
+  it('counts no-match qrels separately from positive retrieval misses', async () => {
+    const report = await evaluateReplayFixture({
+      name: 'negative-qrels-fixture',
+      ks: [1, 3],
+      queries: [
+        {
+          queryId: 'q-positive',
+          query: 'retriever pipeline replay answer',
+          expectation: 'match',
+          expectedIds: ['m-positive'],
+          expectedRelevance: { 'm-positive': 2 },
+          knownAnswer: 'Retriever pipeline replay answer should be found.'
+        },
+        {
+          queryId: 'q-command-artifact-no-match',
+          query: 'local-command-stdout command-name opus',
+          expectation: 'no_match',
+          expectedIds: [],
+          expectedRelevance: {},
+          forbiddenIds: ['m-positive']
+        }
+      ],
+      memories: [
+        {
+          id: 'm-positive',
+          content: 'Retriever pipeline replay answer should be found.'
+        }
+      ]
+    }, {
+      generatedAt: '2026-05-05T00:00:00.000Z',
+      retrievalOptions: { strategy: 'auto' }
+    });
+
+    expect(report.summary).toMatchObject({
+      queryCount: 2,
+      positiveQueryCount: 1,
+      noMatchQueryCount: 1,
+      noMatchCorrect: 1,
+      noMatchAccuracy: 1,
+      failedQueryCount: 0,
+      precisionAtK: { 1: 1, 3: 1 / 3 },
+      recallAtK: { 1: 1, 3: 1 },
+      hitAtK: { 1: 1, 3: 1 }
+    });
+    expect(report.perQuery[1]).toMatchObject({
+      queryId: 'q-command-artifact-no-match',
+      expectation: 'no_match',
+      retrievedIds: [],
+      forbiddenHitIds: [],
+      noMatchSatisfied: true,
+      confidence: 'none'
+    });
+  });
+
+  it('formats markdown reports without raw query or memory content', async () => {
+    const report = await evaluateReplayFixture(fixture, {
+      generatedAt: '2026-05-05T00:00:00.000Z',
+      includePerQuery: false
+    });
+
+    const markdown = formatReplayEvaluationMarkdown(report, {
+      qrelsPath: '.claude-memory/benchmarks/real-session-qrels.json'
+    });
+
+    expect(markdown).toContain('# Retrieval Replay Benchmark Report');
+    expect(markdown).toContain('private-real-session-qrels');
+    expect(markdown).toContain('nDCG@1');
+    expect(markdown).toContain('Hit@1');
+    expect(markdown).toContain('MRR');
+    expect(markdown).toContain('.claude-memory/benchmarks/real-session-qrels.json');
+    expect(markdown).not.toContain('SECRET');
+    expect(markdown).not.toContain('vector search recall regression');
+  });
+});

package/tests/core/retrieval-analytics-service.test.ts

@@ -0,0 +1,210 @@
+import { mkdtempSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { afterEach, describe, expect, it } from 'vitest';
+import { RetrievalAnalyticsService } from '../../src/core/engine/retrieval-analytics-service.js';
+import type {
+  RetrievalAnalyticsStore,
+  RetrievalTrace
+} from '../../src/core/engine/retrieval-analytics-service.js';
+import type { MemoryEvent } from '../../src/core/types.js';
+
+const tempDirs: string[] = [];
+
+afterEach(() => {
+  while (tempDirs.length > 0) {
+    const dir = tempDirs.pop();
+    if (dir) {
+      rmSync(dir, { recursive: true, force: true });
+    }
+  }
+});
+
+function tempStoragePath(): string {
+  const dir = mkdtempSync(join(tmpdir(), 'retrieval-analytics-service-'));
+  tempDirs.push(dir);
+  return dir;
+}
+
+function event(
+  id: string,
+  content: string,
+  timestamp = new Date('2026-02-24T00:00:00.000Z')
+): MemoryEvent & { access_count?: number; last_accessed_at?: string } {
+  return {
+    id,
+    sessionId: 's1',
+    eventType: 'user_prompt',
+    content,
+    canonicalKey: `test/${id}`,
+    dedupeKey: `s1:${id}`,
+    timestamp,
+    metadata: {}
+  };
+}
+
+function baseStore(overrides: Partial<RetrievalAnalyticsStore> = {}): RetrievalAnalyticsStore {
+  return {
+    getRetrievalTraceStats: async () => ({
+      totalQueries: 0,
+      avgCandidateCount: 0,
+      avgSelectedCount: 0,
+      selectionRate: 0
+    }),
+    getRecentRetrievalTraces: async (_limit = 50) => [],
+    getMostAccessed: async (_limit = 10) => [],
+    evaluateSessionHelpfulness: async (_sessionId: string) => {},
+    getUnevaluatedSessions: async (_currentSessionId: string, _limit = 5) => [],
+    getHelpfulMemories: async (_limit = 10) => [],
+    getHelpfulnessStats: async () => ({
+      avgScore: 0,
+      totalEvaluated: 0,
+      totalRetrievals: 0,
+      helpful: 0,
+      neutral: 0,
+      unhelpful: 0
+    }),
+    ...overrides
+  };
+}
+
+describe('RetrievalAnalyticsService', () => {
+  it('maps most-accessed events to dashboard memories and extracts topics from content', async () => {
+    let initialized = 0;
+    let requestedLimit: number | undefined;
+    const accessed = event(
+      'e1',
+      `${'x'.repeat(205)}\n## Thin Core Architecture\nUse **Retrieval Analytics** to keep MemoryService small.`
+    );
+    accessed.access_count = 3;
+    accessed.last_accessed_at = '2026-02-25T00:00:00.000Z';
+    const untouched = event('e2', 'plain content without explicit access metadata');
+
+    const service = new RetrievalAnalyticsService({
+      initialize: async () => { initialized += 1; },
+      retrievalStore: baseStore({
+        getMostAccessed: async (limit = 10) => {
+          requestedLimit = limit;
+          return [accessed, untouched];
+        }
+      })
+    });
+
+    const memories = await service.getMostAccessedMemories(7);
+
+    expect(initialized).toBe(0);
+    expect(requestedLimit).toBe(7);
+    expect(memories).toHaveLength(2);
+    expect(memories[0]).toMatchObject({
+      memoryId: 'e1',
+      summary: `${'x'.repeat(200)}...`,
+      topics: ['Thin Core Architecture', 'Retrieval Analytics'],
+      accessCount: 3,
+      lastAccessed: '2026-02-25T00:00:00.000Z',
+      confidence: 1.0,
+      createdAt: accessed.timestamp
+    });
+    expect(memories[1]).toMatchObject({
+      memoryId: 'e2',
+      accessCount: 0,
+      lastAccessed: null,
+      confidence: 1.0,
+      createdAt: untouched.timestamp
+    });
+  });
+
+  it('evaluates pending sessions best-effort and ignores individual failures', async () => {
+    let initialized = 0;
+    const evaluated: string[] = [];
+
+    const service = new RetrievalAnalyticsService({
+      initialize: async () => { initialized += 1; },
+      retrievalStore: baseStore({
+        getUnevaluatedSessions: async (currentSessionId: string, limit = 5) => {
+          expect(currentSessionId).toBe('current-session');
+          expect(limit).toBe(5);
+          return ['ok-1', 'fails', 'ok-2'];
+        },
+        evaluateSessionHelpfulness: async (sessionId: string) => {
+          if (sessionId === 'fails') {
+            throw new Error('transient evaluation failure');
+          }
+          evaluated.push(sessionId);
+        }
+      })
+    });
+
+    await service.evaluatePendingSessions('current-session');
+
+    expect(initialized).toBe(1);
+    expect(evaluated).toEqual(['ok-1', 'ok-2']);
+  });
+
+  it('delegates trace and helpfulness read-model methods after initialization', async () => {
+    let initialized = 0;
+    const traceStats = {
+      totalQueries: 12,
+      avgCandidateCount: 4,
+      avgSelectedCount: 2,
+      selectionRate: 0.5
+    };
+    const traceRows = [{
+      traceId: 't1',
+      sessionId: 's1',
+      projectHash: 'project-hash',
+      queryText: 'thin core',
+      strategy: 'auto',
+      candidateEventIds: ['e1', 'e2'],
+      selectedEventIds: ['e1'],
+      candidateDetails: [{ eventId: 'e1', score: 0.9 }],
+      selectedDetails: [{ eventId: 'e1', score: 0.9 }],
+      candidateCount: 2,
+      selectedCount: 1,
+      confidence: 'high',
+      fallbackTrace: ['stage:primary:deep'],
+      createdAt: new Date('2026-02-24T01:00:00.000Z')
+    }] satisfies RetrievalTrace[];
+    const helpfulMemories = [{
+      eventId: 'e1',
+      summary: 'helpful memory',
+      helpfulnessScore: 0.8,
+      accessCount: 4,
+      evaluationCount: 2
+    }];
+    const helpfulnessStats = {
+      avgScore: 0.75,
+      totalEvaluated: 8,
+      totalRetrievals: 10,
+      helpful: 6,
+      neutral: 1,
+      unhelpful: 1
+    };
+    const evaluated: string[] = [];
+
+    const service = new RetrievalAnalyticsService({
+      initialize: async () => { initialized += 1; },
+      retrievalStore: baseStore({
+        getRetrievalTraceStats: async () => traceStats,
+        getRecentRetrievalTraces: async (limit = 50) => {
+          expect(limit).toBe(3);
+          return traceRows;
+        },
+        evaluateSessionHelpfulness: async (sessionId: string) => { evaluated.push(sessionId); },
+        getHelpfulMemories: async (limit = 10) => {
+          expect(limit).toBe(2);
+          return helpfulMemories;
+        },
+        getHelpfulnessStats: async () => helpfulnessStats
+      })
+    });
+
+    await expect(service.getRetrievalTraceStats()).resolves.toEqual(traceStats);
+    await expect(service.getRecentRetrievalTraces(3)).resolves.toEqual(traceRows);
+    await service.evaluateSessionHelpfulness('s1');
+    await expect(service.getHelpfulMemories(2)).resolves.toEqual(helpfulMemories);
+    await expect(service.getHelpfulnessStats()).resolves.toEqual(helpfulnessStats);
+
+    expect(evaluated).toEqual(['s1']);
+    expect(initialized).toBe(5);
+  });
+});

package/tests/core/retrieval-benchmark.test.ts

@@ -0,0 +1,93 @@
+import { describe, expect, it } from 'vitest';
+
+import { computePrecisionRecallAtK, summarizeReplayMetrics } from '../../src/core/retrieval-benchmark.js';
+
+describe('retrieval replay benchmark metrics', () => {
+  it('computes Precision@k and Recall@k for replay queries', () => {
+    const queryMetrics = computePrecisionRecallAtK(
+      [
+        { queryId: 'q1', expectedIds: ['a', 'b'], retrievedIds: ['a', 'x', 'b'] },
+        { queryId: 'q2', expectedIds: ['c'], retrievedIds: ['x', 'y', 'z'] }
+      ],
+      [1, 3]
+    );
+
+    expect(queryMetrics[0].at[1]).toMatchObject({ precision: 1, recall: 0.5, hits: 1 });
+    expect(queryMetrics[0].at[3]).toMatchObject({ precision: 2 / 3, recall: 1, hits: 2 });
+    expect(queryMetrics[0].at[1].ndcg).toBe(1);
+    expect(queryMetrics[0].at[3].ndcg).toBeCloseTo(0.91972, 4);
+    expect(queryMetrics[1].at[1]).toMatchObject({ precision: 0, recall: 0, hits: 0 });
+    expect(queryMetrics[1].at[3]).toMatchObject({ precision: 0, recall: 0, hits: 0 });
+
+    const summary = summarizeReplayMetrics(queryMetrics, [1, 3]);
+    expect(summary).toMatchObject({
+      queryCount: 2,
+      precisionAtK: { 1: 0.5, 3: 1 / 3 },
+      recallAtK: { 1: 0.25, 3: 0.5 }
+    });
+    expect(summary.ndcgAtK[1]).toBe(0.5);
+    expect(summary.ndcgAtK[3]).toBeCloseTo(0.45986, 4);
+  });
+
+  it('computes graded nDCG@k from qrels relevance labels', () => {
+    const [queryMetrics] = computePrecisionRecallAtK(
+      [
+        {
+          queryId: 'q-graded',
+          expectedIds: ['a', 'b'],
+          expectedRelevance: { a: 3, b: 1 },
+          retrievedIds: ['b', 'a', 'noise']
+        }
+      ],
+      [2]
+    );
+
+    expect(queryMetrics.at[2]).toMatchObject({ precision: 1, recall: 1, hits: 2 });
+    expect(queryMetrics.at[2].ndcg).toBeCloseTo(0.70981, 4);
+  });
+
+  it('deduplicates retrieved ids so replay metrics cannot over-count repeated hits', () => {
+    const queryMetrics = computePrecisionRecallAtK(
+      [
+        { queryId: 'q-duplicate', expectedIds: ['a'], retrievedIds: ['a', 'a', 'a', 'x'] }
+      ],
+      [1, 3]
+    );
+
+    expect(queryMetrics).toEqual([
+      {
+        queryId: 'q-duplicate',
+        at: {
+          1: { precision: 1, recall: 1, hits: 1, ndcg: 1 },
+          3: { precision: 1 / 3, recall: 1, hits: 1, ndcg: 1 }
+        }
+      }
+    ]);
+    expect(queryMetrics[0].at[3].ndcg).toBe(1);
+  });
+
+  it('normalizes k values without losing zero-result replay rows', () => {
+    const queryMetrics = computePrecisionRecallAtK(
+      [{ queryId: 'q-empty', expectedIds: ['a'], retrievedIds: [] }],
+      [3.9, 1, 1, -2]
+    );
+
+    expect(queryMetrics).toEqual([
+      {
+        queryId: 'q-empty',
+        at: {
+          0: { precision: 0, recall: 0, hits: 0, ndcg: 0 },
+          1: { precision: 0, recall: 0, hits: 0, ndcg: 0 },
+          3: { precision: 0, recall: 0, hits: 0, ndcg: 0 }
+        }
+      }
+    ]);
+
+    expect(summarizeReplayMetrics(queryMetrics, [3.9, 1, 1, -2])).toEqual({
+      queryCount: 1,
+      precisionAtK: { 0: 0, 1: 0, 3: 0 },
+      recallAtK: { 0: 0, 1: 0, 3: 0 },
+      ndcgAtK: { 0: 0, 1: 0, 3: 0 }
+    });
+  });
+});