claude-memory-layer 1.0.27 → 1.0.29

package/tests/core/codex-session-history-importer-validation.test.ts

@@ -0,0 +1,185 @@
+import { mkdtempSync, rmSync, utimesSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { afterEach, describe, expect, it, vi } from 'vitest';
+
+import { createCodexSessionHistoryImporter, validateCodexSessions } from '../../src/services/codex-session-history-importer.js';
+
+const tempDirs: string[] = [];
+
+function tempDir() {
+  const dir = mkdtempSync(join(tmpdir(), 'cml-codex-validation-'));
+  tempDirs.push(dir);
+  return dir;
+}
+
+function writeJsonl(filePath: string, records: Array<string | Record<string, unknown>>) {
+  writeFileSync(
+    filePath,
+    records.map((record) => typeof record === 'string' ? record : JSON.stringify(record)).join('\n') + '\n',
+    'utf8'
+  );
+}
+
+afterEach(() => {
+  for (const dir of tempDirs.splice(0)) {
+    rmSync(dir, { recursive: true, force: true });
+  }
+});
+
+describe('Codex session validation replay', () => {
+  it('dry-runs matching sessions by cwd without exposing transcript content', async () => {
+    const sessionsDir = tempDir();
+    const projectA = join(sessionsDir, 'project-a');
+    const projectB = join(sessionsDir, 'project-b');
+
+    writeJsonl(join(sessionsDir, 'rollout-2026-05-05T00-00-00-session-a.jsonl'), [
+      { type: 'session_meta', payload: { id: 'session-a', cwd: projectA, timestamp: '2026-05-05T00:00:00.000Z' } },
+      { type: 'response_item', timestamp: '2026-05-05T00:00:01.000Z', payload: { type: 'message', role: 'user', content: [{ type: 'input_text', text: 'please implement the Codex validation replay flow' }] } },
+      { type: 'response_item', timestamp: '2026-05-05T00:00:02.000Z', payload: { type: 'function_call', name: 'shell', arguments: '{}' } },
+      { type: 'response_item', timestamp: '2026-05-05T00:00:03.000Z', payload: { type: 'message', role: 'assistant', content: [{ type: 'output_text', text: 'validated answer that should never appear in aggregate reports' }] } },
+      { type: 'response_item', timestamp: '2026-05-05T00:00:04.000Z', payload: { type: 'message', role: 'assistant', content: [] } },
+      '{not valid json'
+    ]);
+
+    writeJsonl(join(sessionsDir, 'rollout-2026-05-05T00-00-00-session-b.jsonl'), [
+      { type: 'session_meta', payload: { id: 'session-b', cwd: projectB } },
+      { type: 'response_item', timestamp: '2026-05-05T00:01:01.000Z', payload: { type: 'message', role: 'user', content: [{ type: 'input_text', text: 'this should not match project a' }] } }
+    ]);
+
+    writeJsonl(join(sessionsDir, 'rollout-2026-05-05T00-00-00-session-missing-cwd.jsonl'), [
+      { type: 'session_meta', payload: { id: 'session-missing-cwd' } },
+      { type: 'response_item', timestamp: '2026-05-05T00:02:01.000Z', payload: { type: 'message', role: 'user', content: [{ type: 'input_text', text: 'missing cwd should be scanned but not project matched' }] } }
+    ]);
+
+    const report = await validateCodexSessions({ sessionsDir, projectPath: projectA });
+
+    expect(report.dryRun).toBe(true);
+    expect(report.willMutate).toBe(false);
+    expect(report.source.sessionsDir).toBe(sessionsDir);
+    expect(report.source.projectPath).toBe(projectA);
+    expect(report.totals.sessionsScanned).toBe(3);
+    expect(report.totals.sessionsMatched).toBe(1);
+    expect(report.totals.userMessages).toBe(1);
+    expect(report.totals.assistantMessages).toBe(1);
+    expect(report.totals.messagesNormalized).toBe(2);
+    expect(report.totals.turnsNormalized).toBe(1);
+    expect(report.totals.skippedUnsupportedRecords).toBe(1);
+    expect(report.totals.emptyAssistantMessages).toBe(1);
+    expect(report.totals.malformedLines).toBe(1);
+    expect(report.totals.missingProjectCwd).toBe(1);
+    expect(report.topProjects[0]).toMatchObject({ sessions: 1, userMessages: 1, assistantMessages: 1 });
+
+    const serialized = JSON.stringify(report);
+    expect(serialized).not.toContain('please implement the Codex validation replay flow');
+    expect(serialized).not.toContain('validated answer that should never appear');
+    expect(serialized).not.toContain('this should not match project a');
+  });
+
+  it('summarizes all sessions and counts malformed, unsupported, empty, and truncated content', async () => {
+    const sessionsDir = tempDir();
+    const projectA = join(sessionsDir, 'project-a');
+    const largeAssistantContent = 'A'.repeat(12_050);
+
+    writeJsonl(join(sessionsDir, 'rollout-2026-05-05T00-00-00-session-a.jsonl'), [
+      { type: 'session_meta', payload: { id: 'session-a', cwd: projectA } },
+      { type: 'response_item', timestamp: '2026-05-05T00:00:01.000Z', payload: { type: 'message', role: 'user', content: [{ type: 'input_text', text: 'normalize all Codex sessions' }] } },
+      { type: 'response_item', timestamp: '2026-05-05T00:00:02.000Z', payload: { type: 'message', role: 'assistant', content: [{ type: 'output_text', text: largeAssistantContent }] } },
+      { type: 'response_item', timestamp: '2026-05-05T00:00:03.000Z', payload: { type: 'message', role: 'assistant', content: [{ type: 'output_text', text: '' }] } },
+      { type: 'response_item', timestamp: '2026-05-05T00:00:04.000Z', payload: { type: 'reasoning', summary: [] } },
+      'not-json'
+    ]);
+
+    writeJsonl(join(sessionsDir, 'rollout-2026-05-05T00-00-00-session-unknown.jsonl'), [
+      { type: 'session_meta', payload: { id: 'session-unknown' } },
+      { type: 'response_item', timestamp: '2026-05-05T00:01:01.000Z', payload: { type: 'message', role: 'user', content: 'string content from real-world Codex JSONL' } }
+    ]);
+
+    const report = await validateCodexSessions({ sessionsDir, maxContentChars: 10_000 });
+
+    expect(report.totals.sessionsScanned).toBe(2);
+    expect(report.totals.sessionsMatched).toBe(2);
+    expect(report.totals.userMessages).toBe(2);
+    expect(report.totals.assistantMessages).toBe(1);
+    expect(report.totals.messagesNormalized).toBe(3);
+    expect(report.totals.turnsNormalized).toBe(2);
+    expect(report.totals.truncatedMessages).toBe(1);
+    expect(report.totals.emptyAssistantMessages).toBe(1);
+    expect(report.totals.skippedUnsupportedRecords).toBe(1);
+    expect(report.totals.malformedLines).toBe(1);
+    expect(report.totals.missingProjectCwd).toBe(1);
+    expect(report.topProjects).toHaveLength(2);
+    expect(report.warnings.some((warning) => warning.includes('missing cwd'))).toBe(true);
+  });
+
+  it('imports only the most recently modified matching Codex session when sessionLimit is set', async () => {
+    const sessionsDir = tempDir();
+    const projectA = join(sessionsDir, 'project-a');
+    const oldFile = join(sessionsDir, 'rollout-2026-05-05T00-00-00-session-old.jsonl');
+    const latestFile = join(sessionsDir, 'rollout-2026-05-05T00-10-00-session-latest.jsonl');
+
+    writeJsonl(oldFile, [
+      { type: 'session_meta', payload: { id: 'session-old', cwd: projectA } },
+      { type: 'response_item', timestamp: '2026-05-05T00:00:01.000Z', payload: { type: 'message', role: 'user', content: 'old Codex project session should not be imported now' } }
+    ]);
+    writeJsonl(latestFile, [
+      { type: 'session_meta', payload: { id: 'session-latest', cwd: projectA } },
+      { type: 'response_item', timestamp: '2026-05-05T00:10:01.000Z', payload: { type: 'message', role: 'user', content: 'latest Codex project session should be imported now' } }
+    ]);
+    utimesSync(oldFile, new Date('2026-05-05T00:00:00.000Z'), new Date('2026-05-05T00:00:00.000Z'));
+    utimesSync(latestFile, new Date('2026-05-05T00:10:00.000Z'), new Date('2026-05-05T00:10:00.000Z'));
+
+    const memoryService = {
+      startSession: vi.fn(async (_sessionId: string, _projectPath?: string) => undefined),
+      endSession: vi.fn(async (_sessionId: string) => undefined),
+      deleteSessionEvents: vi.fn(async (_sessionId: string) => 0),
+      storeUserPrompt: vi.fn(async () => ({ success: true, isDuplicate: false })),
+      storeAgentResponse: vi.fn(async () => ({ success: true, isDuplicate: false }))
+    };
+    const importer = createCodexSessionHistoryImporter(memoryService as never, { sessionsDir });
+
+    const result = await importer.importProject(projectA, { sessionLimit: 1 });
+
+    expect(result.totalSessions).toBe(1);
+    expect(memoryService.startSession).toHaveBeenCalledTimes(1);
+    expect(memoryService.startSession).toHaveBeenCalledWith('session-latest', projectA);
+    expect(memoryService.storeUserPrompt).toHaveBeenCalledTimes(1);
+  });
+
+  it('applies Codex import limit across selected matching sessions', async () => {
+    const sessionsDir = tempDir();
+    const projectA = join(sessionsDir, 'project-a');
+    const oldFile = join(sessionsDir, 'rollout-2026-05-05T00-00-00-session-old.jsonl');
+    const latestFile = join(sessionsDir, 'rollout-2026-05-05T00-10-00-session-latest.jsonl');
+
+    writeJsonl(oldFile, [
+      { type: 'session_meta', payload: { id: 'session-old', cwd: projectA } },
+      { type: 'response_item', timestamp: '2026-05-05T00:00:01.000Z', payload: { type: 'message', role: 'user', content: 'old prompt should remain outside the global Codex import limit' } },
+      { type: 'response_item', timestamp: '2026-05-05T00:00:02.000Z', payload: { type: 'message', role: 'assistant', content: 'old assistant should remain outside the global Codex import limit' } }
+    ]);
+    writeJsonl(latestFile, [
+      { type: 'session_meta', payload: { id: 'session-latest', cwd: projectA } },
+      { type: 'response_item', timestamp: '2026-05-05T00:10:01.000Z', payload: { type: 'message', role: 'user', content: 'latest prompt should be imported first' } },
+      { type: 'response_item', timestamp: '2026-05-05T00:10:02.000Z', payload: { type: 'message', role: 'assistant', content: 'latest assistant should be imported first' } }
+    ]);
+    utimesSync(oldFile, new Date('2026-05-05T00:00:00.000Z'), new Date('2026-05-05T00:00:00.000Z'));
+    utimesSync(latestFile, new Date('2026-05-05T00:10:00.000Z'), new Date('2026-05-05T00:10:00.000Z'));
+
+    const memoryService = {
+      startSession: vi.fn(async (_sessionId: string, _projectPath?: string) => undefined),
+      endSession: vi.fn(async (_sessionId: string) => undefined),
+      deleteSessionEvents: vi.fn(async (_sessionId: string) => 0),
+      storeUserPrompt: vi.fn(async () => ({ success: true, isDuplicate: false })),
+      storeAgentResponse: vi.fn(async () => ({ success: true, isDuplicate: false }))
+    };
+    const importer = createCodexSessionHistoryImporter(memoryService as never, { sessionsDir });
+
+    const result = await importer.importProject(projectA, { sessionLimit: 2, limit: 2 });
+
+    expect(result.totalSessions).toBe(1);
+    expect(memoryService.startSession).toHaveBeenCalledTimes(1);
+    expect(memoryService.startSession).toHaveBeenCalledWith('session-latest', projectA);
+    expect(memoryService.storeUserPrompt).toHaveBeenCalledTimes(1);
+    expect(memoryService.storeAgentResponse).toHaveBeenCalledTimes(1);
+  });
+});

package/tests/{consolidation-worker.test.ts → core/consolidation-worker.test.ts}

@@ -1,7 +1,7 @@
 import { describe, it, expect } from 'vitest';
 
-import { ConsolidationWorker } from '../src/core/consolidation-worker.js';
-import type { EndlessModeConfig, MemoryEvent } from '../src/core/types.js';
+import { ConsolidationWorker } from '../../src/core/consolidation-worker.js';
+import type { EndlessModeConfig, MemoryEvent } from '../../src/core/types.js';
 
 function makeEvent(id: string, content: string, hoursAgo = 20): MemoryEvent {
   return {

package/tests/core/embedding-maintenance-service.test.ts

@@ -0,0 +1,282 @@
+import * as path from 'path';
+import { describe, expect, it, vi } from 'vitest';
+
+import {
+  createEmbeddingMaintenanceService,
+  type EmbeddingMaintenanceFileSystem
+} from '../../src/core/engine/embedding-maintenance-service.js';
+
+function event(id: string, content = `content for ${id}`) {
+  return { id, content };
+}
+
+function createHarness(options?: {
+  currentModel?: string;
+  vectorCount?: number;
+  files?: Record<string, string>;
+  workerRunning?: boolean;
+  nullWorker?: boolean;
+  events?: Array<{ id: string; content: string }>;
+}) {
+  const storagePath = '/memory-root/project-a';
+  const metaPath = path.join(storagePath, 'embedding-meta.json');
+  const files = new Map<string, string>(Object.entries(options?.files ?? {}));
+  const calls: string[] = [];
+
+  const fileSystem: EmbeddingMaintenanceFileSystem = {
+    existsSync: vi.fn((targetPath: string) => files.has(targetPath)),
+    readFileSync: vi.fn((targetPath: string) => files.get(targetPath) ?? ''),
+    writeFileSync: vi.fn((targetPath: string, content: string) => {
+      files.set(targetPath, content);
+    })
+  };
+
+  const initialize = vi.fn(async () => {
+    calls.push('initialize');
+  });
+  const vectorStore = {
+    count: vi.fn(async () => options?.vectorCount ?? 0),
+    clearAll: vi.fn(async () => {
+      calls.push('clear-vectors');
+    })
+  };
+  const events = options?.events ?? [];
+  const eventStore = {
+    clearEmbeddingOutbox: vi.fn(async () => {
+      calls.push('clear-outbox');
+    }),
+    getEventsPage: vi.fn(async (limit: number, offset: number) => events.slice(offset, offset + limit)),
+    enqueueForEmbedding: vi.fn(async (eventId: string, _content: string) => {
+      calls.push(`enqueue:${eventId}`);
+    })
+  };
+  const worker = {
+    isRunning: vi.fn(() => options?.workerRunning ?? false),
+    stop: vi.fn(() => {
+      calls.push('stop-worker');
+    }),
+    start: vi.fn(() => {
+      calls.push('start-worker');
+    })
+  };
+
+  const service = createEmbeddingMaintenanceService({
+    storagePath,
+    initialize,
+    getEmbeddingModelName: vi.fn(() => options?.currentModel ?? 'embedding-model-a'),
+    vectorStore,
+    eventStore,
+    getVectorWorker: () => (options?.nullWorker ? null : worker),
+    fileSystem
+  });
+
+  return {
+    service,
+    storagePath,
+    metaPath,
+    files,
+    calls,
+    fileSystem,
+    initialize,
+    vectorStore,
+    eventStore,
+    worker
+  };
+}
+
+describe('EmbeddingMaintenanceService', () => {
+  it('initializes embedding metadata when no prior vectors exist', async () => {
+    const h = createHarness({ vectorCount: 0 });
+
+    const result = await h.service.ensureEmbeddingModelForImport();
+
+    expect(result).toEqual({
+      changed: false,
+      previousModel: null,
+      currentModel: 'embedding-model-a',
+      enqueued: 0,
+      reason: 'initialized-meta'
+    });
+    expect(h.initialize).toHaveBeenCalledTimes(1);
+    expect(h.vectorStore.count).toHaveBeenCalledTimes(1);
+    expect(h.vectorStore.clearAll).not.toHaveBeenCalled();
+    expect(h.eventStore.clearEmbeddingOutbox).not.toHaveBeenCalled();
+    expect(JSON.parse(h.files.get(h.metaPath)!)).toMatchObject({
+      model: 'embedding-model-a',
+      updatedAt: expect.any(String)
+    });
+  });
+
+  it('returns unchanged when stored metadata already matches the current model', async () => {
+    const metaPath = path.join('/memory-root/project-a', 'embedding-meta.json');
+    const h = createHarness({
+      vectorCount: 5,
+      files: {
+        [metaPath]: JSON.stringify({ model: 'embedding-model-a', updatedAt: '2026-05-02T00:00:00.000Z' })
+      }
+    });
+
+    const result = await h.service.ensureEmbeddingModelForImport();
+
+    expect(result).toEqual({
+      changed: false,
+      previousModel: 'embedding-model-a',
+      currentModel: 'embedding-model-a',
+      enqueued: 0
+    });
+    expect(h.fileSystem.writeFileSync).not.toHaveBeenCalled();
+    expect(h.vectorStore.clearAll).not.toHaveBeenCalled();
+    expect(h.eventStore.clearEmbeddingOutbox).not.toHaveBeenCalled();
+  });
+
+  it('reports a dry-run model mismatch without clearing vectors or outbox state', async () => {
+    const metaPath = path.join('/memory-root/project-a', 'embedding-meta.json');
+    const h = createHarness({
+      currentModel: 'embedding-model-b',
+      vectorCount: 2,
+      files: {
+        [metaPath]: JSON.stringify({ model: 'embedding-model-a' })
+      }
+    });
+
+    const result = await h.service.ensureEmbeddingModelForImport({ autoMigrate: false });
+
+    expect(result).toEqual({
+      changed: true,
+      previousModel: 'embedding-model-a',
+      currentModel: 'embedding-model-b',
+      enqueued: 0,
+      reason: 'model-mismatch'
+    });
+    expect(h.vectorStore.clearAll).not.toHaveBeenCalled();
+    expect(h.eventStore.clearEmbeddingOutbox).not.toHaveBeenCalled();
+    expect(h.eventStore.enqueueForEmbedding).not.toHaveBeenCalled();
+  });
+
+  it('reports legacy vectors without metadata as a dry-run migration requirement', async () => {
+    const h = createHarness({ vectorCount: 3 });
+
+    const result = await h.service.ensureEmbeddingModelForImport({ autoMigrate: false });
+
+    expect(result).toEqual({
+      changed: true,
+      previousModel: null,
+      currentModel: 'embedding-model-a',
+      enqueued: 0,
+      reason: 'legacy-vectors-without-meta'
+    });
+    expect(h.fileSystem.writeFileSync).not.toHaveBeenCalled();
+    expect(h.vectorStore.clearAll).not.toHaveBeenCalled();
+  });
+
+  it('migrates vectors by clearing indexes, re-enqueueing events, and restarting a running worker', async () => {
+    const metaPath = path.join('/memory-root/project-a', 'embedding-meta.json');
+    const h = createHarness({
+      currentModel: 'embedding-model-b',
+      vectorCount: 10,
+      workerRunning: true,
+      files: {
+        [metaPath]: JSON.stringify({ model: 'embedding-model-a' })
+      },
+      events: [event('e1', 'first memory'), event('e2', 'second memory')]
+    });
+
+    const result = await h.service.ensureEmbeddingModelForImport();
+
+    expect(result).toEqual({
+      changed: true,
+      previousModel: 'embedding-model-a',
+      currentModel: 'embedding-model-b',
+      enqueued: 2,
+      reason: 'model-mismatch'
+    });
+    expect(h.calls).toEqual([
+      'initialize',
+      'stop-worker',
+      'clear-vectors',
+      'clear-outbox',
+      'enqueue:e1',
+      'enqueue:e2',
+      'start-worker'
+    ]);
+    expect(h.eventStore.getEventsPage).toHaveBeenCalledWith(1000, 0);
+    expect(h.eventStore.enqueueForEmbedding).toHaveBeenNthCalledWith(1, 'e1', 'first memory');
+    expect(h.eventStore.enqueueForEmbedding).toHaveBeenNthCalledWith(2, 'e2', 'second memory');
+    expect(JSON.parse(h.files.get(h.metaPath)!)).toMatchObject({
+      model: 'embedding-model-b',
+      previousModel: 'embedding-model-a',
+      migratedAt: expect.any(String),
+      enqueued: 2
+    });
+  });
+
+  it('auto-migrates legacy vectors without metadata and does not restart an idle worker', async () => {
+    const h = createHarness({
+      vectorCount: 3,
+      workerRunning: false,
+      events: [event('legacy-1', 'legacy memory')]
+    });
+
+    const result = await h.service.ensureEmbeddingModelForImport();
+
+    expect(result).toEqual({
+      changed: true,
+      previousModel: null,
+      currentModel: 'embedding-model-a',
+      enqueued: 1,
+      reason: 'legacy-vectors-without-meta'
+    });
+    expect(h.worker.stop).not.toHaveBeenCalled();
+    expect(h.worker.start).not.toHaveBeenCalled();
+    expect(h.calls).toEqual(['initialize', 'clear-vectors', 'clear-outbox', 'enqueue:legacy-1']);
+    expect(JSON.parse(h.files.get(h.metaPath)!)).toMatchObject({
+      model: 'embedding-model-a',
+      previousModel: null,
+      migratedAt: expect.any(String),
+      enqueued: 1
+    });
+  });
+
+  it('re-enqueues events across page boundaries during migration', async () => {
+    const metaPath = path.join('/memory-root/project-a', 'embedding-meta.json');
+    const events = Array.from({ length: 1001 }, (_, index) => event(`event-${index + 1}`));
+    const h = createHarness({
+      currentModel: 'embedding-model-b',
+      vectorCount: 1001,
+      files: {
+        [metaPath]: JSON.stringify({ model: 'embedding-model-a' })
+      },
+      events
+    });
+
+    const result = await h.service.ensureEmbeddingModelForImport();
+
+    expect(result.enqueued).toBe(1001);
+    expect(h.eventStore.getEventsPage).toHaveBeenNthCalledWith(1, 1000, 0);
+    expect(h.eventStore.getEventsPage).toHaveBeenNthCalledWith(2, 1000, 1000);
+    expect(h.eventStore.getEventsPage).toHaveBeenCalledTimes(2);
+    expect(h.eventStore.enqueueForEmbedding).toHaveBeenCalledTimes(1001);
+    expect(h.eventStore.enqueueForEmbedding).toHaveBeenNthCalledWith(1001, 'event-1001', 'content for event-1001');
+  });
+
+  it('migrates without worker lifecycle calls when no vector worker is available', async () => {
+    const metaPath = path.join('/memory-root/project-a', 'embedding-meta.json');
+    const h = createHarness({
+      currentModel: 'embedding-model-b',
+      vectorCount: 1,
+      nullWorker: true,
+      files: {
+        [metaPath]: JSON.stringify({ model: 'embedding-model-a' })
+      },
+      events: [event('e1')]
+    });
+
+    const result = await h.service.ensureEmbeddingModelForImport();
+
+    expect(result.enqueued).toBe(1);
+    expect(h.calls).toEqual(['initialize', 'clear-vectors', 'clear-outbox', 'enqueue:e1']);
+    expect(h.worker.isRunning).not.toHaveBeenCalled();
+    expect(h.worker.stop).not.toHaveBeenCalled();
+    expect(h.worker.start).not.toHaveBeenCalled();
+  });
+});

package/tests/{evidence-aligner.test.ts → core/evidence-aligner.test.ts}

@@ -3,7 +3,7 @@
  */
 
 import { describe, it, expect } from 'vitest';
-import { EvidenceAligner } from '../src/core/evidence-aligner.js';
+import { EvidenceAligner } from '../../src/core/evidence-aligner.js';
 
 describe('EvidenceAligner', () => {
   const aligner = new EvidenceAligner();