claude-memory-layer 1.0.8 → 1.0.10

package/src/services/memory-service.ts

@@ -9,6 +9,8 @@ import * as fs from 'fs';
 import * as crypto from 'crypto';
 
 import { EventStore } from '../core/event-store.js';
+import { SQLiteEventStore } from '../core/sqlite-event-store.js';
+import { SyncWorker } from '../core/sync-worker.js';
 import { VectorStore } from '../core/vector-store.js';
 import { Embedder, getDefaultEmbedder } from '../core/embedder.js';
 import { VectorWorker, createVectorWorker } from '../core/vector-worker.js';
@@ -48,6 +50,10 @@ export interface MemoryServiceConfig {
   storagePath: string;
   embeddingModel?: string;
   readOnly?: boolean;
+  /** Enable DuckDB analytics store (default: true for server, false for hooks) */
+  analyticsEnabled?: boolean;
+  /** Lightweight mode for hooks - skip heavy initialization (default: false) */
+  lightweightMode?: boolean;
 }
 
 // ============================================================
@@ -165,7 +171,12 @@ export function getSessionProject(sessionId: string): SessionRegistryEntry | nul
 }
 
 export class MemoryService {
-  private readonly eventStore: EventStore;
+  // Primary store: SQLite (WAL mode) - for hooks, always available
+  private readonly sqliteStore: SQLiteEventStore;
+  // Analytics store: DuckDB - for server reads (optional, synced from SQLite)
+  private readonly analyticsStore: EventStore | null;
+  private syncWorker: SyncWorker | null = null;
+
   private readonly vectorStore: VectorStore;
   private readonly embedder: Embedder;
   private readonly matcher: Matcher;
@@ -191,10 +202,12 @@ export class MemoryService {
   private projectHash: string | null = null;
 
   private readonly readOnly: boolean;
+  private readonly lightweightMode: boolean;
 
   constructor(config: MemoryServiceConfig & { projectHash?: string; sharedStoreConfig?: SharedStoreConfig }) {
     const storagePath = this.expandPath(config.storagePath);
     this.readOnly = config.readOnly ?? false;
+    this.lightweightMode = config.lightweightMode ?? false;
 
     // Ensure storage directory exists (only if not read-only)
     if (!this.readOnly && !fs.existsSync(storagePath)) {
@@ -206,20 +219,52 @@ export class MemoryService {
     // Default: shared store enabled
     this.sharedStoreConfig = config.sharedStoreConfig ?? { enabled: true };
 
-    // Initialize components
-    this.eventStore = new EventStore(path.join(storagePath, 'events.duckdb'), { readOnly: this.readOnly });
+    // Initialize PRIMARY store: SQLite (WAL mode)
+    // This is always used for writes and is the source of truth
+    this.sqliteStore = new SQLiteEventStore(
+      path.join(storagePath, 'events.sqlite'),
+      { readonly: this.readOnly }
+    );
+
+    // Initialize ANALYTICS store: DuckDB (optional, for server reads)
+    // Hooks set analyticsEnabled=false to avoid DuckDB lock conflicts
+    const analyticsEnabled = config.analyticsEnabled ?? this.readOnly; // Default: enabled only for read-only (server)
+
+    if (!analyticsEnabled) {
+      // Hook mode: skip DuckDB entirely to avoid lock conflicts
+      this.analyticsStore = null;
+    } else if (this.readOnly) {
+      // Server mode: try to use DuckDB for analytics, will fallback to SQLite
+      try {
+        this.analyticsStore = new EventStore(
+          path.join(storagePath, 'analytics.duckdb'),
+          { readOnly: true }
+        );
+      } catch {
+        // DuckDB not available, will use SQLite for reads
+        this.analyticsStore = null;
+      }
+    } else {
+      // Writer mode with analytics: create DuckDB for sync target
+      this.analyticsStore = new EventStore(
+        path.join(storagePath, 'analytics.duckdb'),
+        { readOnly: false }
+      );
+    }
+
     this.vectorStore = new VectorStore(path.join(storagePath, 'vectors'));
     this.embedder = config.embeddingModel
       ? new Embedder(config.embeddingModel)
       : getDefaultEmbedder();
     this.matcher = getDefaultMatcher();
+    // Retriever uses SQLite as primary (always available)
     this.retriever = createRetriever(
-      this.eventStore,
+      this.sqliteStore as unknown as EventStore, // Interface compatible
       this.vectorStore,
       this.embedder,
       this.matcher
     );
-    this.graduation = createGraduationPipeline(this.eventStore);
+    this.graduation = createGraduationPipeline(this.sqliteStore as unknown as EventStore);
   }
 
   /**
@@ -228,15 +273,34 @@ export class MemoryService {
   async initialize(): Promise<void> {
     if (this.initialized) return;
 
-    await this.eventStore.initialize();
+    // Initialize PRIMARY store: SQLite (always)
+    await this.sqliteStore.initialize();
+
+    // Lightweight mode: only SQLite, no embedder/vector/workers
+    // Used for hooks that just need to store data quickly
+    if (this.lightweightMode) {
+      this.initialized = true;
+      return;
+    }
+
+    // Initialize analytics store if available (DuckDB)
+    if (this.analyticsStore) {
+      try {
+        await this.analyticsStore.initialize();
+      } catch (error) {
+        console.warn('[MemoryService] Analytics store (DuckDB) initialization failed, using SQLite for reads:', error);
+        // Continue without analytics - SQLite will be used for reads
+      }
+    }
+
     await this.vectorStore.initialize();
     await this.embedder.initialize();
 
     // Skip write-related workers in read-only mode
     if (!this.readOnly) {
-      // Start vector worker
+      // Start vector worker (uses SQLite as source)
       this.vectorWorker = createVectorWorker(
-        this.eventStore,
+        this.sqliteStore as unknown as EventStore,
         this.vectorStore,
         this.embedder
       );
@@ -247,13 +311,23 @@ export class MemoryService {
 
       // Start graduation worker for automatic level promotion
       this.graduationWorker = createGraduationWorker(
-        this.eventStore,
+        this.sqliteStore as unknown as EventStore,
         this.graduation
       );
       this.graduationWorker.start();
 
+      // Start sync worker (SQLite -> DuckDB) if analytics store is available
+      if (this.analyticsStore) {
+        this.syncWorker = new SyncWorker(
+          this.sqliteStore,
+          this.analyticsStore,
+          { intervalMs: 30000, batchSize: 500 }
+        );
+        this.syncWorker.start();
+      }
+
       // Load endless mode setting
-      const savedMode = await this.eventStore.getEndlessConfig('mode') as MemoryMode | null;
+      const savedMode = await this.sqliteStore.getEndlessConfig('mode') as MemoryMode | null;
       if (savedMode === 'endless') {
         this.endlessMode = 'endless';
         await this.initializeEndlessMode();
@@ -309,7 +383,7 @@ export class MemoryService {
   async startSession(sessionId: string, projectPath?: string): Promise<void> {
     await this.initialize();
 
-    await this.eventStore.upsertSession({
+    await this.sqliteStore.upsertSession({
       id: sessionId,
       startedAt: new Date(),
       projectPath
@@ -322,7 +396,7 @@ export class MemoryService {
   async endSession(sessionId: string, summary?: string): Promise<void> {
     await this.initialize();
 
-    await this.eventStore.upsertSession({
+    await this.sqliteStore.upsertSession({
       id: sessionId,
       endedAt: new Date(),
       summary
@@ -339,7 +413,7 @@ export class MemoryService {
   ): Promise<AppendResult> {
     await this.initialize();
 
-    const result = await this.eventStore.append({
+    const result = await this.sqliteStore.append({
       eventType: 'user_prompt',
       sessionId,
       timestamp: new Date(),
@@ -349,7 +423,7 @@ export class MemoryService {
 
     // Enqueue for embedding if new
     if (result.success && !result.isDuplicate) {
-      await this.eventStore.enqueueForEmbedding(result.eventId, content);
+      await this.sqliteStore.enqueueForEmbedding(result.eventId, content);
     }
 
     return result;
@@ -365,7 +439,7 @@ export class MemoryService {
   ): Promise<AppendResult> {
     await this.initialize();
 
-    const result = await this.eventStore.append({
+    const result = await this.sqliteStore.append({
       eventType: 'agent_response',
       sessionId,
       timestamp: new Date(),
@@ -375,7 +449,7 @@ export class MemoryService {
 
     // Enqueue for embedding if new
     if (result.success && !result.isDuplicate) {
-      await this.eventStore.enqueueForEmbedding(result.eventId, content);
+      await this.sqliteStore.enqueueForEmbedding(result.eventId, content);
     }
 
     return result;
@@ -390,7 +464,7 @@ export class MemoryService {
   ): Promise<AppendResult> {
     await this.initialize();
 
-    const result = await this.eventStore.append({
+    const result = await this.sqliteStore.append({
       eventType: 'session_summary',
       sessionId,
       timestamp: new Date(),
@@ -398,7 +472,7 @@ export class MemoryService {
     });
 
     if (result.success && !result.isDuplicate) {
-      await this.eventStore.enqueueForEmbedding(result.eventId, summary);
+      await this.sqliteStore.enqueueForEmbedding(result.eventId, summary);
     }
 
     return result;
@@ -416,7 +490,7 @@ export class MemoryService {
     // Create content for storage (JSON stringified payload)
     const content = JSON.stringify(payload);
 
-    const result = await this.eventStore.append({
+    const result = await this.sqliteStore.append({
       eventType: 'tool_observation',
       sessionId,
       timestamp: new Date(),
@@ -434,7 +508,7 @@ export class MemoryService {
         payload.metadata || {},
         payload.success
       );
-      await this.eventStore.enqueueForEmbedding(result.eventId, embeddingContent);
+      await this.sqliteStore.enqueueForEmbedding(result.eventId, embeddingContent);
     }
 
     return result;
@@ -454,10 +528,8 @@ export class MemoryService {
   ): Promise<UnifiedRetrievalResult> {
     await this.initialize();
 
-    // Process any pending embeddings first
-    if (this.vectorWorker) {
-      await this.vectorWorker.processAll();
-    }
+    // Note: Pending embeddings are processed by the background worker
+    // Don't block retrieval - search with whatever vectors are available
 
     // Use unified retrieval if shared search is requested
     if (options?.includeShared && this.sharedStore) {
@@ -471,12 +543,44 @@ export class MemoryService {
     return this.retriever.retrieve(query, options);
   }
 
+  /**
+   * Fast keyword search using SQLite FTS5
+   * Much faster than vector search - no embedding model needed
+   */
+  async keywordSearch(
+    query: string,
+    options?: { topK?: number; minScore?: number }
+  ): Promise<Array<{event: MemoryEvent; score: number}>> {
+    await this.initialize();
+
+    const results = await this.sqliteStore.keywordSearch(query, options?.topK ?? 10);
+
+    // Normalize FTS5 rank to a score (0-1 range)
+    // FTS5 rank is negative (higher is worse), so we convert it
+    const maxRank = Math.min(...results.map(r => r.rank), -0.001);
+    const minRank = Math.max(...results.map(r => r.rank), -1000);
+    const rankRange = maxRank - minRank || 1;
+
+    return results.map(r => ({
+      event: r.event,
+      score: 1 - (r.rank - minRank) / rankRange  // Normalize to 0-1
+    })).filter(r => !options?.minScore || r.score >= options.minScore);
+  }
+
+  /**
+   * Rebuild FTS index (call after database upgrade)
+   */
+  async rebuildFtsIndex(): Promise<number> {
+    await this.initialize();
+    return this.sqliteStore.rebuildFtsIndex();
+  }
+
   /**
    * Get session history
    */
   async getSessionHistory(sessionId: string): Promise<MemoryEvent[]> {
     await this.initialize();
-    return this.eventStore.getSessionEvents(sessionId);
+    return this.sqliteStore.getSessionEvents(sessionId);
   }
 
   /**
@@ -484,7 +588,7 @@ export class MemoryService {
    */
   async getRecentEvents(limit: number = 100): Promise<MemoryEvent[]> {
     await this.initialize();
-    return this.eventStore.getRecentEvents(limit);
+    return this.sqliteStore.getRecentEvents(limit);
   }
 
   /**
@@ -497,7 +601,7 @@ export class MemoryService {
   }> {
     await this.initialize();
 
-    const recentEvents = await this.eventStore.getRecentEvents(10000);
+    const recentEvents = await this.sqliteStore.getRecentEvents(10000);
     const vectorCount = await this.vectorStore.count();
     const levelStats = await this.graduation.getStats();
 
@@ -523,7 +627,7 @@ export class MemoryService {
    */
   async getEventsByLevel(level: string, options?: { limit?: number; offset?: number }): Promise<MemoryEvent[]> {
     await this.initialize();
-    return this.eventStore.getEventsByLevel(level, options);
+    return this.sqliteStore.getEventsByLevel(level, options);
   }
 
   /**
@@ -531,7 +635,7 @@ export class MemoryService {
    */
   async getEventLevel(eventId: string): Promise<string | null> {
     await this.initialize();
-    return this.eventStore.getEventLevel(eventId);
+    return this.sqliteStore.getEventLevel(eventId);
   }
 
   /**
@@ -644,14 +748,14 @@ export class MemoryService {
   async initializeEndlessMode(): Promise<void> {
     const config = await this.getEndlessConfig();
 
-    this.workingSetStore = createWorkingSetStore(this.eventStore, config);
-    this.consolidatedStore = createConsolidatedStore(this.eventStore);
+    this.workingSetStore = createWorkingSetStore(this.sqliteStore, config);
+    this.consolidatedStore = createConsolidatedStore(this.sqliteStore);
     this.consolidationWorker = createConsolidationWorker(
       this.workingSetStore,
       this.consolidatedStore,
       config
     );
-    this.continuityManager = createContinuityManager(this.eventStore, config);
+    this.continuityManager = createContinuityManager(this.sqliteStore, config);
 
     // Start consolidation worker
     this.consolidationWorker.start();
@@ -661,7 +765,7 @@ export class MemoryService {
    * Get Endless Mode configuration
    */
   async getEndlessConfig(): Promise<EndlessModeConfig> {
-    const savedConfig = await this.eventStore.getEndlessConfig('config') as EndlessModeConfig | null;
+    const savedConfig = await this.sqliteStore.getEndlessConfig('config') as EndlessModeConfig | null;
     return savedConfig || this.getDefaultEndlessConfig();
   }
 
@@ -671,7 +775,7 @@ export class MemoryService {
   async setEndlessConfig(config: Partial<EndlessModeConfig>): Promise<void> {
     const current = await this.getEndlessConfig();
     const merged = { ...current, ...config };
-    await this.eventStore.setEndlessConfig('config', merged);
+    await this.sqliteStore.setEndlessConfig('config', merged);
   }
 
   /**
@@ -683,7 +787,7 @@ export class MemoryService {
     if (mode === this.endlessMode) return;
 
     this.endlessMode = mode;
-    await this.eventStore.setEndlessConfig('mode', mode);
+    await this.sqliteStore.setEndlessConfig('mode', mode);
 
     if (mode === 'endless') {
       await this.initializeEndlessMode();
@@ -749,11 +853,56 @@ export class MemoryService {
   }
 
   /**
-   * Get most accessed consolidated memories
+   * Increment access count for memories that were used in prompts
    */
-  async getMostAccessedMemories(limit: number = 10): Promise<ConsolidatedMemory[]> {
-    if (!this.consolidatedStore) return [];
-    return this.consolidatedStore.getMostAccessed(limit);
+  async incrementMemoryAccess(eventIds: string[]): Promise<void> {
+    if (eventIds.length === 0) return;
+
+    // Use SQLite event store if available
+    if (this.sqliteStore) {
+      await this.sqliteStore.incrementAccessCount(eventIds);
+    } else if (this.eventStore) {
+      // Fallback to regular event store (which has a stub implementation)
+      await this.eventStore.incrementAccessCount(eventIds);
+    }
+  }
+
+  /**
+   * Get most accessed memories from events
+   */
+  async getMostAccessedMemories(limit: number = 10): Promise<any[]> {
+    console.log('[getMostAccessedMemories] sqliteStore available:', !!this.sqliteStore);
+
+    // Try to get from SQLite event store if available
+    if (this.sqliteStore) {
+      const events = await this.sqliteStore.getMostAccessed(limit);
+      console.log('[getMostAccessedMemories] Got events from SQLite:', events.length);
+      return events.map(event => ({
+        memoryId: event.id,
+        summary: event.content.substring(0, 200) + (event.content.length > 200 ? '...' : ''),
+        topics: [], // Could extract topics from content if needed
+        accessCount: (event as any).access_count || 0,
+        lastAccessed: (event as any).last_accessed_at || null,
+        confidence: 1.0,
+        createdAt: event.timestamp
+      }));
+    }
+
+    // Fallback to consolidated store if available
+    if (this.consolidatedStore) {
+      const consolidated = await this.consolidatedStore.getMostAccessed(limit);
+      return consolidated.map(m => ({
+        memoryId: m.memoryId,
+        summary: m.summary,
+        topics: m.topics,
+        accessCount: m.accessCount,
+        lastAccessed: m.accessedAt,
+        confidence: m.confidence,
+        createdAt: m.createdAt
+      }));
+    }
+
+    return [];
   }
 
   /**
@@ -908,12 +1057,23 @@ export class MemoryService {
       this.vectorWorker.stop();
     }
 
+    // Stop sync worker
+    if (this.syncWorker) {
+      this.syncWorker.stop();
+    }
+
     // Close shared store
     if (this.sharedEventStore) {
       await this.sharedEventStore.close();
     }
 
-    await this.eventStore.close();
+    // Close primary store (SQLite)
+    await this.sqliteStore.close();
+
+    // Close analytics store (DuckDB)
+    if (this.analyticsStore) {
+      await this.analyticsStore.close();
+    }
   }
 
   /**
@@ -939,11 +1099,14 @@ const GLOBAL_READONLY_KEY = '__global_readonly__';
 /**
  * Get the global memory service (backward compatibility)
  * Use this for operations not tied to a specific project
+ * Note: analyticsEnabled=false and sharedStore disabled to avoid DuckDB lock conflicts
  */
 export function getDefaultMemoryService(): MemoryService {
   if (!serviceCache.has(GLOBAL_KEY)) {
     serviceCache.set(GLOBAL_KEY, new MemoryService({
-      storagePath: '~/.claude-code/memory'
+      storagePath: '~/.claude-code/memory',
+      analyticsEnabled: false,  // Hooks don't need DuckDB
+      sharedStoreConfig: { enabled: false }  // Shared store uses DuckDB too
     }));
   }
   return serviceCache.get(GLOBAL_KEY)!;
@@ -953,19 +1116,24 @@ export function getDefaultMemoryService(): MemoryService {
  * Get a read-only global memory service
  * Use this for web server/dashboard that only needs to read data
  * Creates a fresh connection each time to avoid blocking the main writer process
+ * Uses SQLite (WAL mode) which supports concurrent readers
  */
 export function getReadOnlyMemoryService(): MemoryService {
   // Don't cache - create fresh instance each time to avoid holding locks
   // The connection will be closed when the request completes
+  // Uses SQLite which supports concurrent readers via WAL mode
   return new MemoryService({
     storagePath: '~/.claude-code/memory',
-    readOnly: true
+    readOnly: true,
+    analyticsEnabled: false,  // Use SQLite for reads (WAL supports concurrent readers)
+    sharedStoreConfig: { enabled: false }  // Skip shared store for now
   });
 }
 
 /**
  * Get memory service for a specific project path
  * Creates isolated storage at ~/.claude-code/memory/projects/{hash}/
+ * Note: analyticsEnabled=false and sharedStore disabled to avoid DuckDB lock conflicts
  */
 export function getMemoryServiceForProject(
   projectPath: string,
@@ -978,7 +1146,9 @@ export function getMemoryServiceForProject(
     serviceCache.set(hash, new MemoryService({
       storagePath,
       projectHash: hash,
-      sharedStoreConfig
+      // Override shared store config - hooks don't need DuckDB
+      sharedStoreConfig: sharedStoreConfig ?? { enabled: false },
+      analyticsEnabled: false  // Hooks don't need DuckDB
     }));
   }
 
@@ -1000,6 +1170,32 @@ export function getMemoryServiceForSession(sessionId: string): MemoryService {
   return getDefaultMemoryService();
 }
 
+/**
+ * Get a lightweight memory service for hooks
+ * Only initializes SQLite - no embedder, no vector store, no workers
+ * This is FAST (<100ms) compared to full initialization (3-5s)
+ */
+export function getLightweightMemoryService(sessionId: string): MemoryService {
+  const projectInfo = getSessionProject(sessionId);
+  const key = projectInfo ? `lightweight_${projectInfo.projectHash}` : 'lightweight_global';
+
+  if (!serviceCache.has(key)) {
+    const storagePath = projectInfo
+      ? getProjectStoragePath(projectInfo.projectPath)
+      : path.join(os.homedir(), '.claude-code', 'memory');
+
+    serviceCache.set(key, new MemoryService({
+      storagePath,
+      projectHash: projectInfo?.projectHash,
+      lightweightMode: true,  // Skip embedder/vector/workers
+      analyticsEnabled: false,
+      sharedStoreConfig: { enabled: false }
+    }));
+  }
+
+  return serviceCache.get(key)!;
+}
+
 export function createMemoryService(config: MemoryServiceConfig): MemoryService {
   return new MemoryService(config);
 }