claude-memory-layer 1.0.9 → 1.0.10

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "claude-memory-layer",
-  "version": "1.0.9",
+  "version": "1.0.10",
   "description": "Claude Code plugin that learns from conversations to provide personalized assistance",
   "main": "dist/index.js",
   "bin": {

package/src/core/sqlite-event-store.ts

@@ -283,6 +283,28 @@ export class SQLiteEventStore {
       CREATE INDEX IF NOT EXISTS idx_consolidated_confidence ON consolidated_memories(confidence);
       CREATE INDEX IF NOT EXISTS idx_continuity_created ON continuity_log(created_at);
       CREATE INDEX IF NOT EXISTS idx_embedding_outbox_status ON embedding_outbox(status);
+
+      -- FTS5 Full-Text Search for fast keyword search
+      CREATE VIRTUAL TABLE IF NOT EXISTS events_fts USING fts5(
+        content,
+        event_id UNINDEXED,
+        content='events',
+        content_rowid='rowid'
+      );
+
+      -- Triggers to keep FTS in sync with events table
+      CREATE TRIGGER IF NOT EXISTS events_fts_insert AFTER INSERT ON events BEGIN
+        INSERT INTO events_fts(rowid, content, event_id) VALUES (NEW.rowid, NEW.content, NEW.id);
+      END;
+
+      CREATE TRIGGER IF NOT EXISTS events_fts_delete AFTER DELETE ON events BEGIN
+        INSERT INTO events_fts(events_fts, rowid, content, event_id) VALUES('delete', OLD.rowid, OLD.content, OLD.id);
+      END;
+
+      CREATE TRIGGER IF NOT EXISTS events_fts_update AFTER UPDATE ON events BEGIN
+        INSERT INTO events_fts(events_fts, rowid, content, event_id) VALUES('delete', OLD.rowid, OLD.content, OLD.id);
+        INSERT INTO events_fts(rowid, content, event_id) VALUES (NEW.rowid, NEW.content, NEW.id);
+      END;
     `);
 
     // Migrate existing events table to add access tracking columns if they don't exist
@@ -807,6 +829,82 @@ export class SQLiteEventStore {
     return rows.map(row => this.rowToEvent(row));
   }
 
+  /**
+   * Fast keyword search using FTS5
+   * Returns events matching the search query, ranked by relevance
+   */
+  async keywordSearch(query: string, limit: number = 10): Promise<Array<{event: MemoryEvent; rank: number}>> {
+    await this.initialize();
+
+    // Escape special FTS5 characters and prepare search terms
+    const searchTerms = query
+      .replace(/['"(){}[\]^~*?:\\/-]/g, ' ')  // Remove special chars
+      .split(/\s+/)
+      .filter(term => term.length > 1)  // Filter short terms
+      .map(term => `"${term}"*`)  // Prefix matching
+      .join(' OR ');
+
+    if (!searchTerms) {
+      return [];
+    }
+
+    try {
+      const rows = sqliteAll<Record<string, unknown>>(
+        this.db,
+        `SELECT e.*, fts.rank
+         FROM events_fts fts
+         JOIN events e ON e.id = fts.event_id
+         WHERE events_fts MATCH ?
+         ORDER BY fts.rank
+         LIMIT ?`,
+        [searchTerms, limit]
+      );
+
+      return rows.map(row => ({
+        event: this.rowToEvent(row),
+        rank: row.rank as number
+      }));
+    } catch (error: any) {
+      // FTS table might not exist yet (old database)
+      // Fallback to LIKE search
+      const likePattern = `%${query}%`;
+      const rows = sqliteAll<Record<string, unknown>>(
+        this.db,
+        `SELECT *, 0 as rank FROM events
+         WHERE content LIKE ?
+         ORDER BY timestamp DESC
+         LIMIT ?`,
+        [likePattern, limit]
+      );
+
+      return rows.map(row => ({
+        event: this.rowToEvent(row),
+        rank: 0
+      }));
+    }
+  }
+
+  /**
+   * Rebuild FTS index from existing events
+   * Call this once after upgrading to FTS5
+   */
+  async rebuildFtsIndex(): Promise<number> {
+    await this.initialize();
+
+    // Get count of events to index
+    const countRow = sqliteGet<{count: number}>(this.db, 'SELECT COUNT(*) as count FROM events', []);
+    const totalEvents = countRow?.count ?? 0;
+
+    // Clear and rebuild FTS index
+    sqliteExec(this.db, `
+      DELETE FROM events_fts;
+      INSERT INTO events_fts(rowid, content, event_id)
+      SELECT rowid, content, id FROM events;
+    `);
+
+    return totalEvents;
+  }
+
   /**
    * Get database instance for direct access
    */

package/src/hooks/user-prompt-submit.ts

@@ -1,93 +1,67 @@
 #!/usr/bin/env node
 /**
  * User Prompt Submit Hook
- * Called when user submits a prompt - retrieves relevant memories
+ * Called when user submits a prompt - retrieves relevant memories using fast keyword search
+ *
+ * Uses SQLite FTS5 for fast keyword-based search (no ML model needed)
+ * Much faster than vector search (~100ms vs 3-5s)
  */
 
-import { getMemoryServiceForSession } from '../services/memory-service.js';
+import { getLightweightMemoryService } from '../services/memory-service.js';
 import type { UserPromptSubmitInput, UserPromptSubmitOutput } from '../core/types.js';
 
+// Configuration
+const MAX_MEMORIES = parseInt(process.env.CLAUDE_MEMORY_MAX_COUNT || '5');
+const MIN_SCORE = parseFloat(process.env.CLAUDE_MEMORY_MIN_SCORE || '0.3');
+const ENABLE_SEARCH = process.env.CLAUDE_MEMORY_SEARCH !== 'false';
+
 async function main(): Promise<void> {
   // Read input from stdin
   const inputData = await readStdin();
   const input: UserPromptSubmitInput = JSON.parse(inputData);
 
-  // Get project-specific memory service via session lookup
-  const memoryService = getMemoryServiceForSession(input.session_id);
-
-  // Configuration from environment variables
-  const config = {
-    candidateCount: parseInt(process.env.CLAUDE_MEMORY_CANDIDATES || '10'),
-    minScore: parseFloat(process.env.CLAUDE_MEMORY_MIN_SCORE || '0.7'),
-    maxMemories: parseInt(process.env.CLAUDE_MEMORY_MAX_COUNT || '5'),
-    dynamicThresholdRatio: parseFloat(process.env.CLAUDE_MEMORY_THRESHOLD_RATIO || '0.85'),
-    strictMinScore: parseFloat(process.env.CLAUDE_MEMORY_STRICT_MIN || '0.75')
-  };
+  // Use lightweight service (SQLite only, no embedder/vector - FAST!)
+  const memoryService = getLightweightMemoryService(input.session_id);
 
   try {
-    // Check if shared store is enabled
-    const includeShared = memoryService.isSharedStoreEnabled();
-
-    // Retrieve relevant memories for the prompt (including shared if enabled)
-    // First, get more candidates to filter from
-    const retrievalResult = await memoryService.retrieveMemories(input.prompt, {
-      topK: config.candidateCount,  // Get more candidates initially
-      minScore: config.minScore,
-      includeShared
-    });
-
     // Store the user prompt for future retrieval
     await memoryService.storeUserPrompt(
       input.session_id,
       input.prompt
     );
 
-    // Filter memories based on relevance and confidence
-    let relevantMemories = [];
-    if (retrievalResult.memories && retrievalResult.memories.length > 0) {
-      // Dynamic threshold based on the best score
-      const bestScore = Math.max(...retrievalResult.memories.map(m => m.score));
-      const dynamicThreshold = Math.max(config.strictMinScore, bestScore * config.dynamicThresholdRatio);
-
-      // Filter memories that meet the dynamic threshold
-      relevantMemories = retrievalResult.memories.filter(m => m.score >= dynamicThreshold);
+    let context = '';
 
-      // Limit to configured max memories
-      relevantMemories = relevantMemories.slice(0, config.maxMemories);
+    // Fast keyword search if enabled
+    if (ENABLE_SEARCH && input.prompt.length > 10) {
+      const results = await memoryService.keywordSearch(input.prompt, {
+        topK: MAX_MEMORIES,
+        minScore: MIN_SCORE
+      });
 
-      // Check if we have enough highly relevant memories
-      if (relevantMemories.length === 0) {
-        // If no memories meet the high threshold, take top 3 with relaxed threshold
-        const fallbackCount = Math.min(3, config.maxMemories);
-        relevantMemories = retrievalResult.memories
-          .filter(m => m.score >= config.minScore)
-          .slice(0, fallbackCount);
-      }
+      if (results.length > 0) {
+        // Increment access count for found memories
+        const eventIds = results.map(r => r.event.id);
+        await memoryService.incrementMemoryAccess(eventIds);
 
-      // Log filtering statistics for debugging
-      if (process.env.CLAUDE_MEMORY_DEBUG) {
-        console.error(`Memory filtering: ${retrievalResult.memories.length} candidates -> ${relevantMemories.length} selected`);
-        console.error(`Threshold: ${dynamicThreshold.toFixed(3)}, Best score: ${bestScore.toFixed(3)}`);
-      }
+        // Format context
+        const memories = results.map(r => {
+          const preview = r.event.content.length > 300
+            ? r.event.content.substring(0, 300) + '...'
+            : r.event.content;
+          return `- [${r.event.eventType}] ${preview}`;
+        });
 
-      // Increment access count only for memories that will actually be included
-      if (relevantMemories.length > 0) {
-        const eventIds = relevantMemories.map(m => m.event.id);
-        await memoryService.incrementMemoryAccess(eventIds);
+        context = `💡 **Related memories found:**\n\n${memories.join('\n\n')}`;
       }
     }
 
-    // Format context for Claude with only the relevant memories
-    const filteredResult = {
-      ...retrievalResult,
-      memories: relevantMemories
-    };
-    const context = memoryService.formatAsContext(filteredResult);
-
     const output: UserPromptSubmitOutput = { context };
     console.log(JSON.stringify(output));
   } catch (error) {
-    console.error('Memory hook error:', error);
+    if (process.env.CLAUDE_MEMORY_DEBUG) {
+      console.error('Memory hook error:', error);
+    }
     console.log(JSON.stringify({ context: '' }));
   }
 }

package/src/services/memory-service.ts

@@ -52,6 +52,8 @@ export interface MemoryServiceConfig {
   readOnly?: boolean;
   /** Enable DuckDB analytics store (default: true for server, false for hooks) */
   analyticsEnabled?: boolean;
+  /** Lightweight mode for hooks - skip heavy initialization (default: false) */
+  lightweightMode?: boolean;
 }
 
 // ============================================================
@@ -200,10 +202,12 @@ export class MemoryService {
   private projectHash: string | null = null;
 
   private readonly readOnly: boolean;
+  private readonly lightweightMode: boolean;
 
   constructor(config: MemoryServiceConfig & { projectHash?: string; sharedStoreConfig?: SharedStoreConfig }) {
     const storagePath = this.expandPath(config.storagePath);
     this.readOnly = config.readOnly ?? false;
+    this.lightweightMode = config.lightweightMode ?? false;
 
     // Ensure storage directory exists (only if not read-only)
     if (!this.readOnly && !fs.existsSync(storagePath)) {
@@ -272,6 +276,13 @@ export class MemoryService {
     // Initialize PRIMARY store: SQLite (always)
     await this.sqliteStore.initialize();
 
+    // Lightweight mode: only SQLite, no embedder/vector/workers
+    // Used for hooks that just need to store data quickly
+    if (this.lightweightMode) {
+      this.initialized = true;
+      return;
+    }
+
     // Initialize analytics store if available (DuckDB)
     if (this.analyticsStore) {
       try {
@@ -517,10 +528,8 @@ export class MemoryService {
   ): Promise<UnifiedRetrievalResult> {
     await this.initialize();
 
-    // Process any pending embeddings first
-    if (this.vectorWorker) {
-      await this.vectorWorker.processAll();
-    }
+    // Note: Pending embeddings are processed by the background worker
+    // Don't block retrieval - search with whatever vectors are available
 
     // Use unified retrieval if shared search is requested
     if (options?.includeShared && this.sharedStore) {
@@ -534,6 +543,38 @@ export class MemoryService {
     return this.retriever.retrieve(query, options);
   }
 
+  /**
+   * Fast keyword search using SQLite FTS5
+   * Much faster than vector search - no embedding model needed
+   */
+  async keywordSearch(
+    query: string,
+    options?: { topK?: number; minScore?: number }
+  ): Promise<Array<{event: MemoryEvent; score: number}>> {
+    await this.initialize();
+
+    const results = await this.sqliteStore.keywordSearch(query, options?.topK ?? 10);
+
+    // Normalize FTS5 rank to a score (0-1 range)
+    // FTS5 rank is negative (higher is worse), so we convert it
+    const maxRank = Math.min(...results.map(r => r.rank), -0.001);
+    const minRank = Math.max(...results.map(r => r.rank), -1000);
+    const rankRange = maxRank - minRank || 1;
+
+    return results.map(r => ({
+      event: r.event,
+      score: 1 - (r.rank - minRank) / rankRange  // Normalize to 0-1
+    })).filter(r => !options?.minScore || r.score >= options.minScore);
+  }
+
+  /**
+   * Rebuild FTS index (call after database upgrade)
+   */
+  async rebuildFtsIndex(): Promise<number> {
+    await this.initialize();
+    return this.sqliteStore.rebuildFtsIndex();
+  }
+
   /**
    * Get session history
    */
@@ -1129,6 +1170,32 @@ export function getMemoryServiceForSession(sessionId: string): MemoryService {
   return getDefaultMemoryService();
 }
 
+/**
+ * Get a lightweight memory service for hooks
+ * Only initializes SQLite - no embedder, no vector store, no workers
+ * This is FAST (<100ms) compared to full initialization (3-5s)
+ */
+export function getLightweightMemoryService(sessionId: string): MemoryService {
+  const projectInfo = getSessionProject(sessionId);
+  const key = projectInfo ? `lightweight_${projectInfo.projectHash}` : 'lightweight_global';
+
+  if (!serviceCache.has(key)) {
+    const storagePath = projectInfo
+      ? getProjectStoragePath(projectInfo.projectPath)
+      : path.join(os.homedir(), '.claude-code', 'memory');
+
+    serviceCache.set(key, new MemoryService({
+      storagePath,
+      projectHash: projectInfo?.projectHash,
+      lightweightMode: true,  // Skip embedder/vector/workers
+      analyticsEnabled: false,
+      sharedStoreConfig: { enabled: false }
+    }));
+  }
+
+  return serviceCache.get(key)!;
+}
+
 export function createMemoryService(config: MemoryServiceConfig): MemoryService {
   return new MemoryService(config);
 }