claude-memory-layer 1.0.7 → 1.0.9

package/src/core/sqlite-wrapper.ts

@@ -0,0 +1,108 @@
+/**
+ * SQLite Wrapper with WAL Mode Support
+ * Primary store for hooks - always available, no lock conflicts
+ */
+
+import Database from 'better-sqlite3';
+
+export type SQLiteDatabase = Database.Database;
+
+export interface SQLiteOptions {
+  readonly?: boolean;
+  walMode?: boolean;
+}
+
+/**
+ * Creates a new SQLite database with WAL mode
+ */
+export function createSQLiteDatabase(path: string, options?: SQLiteOptions): SQLiteDatabase {
+  const db = new Database(path, {
+    readonly: options?.readonly ?? false,
+  });
+
+  // Enable WAL mode for concurrent access (unless read-only)
+  if (!options?.readonly && (options?.walMode ?? true)) {
+    db.pragma('journal_mode = WAL');
+    db.pragma('synchronous = NORMAL');
+    db.pragma('busy_timeout = 5000');
+  }
+
+  return db;
+}
+
+/**
+ * Execute a statement that doesn't return rows (INSERT, UPDATE, DELETE)
+ */
+export function sqliteRun(
+  db: SQLiteDatabase,
+  sql: string,
+  params: unknown[] = []
+): Database.RunResult {
+  const stmt = db.prepare(sql);
+  return stmt.run(...params);
+}
+
+/**
+ * Execute a query and return all rows
+ */
+export function sqliteAll<T = Record<string, unknown>>(
+  db: SQLiteDatabase,
+  sql: string,
+  params: unknown[] = []
+): T[] {
+  const stmt = db.prepare(sql);
+  return stmt.all(...params) as T[];
+}
+
+/**
+ * Execute a query and return first row
+ */
+export function sqliteGet<T = Record<string, unknown>>(
+  db: SQLiteDatabase,
+  sql: string,
+  params: unknown[] = []
+): T | undefined {
+  const stmt = db.prepare(sql);
+  return stmt.get(...params) as T | undefined;
+}
+
+/**
+ * Execute multiple statements (for schema creation)
+ */
+export function sqliteExec(db: SQLiteDatabase, sql: string): void {
+  db.exec(sql);
+}
+
+/**
+ * Close database connection
+ */
+export function sqliteClose(db: SQLiteDatabase): void {
+  db.close();
+}
+
+/**
+ * Run multiple statements in a transaction
+ */
+export function sqliteTransaction<T>(
+  db: SQLiteDatabase,
+  fn: () => T
+): T {
+  return db.transaction(fn)();
+}
+
+/**
+ * Safely converts a value to a Date object
+ */
+export function toDateFromSQLite(value: unknown): Date {
+  if (value instanceof Date) return value;
+  if (typeof value === 'string') return new Date(value);
+  if (typeof value === 'number') return new Date(value);
+  return new Date(String(value));
+}
+
+/**
+ * Convert Date to ISO string for SQLite storage
+ */
+export function toSQLiteTimestamp(date: Date): string {
+  return date.toISOString();
+}

package/src/core/sync-worker.ts

@@ -0,0 +1,228 @@
+/**
+ * Sync Worker - SQLite to DuckDB synchronization
+ * Runs periodically to sync primary store (SQLite) to analytics store (DuckDB)
+ */
+
+import { SQLiteEventStore } from './sqlite-event-store.js';
+import { EventStore } from './event-store.js';
+import { MemoryEvent } from './types.js';
+
+export interface SyncWorkerConfig {
+  intervalMs: number;      // Sync interval (default: 30000 = 30 seconds)
+  batchSize: number;       // Events per batch (default: 500)
+  maxRetries: number;      // Max retries on failure (default: 3)
+  retryDelayMs: number;    // Delay between retries (default: 5000)
+}
+
+const DEFAULT_CONFIG: SyncWorkerConfig = {
+  intervalMs: 30000,
+  batchSize: 500,
+  maxRetries: 3,
+  retryDelayMs: 5000
+};
+
+export interface SyncStats {
+  lastSyncAt: Date | null;
+  eventsSynced: number;
+  sessionsSynced: number;
+  errors: number;
+  status: 'idle' | 'syncing' | 'error' | 'stopped';
+}
+
+export class SyncWorker {
+  private config: SyncWorkerConfig;
+  private intervalHandle: NodeJS.Timeout | null = null;
+  private running = false;
+  private stats: SyncStats = {
+    lastSyncAt: null,
+    eventsSynced: 0,
+    sessionsSynced: 0,
+    errors: 0,
+    status: 'idle'
+  };
+
+  constructor(
+    private sqliteStore: SQLiteEventStore,
+    private duckdbStore: EventStore,
+    config?: Partial<SyncWorkerConfig>
+  ) {
+    this.config = { ...DEFAULT_CONFIG, ...config };
+  }
+
+  /**
+   * Start the sync worker
+   */
+  start(): void {
+    if (this.running) return;
+
+    this.running = true;
+    this.stats.status = 'idle';
+
+    // Run initial sync
+    this.syncNow().catch(err => {
+      console.error('[SyncWorker] Initial sync failed:', err);
+    });
+
+    // Schedule periodic sync
+    this.intervalHandle = setInterval(() => {
+      this.syncNow().catch(err => {
+        console.error('[SyncWorker] Periodic sync failed:', err);
+      });
+    }, this.config.intervalMs);
+  }
+
+  /**
+   * Stop the sync worker
+   */
+  stop(): void {
+    this.running = false;
+    this.stats.status = 'stopped';
+
+    if (this.intervalHandle) {
+      clearInterval(this.intervalHandle);
+      this.intervalHandle = null;
+    }
+  }
+
+  /**
+   * Trigger immediate sync
+   */
+  async syncNow(): Promise<void> {
+    if (this.stats.status === 'syncing') {
+      return; // Already syncing
+    }
+
+    this.stats.status = 'syncing';
+
+    try {
+      await this.syncEvents();
+      await this.syncSessions();
+      this.stats.lastSyncAt = new Date();
+      this.stats.status = 'idle';
+    } catch (error) {
+      this.stats.errors++;
+      this.stats.status = 'error';
+      throw error;
+    }
+  }
+
+  /**
+   * Sync events from SQLite to DuckDB
+   */
+  private async syncEvents(): Promise<void> {
+    const targetName = 'duckdb_analytics';
+
+    // Get last sync position from SQLite
+    const position = await this.sqliteStore.getSyncPosition(targetName);
+    const lastTimestamp = position.lastTimestamp || '1970-01-01T00:00:00.000Z';
+
+    let hasMore = true;
+    let totalSynced = 0;
+
+    while (hasMore) {
+      // Get batch of events since last sync
+      const events = await this.sqliteStore.getEventsSince(lastTimestamp, this.config.batchSize);
+
+      if (events.length === 0) {
+        hasMore = false;
+        break;
+      }
+
+      // Insert into DuckDB with retry
+      await this.retryWithBackoff(async () => {
+        for (const event of events) {
+          await this.insertEventToDuckDB(event);
+        }
+      });
+
+      totalSynced += events.length;
+
+      // Update sync position
+      const lastEvent = events[events.length - 1];
+      await this.sqliteStore.updateSyncPosition(
+        targetName,
+        lastEvent.id,
+        lastEvent.timestamp.toISOString()
+      );
+
+      // Check if we got a full batch (more to sync)
+      hasMore = events.length === this.config.batchSize;
+    }
+
+    this.stats.eventsSynced += totalSynced;
+  }
+
+  /**
+   * Sync sessions from SQLite to DuckDB
+   */
+  private async syncSessions(): Promise<void> {
+    // Get all sessions from SQLite
+    const sessions = await this.sqliteStore.getAllSessions();
+
+    // Upsert each session to DuckDB
+    for (const session of sessions) {
+      await this.retryWithBackoff(async () => {
+        await this.duckdbStore.upsertSession(session);
+      });
+    }
+
+    this.stats.sessionsSynced = sessions.length;
+  }
+
+  /**
+   * Insert a single event into DuckDB
+   */
+  private async insertEventToDuckDB(event: MemoryEvent): Promise<void> {
+    // Use append which handles deduplication
+    await this.duckdbStore.append({
+      eventType: event.eventType,
+      sessionId: event.sessionId,
+      timestamp: event.timestamp,
+      content: event.content,
+      metadata: event.metadata
+    });
+  }
+
+  /**
+   * Retry operation with exponential backoff
+   */
+  private async retryWithBackoff<T>(fn: () => Promise<T>): Promise<T> {
+    let lastError: Error | null = null;
+
+    for (let attempt = 0; attempt < this.config.maxRetries; attempt++) {
+      try {
+        return await fn();
+      } catch (error) {
+        lastError = error instanceof Error ? error : new Error(String(error));
+
+        if (attempt < this.config.maxRetries - 1) {
+          const delay = this.config.retryDelayMs * Math.pow(2, attempt);
+          await this.sleep(delay);
+        }
+      }
+    }
+
+    throw lastError;
+  }
+
+  /**
+   * Sleep utility
+   */
+  private sleep(ms: number): Promise<void> {
+    return new Promise(resolve => setTimeout(resolve, ms));
+  }
+
+  /**
+   * Get sync statistics
+   */
+  getStats(): SyncStats {
+    return { ...this.stats };
+  }
+
+  /**
+   * Check if worker is running
+   */
+  isRunning(): boolean {
+    return this.running;
+  }
+}

package/src/core/vector-worker.ts

@@ -26,6 +26,7 @@ export class VectorWorker {
   private readonly embedder: Embedder;
   private readonly config: WorkerConfig;
   private running = false;
+  private stopping = false;
   private pollTimeout: NodeJS.Timeout | null = null;
 
   constructor(
@@ -46,6 +47,7 @@ export class VectorWorker {
   start(): void {
     if (this.running) return;
     this.running = true;
+    this.stopping = false;
     this.poll();
   }
 
@@ -54,6 +56,7 @@ export class VectorWorker {
    */
   stop(): void {
     this.running = false;
+    this.stopping = true;
     if (this.pollTimeout) {
       clearTimeout(this.pollTimeout);
       this.pollTimeout = null;
@@ -122,10 +125,17 @@ export class VectorWorker {
 
       return successful.length;
     } catch (error) {
-      // Mark all items as failed
-      const allIds = items.map(i => i.id);
-      const errorMessage = error instanceof Error ? error.message : String(error);
-      await this.eventStore.failOutboxItems(allIds, errorMessage);
+      // Mark all items as failed, but only if not stopping (DB might be closed)
+      if (!this.stopping) {
+        try {
+          const allIds = items.map(i => i.id);
+          const errorMessage = error instanceof Error ? error.message : String(error);
+          await this.eventStore.failOutboxItems(allIds, errorMessage);
+        } catch (failError) {
+          // Database might be closed during shutdown, ignore
+          console.warn('Could not mark outbox items as failed (database may be closed)');
+        }
+      }
       throw error;
     }
   }
@@ -134,16 +144,21 @@ export class VectorWorker {
    * Poll for new items
    */
   private async poll(): Promise<void> {
-    if (!this.running) return;
+    if (!this.running || this.stopping) return;
 
     try {
       await this.processBatch();
     } catch (error) {
-      console.error('Vector worker error:', error);
+      // Only log if not stopping (error during shutdown is expected)
+      if (!this.stopping) {
+        console.error('Vector worker error:', error);
+      }
     }
 
-    // Schedule next poll
-    this.pollTimeout = setTimeout(() => this.poll(), this.config.pollIntervalMs);
+    // Schedule next poll only if still running
+    if (this.running && !this.stopping) {
+      this.pollTimeout = setTimeout(() => this.poll(), this.config.pollIntervalMs);
+    }
   }
 
   /**
@@ -319,6 +334,7 @@ export class VectorWorkerV2 {
   private readonly contentProvider: ContentProvider;
   private readonly config: WorkerConfigV2;
   private running = false;
+  private stopping = false;
   private pollTimeout: NodeJS.Timeout | null = null;
 
   constructor(
@@ -344,6 +360,7 @@ export class VectorWorkerV2 {
   start(): void {
     if (this.running) return;
     this.running = true;
+    this.stopping = false;
     this.poll();
   }
 
@@ -352,6 +369,7 @@ export class VectorWorkerV2 {
    */
   stop(): void {
     this.running = false;
+    this.stopping = true;
     if (this.pollTimeout) {
       clearTimeout(this.pollTimeout);
       this.pollTimeout = null;
@@ -376,8 +394,15 @@ export class VectorWorkerV2 {
         await this.outbox.markDone(job.jobId);
         successCount++;
       } catch (error) {
-        const errorMessage = error instanceof Error ? error.message : String(error);
-        await this.outbox.markFailed(job.jobId, errorMessage);
+        // Only try to mark as failed if not stopping (DB might be closed)
+        if (!this.stopping) {
+          try {
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            await this.outbox.markFailed(job.jobId, errorMessage);
+          } catch {
+            // Database might be closed during shutdown, ignore
+          }
+        }
       }
     }
 
@@ -422,16 +447,21 @@ export class VectorWorkerV2 {
    * Poll for new jobs
    */
   private async poll(): Promise<void> {
-    if (!this.running) return;
+    if (!this.running || this.stopping) return;
 
     try {
       await this.processBatch();
     } catch (error) {
-      console.error('Vector worker V2 error:', error);
+      // Only log if not stopping (error during shutdown is expected)
+      if (!this.stopping) {
+        console.error('Vector worker V2 error:', error);
+      }
     }
 
-    // Schedule next poll
-    this.pollTimeout = setTimeout(() => this.poll(), this.config.pollIntervalMs);
+    // Schedule next poll only if still running
+    if (this.running && !this.stopping) {
+      this.pollTimeout = setTimeout(() => this.poll(), this.config.pollIntervalMs);
+    }
   }
 
   /**

package/src/hooks/user-prompt-submit.ts

@@ -15,14 +15,24 @@ async function main(): Promise<void> {
   // Get project-specific memory service via session lookup
   const memoryService = getMemoryServiceForSession(input.session_id);
 
+  // Configuration from environment variables
+  const config = {
+    candidateCount: parseInt(process.env.CLAUDE_MEMORY_CANDIDATES || '10'),
+    minScore: parseFloat(process.env.CLAUDE_MEMORY_MIN_SCORE || '0.7'),
+    maxMemories: parseInt(process.env.CLAUDE_MEMORY_MAX_COUNT || '5'),
+    dynamicThresholdRatio: parseFloat(process.env.CLAUDE_MEMORY_THRESHOLD_RATIO || '0.85'),
+    strictMinScore: parseFloat(process.env.CLAUDE_MEMORY_STRICT_MIN || '0.75')
+  };
+
   try {
     // Check if shared store is enabled
     const includeShared = memoryService.isSharedStoreEnabled();
 
     // Retrieve relevant memories for the prompt (including shared if enabled)
+    // First, get more candidates to filter from
     const retrievalResult = await memoryService.retrieveMemories(input.prompt, {
-      topK: 5,
-      minScore: 0.7,
+      topK: config.candidateCount,  // Get more candidates initially
+      minScore: config.minScore,
       includeShared
     });
 
@@ -32,8 +42,47 @@ async function main(): Promise<void> {
       input.prompt
     );
 
-    // Format context for Claude
-    const context = memoryService.formatAsContext(retrievalResult);
+    // Filter memories based on relevance and confidence
+    let relevantMemories = [];
+    if (retrievalResult.memories && retrievalResult.memories.length > 0) {
+      // Dynamic threshold based on the best score
+      const bestScore = Math.max(...retrievalResult.memories.map(m => m.score));
+      const dynamicThreshold = Math.max(config.strictMinScore, bestScore * config.dynamicThresholdRatio);
+
+      // Filter memories that meet the dynamic threshold
+      relevantMemories = retrievalResult.memories.filter(m => m.score >= dynamicThreshold);
+
+      // Limit to configured max memories
+      relevantMemories = relevantMemories.slice(0, config.maxMemories);
+
+      // Check if we have enough highly relevant memories
+      if (relevantMemories.length === 0) {
+        // If no memories meet the high threshold, take top 3 with relaxed threshold
+        const fallbackCount = Math.min(3, config.maxMemories);
+        relevantMemories = retrievalResult.memories
+          .filter(m => m.score >= config.minScore)
+          .slice(0, fallbackCount);
+      }
+
+      // Log filtering statistics for debugging
+      if (process.env.CLAUDE_MEMORY_DEBUG) {
+        console.error(`Memory filtering: ${retrievalResult.memories.length} candidates -> ${relevantMemories.length} selected`);
+        console.error(`Threshold: ${dynamicThreshold.toFixed(3)}, Best score: ${bestScore.toFixed(3)}`);
+      }
+
+      // Increment access count only for memories that will actually be included
+      if (relevantMemories.length > 0) {
+        const eventIds = relevantMemories.map(m => m.event.id);
+        await memoryService.incrementMemoryAccess(eventIds);
+      }
+    }
+
+    // Format context for Claude with only the relevant memories
+    const filteredResult = {
+      ...retrievalResult,
+      memories: relevantMemories
+    };
+    const context = memoryService.formatAsContext(filteredResult);
 
     const output: UserPromptSubmitOutput = { context };
     console.log(JSON.stringify(output));

package/src/server/api/citations.ts

@@ -4,7 +4,7 @@
  */
 
 import { Hono } from 'hono';
-import { getDefaultMemoryService } from '../../services/memory-service.js';
+import { getReadOnlyMemoryService } from '../../services/memory-service.js';
 import { generateCitationId, parseCitationId } from '../../core/citation-generator.js';
 
 export const citationsRouter = new Hono();
@@ -15,9 +15,9 @@ citationsRouter.get('/:id', async (c) => {
 
   // Support both formats: "a7Bc3x" or "mem:a7Bc3x"
   const citationId = parseCitationId(id) || id;
+  const memoryService = getReadOnlyMemoryService();
 
   try {
-    const memoryService = getDefaultMemoryService();
     await memoryService.initialize();
 
     // Search through recent events to find the one matching this citation ID
@@ -48,6 +48,8 @@ citationsRouter.get('/:id', async (c) => {
     });
   } catch (error) {
     return c.json({ error: (error as Error).message }, 500);
+  } finally {
+    await memoryService.shutdown();
   }
 });
 
@@ -55,9 +57,9 @@ citationsRouter.get('/:id', async (c) => {
 citationsRouter.get('/:id/related', async (c) => {
   const { id } = c.req.param();
   const citationId = parseCitationId(id) || id;
+  const memoryService = getReadOnlyMemoryService();
 
   try {
-    const memoryService = getDefaultMemoryService();
     await memoryService.initialize();
 
     const recentEvents = await memoryService.getRecentEvents(10000);
@@ -97,5 +99,7 @@ citationsRouter.get('/:id/related', async (c) => {
     });
   } catch (error) {
     return c.json({ error: (error as Error).message }, 500);
+  } finally {
+    await memoryService.shutdown();
   }
 });

package/src/server/api/events.ts

@@ -4,7 +4,7 @@
  */
 
 import { Hono } from 'hono';
-import { getDefaultMemoryService } from '../../services/memory-service.js';
+import { getReadOnlyMemoryService } from '../../services/memory-service.js';
 
 export const eventsRouter = new Hono();
 
@@ -14,9 +14,9 @@ eventsRouter.get('/', async (c) => {
   const eventType = c.req.query('type');
   const limit = parseInt(c.req.query('limit') || '100', 10);
   const offset = parseInt(c.req.query('offset') || '0', 10);
+  const memoryService = getReadOnlyMemoryService();
 
   try {
-    const memoryService = getDefaultMemoryService();
     await memoryService.initialize();
 
     let events = await memoryService.getRecentEvents(limit + offset + 1000);
@@ -51,15 +51,17 @@ eventsRouter.get('/', async (c) => {
     });
   } catch (error) {
     return c.json({ error: (error as Error).message }, 500);
+  } finally {
+    await memoryService.shutdown();
   }
 });
 
 // GET /api/events/:id - Get event details
 eventsRouter.get('/:id', async (c) => {
   const { id } = c.req.param();
+  const memoryService = getReadOnlyMemoryService();
 
   try {
-    const memoryService = getDefaultMemoryService();
     await memoryService.initialize();
 
     const recentEvents = await memoryService.getRecentEvents(10000);
@@ -97,5 +99,7 @@ eventsRouter.get('/:id', async (c) => {
     });
   } catch (error) {
     return c.json({ error: (error as Error).message }, 500);
+  } finally {
+    await memoryService.shutdown();
   }
 });

package/src/server/api/search.ts

@@ -4,7 +4,7 @@
  */
 
 import { Hono } from 'hono';
-import { getDefaultMemoryService } from '../../services/memory-service.js';
+import { getReadOnlyMemoryService } from '../../services/memory-service.js';
 
 export const searchRouter = new Hono();
 
@@ -20,6 +20,7 @@ interface SearchRequest {
 
 // POST /api/search - Search memories
 searchRouter.post('/', async (c) => {
+  const memoryService = getReadOnlyMemoryService();
   try {
     const body = await c.req.json<SearchRequest>();
 
@@ -27,7 +28,6 @@ searchRouter.post('/', async (c) => {
       return c.json({ error: 'Query is required' }, 400);
     }
 
-    const memoryService = getDefaultMemoryService();
     await memoryService.initialize();
 
     const startTime = Date.now();
@@ -60,6 +60,8 @@ searchRouter.post('/', async (c) => {
     });
   } catch (error) {
     return c.json({ error: (error as Error).message }, 500);
+  } finally {
+    await memoryService.shutdown();
   }
 });
 
@@ -72,9 +74,9 @@ searchRouter.get('/', async (c) => {
   }
 
   const topK = parseInt(c.req.query('topK') || '5', 10);
+  const memoryService = getReadOnlyMemoryService();
 
   try {
-    const memoryService = getDefaultMemoryService();
     await memoryService.initialize();
 
     const result = await memoryService.retrieveMemories(query, { topK });
@@ -94,5 +96,7 @@ searchRouter.get('/', async (c) => {
     });
   } catch (error) {
     return c.json({ error: (error as Error).message }, 500);
+  } finally {
+    await memoryService.shutdown();
   }
 });