claude-memory-layer 1.0.8 → 1.0.9

package/src/core/sqlite-wrapper.ts

@@ -0,0 +1,108 @@
+/**
+ * SQLite Wrapper with WAL Mode Support
+ * Primary store for hooks - always available, no lock conflicts
+ */
+
+import Database from 'better-sqlite3';
+
+export type SQLiteDatabase = Database.Database;
+
+export interface SQLiteOptions {
+  readonly?: boolean;
+  walMode?: boolean;
+}
+
+/**
+ * Creates a new SQLite database with WAL mode
+ */
+export function createSQLiteDatabase(path: string, options?: SQLiteOptions): SQLiteDatabase {
+  const db = new Database(path, {
+    readonly: options?.readonly ?? false,
+  });
+
+  // Enable WAL mode for concurrent access (unless read-only)
+  if (!options?.readonly && (options?.walMode ?? true)) {
+    db.pragma('journal_mode = WAL');
+    db.pragma('synchronous = NORMAL');
+    db.pragma('busy_timeout = 5000');
+  }
+
+  return db;
+}
+
+/**
+ * Execute a statement that doesn't return rows (INSERT, UPDATE, DELETE)
+ */
+export function sqliteRun(
+  db: SQLiteDatabase,
+  sql: string,
+  params: unknown[] = []
+): Database.RunResult {
+  const stmt = db.prepare(sql);
+  return stmt.run(...params);
+}
+
+/**
+ * Execute a query and return all rows
+ */
+export function sqliteAll<T = Record<string, unknown>>(
+  db: SQLiteDatabase,
+  sql: string,
+  params: unknown[] = []
+): T[] {
+  const stmt = db.prepare(sql);
+  return stmt.all(...params) as T[];
+}
+
+/**
+ * Execute a query and return first row
+ */
+export function sqliteGet<T = Record<string, unknown>>(
+  db: SQLiteDatabase,
+  sql: string,
+  params: unknown[] = []
+): T | undefined {
+  const stmt = db.prepare(sql);
+  return stmt.get(...params) as T | undefined;
+}
+
+/**
+ * Execute multiple statements (for schema creation)
+ */
+export function sqliteExec(db: SQLiteDatabase, sql: string): void {
+  db.exec(sql);
+}
+
+/**
+ * Close database connection
+ */
+export function sqliteClose(db: SQLiteDatabase): void {
+  db.close();
+}
+
+/**
+ * Run multiple statements in a transaction
+ */
+export function sqliteTransaction<T>(
+  db: SQLiteDatabase,
+  fn: () => T
+): T {
+  return db.transaction(fn)();
+}
+
+/**
+ * Safely converts a value to a Date object
+ */
+export function toDateFromSQLite(value: unknown): Date {
+  if (value instanceof Date) return value;
+  if (typeof value === 'string') return new Date(value);
+  if (typeof value === 'number') return new Date(value);
+  return new Date(String(value));
+}
+
+/**
+ * Convert Date to ISO string for SQLite storage
+ */
+export function toSQLiteTimestamp(date: Date): string {
+  return date.toISOString();
+}

package/src/core/sync-worker.ts

@@ -0,0 +1,228 @@
+/**
+ * Sync Worker - SQLite to DuckDB synchronization
+ * Runs periodically to sync primary store (SQLite) to analytics store (DuckDB)
+ */
+
+import { SQLiteEventStore } from './sqlite-event-store.js';
+import { EventStore } from './event-store.js';
+import { MemoryEvent } from './types.js';
+
+export interface SyncWorkerConfig {
+  intervalMs: number;      // Sync interval (default: 30000 = 30 seconds)
+  batchSize: number;       // Events per batch (default: 500)
+  maxRetries: number;      // Max retries on failure (default: 3)
+  retryDelayMs: number;    // Delay between retries (default: 5000)
+}
+
+const DEFAULT_CONFIG: SyncWorkerConfig = {
+  intervalMs: 30000,
+  batchSize: 500,
+  maxRetries: 3,
+  retryDelayMs: 5000
+};
+
+export interface SyncStats {
+  lastSyncAt: Date | null;
+  eventsSynced: number;
+  sessionsSynced: number;
+  errors: number;
+  status: 'idle' | 'syncing' | 'error' | 'stopped';
+}
+
+export class SyncWorker {
+  private config: SyncWorkerConfig;
+  private intervalHandle: NodeJS.Timeout | null = null;
+  private running = false;
+  private stats: SyncStats = {
+    lastSyncAt: null,
+    eventsSynced: 0,
+    sessionsSynced: 0,
+    errors: 0,
+    status: 'idle'
+  };
+
+  constructor(
+    private sqliteStore: SQLiteEventStore,
+    private duckdbStore: EventStore,
+    config?: Partial<SyncWorkerConfig>
+  ) {
+    this.config = { ...DEFAULT_CONFIG, ...config };
+  }
+
+  /**
+   * Start the sync worker
+   */
+  start(): void {
+    if (this.running) return;
+
+    this.running = true;
+    this.stats.status = 'idle';
+
+    // Run initial sync
+    this.syncNow().catch(err => {
+      console.error('[SyncWorker] Initial sync failed:', err);
+    });
+
+    // Schedule periodic sync
+    this.intervalHandle = setInterval(() => {
+      this.syncNow().catch(err => {
+        console.error('[SyncWorker] Periodic sync failed:', err);
+      });
+    }, this.config.intervalMs);
+  }
+
+  /**
+   * Stop the sync worker
+   */
+  stop(): void {
+    this.running = false;
+    this.stats.status = 'stopped';
+
+    if (this.intervalHandle) {
+      clearInterval(this.intervalHandle);
+      this.intervalHandle = null;
+    }
+  }
+
+  /**
+   * Trigger immediate sync
+   */
+  async syncNow(): Promise<void> {
+    if (this.stats.status === 'syncing') {
+      return; // Already syncing
+    }
+
+    this.stats.status = 'syncing';
+
+    try {
+      await this.syncEvents();
+      await this.syncSessions();
+      this.stats.lastSyncAt = new Date();
+      this.stats.status = 'idle';
+    } catch (error) {
+      this.stats.errors++;
+      this.stats.status = 'error';
+      throw error;
+    }
+  }
+
+  /**
+   * Sync events from SQLite to DuckDB
+   */
+  private async syncEvents(): Promise<void> {
+    const targetName = 'duckdb_analytics';
+
+    // Get last sync position from SQLite
+    const position = await this.sqliteStore.getSyncPosition(targetName);
+    const lastTimestamp = position.lastTimestamp || '1970-01-01T00:00:00.000Z';
+
+    let hasMore = true;
+    let totalSynced = 0;
+
+    while (hasMore) {
+      // Get batch of events since last sync
+      const events = await this.sqliteStore.getEventsSince(lastTimestamp, this.config.batchSize);
+
+      if (events.length === 0) {
+        hasMore = false;
+        break;
+      }
+
+      // Insert into DuckDB with retry
+      await this.retryWithBackoff(async () => {
+        for (const event of events) {
+          await this.insertEventToDuckDB(event);
+        }
+      });
+
+      totalSynced += events.length;
+
+      // Update sync position
+      const lastEvent = events[events.length - 1];
+      await this.sqliteStore.updateSyncPosition(
+        targetName,
+        lastEvent.id,
+        lastEvent.timestamp.toISOString()
+      );
+
+      // Check if we got a full batch (more to sync)
+      hasMore = events.length === this.config.batchSize;
+    }
+
+    this.stats.eventsSynced += totalSynced;
+  }
+
+  /**
+   * Sync sessions from SQLite to DuckDB
+   */
+  private async syncSessions(): Promise<void> {
+    // Get all sessions from SQLite
+    const sessions = await this.sqliteStore.getAllSessions();
+
+    // Upsert each session to DuckDB
+    for (const session of sessions) {
+      await this.retryWithBackoff(async () => {
+        await this.duckdbStore.upsertSession(session);
+      });
+    }
+
+    this.stats.sessionsSynced = sessions.length;
+  }
+
+  /**
+   * Insert a single event into DuckDB
+   */
+  private async insertEventToDuckDB(event: MemoryEvent): Promise<void> {
+    // Use append which handles deduplication
+    await this.duckdbStore.append({
+      eventType: event.eventType,
+      sessionId: event.sessionId,
+      timestamp: event.timestamp,
+      content: event.content,
+      metadata: event.metadata
+    });
+  }
+
+  /**
+   * Retry operation with exponential backoff
+   */
+  private async retryWithBackoff<T>(fn: () => Promise<T>): Promise<T> {
+    let lastError: Error | null = null;
+
+    for (let attempt = 0; attempt < this.config.maxRetries; attempt++) {
+      try {
+        return await fn();
+      } catch (error) {
+        lastError = error instanceof Error ? error : new Error(String(error));
+
+        if (attempt < this.config.maxRetries - 1) {
+          const delay = this.config.retryDelayMs * Math.pow(2, attempt);
+          await this.sleep(delay);
+        }
+      }
+    }
+
+    throw lastError;
+  }
+
+  /**
+   * Sleep utility
+   */
+  private sleep(ms: number): Promise<void> {
+    return new Promise(resolve => setTimeout(resolve, ms));
+  }
+
+  /**
+   * Get sync statistics
+   */
+  getStats(): SyncStats {
+    return { ...this.stats };
+  }
+
+  /**
+   * Check if worker is running
+   */
+  isRunning(): boolean {
+    return this.running;
+  }
+}

package/src/core/vector-worker.ts

@@ -26,6 +26,7 @@ export class VectorWorker {
   private readonly embedder: Embedder;
   private readonly config: WorkerConfig;
   private running = false;
+  private stopping = false;
   private pollTimeout: NodeJS.Timeout | null = null;
 
   constructor(
@@ -46,6 +47,7 @@ export class VectorWorker {
   start(): void {
     if (this.running) return;
     this.running = true;
+    this.stopping = false;
     this.poll();
   }
 
@@ -54,6 +56,7 @@ export class VectorWorker {
    */
   stop(): void {
     this.running = false;
+    this.stopping = true;
     if (this.pollTimeout) {
       clearTimeout(this.pollTimeout);
       this.pollTimeout = null;
@@ -122,10 +125,17 @@ export class VectorWorker {
 
       return successful.length;
     } catch (error) {
-      // Mark all items as failed
-      const allIds = items.map(i => i.id);
-      const errorMessage = error instanceof Error ? error.message : String(error);
-      await this.eventStore.failOutboxItems(allIds, errorMessage);
+      // Mark all items as failed, but only if not stopping (DB might be closed)
+      if (!this.stopping) {
+        try {
+          const allIds = items.map(i => i.id);
+          const errorMessage = error instanceof Error ? error.message : String(error);
+          await this.eventStore.failOutboxItems(allIds, errorMessage);
+        } catch (failError) {
+          // Database might be closed during shutdown, ignore
+          console.warn('Could not mark outbox items as failed (database may be closed)');
+        }
+      }
       throw error;
     }
   }
@@ -134,16 +144,21 @@ export class VectorWorker {
    * Poll for new items
    */
   private async poll(): Promise<void> {
-    if (!this.running) return;
+    if (!this.running || this.stopping) return;
 
     try {
       await this.processBatch();
     } catch (error) {
-      console.error('Vector worker error:', error);
+      // Only log if not stopping (error during shutdown is expected)
+      if (!this.stopping) {
+        console.error('Vector worker error:', error);
+      }
     }
 
-    // Schedule next poll
-    this.pollTimeout = setTimeout(() => this.poll(), this.config.pollIntervalMs);
+    // Schedule next poll only if still running
+    if (this.running && !this.stopping) {
+      this.pollTimeout = setTimeout(() => this.poll(), this.config.pollIntervalMs);
+    }
   }
 
   /**
@@ -319,6 +334,7 @@ export class VectorWorkerV2 {
   private readonly contentProvider: ContentProvider;
   private readonly config: WorkerConfigV2;
   private running = false;
+  private stopping = false;
   private pollTimeout: NodeJS.Timeout | null = null;
 
   constructor(
@@ -344,6 +360,7 @@ export class VectorWorkerV2 {
   start(): void {
     if (this.running) return;
     this.running = true;
+    this.stopping = false;
     this.poll();
   }
 
@@ -352,6 +369,7 @@ export class VectorWorkerV2 {
    */
   stop(): void {
     this.running = false;
+    this.stopping = true;
     if (this.pollTimeout) {
       clearTimeout(this.pollTimeout);
       this.pollTimeout = null;
@@ -376,8 +394,15 @@ export class VectorWorkerV2 {
         await this.outbox.markDone(job.jobId);
         successCount++;
       } catch (error) {
-        const errorMessage = error instanceof Error ? error.message : String(error);
-        await this.outbox.markFailed(job.jobId, errorMessage);
+        // Only try to mark as failed if not stopping (DB might be closed)
+        if (!this.stopping) {
+          try {
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            await this.outbox.markFailed(job.jobId, errorMessage);
+          } catch {
+            // Database might be closed during shutdown, ignore
+          }
+        }
       }
     }
 
@@ -422,16 +447,21 @@ export class VectorWorkerV2 {
    * Poll for new jobs
    */
   private async poll(): Promise<void> {
-    if (!this.running) return;
+    if (!this.running || this.stopping) return;
 
     try {
       await this.processBatch();
     } catch (error) {
-      console.error('Vector worker V2 error:', error);
+      // Only log if not stopping (error during shutdown is expected)
+      if (!this.stopping) {
+        console.error('Vector worker V2 error:', error);
+      }
     }
 
-    // Schedule next poll
-    this.pollTimeout = setTimeout(() => this.poll(), this.config.pollIntervalMs);
+    // Schedule next poll only if still running
+    if (this.running && !this.stopping) {
+      this.pollTimeout = setTimeout(() => this.poll(), this.config.pollIntervalMs);
+    }
   }
 
   /**

package/src/hooks/user-prompt-submit.ts

@@ -15,14 +15,24 @@ async function main(): Promise<void> {
   // Get project-specific memory service via session lookup
   const memoryService = getMemoryServiceForSession(input.session_id);
 
+  // Configuration from environment variables
+  const config = {
+    candidateCount: parseInt(process.env.CLAUDE_MEMORY_CANDIDATES || '10'),
+    minScore: parseFloat(process.env.CLAUDE_MEMORY_MIN_SCORE || '0.7'),
+    maxMemories: parseInt(process.env.CLAUDE_MEMORY_MAX_COUNT || '5'),
+    dynamicThresholdRatio: parseFloat(process.env.CLAUDE_MEMORY_THRESHOLD_RATIO || '0.85'),
+    strictMinScore: parseFloat(process.env.CLAUDE_MEMORY_STRICT_MIN || '0.75')
+  };
+
   try {
     // Check if shared store is enabled
     const includeShared = memoryService.isSharedStoreEnabled();
 
     // Retrieve relevant memories for the prompt (including shared if enabled)
+    // First, get more candidates to filter from
     const retrievalResult = await memoryService.retrieveMemories(input.prompt, {
-      topK: 5,
-      minScore: 0.7,
+      topK: config.candidateCount,  // Get more candidates initially
+      minScore: config.minScore,
       includeShared
     });
 
@@ -32,8 +42,47 @@ async function main(): Promise<void> {
       input.prompt
     );
 
-    // Format context for Claude
-    const context = memoryService.formatAsContext(retrievalResult);
+    // Filter memories based on relevance and confidence
+    let relevantMemories = [];
+    if (retrievalResult.memories && retrievalResult.memories.length > 0) {
+      // Dynamic threshold based on the best score
+      const bestScore = Math.max(...retrievalResult.memories.map(m => m.score));
+      const dynamicThreshold = Math.max(config.strictMinScore, bestScore * config.dynamicThresholdRatio);
+
+      // Filter memories that meet the dynamic threshold
+      relevantMemories = retrievalResult.memories.filter(m => m.score >= dynamicThreshold);
+
+      // Limit to configured max memories
+      relevantMemories = relevantMemories.slice(0, config.maxMemories);
+
+      // Check if we have enough highly relevant memories
+      if (relevantMemories.length === 0) {
+        // If no memories meet the high threshold, take top 3 with relaxed threshold
+        const fallbackCount = Math.min(3, config.maxMemories);
+        relevantMemories = retrievalResult.memories
+          .filter(m => m.score >= config.minScore)
+          .slice(0, fallbackCount);
+      }
+
+      // Log filtering statistics for debugging
+      if (process.env.CLAUDE_MEMORY_DEBUG) {
+        console.error(`Memory filtering: ${retrievalResult.memories.length} candidates -> ${relevantMemories.length} selected`);
+        console.error(`Threshold: ${dynamicThreshold.toFixed(3)}, Best score: ${bestScore.toFixed(3)}`);
+      }
+
+      // Increment access count only for memories that will actually be included
+      if (relevantMemories.length > 0) {
+        const eventIds = relevantMemories.map(m => m.event.id);
+        await memoryService.incrementMemoryAccess(eventIds);
+      }
+    }
+
+    // Format context for Claude with only the relevant memories
+    const filteredResult = {
+      ...retrievalResult,
+      memories: relevantMemories
+    };
+    const context = memoryService.formatAsContext(filteredResult);
 
     const output: UserPromptSubmitOutput = { context };
     console.log(JSON.stringify(output));

package/src/server/api/stats.ts

@@ -66,6 +66,7 @@ statsRouter.get('/levels/:level', async (c) => {
   const { level } = c.req.param();
   const limit = parseInt(c.req.query('limit') || '20', 10);
   const offset = parseInt(c.req.query('offset') || '0', 10);
+  const sort = c.req.query('sort') || 'recent';
 
   // Validate level
   const validLevels = ['L0', 'L1', 'L2', 'L3', 'L4'];
@@ -76,19 +77,45 @@ statsRouter.get('/levels/:level', async (c) => {
   const memoryService = getReadOnlyMemoryService();
   try {
     await memoryService.initialize();
-    const events = await memoryService.getEventsByLevel(level, { limit, offset });
+    let events = await memoryService.getEventsByLevel(level, { limit: limit * 2, offset });
     const stats = await memoryService.getStats();
     const levelStat = stats.levelStats.find(s => s.level === level);
 
+    // Apply sorting
+    if (sort === 'accessed') {
+      // Sort by access count (will need to get from SQLite)
+      // For now, add access count from SQLite if available
+      const sqliteStore = (memoryService as any).sqliteEventStore;
+      if (sqliteStore) {
+        const eventIds = events.map(e => e.id);
+        const accessedEvents = await sqliteStore.getMostAccessed(1000);
+        const accessMap = new Map(accessedEvents.map((e: any) => [e.id, e.access_count || 0]));
+        events = events.map((e: any) => ({
+          ...e,
+          accessCount: accessMap.get(e.id) || 0
+        }));
+        events.sort((a: any, b: any) => b.accessCount - a.accessCount);
+      }
+    } else if (sort === 'oldest') {
+      events.sort((a, b) => a.timestamp.getTime() - b.timestamp.getTime());
+    } else {
+      // 'recent' - default sorting (newest first)
+      events.sort((a, b) => b.timestamp.getTime() - a.timestamp.getTime());
+    }
+
+    // Apply limit after sorting
+    events = events.slice(0, limit);
+
     return c.json({
       level,
-      events: events.map(e => ({
+      events: events.map((e: any) => ({
         id: e.id,
         eventType: e.eventType,
         sessionId: e.sessionId,
         timestamp: e.timestamp.toISOString(),
         content: e.content.slice(0, 500) + (e.content.length > 500 ? '...' : ''),
-        metadata: e.metadata
+        metadata: e.metadata,
+        accessCount: e.accessCount || 0
       })),
       total: levelStat?.count || 0,
       limit,
@@ -159,12 +186,14 @@ statsRouter.get('/', async (c) => {
 // GET /api/stats/most-accessed - Get most accessed memories
 statsRouter.get('/most-accessed', async (c) => {
   const limit = parseInt(c.req.query('limit') || '10', 10);
-  const projectPath = c.req.query('project') || process.cwd();
-  const memoryService = getMemoryServiceForProject(projectPath);
+  // Use the same read-only service that other stats endpoints use
+  const memoryService = getReadOnlyMemoryService();
 
   try {
     await memoryService.initialize();
+    console.log('[most-accessed] Fetching most accessed memories, limit:', limit);
     const memories = await memoryService.getMostAccessedMemories(limit);
+    console.log('[most-accessed] Got memories:', memories.length);
 
     return c.json({
       memories: memories.map(m => ({
@@ -172,13 +201,14 @@ statsRouter.get('/most-accessed', async (c) => {
         summary: m.summary,
         topics: m.topics,
         accessCount: m.accessCount,
-        lastAccessed: m.accessedAt?.toISOString() || null,
+        lastAccessed: m.lastAccessed || null,
         confidence: m.confidence,
-        createdAt: m.createdAt.toISOString()
+        createdAt: m.createdAt instanceof Date ? m.createdAt.toISOString() : m.createdAt
       })),
       total: memories.length
     });
   } catch (error) {
+    console.error('[most-accessed] Error:', error);
     return c.json({
       memories: [],
       total: 0,