claude-memory-layer 1.0.9 → 1.0.11

package/src/core/sqlite-event-store.ts

@@ -258,6 +258,23 @@ export class SQLiteEventStore {
         updated_at TEXT DEFAULT (datetime('now'))
       );
 
+      -- Memory Helpfulness tracking
+      CREATE TABLE IF NOT EXISTS memory_helpfulness (
+        id TEXT PRIMARY KEY,
+        event_id TEXT NOT NULL,
+        session_id TEXT NOT NULL,
+        retrieval_score REAL DEFAULT 0,
+        query_preview TEXT,
+        session_continued INTEGER DEFAULT 0,
+        prompt_count_after INTEGER DEFAULT 0,
+        tool_success_count INTEGER DEFAULT 0,
+        tool_total_count INTEGER DEFAULT 0,
+        was_reasked INTEGER DEFAULT 0,
+        helpfulness_score REAL DEFAULT 0.5,
+        created_at TEXT DEFAULT (datetime('now')),
+        measured_at TEXT
+      );
+
       -- Sync position tracking (for SQLite -> DuckDB sync)
       CREATE TABLE IF NOT EXISTS sync_positions (
         target_name TEXT PRIMARY KEY,
@@ -283,9 +300,34 @@ export class SQLiteEventStore {
       CREATE INDEX IF NOT EXISTS idx_consolidated_confidence ON consolidated_memories(confidence);
       CREATE INDEX IF NOT EXISTS idx_continuity_created ON continuity_log(created_at);
       CREATE INDEX IF NOT EXISTS idx_embedding_outbox_status ON embedding_outbox(status);
+      CREATE INDEX IF NOT EXISTS idx_helpfulness_event ON memory_helpfulness(event_id);
+      CREATE INDEX IF NOT EXISTS idx_helpfulness_session ON memory_helpfulness(session_id);
+      CREATE INDEX IF NOT EXISTS idx_helpfulness_score ON memory_helpfulness(helpfulness_score DESC);
+
+      -- FTS5 Full-Text Search for fast keyword search
+      CREATE VIRTUAL TABLE IF NOT EXISTS events_fts USING fts5(
+        content,
+        event_id UNINDEXED,
+        content='events',
+        content_rowid='rowid'
+      );
+
+      -- Triggers to keep FTS in sync with events table
+      CREATE TRIGGER IF NOT EXISTS events_fts_insert AFTER INSERT ON events BEGIN
+        INSERT INTO events_fts(rowid, content, event_id) VALUES (NEW.rowid, NEW.content, NEW.id);
+      END;
+
+      CREATE TRIGGER IF NOT EXISTS events_fts_delete AFTER DELETE ON events BEGIN
+        INSERT INTO events_fts(events_fts, rowid, content, event_id) VALUES('delete', OLD.rowid, OLD.content, OLD.id);
+      END;
+
+      CREATE TRIGGER IF NOT EXISTS events_fts_update AFTER UPDATE ON events BEGIN
+        INSERT INTO events_fts(events_fts, rowid, content, event_id) VALUES('delete', OLD.rowid, OLD.content, OLD.id);
+        INSERT INTO events_fts(rowid, content, event_id) VALUES (NEW.rowid, NEW.content, NEW.id);
+      END;
     `);
 
-    // Migrate existing events table to add access tracking columns if they don't exist
+    // Migrate existing events table to add new columns if they don't exist
     // Check if columns exist before trying to add them
     const tableInfo = sqliteAll(this.db, "PRAGMA table_info(events)", []);
     const columnNames = tableInfo.map((col: any) => col.name);
@@ -310,6 +352,17 @@ export class SQLiteEventStore {
       }
     }
 
+    // Add turn_id column for grouping events within a conversation turn
+    if (!columnNames.includes('turn_id')) {
+      try {
+        sqliteExec(this.db, `
+          ALTER TABLE events ADD COLUMN turn_id TEXT;
+        `);
+      } catch (err: any) {
+        console.error('Error adding turn_id column:', err);
+      }
+    }
+
     // Create indexes for new columns if they don't exist
     try {
       sqliteExec(this.db, `
@@ -327,6 +380,14 @@ export class SQLiteEventStore {
       // Index may already exist, ignore
     }
 
+    try {
+      sqliteExec(this.db, `
+        CREATE INDEX IF NOT EXISTS idx_events_turn_id ON events(turn_id);
+      `);
+    } catch (err: any) {
+      // Index may already exist, ignore
+    }
+
     this.initialized = true;
   }
 
@@ -358,10 +419,14 @@ export class SQLiteEventStore {
     const timestamp = toSQLiteTimestamp(input.timestamp);
 
     try {
+      // Extract turnId from metadata if present
+      const metadata = input.metadata || {};
+      const turnId = (metadata.turnId as string) || null;
+
       // Use transaction for atomicity
       const insertEvent = this.db.prepare(`
-        INSERT INTO events (id, event_type, session_id, timestamp, content, canonical_key, dedupe_key, metadata)
-        VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+        INSERT INTO events (id, event_type, session_id, timestamp, content, canonical_key, dedupe_key, metadata, turn_id)
+        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
       `);
 
       const insertDedup = this.db.prepare(`
@@ -381,7 +446,8 @@ export class SQLiteEventStore {
           input.content,
           canonicalKey,
           dedupeKey,
-          JSON.stringify(input.metadata || {})
+          JSON.stringify(metadata),
+          turnId
         );
         insertDedup.run(dedupeKey, id);
         insertLevel.run(id);
@@ -790,12 +856,13 @@ export class SQLiteEventStore {
   }
 
   /**
-   * Get most accessed memories
+   * Get most accessed memories (falls back to recent events if none accessed)
    */
   async getMostAccessed(limit: number = 10): Promise<MemoryEvent[]> {
     await this.initialize();
 
-    const rows = sqliteAll<Record<string, unknown>>(
+    // First try events with access_count > 0
+    let rows = sqliteAll<Record<string, unknown>>(
       this.db,
       `SELECT * FROM events
        WHERE access_count > 0
@@ -804,9 +871,279 @@ export class SQLiteEventStore {
       [limit]
     );
 
+    // Fallback: if no accessed events, show recent events
+    if (rows.length === 0) {
+      rows = sqliteAll<Record<string, unknown>>(
+        this.db,
+        `SELECT * FROM events
+         ORDER BY timestamp DESC
+         LIMIT ?`,
+        [limit]
+      );
+    }
+
     return rows.map(row => this.rowToEvent(row));
   }
 
+  /**
+   * Record a memory retrieval for helpfulness tracking
+   */
+  async recordRetrieval(eventId: string, sessionId: string, score: number, query: string): Promise<void> {
+    if (this.readOnly) return;
+    await this.initialize();
+
+    const id = randomUUID();
+    sqliteRun(
+      this.db,
+      `INSERT INTO memory_helpfulness (id, event_id, session_id, retrieval_score, query_preview, created_at)
+       VALUES (?, ?, ?, ?, ?, datetime('now'))`,
+      [id, eventId, sessionId, score, query.slice(0, 100)]
+    );
+  }
+
+  /**
+   * Evaluate helpfulness for all retrievals in a session
+   * Called at session end - uses behavioral signals to compute score
+   */
+  async evaluateSessionHelpfulness(sessionId: string): Promise<void> {
+    if (this.readOnly) return;
+    await this.initialize();
+
+    // Get all retrieval records for this session
+    const retrievals = sqliteAll<Record<string, unknown>>(
+      this.db,
+      `SELECT * FROM memory_helpfulness WHERE session_id = ? AND measured_at IS NULL`,
+      [sessionId]
+    );
+
+    if (retrievals.length === 0) return;
+
+    // Get session events to analyze behavior after retrieval
+    const sessionEvents = sqliteAll<Record<string, unknown>>(
+      this.db,
+      `SELECT * FROM events WHERE session_id = ? ORDER BY timestamp ASC`,
+      [sessionId]
+    );
+
+    const promptEvents = sessionEvents.filter((e: any) => e.event_type === 'user_prompt');
+    const toolEvents = sessionEvents.filter((e: any) => e.event_type === 'tool_observation');
+
+    // Count successful vs failed tools
+    let toolSuccessCount = 0;
+    let toolTotalCount = toolEvents.length;
+    for (const t of toolEvents) {
+      try {
+        const content = JSON.parse(t.content as string);
+        if (content.success !== false) toolSuccessCount++;
+      } catch {
+        toolSuccessCount++; // Assume success if can't parse
+      }
+    }
+    const toolSuccessRatio = toolTotalCount > 0 ? toolSuccessCount / toolTotalCount : 0.5;
+
+    for (const retrieval of retrievals) {
+      const retrievalTime = retrieval.created_at as string;
+
+      // 1. Session continued after retrieval?
+      const eventsAfter = sessionEvents.filter((e: any) => e.timestamp > retrievalTime);
+      const sessionContinued = eventsAfter.length > 0 ? 1 : 0;
+
+      // 2. How many prompts came after?
+      const promptsAfter = promptEvents.filter((e: any) => e.timestamp > retrievalTime);
+      const promptCountAfter = promptsAfter.length;
+
+      // 3. Was a similar query asked again? (simple word overlap check)
+      const queryWords = new Set((retrieval.query_preview as string || '').toLowerCase().split(/\s+/).filter(w => w.length > 2));
+      let wasReasked = 0;
+      for (const p of promptsAfter) {
+        const pWords = new Set((p.content as string).toLowerCase().split(/\s+/).filter((w: string) => w.length > 2));
+        let overlap = 0;
+        for (const w of queryWords) {
+          if (pWords.has(w)) overlap++;
+        }
+        if (queryWords.size > 0 && overlap / queryWords.size > 0.5) {
+          wasReasked = 1;
+          break;
+        }
+      }
+
+      // Calculate helpfulness score
+      const retrievalScore = retrieval.retrieval_score as number || 0;
+      const helpfulnessScore = (
+        0.30 * Math.min(retrievalScore, 1.0) +
+        0.25 * (sessionContinued ? 1.0 : 0.0) +
+        0.25 * toolSuccessRatio +
+        0.20 * (wasReasked ? 0.0 : 1.0)
+      );
+
+      sqliteRun(
+        this.db,
+        `UPDATE memory_helpfulness
+         SET session_continued = ?, prompt_count_after = ?,
+             tool_success_count = ?, tool_total_count = ?,
+             was_reasked = ?, helpfulness_score = ?,
+             measured_at = datetime('now')
+         WHERE id = ?`,
+        [sessionContinued, promptCountAfter, toolSuccessCount, toolTotalCount,
+         wasReasked, helpfulnessScore, retrieval.id]
+      );
+    }
+  }
+
+  /**
+   * Get most helpful memories ranked by helpfulness score
+   */
+  async getHelpfulMemories(limit: number = 10): Promise<Array<{
+    eventId: string;
+    summary: string;
+    helpfulnessScore: number;
+    accessCount: number;
+    evaluationCount: number;
+  }>> {
+    await this.initialize();
+
+    const rows = sqliteAll<Record<string, unknown>>(
+      this.db,
+      `SELECT
+         mh.event_id,
+         AVG(mh.helpfulness_score) as avg_score,
+         COUNT(*) as eval_count,
+         e.content,
+         e.access_count
+       FROM memory_helpfulness mh
+       JOIN events e ON e.id = mh.event_id
+       WHERE mh.measured_at IS NOT NULL
+       GROUP BY mh.event_id
+       ORDER BY avg_score DESC
+       LIMIT ?`,
+      [limit]
+    );
+
+    return rows.map(r => ({
+      eventId: r.event_id as string,
+      summary: (r.content as string).substring(0, 200) + ((r.content as string).length > 200 ? '...' : ''),
+      helpfulnessScore: Math.round((r.avg_score as number) * 100) / 100,
+      accessCount: (r.access_count as number) || 0,
+      evaluationCount: r.eval_count as number
+    }));
+  }
+
+  /**
+   * Get helpfulness statistics for dashboard
+   */
+  async getHelpfulnessStats(): Promise<{
+    avgScore: number;
+    totalEvaluated: number;
+    totalRetrievals: number;
+    helpful: number;
+    neutral: number;
+    unhelpful: number;
+  }> {
+    await this.initialize();
+
+    const stats = sqliteGet<Record<string, unknown>>(
+      this.db,
+      `SELECT
+         AVG(helpfulness_score) as avg_score,
+         COUNT(*) as total_evaluated,
+         SUM(CASE WHEN helpfulness_score >= 0.7 THEN 1 ELSE 0 END) as helpful,
+         SUM(CASE WHEN helpfulness_score >= 0.4 AND helpfulness_score < 0.7 THEN 1 ELSE 0 END) as neutral,
+         SUM(CASE WHEN helpfulness_score < 0.4 THEN 1 ELSE 0 END) as unhelpful
+       FROM memory_helpfulness
+       WHERE measured_at IS NOT NULL`
+    );
+
+    const totalRow = sqliteGet<Record<string, unknown>>(
+      this.db,
+      `SELECT COUNT(*) as total FROM memory_helpfulness`
+    );
+
+    return {
+      avgScore: Math.round(((stats?.avg_score as number) || 0) * 100) / 100,
+      totalEvaluated: (stats?.total_evaluated as number) || 0,
+      totalRetrievals: (totalRow?.total as number) || 0,
+      helpful: (stats?.helpful as number) || 0,
+      neutral: (stats?.neutral as number) || 0,
+      unhelpful: (stats?.unhelpful as number) || 0
+    };
+  }
+
+  /**
+   * Fast keyword search using FTS5
+   * Returns events matching the search query, ranked by relevance
+   */
+  async keywordSearch(query: string, limit: number = 10): Promise<Array<{event: MemoryEvent; rank: number}>> {
+    await this.initialize();
+
+    // Escape special FTS5 characters and prepare search terms
+    const searchTerms = query
+      .replace(/['"(){}[\]^~*?:\\/-]/g, ' ')  // Remove special chars
+      .split(/\s+/)
+      .filter(term => term.length > 1)  // Filter short terms
+      .map(term => `"${term}"*`)  // Prefix matching
+      .join(' OR ');
+
+    if (!searchTerms) {
+      return [];
+    }
+
+    try {
+      const rows = sqliteAll<Record<string, unknown>>(
+        this.db,
+        `SELECT e.*, fts.rank
+         FROM events_fts fts
+         JOIN events e ON e.id = fts.event_id
+         WHERE events_fts MATCH ?
+         ORDER BY fts.rank
+         LIMIT ?`,
+        [searchTerms, limit]
+      );
+
+      return rows.map(row => ({
+        event: this.rowToEvent(row),
+        rank: row.rank as number
+      }));
+    } catch (error: any) {
+      // FTS table might not exist yet (old database)
+      // Fallback to LIKE search
+      const likePattern = `%${query}%`;
+      const rows = sqliteAll<Record<string, unknown>>(
+        this.db,
+        `SELECT *, 0 as rank FROM events
+         WHERE content LIKE ?
+         ORDER BY timestamp DESC
+         LIMIT ?`,
+        [likePattern, limit]
+      );
+
+      return rows.map(row => ({
+        event: this.rowToEvent(row),
+        rank: 0
+      }));
+    }
+  }
+
+  /**
+   * Rebuild FTS index from existing events
+   * Call this once after upgrading to FTS5
+   */
+  async rebuildFtsIndex(): Promise<number> {
+    await this.initialize();
+
+    // Get count of events to index
+    const countRow = sqliteGet<{count: number}>(this.db, 'SELECT COUNT(*) as count FROM events', []);
+    const totalEvents = countRow?.count ?? 0;
+
+    // Clear and rebuild FTS index
+    sqliteExec(this.db, `
+      DELETE FROM events_fts;
+      INSERT INTO events_fts(rowid, content, event_id)
+      SELECT rowid, content, id FROM events;
+    `);
+
+    return totalEvents;
+  }
+
   /**
    * Get database instance for direct access
    */
@@ -821,6 +1158,201 @@ export class SQLiteEventStore {
     sqliteClose(this.db);
   }
 
+  /**
+   * Get events grouped by turn_id for a session
+   * Returns turns ordered by first event timestamp (newest first)
+   */
+  async getSessionTurns(sessionId: string, options?: { limit?: number; offset?: number }): Promise<Array<{
+    turnId: string;
+    events: MemoryEvent[];
+    startedAt: Date;
+    promptPreview: string;
+    eventCount: number;
+    toolCount: number;
+    hasResponse: boolean;
+  }>> {
+    await this.initialize();
+
+    const limit = options?.limit || 20;
+    const offset = options?.offset || 0;
+
+    // Get distinct turn_ids for this session, ordered by first event timestamp
+    const turnRows = sqliteAll<{ turn_id: string; min_ts: string }>(
+      this.db,
+      `SELECT turn_id, MIN(timestamp) as min_ts
+       FROM events
+       WHERE session_id = ? AND turn_id IS NOT NULL
+       GROUP BY turn_id
+       ORDER BY min_ts DESC
+       LIMIT ? OFFSET ?`,
+      [sessionId, limit, offset]
+    );
+
+    const turns: Array<{
+      turnId: string;
+      events: MemoryEvent[];
+      startedAt: Date;
+      promptPreview: string;
+      eventCount: number;
+      toolCount: number;
+      hasResponse: boolean;
+    }> = [];
+
+    for (const turnRow of turnRows) {
+      const events = await this.getEventsByTurn(turnRow.turn_id);
+
+      const promptEvent = events.find(e => e.eventType === 'user_prompt');
+      const toolEvents = events.filter(e => e.eventType === 'tool_observation');
+      const hasResponse = events.some(e => e.eventType === 'agent_response');
+
+      turns.push({
+        turnId: turnRow.turn_id,
+        events,
+        startedAt: toDateFromSQLite(turnRow.min_ts),
+        promptPreview: promptEvent
+          ? promptEvent.content.slice(0, 200) + (promptEvent.content.length > 200 ? '...' : '')
+          : '(no prompt)',
+        eventCount: events.length,
+        toolCount: toolEvents.length,
+        hasResponse
+      });
+    }
+
+    return turns;
+  }
+
+  /**
+   * Get all events for a specific turn_id
+   */
+  async getEventsByTurn(turnId: string): Promise<MemoryEvent[]> {
+    await this.initialize();
+
+    const rows = sqliteAll<Record<string, unknown>>(
+      this.db,
+      `SELECT * FROM events WHERE turn_id = ? ORDER BY timestamp ASC`,
+      [turnId]
+    );
+
+    return rows.map(this.rowToEvent);
+  }
+
+  /**
+   * Count total turns for a session
+   */
+  async countSessionTurns(sessionId: string): Promise<number> {
+    await this.initialize();
+
+    const row = sqliteGet<{ count: number }>(
+      this.db,
+      `SELECT COUNT(DISTINCT turn_id) as count
+       FROM events
+       WHERE session_id = ? AND turn_id IS NOT NULL`,
+      [sessionId]
+    );
+
+    return row?.count || 0;
+  }
+
+  /**
+   * Migrate existing events: backfill turn_id for events that have turnId in metadata
+   * but no turn_id column value (for events stored before this migration)
+   */
+  async backfillTurnIds(): Promise<number> {
+    await this.initialize();
+
+    // Find events with turnId in metadata JSON but no turn_id column value
+    const rows = sqliteAll<{ id: string; metadata: string }>(
+      this.db,
+      `SELECT id, metadata FROM events
+       WHERE turn_id IS NULL AND metadata IS NOT NULL AND metadata LIKE '%turnId%'`
+    );
+
+    let updated = 0;
+    for (const row of rows) {
+      try {
+        const metadata = JSON.parse(row.metadata);
+        if (metadata.turnId) {
+          sqliteRun(
+            this.db,
+            `UPDATE events SET turn_id = ? WHERE id = ?`,
+            [metadata.turnId, row.id]
+          );
+          updated++;
+        }
+      } catch {
+        // Skip rows with invalid JSON
+      }
+    }
+
+    return updated;
+  }
+
+  /**
+   * Delete all events for a session (for force reimport)
+   */
+  async deleteSessionEvents(sessionId: string): Promise<number> {
+    await this.initialize();
+
+    // Get event IDs first for cascading deletes
+    const events = sqliteAll<{ id: string }>(
+      this.db,
+      `SELECT id FROM events WHERE session_id = ?`,
+      [sessionId]
+    );
+
+    if (events.length === 0) return 0;
+
+    const eventIds = events.map(e => e.id);
+    const placeholders = eventIds.map(() => '?').join(',');
+
+    // Drop FTS triggers to prevent SQLITE_CORRUPT_VTAB during bulk delete
+    const ftsTriggersDropped: string[] = [];
+    for (const triggerName of ['events_fts_delete', 'events_fts_update', 'events_fts_insert']) {
+      try {
+        sqliteRun(this.db, `DROP TRIGGER IF EXISTS ${triggerName}`);
+        ftsTriggersDropped.push(triggerName);
+      } catch {
+        // Trigger may not exist
+      }
+    }
+
+    // Delete from related tables first (some may not exist depending on DB version)
+    for (const table of ['event_dedup', 'memory_levels', 'embedding_queue', 'embedding_outbox', 'vector_outbox']) {
+      try {
+        sqliteRun(this.db, `DELETE FROM ${table} WHERE event_id IN (${placeholders})`, eventIds);
+      } catch {
+        // Table may not exist
+      }
+    }
+
+    // Delete events
+    const result = sqliteRun(this.db, `DELETE FROM events WHERE session_id = ?`, [sessionId]);
+
+    // Rebuild FTS index if we dropped triggers
+    if (ftsTriggersDropped.length > 0) {
+      try {
+        // Rebuild FTS from remaining events
+        sqliteRun(this.db, `INSERT INTO events_fts(events_fts) VALUES('rebuild')`);
+
+        // Recreate triggers
+        sqliteRun(this.db, `CREATE TRIGGER IF NOT EXISTS events_fts_insert AFTER INSERT ON events BEGIN
+          INSERT INTO events_fts(rowid, content) VALUES (NEW.rowid, NEW.content);
+        END`);
+        sqliteRun(this.db, `CREATE TRIGGER IF NOT EXISTS events_fts_delete AFTER DELETE ON events BEGIN
+          INSERT INTO events_fts(events_fts, rowid, content) VALUES('delete', OLD.rowid, OLD.content);
+        END`);
+        sqliteRun(this.db, `CREATE TRIGGER IF NOT EXISTS events_fts_update AFTER UPDATE ON events BEGIN
+          INSERT INTO events_fts(events_fts, rowid, content) VALUES('delete', OLD.rowid, OLD.content);
+          INSERT INTO events_fts(rowid, content) VALUES (NEW.rowid, NEW.content);
+        END`);
+      } catch {
+        // FTS rebuild failed - non-critical, will be rebuilt on next initialize
+      }
+    }
+
+    return result.changes || 0;
+  }
+
   /**
    * Convert database row to MemoryEvent
    */
@@ -843,6 +1375,10 @@ export class SQLiteEventStore {
     if (row.last_accessed_at !== undefined) {
       event.last_accessed_at = row.last_accessed_at;
     }
+    // Include turn_id if present
+    if (row.turn_id !== undefined && row.turn_id !== null) {
+      event.turn_id = row.turn_id;
+    }
 
     return event;
   }

package/src/core/sqlite-wrapper.ts

@@ -4,6 +4,8 @@
  */
 
 import Database from 'better-sqlite3';
+import * as fs from 'fs';
+import * as nodePath from 'path';
 
 export type SQLiteDatabase = Database.Database;
 
@@ -16,6 +18,12 @@ export interface SQLiteOptions {
  * Creates a new SQLite database with WAL mode
  */
 export function createSQLiteDatabase(path: string, options?: SQLiteOptions): SQLiteDatabase {
+  // Ensure parent directory exists
+  const dir = nodePath.dirname(path);
+  if (!fs.existsSync(dir)) {
+    fs.mkdirSync(dir, { recursive: true });
+  }
+
   const db = new Database(path, {
     readonly: options?.readonly ?? false,
   });