claude-memory-layer 1.0.10 → 1.0.12

package/src/core/sqlite-event-store.ts

@@ -24,15 +24,17 @@ import {
   type SQLiteDatabase,
   type SQLiteOptions
 } from './sqlite-wrapper.js';
+import { MarkdownMirror } from './markdown-mirror.js';
 
 export interface SQLiteEventStoreOptions extends SQLiteOptions {
-  // Additional options can be added here
+  markdownMirrorRoot?: string;
 }
 
 export class SQLiteEventStore {
   private db: SQLiteDatabase;
   private initialized = false;
   private readonly readOnly: boolean;
+  private readonly markdownMirror: MarkdownMirror | null;
 
   constructor(private dbPath: string, options?: SQLiteEventStoreOptions) {
     this.readOnly = options?.readonly ?? false;
@@ -40,6 +42,9 @@ export class SQLiteEventStore {
       readonly: this.readOnly,
       walMode: !this.readOnly
     });
+    this.markdownMirror = this.readOnly || !options?.markdownMirrorRoot
+      ? null
+      : new MarkdownMirror(options.markdownMirrorRoot);
   }
 
   /**
@@ -251,6 +256,17 @@ export class SQLiteEventStore {
         created_at TEXT DEFAULT (datetime('now'))
       );
 
+      -- Consolidated Rules table (long-term stable memory)
+      CREATE TABLE IF NOT EXISTS consolidated_rules (
+        rule_id TEXT PRIMARY KEY,
+        rule TEXT NOT NULL,
+        topics TEXT,
+        source_memory_ids TEXT,
+        source_events TEXT,
+        confidence REAL DEFAULT 0.5,
+        created_at TEXT DEFAULT (datetime('now'))
+      );
+
       -- Endless Mode Config table
       CREATE TABLE IF NOT EXISTS endless_config (
         key TEXT PRIMARY KEY,
@@ -258,6 +274,41 @@ export class SQLiteEventStore {
         updated_at TEXT DEFAULT (datetime('now'))
       );
 
+      -- Memory Helpfulness tracking
+      CREATE TABLE IF NOT EXISTS memory_helpfulness (
+        id TEXT PRIMARY KEY,
+        event_id TEXT NOT NULL,
+        session_id TEXT NOT NULL,
+        retrieval_score REAL DEFAULT 0,
+        query_preview TEXT,
+        session_continued INTEGER DEFAULT 0,
+        prompt_count_after INTEGER DEFAULT 0,
+        tool_success_count INTEGER DEFAULT 0,
+        tool_total_count INTEGER DEFAULT 0,
+        was_reasked INTEGER DEFAULT 0,
+        helpfulness_score REAL DEFAULT 0.5,
+        created_at TEXT DEFAULT (datetime('now')),
+        measured_at TEXT
+      );
+
+      -- Retrieval trace log (query -> candidates -> selected for context)
+      CREATE TABLE IF NOT EXISTS retrieval_traces (
+        trace_id TEXT PRIMARY KEY,
+        session_id TEXT,
+        project_hash TEXT,
+        query_text TEXT NOT NULL,
+        strategy TEXT,
+        candidate_event_ids TEXT,
+        selected_event_ids TEXT,
+        candidate_details_json TEXT,
+        selected_details_json TEXT,
+        candidate_count INTEGER DEFAULT 0,
+        selected_count INTEGER DEFAULT 0,
+        confidence TEXT,
+        fallback_trace TEXT,
+        created_at TEXT DEFAULT (datetime('now'))
+      );
+
       -- Sync position tracking (for SQLite -> DuckDB sync)
       CREATE TABLE IF NOT EXISTS sync_positions (
         target_name TEXT PRIMARY KEY,
@@ -282,7 +333,14 @@ export class SQLiteEventStore {
       CREATE INDEX IF NOT EXISTS idx_working_set_relevance ON working_set(relevance_score);
       CREATE INDEX IF NOT EXISTS idx_consolidated_confidence ON consolidated_memories(confidence);
       CREATE INDEX IF NOT EXISTS idx_continuity_created ON continuity_log(created_at);
+      CREATE INDEX IF NOT EXISTS idx_consolidated_rules_confidence ON consolidated_rules(confidence);
       CREATE INDEX IF NOT EXISTS idx_embedding_outbox_status ON embedding_outbox(status);
+      CREATE INDEX IF NOT EXISTS idx_helpfulness_event ON memory_helpfulness(event_id);
+      CREATE INDEX IF NOT EXISTS idx_helpfulness_session ON memory_helpfulness(session_id);
+      CREATE INDEX IF NOT EXISTS idx_helpfulness_score ON memory_helpfulness(helpfulness_score DESC);
+      CREATE INDEX IF NOT EXISTS idx_retrieval_traces_created_at ON retrieval_traces(created_at DESC);
+      CREATE INDEX IF NOT EXISTS idx_retrieval_traces_project_hash ON retrieval_traces(project_hash);
+      CREATE INDEX IF NOT EXISTS idx_retrieval_traces_session_id ON retrieval_traces(session_id);
 
       -- FTS5 Full-Text Search for fast keyword search
       CREATE VIRTUAL TABLE IF NOT EXISTS events_fts USING fts5(
@@ -307,7 +365,20 @@ export class SQLiteEventStore {
       END;
     `);
 
-    // Migrate existing events table to add access tracking columns if they don't exist
+
+    // Best-effort forward migration for retrieval trace detail column
+    try {
+      sqliteExec(this.db, `ALTER TABLE retrieval_traces ADD COLUMN selected_details_json TEXT;`);
+    } catch {
+      // column may already exist
+    }
+    try {
+      sqliteExec(this.db, `ALTER TABLE retrieval_traces ADD COLUMN candidate_details_json TEXT;`);
+    } catch {
+      // column may already exist
+    }
+
+    // Migrate existing events table to add new columns if they don't exist
     // Check if columns exist before trying to add them
     const tableInfo = sqliteAll(this.db, "PRAGMA table_info(events)", []);
     const columnNames = tableInfo.map((col: any) => col.name);
@@ -332,6 +403,17 @@ export class SQLiteEventStore {
       }
     }
 
+    // Add turn_id column for grouping events within a conversation turn
+    if (!columnNames.includes('turn_id')) {
+      try {
+        sqliteExec(this.db, `
+          ALTER TABLE events ADD COLUMN turn_id TEXT;
+        `);
+      } catch (err: any) {
+        console.error('Error adding turn_id column:', err);
+      }
+    }
+
     // Create indexes for new columns if they don't exist
     try {
       sqliteExec(this.db, `
@@ -349,6 +431,14 @@ export class SQLiteEventStore {
       // Index may already exist, ignore
     }
 
+    try {
+      sqliteExec(this.db, `
+        CREATE INDEX IF NOT EXISTS idx_events_turn_id ON events(turn_id);
+      `);
+    } catch (err: any) {
+      // Index may already exist, ignore
+    }
+
     this.initialized = true;
   }
 
@@ -380,10 +470,14 @@ export class SQLiteEventStore {
     const timestamp = toSQLiteTimestamp(input.timestamp);
 
     try {
+      // Extract turnId from metadata if present
+      const metadata = input.metadata || {};
+      const turnId = (metadata.turnId as string) || null;
+
       // Use transaction for atomicity
       const insertEvent = this.db.prepare(`
-        INSERT INTO events (id, event_type, session_id, timestamp, content, canonical_key, dedupe_key, metadata)
-        VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+        INSERT INTO events (id, event_type, session_id, timestamp, content, canonical_key, dedupe_key, metadata, turn_id)
+        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
       `);
 
       const insertDedup = this.db.prepare(`
@@ -403,7 +497,8 @@ export class SQLiteEventStore {
           input.content,
           canonicalKey,
           dedupeKey,
-          JSON.stringify(input.metadata || {})
+          JSON.stringify(metadata),
+          turnId
         );
         insertDedup.run(dedupeKey, id);
         insertLevel.run(id);
@@ -411,6 +506,22 @@ export class SQLiteEventStore {
 
       transaction();
 
+      if (this.markdownMirror) {
+        const event: MemoryEvent = {
+          id,
+          eventType: input.eventType,
+          sessionId: input.sessionId,
+          timestamp: input.timestamp,
+          content: input.content,
+          canonicalKey,
+          dedupeKey,
+          metadata
+        };
+        this.markdownMirror.append(event).catch((err) => {
+          console.warn('[SQLiteEventStore] markdown mirror append failed:', err);
+        });
+      }
+
       return { success: true, eventId: id, isDuplicate: false };
     } catch (error) {
       return {
@@ -481,6 +592,114 @@ export class SQLiteEventStore {
     return rows.map(this.rowToEvent);
   }
 
+  /**
+   * Get events since a SQLite rowid (for robust incremental replication).
+   * Rowid is monotonic for append-only tables, independent of client timestamps.
+   */
+  async getEventsSinceRowid(
+    lastRowid: number,
+    limit: number = 1000
+  ): Promise<Array<{ rowid: number; event: MemoryEvent }>> {
+    await this.initialize();
+
+    const rows = sqliteAll<Record<string, unknown>>(
+      this.db,
+      `SELECT rowid as _rowid, * FROM events WHERE rowid > ? ORDER BY rowid ASC LIMIT ?`,
+      [lastRowid, limit]
+    );
+
+    return rows.map(row => ({
+      rowid: row._rowid as number,
+      event: this.rowToEvent(row)
+    }));
+  }
+
+  /**
+   * Import events with fixed IDs (used for cross-machine replication).
+   * Idempotent: skips if event id or dedupeKey already exists.
+   *
+   * NOTE: This bypasses the append() id generation to preserve stable IDs.
+   */
+  async importEvents(events: MemoryEvent[]): Promise<{ inserted: number; skipped: number }> {
+    if (events.length === 0) return { inserted: 0, skipped: 0 };
+    if (this.readOnly) return { inserted: 0, skipped: events.length };
+
+    await this.initialize();
+
+    const getById = this.db.prepare(`SELECT id FROM events WHERE id = ?`);
+    const getByDedupe = this.db.prepare(`SELECT event_id FROM event_dedup WHERE dedupe_key = ?`);
+
+    const insertEvent = this.db.prepare(`
+      INSERT INTO events (id, event_type, session_id, timestamp, content, canonical_key, dedupe_key, metadata, turn_id)
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+    `);
+
+    const insertDedup = this.db.prepare(`
+      INSERT INTO event_dedup (dedupe_key, event_id) VALUES (?, ?)
+    `);
+
+    const insertLevel = this.db.prepare(`
+      INSERT INTO memory_levels (event_id, level) VALUES (?, 'L0')
+    `);
+
+    let inserted = 0;
+    let skipped = 0;
+    const insertedEvents: MemoryEvent[] = [];
+
+    const tx = this.db.transaction((batch: MemoryEvent[]) => {
+      for (const ev of batch) {
+        // Skip if already present by id
+        const existingById = getById.get(ev.id) as { id: string } | undefined;
+        if (existingById) {
+          skipped++;
+          continue;
+        }
+
+        const canonicalKey = ev.canonicalKey || makeCanonicalKey(ev.content);
+        const dedupeKey = ev.dedupeKey || makeDedupeKey(ev.content, ev.sessionId);
+
+        // Skip if already present by dedupe key
+        const existingByDedupe = getByDedupe.get(dedupeKey) as { event_id: string } | undefined;
+        if (existingByDedupe) {
+          skipped++;
+          continue;
+        }
+
+        const metadata = ev.metadata || {};
+        const turnId = (metadata as any).turnId as string | undefined;
+
+        insertEvent.run(
+          ev.id,
+          ev.eventType,
+          ev.sessionId,
+          toSQLiteTimestamp(ev.timestamp),
+          ev.content,
+          canonicalKey,
+          dedupeKey,
+          JSON.stringify(metadata),
+          turnId ?? null
+        );
+
+        insertDedup.run(dedupeKey, ev.id);
+        insertLevel.run(ev.id);
+        inserted++;
+        insertedEvents.push(ev);
+      }
+    });
+
+    tx(events);
+
+    if (this.markdownMirror && insertedEvents.length > 0) {
+      for (const ev of insertedEvents) {
+        this.markdownMirror.append(ev).catch((err) => {
+          console.warn('[SQLiteEventStore] markdown mirror append failed:', err);
+        });
+      }
+    }
+
+    return { inserted, skipped };
+  }
+
   /**
    * Create or update session
    */
@@ -664,6 +883,43 @@ export class SQLiteEventStore {
     );
   }
 
+
+  /**
+   * Get embedding/vector outbox health statistics
+   */
+  async getOutboxStats(): Promise<{
+    embedding: { pending: number; processing: number; failed: number; total: number };
+    vector: { pending: number; processing: number; failed: number; total: number };
+  }> {
+    await this.initialize();
+
+    const embeddingRows = sqliteAll<{ status: string; count: number }>(
+      this.db,
+      `SELECT status, COUNT(*) as count FROM embedding_outbox GROUP BY status`
+    );
+    const vectorRows = sqliteAll<{ status: string; count: number }>(
+      this.db,
+      `SELECT status, COUNT(*) as count FROM vector_outbox GROUP BY status`
+    );
+
+    const fromRows = (rows: Array<{ status: string; count: number }>) => {
+      const out = { pending: 0, processing: 0, failed: 0, total: 0 };
+      for (const row of rows) {
+        const key = row.status as 'pending' | 'processing' | 'failed' | 'done';
+        if (key === 'pending' || key === 'processing' || key === 'failed') {
+          out[key] += row.count;
+        }
+        out.total += row.count;
+      }
+      return out;
+    };
+
+    return {
+      embedding: fromRows(embeddingRows),
+      vector: fromRows(vectorRows)
+    };
+  }
+
   /**
    * Update memory level
    */
@@ -812,12 +1068,13 @@ export class SQLiteEventStore {
   }
 
   /**
-   * Get most accessed memories
+   * Get most accessed memories (falls back to recent events if none accessed)
    */
   async getMostAccessed(limit: number = 10): Promise<MemoryEvent[]> {
     await this.initialize();
 
-    const rows = sqliteAll<Record<string, unknown>>(
+    // First try events with access_count > 0
+    let rows = sqliteAll<Record<string, unknown>>(
       this.db,
       `SELECT * FROM events
        WHERE access_count > 0
@@ -826,9 +1083,203 @@ export class SQLiteEventStore {
       [limit]
     );
 
+    // Fallback: if no accessed events, show recent events
+    if (rows.length === 0) {
+      rows = sqliteAll<Record<string, unknown>>(
+        this.db,
+        `SELECT * FROM events
+         ORDER BY timestamp DESC
+         LIMIT ?`,
+        [limit]
+      );
+    }
+
     return rows.map(row => this.rowToEvent(row));
   }
 
+  /**
+   * Record a memory retrieval for helpfulness tracking
+   */
+  async recordRetrieval(eventId: string, sessionId: string, score: number, query: string): Promise<void> {
+    if (this.readOnly) return;
+    await this.initialize();
+
+    const id = randomUUID();
+    sqliteRun(
+      this.db,
+      `INSERT INTO memory_helpfulness (id, event_id, session_id, retrieval_score, query_preview, created_at)
+       VALUES (?, ?, ?, ?, ?, datetime('now'))`,
+      [id, eventId, sessionId, score, query.slice(0, 100)]
+    );
+  }
+
+  /**
+   * Evaluate helpfulness for all retrievals in a session
+   * Called at session end - uses behavioral signals to compute score
+   */
+  async evaluateSessionHelpfulness(sessionId: string): Promise<void> {
+    if (this.readOnly) return;
+    await this.initialize();
+
+    // Get all retrieval records for this session
+    const retrievals = sqliteAll<Record<string, unknown>>(
+      this.db,
+      `SELECT * FROM memory_helpfulness WHERE session_id = ? AND measured_at IS NULL`,
+      [sessionId]
+    );
+
+    if (retrievals.length === 0) return;
+
+    // Get session events to analyze behavior after retrieval
+    const sessionEvents = sqliteAll<Record<string, unknown>>(
+      this.db,
+      `SELECT * FROM events WHERE session_id = ? ORDER BY timestamp ASC`,
+      [sessionId]
+    );
+
+    const promptEvents = sessionEvents.filter((e: any) => e.event_type === 'user_prompt');
+    const toolEvents = sessionEvents.filter((e: any) => e.event_type === 'tool_observation');
+
+    // Count successful vs failed tools
+    let toolSuccessCount = 0;
+    let toolTotalCount = toolEvents.length;
+    for (const t of toolEvents) {
+      try {
+        const content = JSON.parse(t.content as string);
+        if (content.success !== false) toolSuccessCount++;
+      } catch {
+        toolSuccessCount++; // Assume success if can't parse
+      }
+    }
+    const toolSuccessRatio = toolTotalCount > 0 ? toolSuccessCount / toolTotalCount : 0.5;
+
+    for (const retrieval of retrievals) {
+      const retrievalTime = retrieval.created_at as string;
+
+      // 1. Session continued after retrieval?
+      const eventsAfter = sessionEvents.filter((e: any) => e.timestamp > retrievalTime);
+      const sessionContinued = eventsAfter.length > 0 ? 1 : 0;
+
+      // 2. How many prompts came after?
+      const promptsAfter = promptEvents.filter((e: any) => e.timestamp > retrievalTime);
+      const promptCountAfter = promptsAfter.length;
+
+      // 3. Was a similar query asked again? (simple word overlap check)
+      const queryWords = new Set((retrieval.query_preview as string || '').toLowerCase().split(/\s+/).filter(w => w.length > 2));
+      let wasReasked = 0;
+      for (const p of promptsAfter) {
+        const pWords = new Set((p.content as string).toLowerCase().split(/\s+/).filter((w: string) => w.length > 2));
+        let overlap = 0;
+        for (const w of queryWords) {
+          if (pWords.has(w)) overlap++;
+        }
+        if (queryWords.size > 0 && overlap / queryWords.size > 0.5) {
+          wasReasked = 1;
+          break;
+        }
+      }
+
+      // Calculate helpfulness score
+      const retrievalScore = retrieval.retrieval_score as number || 0;
+      const helpfulnessScore = (
+        0.30 * Math.min(retrievalScore, 1.0) +
+        0.25 * (sessionContinued ? 1.0 : 0.0) +
+        0.25 * toolSuccessRatio +
+        0.20 * (wasReasked ? 0.0 : 1.0)
+      );
+
+      sqliteRun(
+        this.db,
+        `UPDATE memory_helpfulness
+         SET session_continued = ?, prompt_count_after = ?,
+             tool_success_count = ?, tool_total_count = ?,
+             was_reasked = ?, helpfulness_score = ?,
+             measured_at = datetime('now')
+         WHERE id = ?`,
+        [sessionContinued, promptCountAfter, toolSuccessCount, toolTotalCount,
+         wasReasked, helpfulnessScore, retrieval.id]
+      );
+    }
+  }
+
+  /**
+   * Get most helpful memories ranked by helpfulness score
+   */
+  async getHelpfulMemories(limit: number = 10): Promise<Array<{
+    eventId: string;
+    summary: string;
+    helpfulnessScore: number;
+    accessCount: number;
+    evaluationCount: number;
+  }>> {
+    await this.initialize();
+
+    const rows = sqliteAll<Record<string, unknown>>(
+      this.db,
+      `SELECT
+         mh.event_id,
+         AVG(mh.helpfulness_score) as avg_score,
+         COUNT(*) as eval_count,
+         e.content,
+         e.access_count
+       FROM memory_helpfulness mh
+       JOIN events e ON e.id = mh.event_id
+       WHERE mh.measured_at IS NOT NULL
+       GROUP BY mh.event_id
+       ORDER BY avg_score DESC
+       LIMIT ?`,
+      [limit]
+    );
+
+    return rows.map(r => ({
+      eventId: r.event_id as string,
+      summary: (r.content as string).substring(0, 200) + ((r.content as string).length > 200 ? '...' : ''),
+      helpfulnessScore: Math.round((r.avg_score as number) * 100) / 100,
+      accessCount: (r.access_count as number) || 0,
+      evaluationCount: r.eval_count as number
+    }));
+  }
+
+  /**
+   * Get helpfulness statistics for dashboard
+   */
+  async getHelpfulnessStats(): Promise<{
+    avgScore: number;
+    totalEvaluated: number;
+    totalRetrievals: number;
+    helpful: number;
+    neutral: number;
+    unhelpful: number;
+  }> {
+    await this.initialize();
+
+    const stats = sqliteGet<Record<string, unknown>>(
+      this.db,
+      `SELECT
+         AVG(helpfulness_score) as avg_score,
+         COUNT(*) as total_evaluated,
+         SUM(CASE WHEN helpfulness_score >= 0.7 THEN 1 ELSE 0 END) as helpful,
+         SUM(CASE WHEN helpfulness_score >= 0.4 AND helpfulness_score < 0.7 THEN 1 ELSE 0 END) as neutral,
+         SUM(CASE WHEN helpfulness_score < 0.4 THEN 1 ELSE 0 END) as unhelpful
+       FROM memory_helpfulness
+       WHERE measured_at IS NOT NULL`
+    );
+
+    const totalRow = sqliteGet<Record<string, unknown>>(
+      this.db,
+      `SELECT COUNT(*) as total FROM memory_helpfulness`
+    );
+
+    return {
+      avgScore: Math.round(((stats?.avg_score as number) || 0) * 100) / 100,
+      totalEvaluated: (stats?.total_evaluated as number) || 0,
+      totalRetrievals: (totalRow?.total as number) || 0,
+      helpful: (stats?.helpful as number) || 0,
+      neutral: (stats?.neutral as number) || 0,
+      unhelpful: (stats?.unhelpful as number) || 0
+    };
+  }
+
   /**
    * Fast keyword search using FTS5
    * Returns events matching the search query, ranked by relevance
@@ -912,6 +1363,143 @@ export class SQLiteEventStore {
     return this.db;
   }
 
+
+  async recordRetrievalTrace(input: {
+    sessionId?: string;
+    projectHash?: string;
+    queryText: string;
+    strategy?: string;
+    candidateEventIds: string[];
+    selectedEventIds: string[];
+    candidateDetails?: Array<{
+      eventId: string;
+      score: number;
+      semanticScore?: number;
+      lexicalScore?: number;
+      recencyScore?: number;
+    }>;
+    selectedDetails?: Array<{
+      eventId: string;
+      score: number;
+      semanticScore?: number;
+      lexicalScore?: number;
+      recencyScore?: number;
+    }>;
+    confidence?: string;
+    fallbackTrace?: string[];
+  }): Promise<void> {
+    await this.initialize();
+
+    const traceId = randomUUID();
+    sqliteRun(
+      this.db,
+      `INSERT INTO retrieval_traces (
+        trace_id, session_id, project_hash, query_text, strategy,
+        candidate_event_ids, selected_event_ids, candidate_details_json, selected_details_json,
+        candidate_count, selected_count, confidence, fallback_trace
+      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+      [
+        traceId,
+        input.sessionId || null,
+        input.projectHash || null,
+        input.queryText,
+        input.strategy || null,
+        JSON.stringify(input.candidateEventIds || []),
+        JSON.stringify(input.selectedEventIds || []),
+        JSON.stringify(input.candidateDetails || []),
+        JSON.stringify(input.selectedDetails || []),
+        (input.candidateEventIds || []).length,
+        (input.selectedEventIds || []).length,
+        input.confidence || null,
+        JSON.stringify(input.fallbackTrace || [])
+      ]
+    );
+  }
+
+  async getRecentRetrievalTraces(limit: number = 50): Promise<Array<{
+    traceId: string;
+    sessionId?: string;
+    projectHash?: string;
+    queryText: string;
+    strategy?: string;
+    candidateEventIds: string[];
+    selectedEventIds: string[];
+    candidateDetails: Array<{
+      eventId: string;
+      score: number;
+      semanticScore?: number;
+      lexicalScore?: number;
+      recencyScore?: number;
+    }>;
+    selectedDetails: Array<{
+      eventId: string;
+      score: number;
+      semanticScore?: number;
+      lexicalScore?: number;
+      recencyScore?: number;
+    }>;
+    candidateCount: number;
+    selectedCount: number;
+    confidence?: string;
+    fallbackTrace: string[];
+    createdAt: Date;
+  }>> {
+    await this.initialize();
+
+    const rows = sqliteAll<Record<string, unknown>>(
+      this.db,
+      `SELECT * FROM retrieval_traces ORDER BY created_at DESC LIMIT ?`,
+      [limit]
+    );
+
+    return rows.map((row) => ({
+      traceId: row.trace_id as string,
+      sessionId: (row.session_id as string) || undefined,
+      projectHash: (row.project_hash as string) || undefined,
+      queryText: row.query_text as string,
+      strategy: (row.strategy as string) || undefined,
+      candidateEventIds: row.candidate_event_ids ? JSON.parse(row.candidate_event_ids as string) : [],
+      selectedEventIds: row.selected_event_ids ? JSON.parse(row.selected_event_ids as string) : [],
+      candidateDetails: row.candidate_details_json ? JSON.parse(row.candidate_details_json as string) : [],
+      selectedDetails: row.selected_details_json ? JSON.parse(row.selected_details_json as string) : [],
+      candidateCount: Number(row.candidate_count || 0),
+      selectedCount: Number(row.selected_count || 0),
+      confidence: (row.confidence as string) || undefined,
+      fallbackTrace: row.fallback_trace ? JSON.parse(row.fallback_trace as string) : [],
+      createdAt: toDateFromSQLite(row.created_at),
+    }));
+  }
+
+  async getRetrievalTraceStats(): Promise<{
+    totalQueries: number;
+    avgCandidateCount: number;
+    avgSelectedCount: number;
+    selectionRate: number;
+  }> {
+    await this.initialize();
+
+    const row = sqliteGet<Record<string, unknown>>(
+      this.db,
+      `SELECT
+        COUNT(*) as total_queries,
+        AVG(candidate_count) as avg_candidate_count,
+        AVG(selected_count) as avg_selected_count,
+        CASE
+          WHEN SUM(candidate_count) > 0 THEN (SUM(selected_count) * 1.0 / SUM(candidate_count))
+          ELSE 0
+        END as selection_rate
+       FROM retrieval_traces`,
+      []
+    );
+
+    return {
+      totalQueries: Number(row?.total_queries || 0),
+      avgCandidateCount: Number(row?.avg_candidate_count || 0),
+      avgSelectedCount: Number(row?.avg_selected_count || 0),
+      selectionRate: Number(row?.selection_rate || 0),
+    };
+  }
+
   /**
    * Close database connection
    */
@@ -919,6 +1507,201 @@ export class SQLiteEventStore {
     sqliteClose(this.db);
   }
 
+  /**
+   * Get events grouped by turn_id for a session
+   * Returns turns ordered by first event timestamp (newest first)
+   */
+  async getSessionTurns(sessionId: string, options?: { limit?: number; offset?: number }): Promise<Array<{
+    turnId: string;
+    events: MemoryEvent[];
+    startedAt: Date;
+    promptPreview: string;
+    eventCount: number;
+    toolCount: number;
+    hasResponse: boolean;
+  }>> {
+    await this.initialize();
+
+    const limit = options?.limit || 20;
+    const offset = options?.offset || 0;
+
+    // Get distinct turn_ids for this session, ordered by first event timestamp
+    const turnRows = sqliteAll<{ turn_id: string; min_ts: string }>(
+      this.db,
+      `SELECT turn_id, MIN(timestamp) as min_ts
+       FROM events
+       WHERE session_id = ? AND turn_id IS NOT NULL
+       GROUP BY turn_id
+       ORDER BY min_ts DESC
+       LIMIT ? OFFSET ?`,
+      [sessionId, limit, offset]
+    );
+
+    const turns: Array<{
+      turnId: string;
+      events: MemoryEvent[];
+      startedAt: Date;
+      promptPreview: string;
+      eventCount: number;
+      toolCount: number;
+      hasResponse: boolean;
+    }> = [];
+
+    for (const turnRow of turnRows) {
+      const events = await this.getEventsByTurn(turnRow.turn_id);
+
+      const promptEvent = events.find(e => e.eventType === 'user_prompt');
+      const toolEvents = events.filter(e => e.eventType === 'tool_observation');
+      const hasResponse = events.some(e => e.eventType === 'agent_response');
+
+      turns.push({
+        turnId: turnRow.turn_id,
+        events,
+        startedAt: toDateFromSQLite(turnRow.min_ts),
+        promptPreview: promptEvent
+          ? promptEvent.content.slice(0, 200) + (promptEvent.content.length > 200 ? '...' : '')
+          : '(no prompt)',
+        eventCount: events.length,
+        toolCount: toolEvents.length,
+        hasResponse
+      });
+    }
+
+    return turns;
+  }
+
+  /**
+   * Get all events for a specific turn_id
+   */
+  async getEventsByTurn(turnId: string): Promise<MemoryEvent[]> {
+    await this.initialize();
+
+    const rows = sqliteAll<Record<string, unknown>>(
+      this.db,
+      `SELECT * FROM events WHERE turn_id = ? ORDER BY timestamp ASC`,
+      [turnId]
+    );
+
+    return rows.map(this.rowToEvent);
+  }
+
+  /**
+   * Count total turns for a session
+   */
+  async countSessionTurns(sessionId: string): Promise<number> {
+    await this.initialize();
+
+    const row = sqliteGet<{ count: number }>(
+      this.db,
+      `SELECT COUNT(DISTINCT turn_id) as count
+       FROM events
+       WHERE session_id = ? AND turn_id IS NOT NULL`,
+      [sessionId]
+    );
+
+    return row?.count || 0;
+  }
+
+  /**
+   * Migrate existing events: backfill turn_id for events that have turnId in metadata
+   * but no turn_id column value (for events stored before this migration)
+   */
+  async backfillTurnIds(): Promise<number> {
+    await this.initialize();
+
+    // Find events with turnId in metadata JSON but no turn_id column value
+    const rows = sqliteAll<{ id: string; metadata: string }>(
+      this.db,
+      `SELECT id, metadata FROM events
+       WHERE turn_id IS NULL AND metadata IS NOT NULL AND metadata LIKE '%turnId%'`
+    );
+
+    let updated = 0;
+    for (const row of rows) {
+      try {
+        const metadata = JSON.parse(row.metadata);
+        if (metadata.turnId) {
+          sqliteRun(
+            this.db,
+            `UPDATE events SET turn_id = ? WHERE id = ?`,
+            [metadata.turnId, row.id]
+          );
+          updated++;
+        }
+      } catch {
+        // Skip rows with invalid JSON
+      }
+    }
+
+    return updated;
+  }
+
+  /**
+   * Delete all events for a session (for force reimport)
+   */
+  async deleteSessionEvents(sessionId: string): Promise<number> {
+    await this.initialize();
+
+    // Get event IDs first for cascading deletes
+    const events = sqliteAll<{ id: string }>(
+      this.db,
+      `SELECT id FROM events WHERE session_id = ?`,
+      [sessionId]
+    );
+
+    if (events.length === 0) return 0;
+
+    const eventIds = events.map(e => e.id);
+    const placeholders = eventIds.map(() => '?').join(',');
+
+    // Drop FTS triggers to prevent SQLITE_CORRUPT_VTAB during bulk delete
+    const ftsTriggersDropped: string[] = [];
+    for (const triggerName of ['events_fts_delete', 'events_fts_update', 'events_fts_insert']) {
+      try {
+        sqliteRun(this.db, `DROP TRIGGER IF EXISTS ${triggerName}`);
+        ftsTriggersDropped.push(triggerName);
+      } catch {
+        // Trigger may not exist
+      }
+    }
+
+    // Delete from related tables first (some may not exist depending on DB version)
+    for (const table of ['event_dedup', 'memory_levels', 'embedding_queue', 'embedding_outbox', 'vector_outbox']) {
+      try {
+        sqliteRun(this.db, `DELETE FROM ${table} WHERE event_id IN (${placeholders})`, eventIds);
+      } catch {
+        // Table may not exist
+      }
+    }
+
+    // Delete events
+    const result = sqliteRun(this.db, `DELETE FROM events WHERE session_id = ?`, [sessionId]);
+
+    // Rebuild FTS index if we dropped triggers
+    if (ftsTriggersDropped.length > 0) {
+      try {
+        // Rebuild FTS from remaining events
+        sqliteRun(this.db, `INSERT INTO events_fts(events_fts) VALUES('rebuild')`);
+
+        // Recreate triggers
+        sqliteRun(this.db, `CREATE TRIGGER IF NOT EXISTS events_fts_insert AFTER INSERT ON events BEGIN
+          INSERT INTO events_fts(rowid, content) VALUES (NEW.rowid, NEW.content);
+        END`);
+        sqliteRun(this.db, `CREATE TRIGGER IF NOT EXISTS events_fts_delete AFTER DELETE ON events BEGIN
+          INSERT INTO events_fts(events_fts, rowid, content) VALUES('delete', OLD.rowid, OLD.content);
+        END`);
+        sqliteRun(this.db, `CREATE TRIGGER IF NOT EXISTS events_fts_update AFTER UPDATE ON events BEGIN
+          INSERT INTO events_fts(events_fts, rowid, content) VALUES('delete', OLD.rowid, OLD.content);
+          INSERT INTO events_fts(rowid, content) VALUES (NEW.rowid, NEW.content);
+        END`);
+      } catch {
+        // FTS rebuild failed - non-critical, will be rebuilt on next initialize
+      }
+    }
+
+    return result.changes || 0;
+  }
+
   /**
    * Convert database row to MemoryEvent
    */
@@ -941,6 +1724,10 @@ export class SQLiteEventStore {
     if (row.last_accessed_at !== undefined) {
       event.last_accessed_at = row.last_accessed_at;
     }
+    // Include turn_id if present
+    if (row.turn_id !== undefined && row.turn_id !== null) {
+      event.turn_id = row.turn_id;
+    }
 
     return event;
   }