claude-memory-layer 1.0.11 → 1.0.13

package/src/core/sqlite-event-store.ts

@@ -24,15 +24,17 @@ import {
   type SQLiteDatabase,
   type SQLiteOptions
 } from './sqlite-wrapper.js';
+import { MarkdownMirror } from './markdown-mirror.js';
 
 export interface SQLiteEventStoreOptions extends SQLiteOptions {
-  // Additional options can be added here
+  markdownMirrorRoot?: string;
 }
 
 export class SQLiteEventStore {
   private db: SQLiteDatabase;
   private initialized = false;
   private readonly readOnly: boolean;
+  private readonly markdownMirror: MarkdownMirror | null;
 
   constructor(private dbPath: string, options?: SQLiteEventStoreOptions) {
     this.readOnly = options?.readonly ?? false;
@@ -40,6 +42,9 @@ export class SQLiteEventStore {
       readonly: this.readOnly,
       walMode: !this.readOnly
     });
+    this.markdownMirror = this.readOnly || !options?.markdownMirrorRoot
+      ? null
+      : new MarkdownMirror(options.markdownMirrorRoot);
   }
 
   /**
@@ -251,6 +256,17 @@ export class SQLiteEventStore {
         created_at TEXT DEFAULT (datetime('now'))
       );
 
+      -- Consolidated Rules table (long-term stable memory)
+      CREATE TABLE IF NOT EXISTS consolidated_rules (
+        rule_id TEXT PRIMARY KEY,
+        rule TEXT NOT NULL,
+        topics TEXT,
+        source_memory_ids TEXT,
+        source_events TEXT,
+        confidence REAL DEFAULT 0.5,
+        created_at TEXT DEFAULT (datetime('now'))
+      );
+
       -- Endless Mode Config table
       CREATE TABLE IF NOT EXISTS endless_config (
         key TEXT PRIMARY KEY,
@@ -275,6 +291,24 @@ export class SQLiteEventStore {
         measured_at TEXT
       );
 
+      -- Retrieval trace log (query -> candidates -> selected for context)
+      CREATE TABLE IF NOT EXISTS retrieval_traces (
+        trace_id TEXT PRIMARY KEY,
+        session_id TEXT,
+        project_hash TEXT,
+        query_text TEXT NOT NULL,
+        strategy TEXT,
+        candidate_event_ids TEXT,
+        selected_event_ids TEXT,
+        candidate_details_json TEXT,
+        selected_details_json TEXT,
+        candidate_count INTEGER DEFAULT 0,
+        selected_count INTEGER DEFAULT 0,
+        confidence TEXT,
+        fallback_trace TEXT,
+        created_at TEXT DEFAULT (datetime('now'))
+      );
+
       -- Sync position tracking (for SQLite -> DuckDB sync)
       CREATE TABLE IF NOT EXISTS sync_positions (
         target_name TEXT PRIMARY KEY,
@@ -299,10 +333,14 @@ export class SQLiteEventStore {
       CREATE INDEX IF NOT EXISTS idx_working_set_relevance ON working_set(relevance_score);
       CREATE INDEX IF NOT EXISTS idx_consolidated_confidence ON consolidated_memories(confidence);
       CREATE INDEX IF NOT EXISTS idx_continuity_created ON continuity_log(created_at);
+      CREATE INDEX IF NOT EXISTS idx_consolidated_rules_confidence ON consolidated_rules(confidence);
       CREATE INDEX IF NOT EXISTS idx_embedding_outbox_status ON embedding_outbox(status);
       CREATE INDEX IF NOT EXISTS idx_helpfulness_event ON memory_helpfulness(event_id);
       CREATE INDEX IF NOT EXISTS idx_helpfulness_session ON memory_helpfulness(session_id);
       CREATE INDEX IF NOT EXISTS idx_helpfulness_score ON memory_helpfulness(helpfulness_score DESC);
+      CREATE INDEX IF NOT EXISTS idx_retrieval_traces_created_at ON retrieval_traces(created_at DESC);
+      CREATE INDEX IF NOT EXISTS idx_retrieval_traces_project_hash ON retrieval_traces(project_hash);
+      CREATE INDEX IF NOT EXISTS idx_retrieval_traces_session_id ON retrieval_traces(session_id);
 
       -- FTS5 Full-Text Search for fast keyword search
       CREATE VIRTUAL TABLE IF NOT EXISTS events_fts USING fts5(
@@ -327,6 +365,19 @@ export class SQLiteEventStore {
       END;
     `);
 
+
+    // Best-effort forward migration for retrieval trace detail column
+    try {
+      sqliteExec(this.db, `ALTER TABLE retrieval_traces ADD COLUMN selected_details_json TEXT;`);
+    } catch {
+      // column may already exist
+    }
+    try {
+      sqliteExec(this.db, `ALTER TABLE retrieval_traces ADD COLUMN candidate_details_json TEXT;`);
+    } catch {
+      // column may already exist
+    }
+
     // Migrate existing events table to add new columns if they don't exist
     // Check if columns exist before trying to add them
     const tableInfo = sqliteAll(this.db, "PRAGMA table_info(events)", []);
@@ -455,6 +506,22 @@ export class SQLiteEventStore {
 
       transaction();
 
+      if (this.markdownMirror) {
+        const event: MemoryEvent = {
+          id,
+          eventType: input.eventType,
+          sessionId: input.sessionId,
+          timestamp: input.timestamp,
+          content: input.content,
+          canonicalKey,
+          dedupeKey,
+          metadata
+        };
+        this.markdownMirror.append(event).catch((err) => {
+          console.warn('[SQLiteEventStore] markdown mirror append failed:', err);
+        });
+      }
+
       return { success: true, eventId: id, isDuplicate: false };
     } catch (error) {
       return {
@@ -525,6 +592,114 @@ export class SQLiteEventStore {
     return rows.map(this.rowToEvent);
   }
 
+  /**
+   * Get events since a SQLite rowid (for robust incremental replication).
+   * Rowid is monotonic for append-only tables, independent of client timestamps.
+   */
+  async getEventsSinceRowid(
+    lastRowid: number,
+    limit: number = 1000
+  ): Promise<Array<{ rowid: number; event: MemoryEvent }>> {
+    await this.initialize();
+
+    const rows = sqliteAll<Record<string, unknown>>(
+      this.db,
+      `SELECT rowid as _rowid, * FROM events WHERE rowid > ? ORDER BY rowid ASC LIMIT ?`,
+      [lastRowid, limit]
+    );
+
+    return rows.map(row => ({
+      rowid: row._rowid as number,
+      event: this.rowToEvent(row)
+    }));
+  }
+
+  /**
+   * Import events with fixed IDs (used for cross-machine replication).
+   * Idempotent: skips if event id or dedupeKey already exists.
+   *
+   * NOTE: This bypasses the append() id generation to preserve stable IDs.
+   */
+  async importEvents(events: MemoryEvent[]): Promise<{ inserted: number; skipped: number }> {
+    if (events.length === 0) return { inserted: 0, skipped: 0 };
+    if (this.readOnly) return { inserted: 0, skipped: events.length };
+
+    await this.initialize();
+
+    const getById = this.db.prepare(`SELECT id FROM events WHERE id = ?`);
+    const getByDedupe = this.db.prepare(`SELECT event_id FROM event_dedup WHERE dedupe_key = ?`);
+
+    const insertEvent = this.db.prepare(`
+      INSERT INTO events (id, event_type, session_id, timestamp, content, canonical_key, dedupe_key, metadata, turn_id)
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+    `);
+
+    const insertDedup = this.db.prepare(`
+      INSERT INTO event_dedup (dedupe_key, event_id) VALUES (?, ?)
+    `);
+
+    const insertLevel = this.db.prepare(`
+      INSERT INTO memory_levels (event_id, level) VALUES (?, 'L0')
+    `);
+
+    let inserted = 0;
+    let skipped = 0;
+    const insertedEvents: MemoryEvent[] = [];
+
+    const tx = this.db.transaction((batch: MemoryEvent[]) => {
+      for (const ev of batch) {
+        // Skip if already present by id
+        const existingById = getById.get(ev.id) as { id: string } | undefined;
+        if (existingById) {
+          skipped++;
+          continue;
+        }
+
+        const canonicalKey = ev.canonicalKey || makeCanonicalKey(ev.content);
+        const dedupeKey = ev.dedupeKey || makeDedupeKey(ev.content, ev.sessionId);
+
+        // Skip if already present by dedupe key
+        const existingByDedupe = getByDedupe.get(dedupeKey) as { event_id: string } | undefined;
+        if (existingByDedupe) {
+          skipped++;
+          continue;
+        }
+
+        const metadata = ev.metadata || {};
+        const turnId = (metadata as any).turnId as string | undefined;
+
+        insertEvent.run(
+          ev.id,
+          ev.eventType,
+          ev.sessionId,
+          toSQLiteTimestamp(ev.timestamp),
+          ev.content,
+          canonicalKey,
+          dedupeKey,
+          JSON.stringify(metadata),
+          turnId ?? null
+        );
+
+        insertDedup.run(dedupeKey, ev.id);
+        insertLevel.run(ev.id);
+        inserted++;
+        insertedEvents.push(ev);
+      }
+    });
+
+    tx(events);
+
+    if (this.markdownMirror && insertedEvents.length > 0) {
+      for (const ev of insertedEvents) {
+        this.markdownMirror.append(ev).catch((err) => {
+          console.warn('[SQLiteEventStore] markdown mirror append failed:', err);
+        });
+      }
+    }
+
+    return { inserted, skipped };
+  }
+
   /**
    * Create or update session
    */
@@ -708,6 +883,43 @@ export class SQLiteEventStore {
     );
   }
 
+
+  /**
+   * Get embedding/vector outbox health statistics
+   */
+  async getOutboxStats(): Promise<{
+    embedding: { pending: number; processing: number; failed: number; total: number };
+    vector: { pending: number; processing: number; failed: number; total: number };
+  }> {
+    await this.initialize();
+
+    const embeddingRows = sqliteAll<{ status: string; count: number }>(
+      this.db,
+      `SELECT status, COUNT(*) as count FROM embedding_outbox GROUP BY status`
+    );
+    const vectorRows = sqliteAll<{ status: string; count: number }>(
+      this.db,
+      `SELECT status, COUNT(*) as count FROM vector_outbox GROUP BY status`
+    );
+
+    const fromRows = (rows: Array<{ status: string; count: number }>) => {
+      const out = { pending: 0, processing: 0, failed: 0, total: 0 };
+      for (const row of rows) {
+        const key = row.status as 'pending' | 'processing' | 'failed' | 'done';
+        if (key === 'pending' || key === 'processing' || key === 'failed') {
+          out[key] += row.count;
+        }
+        out.total += row.count;
+      }
+      return out;
+    };
+
+    return {
+      embedding: fromRows(embeddingRows),
+      vector: fromRows(vectorRows)
+    };
+  }
+
   /**
    * Update memory level
    */
@@ -1151,6 +1363,143 @@ export class SQLiteEventStore {
     return this.db;
   }
 
+
+  async recordRetrievalTrace(input: {
+    sessionId?: string;
+    projectHash?: string;
+    queryText: string;
+    strategy?: string;
+    candidateEventIds: string[];
+    selectedEventIds: string[];
+    candidateDetails?: Array<{
+      eventId: string;
+      score: number;
+      semanticScore?: number;
+      lexicalScore?: number;
+      recencyScore?: number;
+    }>;
+    selectedDetails?: Array<{
+      eventId: string;
+      score: number;
+      semanticScore?: number;
+      lexicalScore?: number;
+      recencyScore?: number;
+    }>;
+    confidence?: string;
+    fallbackTrace?: string[];
+  }): Promise<void> {
+    await this.initialize();
+
+    const traceId = randomUUID();
+    sqliteRun(
+      this.db,
+      `INSERT INTO retrieval_traces (
+        trace_id, session_id, project_hash, query_text, strategy,
+        candidate_event_ids, selected_event_ids, candidate_details_json, selected_details_json,
+        candidate_count, selected_count, confidence, fallback_trace
+      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+      [
+        traceId,
+        input.sessionId || null,
+        input.projectHash || null,
+        input.queryText,
+        input.strategy || null,
+        JSON.stringify(input.candidateEventIds || []),
+        JSON.stringify(input.selectedEventIds || []),
+        JSON.stringify(input.candidateDetails || []),
+        JSON.stringify(input.selectedDetails || []),
+        (input.candidateEventIds || []).length,
+        (input.selectedEventIds || []).length,
+        input.confidence || null,
+        JSON.stringify(input.fallbackTrace || [])
+      ]
+    );
+  }
+
+  async getRecentRetrievalTraces(limit: number = 50): Promise<Array<{
+    traceId: string;
+    sessionId?: string;
+    projectHash?: string;
+    queryText: string;
+    strategy?: string;
+    candidateEventIds: string[];
+    selectedEventIds: string[];
+    candidateDetails: Array<{
+      eventId: string;
+      score: number;
+      semanticScore?: number;
+      lexicalScore?: number;
+      recencyScore?: number;
+    }>;
+    selectedDetails: Array<{
+      eventId: string;
+      score: number;
+      semanticScore?: number;
+      lexicalScore?: number;
+      recencyScore?: number;
+    }>;
+    candidateCount: number;
+    selectedCount: number;
+    confidence?: string;
+    fallbackTrace: string[];
+    createdAt: Date;
+  }>> {
+    await this.initialize();
+
+    const rows = sqliteAll<Record<string, unknown>>(
+      this.db,
+      `SELECT * FROM retrieval_traces ORDER BY created_at DESC LIMIT ?`,
+      [limit]
+    );
+
+    return rows.map((row) => ({
+      traceId: row.trace_id as string,
+      sessionId: (row.session_id as string) || undefined,
+      projectHash: (row.project_hash as string) || undefined,
+      queryText: row.query_text as string,
+      strategy: (row.strategy as string) || undefined,
+      candidateEventIds: row.candidate_event_ids ? JSON.parse(row.candidate_event_ids as string) : [],
+      selectedEventIds: row.selected_event_ids ? JSON.parse(row.selected_event_ids as string) : [],
+      candidateDetails: row.candidate_details_json ? JSON.parse(row.candidate_details_json as string) : [],
+      selectedDetails: row.selected_details_json ? JSON.parse(row.selected_details_json as string) : [],
+      candidateCount: Number(row.candidate_count || 0),
+      selectedCount: Number(row.selected_count || 0),
+      confidence: (row.confidence as string) || undefined,
+      fallbackTrace: row.fallback_trace ? JSON.parse(row.fallback_trace as string) : [],
+      createdAt: toDateFromSQLite(row.created_at),
+    }));
+  }
+
+  async getRetrievalTraceStats(): Promise<{
+    totalQueries: number;
+    avgCandidateCount: number;
+    avgSelectedCount: number;
+    selectionRate: number;
+  }> {
+    await this.initialize();
+
+    const row = sqliteGet<Record<string, unknown>>(
+      this.db,
+      `SELECT
+        COUNT(*) as total_queries,
+        AVG(candidate_count) as avg_candidate_count,
+        AVG(selected_count) as avg_selected_count,
+        CASE
+          WHEN SUM(candidate_count) > 0 THEN (SUM(selected_count) * 1.0 / SUM(candidate_count))
+          ELSE 0
+        END as selection_rate
+       FROM retrieval_traces`,
+      []
+    );
+
+    return {
+      totalQueries: Number(row?.total_queries || 0),
+      avgCandidateCount: Number(row?.avg_candidate_count || 0),
+      avgSelectedCount: Number(row?.avg_selected_count || 0),
+      selectionRate: Number(row?.selection_rate || 0),
+    };
+  }
+
   /**
    * Close database connection
    */

package/src/core/tag-taxonomy.ts

@@ -0,0 +1,51 @@
+export const TAG_NAMESPACES = {
+  SYSTEM: 'sys:',
+  QUALITY: 'q:',
+  PROJECT: 'proj:',
+  TOPIC: 'topic:',
+  TEMPORAL: 't:',
+  USER: 'user:',
+  AGENT: 'agent:'
+} as const;
+
+export const VALID_TAG_NAMESPACES = new Set<string>(Object.values(TAG_NAMESPACES));
+
+export function parseTag(tag: string): { namespace?: string; value: string } {
+  const value = (tag || '').trim();
+  const idx = value.indexOf(':');
+  if (idx <= 0) return { value };
+
+  const namespace = `${value.slice(0, idx)}:`;
+  const tagValue = value.slice(idx + 1);
+  if (!tagValue) return { value };
+
+  return { namespace, value: tagValue };
+}
+
+export function validateTag(tag: string): boolean {
+  const normalized = (tag || '').trim();
+  if (!normalized) return false;
+
+  const { namespace } = parseTag(normalized);
+  if (!namespace) return true; // backward compatibility for legacy tags
+  return VALID_TAG_NAMESPACES.has(namespace);
+}
+
+export function withNamespace(value: string, namespace: string): string {
+  const clean = parseTag(value).value.trim();
+  return `${namespace}${clean}`;
+}
+
+export function normalizeTags(tags: unknown): string[] {
+  if (!Array.isArray(tags)) return [];
+
+  const dedup = new Set<string>();
+  for (const item of tags) {
+    if (typeof item !== 'string') continue;
+    const normalized = item.trim();
+    if (!validateTag(normalized)) continue;
+    dedup.add(normalized);
+  }
+
+  return [...dedup];
+}

package/src/core/types.ts

@@ -818,6 +818,34 @@ export interface ConsolidatedMemoryInput {
   confidence: number;
 }
 
+// Long-term Rule (promoted from stable summaries)
+export const ConsolidationRuleSchema = z.object({
+  ruleId: z.string(),
+  rule: z.string(),
+  topics: z.array(z.string()),
+  sourceMemoryIds: z.array(z.string()),
+  sourceEvents: z.array(z.string()),
+  confidence: z.number(),
+  createdAt: z.date()
+});
+export type ConsolidationRule = z.infer<typeof ConsolidationRuleSchema>;
+
+export interface ConsolidationRuleInput {
+  rule: string;
+  topics: string[];
+  sourceMemoryIds: string[];
+  sourceEvents: string[];
+  confidence: number;
+}
+
+export interface ConsolidationCostQualityReport {
+  beforeTokenEstimate: number;
+  afterTokenEstimate: number;
+  reductionRatio: number;
+  qualityGuardPassed: boolean;
+  details: string;
+}
+
 // Event Group (for consolidation)
 export interface EventGroup {
   topics: string[];

package/src/server/api/health.ts

@@ -0,0 +1,53 @@
+/**
+ * Health API
+ * Operational health checks including outbox backlog/failures
+ */
+
+import { Hono } from 'hono';
+import { getServiceFromQuery } from './utils.js';
+
+export const healthRouter = new Hono();
+
+// GET /api/health
+healthRouter.get('/', async (c) => {
+  const memoryService = getServiceFromQuery(c);
+  try {
+    await memoryService.initialize();
+
+    const [stats, outbox] = await Promise.all([
+      memoryService.getStats(),
+      memoryService.getOutboxStats()
+    ]);
+
+    const outboxPending = outbox.embedding.pending + outbox.vector.pending;
+    const outboxFailed = outbox.embedding.failed + outbox.vector.failed;
+
+    const status = outboxFailed > 0 ? 'needs-attention' : 'ok';
+
+    return c.json({
+      status,
+      timestamp: new Date().toISOString(),
+      storage: {
+        totalEvents: stats.totalEvents,
+        vectorCount: stats.vectorCount
+      },
+      outbox: {
+        embedding: outbox.embedding,
+        vector: outbox.vector,
+        totals: {
+          pending: outboxPending,
+          failed: outboxFailed
+        }
+      },
+      levelStats: stats.levelStats
+    });
+  } catch (error) {
+    return c.json({
+      status: 'error',
+      timestamp: new Date().toISOString(),
+      error: (error as Error).message
+    }, 500);
+  } finally {
+    await memoryService.shutdown();
+  }
+});

package/src/server/api/index.ts

@@ -12,6 +12,7 @@ import { citationsRouter } from './citations.js';
 import { turnsRouter } from './turns.js';
 import { projectsRouter } from './projects.js';
 import { chatRouter } from './chat.js';
+import { healthRouter } from './health.js';
 
 export const apiRouter = new Hono()
   .route('/sessions', sessionsRouter)
@@ -21,4 +22,5 @@ export const apiRouter = new Hono()
   .route('/citations', citationsRouter)
   .route('/turns', turnsRouter)
   .route('/projects', projectsRouter)
-  .route('/chat', chatRouter);
+  .route('/chat', chatRouter)
+  .route('/health', healthRouter);

package/src/server/api/stats.ts

@@ -158,6 +158,8 @@ statsRouter.get('/', async (c) => {
         return acc;
       }, {} as Record<string, number>);
 
+    const retrievalTrace = await memoryService.getRetrievalTraceStats();
+
     return c.json({
       storage: {
         eventCount: stats.totalEvents,
@@ -175,7 +177,8 @@ statsRouter.get('/', async (c) => {
         heapUsed: Math.round(process.memoryUsage().heapUsed / 1024 / 1024),
         heapTotal: Math.round(process.memoryUsage().heapTotal / 1024 / 1024)
       },
-      levelStats: stats.levelStats
+      levelStats: stats.levelStats,
+      retrievalTrace
     });
   } catch (error) {
     return c.json({ error: (error as Error).message }, 500);
@@ -291,6 +294,48 @@ statsRouter.get('/helpfulness', async (c) => {
   }
 });
 
+
+
+// GET /api/stats/retrieval-traces - Get recent retrieval traces (query -> selected context)
+statsRouter.get('/retrieval-traces', async (c) => {
+  const limit = parseInt(c.req.query('limit') || '50', 10);
+  const memoryService = getServiceFromQuery(c);
+
+  try {
+    await memoryService.initialize();
+    const traces = await memoryService.getRecentRetrievalTraces(limit);
+    const traceStats = await memoryService.getRetrievalTraceStats();
+
+    return c.json({
+      stats: traceStats,
+      traces: traces.map((t) => ({
+        traceId: t.traceId,
+        sessionId: t.sessionId || null,
+        projectHash: t.projectHash || null,
+        queryText: t.queryText,
+        strategy: t.strategy || null,
+        candidateEventIds: t.candidateEventIds,
+        selectedEventIds: t.selectedEventIds,
+        candidateDetails: t.candidateDetails || [],
+        selectedDetails: t.selectedDetails || [],
+        candidateCount: t.candidateCount,
+        selectedCount: t.selectedCount,
+        confidence: t.confidence || null,
+        fallbackTrace: t.fallbackTrace,
+        createdAt: t.createdAt.toISOString()
+      }))
+    });
+  } catch (error) {
+    return c.json({
+      stats: { totalQueries: 0, avgCandidateCount: 0, avgSelectedCount: 0, selectionRate: 0 },
+      traces: [],
+      error: (error as Error).message
+    }, 500);
+  } finally {
+    await memoryService.shutdown();
+  }
+});
+
 // POST /api/stats/graduation/run - Force graduation evaluation
 statsRouter.post('/graduation/run', async (c) => {
   const memoryService = getServiceFromQuery(c);