agent-working-memory 0.7.8 → 0.7.10

package/src/storage/sqlite.ts

@@ -28,10 +28,33 @@ const DEFAULT_SALIENCE_FEATURES: SalienceFeatures = {
   surprise: 0, decisionMade: false, causalDepth: 0, resolutionEffort: 0, eventType: 'observation',
 };
 
+/**
+ * In-memory slim entry — the minimum data the activation pipeline's pre-filter
+ * pass reads. Lives in EngramStore.slimCache to skip the SQL fetch + Buffer→
+ * Float32Array conversion on every recall. ~22 bytes overhead per entry plus
+ * the embedding (~1.5KB), so ~15MB at 10K engrams.
+ */
+type SlimCacheEntry = {
+  id: string;
+  agentId: string;
+  concept: string;
+  embedding: number[] | null;
+  stage: EngramStage;
+  retracted: boolean;
+};
+
 export class EngramStore {
   private db: Database.Database;
   private walTimer: ReturnType<typeof setInterval> | null = null;
 
+  // Slim cache for the activation pipeline pre-filter. Populated lazily on
+  // first call to getEngramsByAgentSlim(). Mutations to engrams keep this in
+  // sync via private cache* helpers. Disable via AWM_DISABLE_SLIM_CACHE=1
+  // (for A/B testing or if a regression appears).
+  private slimCache: Map<string, SlimCacheEntry> = new Map();
+  private slimCachePopulated: boolean = false;
+  private slimCacheEnabled: boolean = process.env.AWM_DISABLE_SLIM_CACHE !== '1';
+
   constructor(dbPath: string = 'memory.db') {
     this.db = new Database(dbPath);
     this.db.pragma('journal_mode = WAL');
@@ -43,6 +66,62 @@ export class EngramStore {
     this.startWalCheckpointTimer();
   }
 
+  // --- Slim cache management ---
+
+  /** Lazy-populate the slim cache from the engrams table. Called on first slim fetch. */
+  private ensureSlimCachePopulated(): void {
+    if (this.slimCachePopulated || !this.slimCacheEnabled) return;
+    const rows = this.db.prepare(
+      'SELECT id, agent_id, concept, embedding, stage, retracted FROM engrams'
+    ).all() as any[];
+    for (const r of rows) {
+      this.slimCache.set(r.id as string, {
+        id: r.id as string,
+        agentId: r.agent_id as string,
+        concept: r.concept as string,
+        embedding: r.embedding ? Array.from(bufferToFloat32Array(r.embedding)) : null,
+        stage: r.stage as EngramStage,
+        retracted: !!r.retracted,
+      });
+    }
+    this.slimCachePopulated = true;
+  }
+
+  /** Add a new engram to the slim cache. Called from createEngram. */
+  private cacheAdd(entry: SlimCacheEntry): void {
+    if (!this.slimCacheEnabled) return;
+    this.slimCache.set(entry.id, entry);
+  }
+
+  private cacheUpdateStage(id: string, stage: EngramStage): void {
+    if (!this.slimCacheEnabled) return;
+    const e = this.slimCache.get(id);
+    if (e) e.stage = stage;
+  }
+
+  private cacheUpdateEmbedding(id: string, embedding: number[]): void {
+    if (!this.slimCacheEnabled) return;
+    const e = this.slimCache.get(id);
+    if (e) e.embedding = embedding;
+  }
+
+  private cacheRetract(id: string): void {
+    if (!this.slimCacheEnabled) return;
+    const e = this.slimCache.get(id);
+    if (e) e.retracted = true;
+  }
+
+  private cacheRemove(id: string): void {
+    if (!this.slimCacheEnabled) return;
+    this.slimCache.delete(id);
+  }
+
+  /** Reset cache (used by tests + after timeWarp/bulk operations). */
+  resetSlimCache(): void {
+    this.slimCache.clear();
+    this.slimCachePopulated = false;
+  }
+
   /** Expose the raw database handle for the coordination module. */
   getDb(): Database.Database {
     return this.db;
@@ -306,6 +385,18 @@ export class EngramStore {
       input.memoryType ?? 'unclassified',
     );
 
+    // Add to slim cache (skip if not yet populated — first slim fetch will load it)
+    if (this.slimCachePopulated) {
+      this.cacheAdd({
+        id,
+        agentId: input.agentId,
+        concept: input.concept,
+        embedding: input.embedding ?? null,
+        stage: 'active',
+        retracted: false,
+      });
+    }
+
     return this.getEngram(id)!;
   }
 
@@ -329,6 +420,112 @@ export class EngramStore {
     return (this.db.prepare(query).all(...params) as any[]).map(r => this.rowToEngram(r));
   }
 
+  /**
+   * Slim variant that returns only (id, concept, embedding) — the minimum needed
+   * for the activation pipeline's pre-filter pass (cosine sim + concept-jaccard
+   * survival check). Avoids materializing the content blob, tag JSON, salience
+   * features JSON, etc. for ~10K rows when only ~200 will be deep-scored.
+   *
+   * Why: phase-breakdown spike (2026-05-08) showed the full SELECT * over 10K
+   * engrams costs 440ms on a 17K-engram corpus — 40% of recall latency. Most
+   * of that is row materialization of fields we don't read in the filter pass.
+   */
+  getEngramsByAgentSlim(
+    agentId: string,
+    stage?: EngramStage,
+    includeRetracted: boolean = false
+  ): Array<{ id: string; concept: string; embedding: number[] | null }> {
+    if (this.slimCacheEnabled) {
+      this.ensureSlimCachePopulated();
+      const result: Array<{ id: string; concept: string; embedding: number[] | null }> = [];
+      for (const entry of this.slimCache.values()) {
+        if (entry.agentId !== agentId) continue;
+        if (stage && entry.stage !== stage) continue;
+        if (!includeRetracted && entry.retracted) continue;
+        result.push({ id: entry.id, concept: entry.concept, embedding: entry.embedding });
+      }
+      return result;
+    }
+    // Cache disabled — fall back to direct SQL
+    let query = 'SELECT id, concept, embedding FROM engrams WHERE agent_id = ?';
+    const params: any[] = [agentId];
+
+    if (stage) {
+      query += ' AND stage = ?';
+      params.push(stage);
+    }
+    if (!includeRetracted) {
+      query += ' AND retracted = 0';
+    }
+
+    return (this.db.prepare(query).all(...params) as any[]).map(r => ({
+      id: r.id as string,
+      concept: r.concept as string,
+      embedding: r.embedding ? Array.from(bufferToFloat32Array(r.embedding)) : null,
+    }));
+  }
+
+  /** Slim variant for multi-agent (workspace-scoped) pre-filter. */
+  getEngramsByAgentsSlim(
+    agentIds: string[],
+    stage?: EngramStage,
+    includeRetracted: boolean = false
+  ): Array<{ id: string; concept: string; embedding: number[] | null }> {
+    if (agentIds.length === 0) return [];
+    if (agentIds.length === 1) return this.getEngramsByAgentSlim(agentIds[0], stage, includeRetracted);
+
+    if (this.slimCacheEnabled) {
+      this.ensureSlimCachePopulated();
+      const agentSet = new Set(agentIds);
+      const result: Array<{ id: string; concept: string; embedding: number[] | null }> = [];
+      for (const entry of this.slimCache.values()) {
+        if (!agentSet.has(entry.agentId)) continue;
+        if (stage && entry.stage !== stage) continue;
+        if (!includeRetracted && entry.retracted) continue;
+        result.push({ id: entry.id, concept: entry.concept, embedding: entry.embedding });
+      }
+      return result;
+    }
+
+    const placeholders = agentIds.map(() => '?').join(',');
+    let query = `SELECT id, concept, embedding FROM engrams WHERE agent_id IN (${placeholders})`;
+    const params: any[] = [...agentIds];
+
+    if (stage) {
+      query += ' AND stage = ?';
+      params.push(stage);
+    }
+    if (!includeRetracted) {
+      query += ' AND retracted = 0';
+    }
+
+    return (this.db.prepare(query).all(...params) as any[]).map(r => ({
+      id: r.id as string,
+      concept: r.concept as string,
+      embedding: r.embedding ? Array.from(bufferToFloat32Array(r.embedding)) : null,
+    }));
+  }
+
+  /**
+   * Fetch full Engram rows for a list of IDs. Used after the pre-filter to hydrate
+   * only the survivors that need deep scoring. Chunks IN-clause queries to stay
+   * under SQLITE_LIMIT_VARIABLE_NUMBER (default 999).
+   */
+  getEngramsByIds(ids: string[]): Engram[] {
+    if (ids.length === 0) return [];
+    const CHUNK = 800;
+    const result: Engram[] = [];
+    for (let i = 0; i < ids.length; i += CHUNK) {
+      const chunk = ids.slice(i, i + CHUNK);
+      const placeholders = chunk.map(() => '?').join(',');
+      const rows = this.db.prepare(
+        `SELECT * FROM engrams WHERE id IN (${placeholders})`
+      ).all(...chunk) as any[];
+      for (const r of rows) result.push(this.rowToEngram(r));
+    }
+    return result;
+  }
+
   /**
    * Get engrams across multiple agents (workspace-scoped recall).
    * Used when workspace mode is enabled for hive memory sharing.
@@ -432,6 +629,7 @@ export class EngramStore {
 
   updateStage(id: string, stage: EngramStage): void {
     this.db.prepare('UPDATE engrams SET stage = ? WHERE id = ?').run(stage, id);
+    this.cacheUpdateStage(id, stage);
   }
 
   updateConfidence(id: string, confidence: number): void {
@@ -447,16 +645,19 @@ export class EngramStore {
     } else {
       this.db.prepare('UPDATE engrams SET embedding = ? WHERE id = ?').run(blob, id);
     }
+    this.cacheUpdateEmbedding(id, embedding);
   }
 
   retractEngram(id: string, retractedBy: string | null): void {
     this.db.prepare(`
       UPDATE engrams SET retracted = 1, retracted_by = ?, retracted_at = ? WHERE id = ?
     `).run(retractedBy, new Date().toISOString(), id);
+    this.cacheRetract(id);
   }
 
   deleteEngram(id: string): void {
     this.db.prepare('DELETE FROM engrams WHERE id = ?').run(id);
+    this.cacheRemove(id);
   }
 
   /**