@getplumb/core 0.1.6 → 0.4.0

package/dist/local-store.js

@@ -4,19 +4,111 @@ import { mkdirSync } from 'node:fs';
 import { join, dirname } from 'node:path';
 import { openDb } from './wasm-db.js';
 import { applySchema } from './schema.js';
-import { extractFacts } from './extractor.js';
-import { callLLMWithConfig } from './llm-client.js';
-import { embed } from './embedder.js';
+import { embed, warmEmbedder, warmReranker } from './embedder.js';
 import { formatExchange } from './chunker.js';
 import { searchRawLog } from './raw-log-search.js';
-import { searchFacts } from './fact-search.js';
-import { ExtractionQueue } from './extraction-queue.js';
-import { serializeEmbedding } from './vector-search.js';
+import { serializeEmbedding, deserializeEmbedding } from './vector-search.js';
+/**
+ * Split text into overlapping child chunks for parent-child chunking (T-108).
+ * Target: ~250 chars per chunk with ~50 char overlap.
+ * Prefers sentence boundaries, falls back to word boundaries, hard-cuts at 300 chars max.
+ *
+ * Uses a generator to avoid materializing the full chunk array in memory,
+ * which prevents OOM crashes on large inputs (fix for splitIntoChildren array limit bug).
+ */
+function* splitIntoChildren(text) {
+    const TARGET_SIZE = 250;
+    const OVERLAP = 50;
+    const MAX_SIZE = 300;
+    const SENTENCE_ENDINGS = /[.!?]\s+/g;
+    if (text.length <= TARGET_SIZE) {
+        // Text is already small enough — yield as single child
+        if (text.trim().length > 0)
+            yield text;
+        return;
+    }
+    let pos = 0;
+    while (pos < text.length) {
+        let endPos = Math.min(pos + TARGET_SIZE, text.length);
+        // If we're at the end of the text, take the rest
+        if (endPos >= text.length) {
+            const last = text.slice(pos).trim();
+            if (last.length > 0)
+                yield last;
+            break;
+        }
+        // Try to find a sentence boundary within the target range
+        const segment = text.slice(pos, Math.min(pos + MAX_SIZE, text.length));
+        const sentenceMatches = Array.from(segment.matchAll(SENTENCE_ENDINGS));
+        if (sentenceMatches.length > 0) {
+            // Find the last sentence boundary before TARGET_SIZE
+            let bestMatch = sentenceMatches[0]; // Safe: array is non-empty
+            for (const match of sentenceMatches) {
+                if (match.index !== undefined && match.index <= TARGET_SIZE) {
+                    bestMatch = match;
+                }
+                else {
+                    break;
+                }
+            }
+            if (bestMatch.index !== undefined && bestMatch[0] !== undefined) {
+                endPos = pos + bestMatch.index + bestMatch[0].length;
+            }
+            else {
+                // Fall back to word boundary
+                endPos = findWordBoundary(text, pos, TARGET_SIZE, MAX_SIZE);
+            }
+        }
+        else {
+            // No sentence boundary found — fall back to word boundary
+            endPos = findWordBoundary(text, pos, TARGET_SIZE, MAX_SIZE);
+        }
+        const chunk = text.slice(pos, endPos).trim();
+        if (chunk.length > 0)
+            yield chunk;
+        // Move position forward, with overlap
+        pos = endPos - OVERLAP;
+        if (pos < 0)
+            pos = endPos; // Safety: don't go negative
+    }
+}
+/**
+ * Find a word boundary near the target position.
+ * Prefers breaking at TARGET_SIZE, but will extend up to MAX_SIZE if needed.
+ */
+function findWordBoundary(text, start, targetSize, maxSize) {
+    const targetPos = start + targetSize;
+    const maxPos = Math.min(start + maxSize, text.length);
+    // Look for whitespace near the target position
+    let endPos = targetPos;
+    // First try: find whitespace after targetPos
+    for (let i = targetPos; i < maxPos; i++) {
+        if (/\s/.test(text[i] ?? '')) {
+            endPos = i + 1; // Include the whitespace
+            break;
+        }
+    }
+    // If we hit maxPos without finding whitespace, hard cut at maxPos
+    if (endPos === targetPos && targetPos < maxPos) {
+        endPos = maxPos;
+    }
+    return endPos;
+}
 export class LocalStore {
     #db;
     #userId;
-    #llmConfig;
-    #extractionQueue;
+    // Backlog processor state (T-095: drain loop)
+    #embedDrainStopped = false;
+    #embedDrainPromise = null;
+    #embedIdleMs;
+    // T-103: In-memory embedding cache for vec_raw_log (eliminates ~3,700ms SQLite load on each query)
+    #rawLogEmbeddingCache = [];
+    // FIX 3: WAL checkpoint throttling to prevent unbounded WAL growth
+    #lastCheckpoint = Date.now();
+    #checkpointIntervalMs = 60000; // Checkpoint every minute
+    // FIX 4: Health check to detect stuck drain loops
+    #lastActivityTimestamp = Date.now();
+    #healthCheckInterval = null;
     /** Expose database for plugin use (e.g., NudgeManager) */
     get db() {
         return this.#db;
@@ -25,15 +117,11 @@ export class LocalStore {
     get userId() {
         return this.#userId;
     }
-    /** Expose extraction queue for lifecycle management (start/stop) */
-    get extractionQueue() {
-        return this.#extractionQueue;
-    }
-    constructor(db, userId, llmConfig, extractionQueue) {
+    constructor(db, userId, backlog) {
         this.#db = db;
         this.#userId = userId;
-        this.#llmConfig = llmConfig;
-        this.#extractionQueue = extractionQueue;
+        // Initialize backlog processor config
+        this.#embedIdleMs = backlog?.embedIdleMs ?? 5000;
     }
     /**
      * Create a new LocalStore instance (async factory).
@@ -42,99 +130,38 @@ export class LocalStore {
     static async create(options = {}) {
         const dbPath = options.dbPath ?? join(homedir(), '.plumb', 'memory.db');
         const userId = options.userId ?? 'default';
-        const llmConfig = options.llmConfig;
         mkdirSync(dirname(dbPath), { recursive: true });
         const db = await openDb(dbPath);
         // Enable WAL mode and foreign keys
         db.exec('PRAGMA journal_mode = WAL');
         db.exec('PRAGMA foreign_keys = ON');
         applySchema(db);
-        // Use a mutable cell to hold the store reference (needed for circular dependency)
-        let storeRef = null;
-        // Initialize extraction queue with deferred store lookup
-        const extractFn = (exchange, userId) => {
-            if (!storeRef)
-                throw new Error('Store not initialized');
-            const llmFn = llmConfig
-                ? (prompt) => callLLMWithConfig(prompt, llmConfig)
-                : undefined;
-            return extractFacts(exchange, userId, storeRef, llmFn);
-        };
-        const extractionQueue = options.extractionQueue ?? new ExtractionQueue(extractFn);
-        // Create store and assign to ref
-        const store = new LocalStore(db, userId, llmConfig, extractionQueue);
-        storeRef = store;
-        return store;
-    }
-    async store(fact) {
-        const id = crypto.randomUUID();
-        // Embed concatenated fact text for vector search.
-        const text = `${fact.subject} ${fact.predicate} ${fact.object} ${fact.context ?? ''}`.trim();
-        const embedding = await embed(text);
-        const embeddingJson = serializeEmbedding(embedding);
-        // Begin transaction
-        this.#db.exec('BEGIN');
-        try {
-            // Insert fact
-            const factStmt = this.#db.prepare(`
-        INSERT INTO facts
-          (id, user_id, subject, predicate, object,
-           confidence, decay_rate, timestamp, source_session_id,
-           source_session_label, context)
-        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
-      `);
-            factStmt.bind([
-                id,
-                this.#userId,
-                fact.subject,
-                fact.predicate,
-                fact.object,
-                fact.confidence,
-                fact.decayRate,
-                fact.timestamp.toISOString(),
-                fact.sourceSessionId,
-                fact.sourceSessionLabel ?? null,
-                fact.context ?? null,
-            ]);
-            factStmt.step();
-            factStmt.finalize();
-            // Insert embedding into vec_facts (auto-assigned id).
-            const vecStmt = this.#db.prepare(`INSERT INTO vec_facts(embedding) VALUES (?)`);
-            vecStmt.bind([embeddingJson]);
-            vecStmt.step();
-            vecStmt.finalize();
-            const vecRowid = this.#db.selectValue('SELECT last_insert_rowid()');
-            // Back-fill vec_rowid so fact-search can join without a mapping table.
-            const updateStmt = this.#db.prepare(`UPDATE facts SET vec_rowid = ? WHERE id = ?`);
-            updateStmt.bind([vecRowid, id]);
-            updateStmt.step();
-            updateStmt.finalize();
-            this.#db.exec('COMMIT');
-        }
-        catch (err) {
-            this.#db.exec('ROLLBACK');
-            throw err;
-        }
-        return id;
-    }
-    async search(query, limit = 20) {
-        return searchFacts(this.#db, this.#userId, query, limit);
-    }
-    async delete(id) {
-        // Soft delete only — never hard delete.
-        const stmt = this.#db.prepare(`
-      UPDATE facts SET deleted_at = ? WHERE id = ? AND user_id = ?
+        // Create store
+        const store = new LocalStore(db, userId, options.backlog);
+        // T-096: Warm embedder pipeline to eliminate 365ms cold-start on first query
+        await warmEmbedder();
+        // T-101: Warm reranker pipeline to eliminate ~200ms cold-start on first query
+        // (intentionally loads ~80MB model at init for consistent <250ms query performance)
+        await warmReranker();
+        // T-103/T-108: Load vec_raw_log embeddings for child rows only (eliminates ~3,700ms SQLite load per query)
+        // Child rows have parent_id IS NOT NULL. Parent rows are not embedded (embed_status='no_embed').
+        const rawLogVecStmt = db.prepare(`
+      SELECT v.rowid, v.embedding
+      FROM vec_raw_log v
+      JOIN raw_log r ON r.vec_rowid = v.rowid
+      WHERE r.parent_id IS NOT NULL
     `);
-        stmt.bind([new Date().toISOString(), id, this.#userId]);
-        stmt.step();
-        stmt.finalize();
+        while (rawLogVecStmt.step()) {
+            const row = rawLogVecStmt.get({});
+            store.#rawLogEmbeddingCache.push({
+                rowid: row.rowid,
+                embedding: deserializeEmbedding(row.embedding),
+            });
+        }
+        rawLogVecStmt.finalize();
+        return store;
     }
     async status() {
-        const factStmt = this.#db.prepare(`SELECT COUNT(*) AS c FROM facts WHERE user_id = ? AND deleted_at IS NULL`);
-        factStmt.bind([this.#userId]);
-        factStmt.step();
-        const factCount = factStmt.get(0);
-        factStmt.finalize();
         const rawLogStmt = this.#db.prepare(`SELECT COUNT(*) AS c FROM raw_log WHERE user_id = ?`);
         rawLogStmt.bind([this.#userId]);
         rawLogStmt.step();
@@ -148,7 +175,6 @@ export class LocalStore {
         const pageCount = this.#db.selectValue('PRAGMA page_count');
         const pageSize = this.#db.selectValue('PRAGMA page_size');
         return {
-            factCount,
             rawLogCount,
             lastIngestion: lastIngestionTs !== null ? new Date(lastIngestionTs) : null,
             storageBytes: pageCount * pageSize,
@@ -159,18 +185,16 @@ export class LocalStore {
         const chunkText = formatExchange(exchange);
         // Compute content hash for deduplication (scoped per userId).
         const contentHash = createHash('sha256').update(chunkText).digest('hex');
-        // Embed before opening the DB transaction.
-        const embedding = await embed(chunkText);
-        const embeddingJson = serializeEmbedding(embedding);
         // Attempt insert — catch UNIQUE constraint violations (duplicate content_hash).
         try {
             this.#db.exec('BEGIN');
-            // Insert into raw_log
+            // T-108: Insert parent row (no embedding, no vec_rowid).
             const rawLogStmt = this.#db.prepare(`
         INSERT INTO raw_log
           (id, user_id, session_id, session_label,
-           user_message, agent_response, timestamp, source, chunk_text, chunk_index, content_hash)
-        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+           user_message, agent_response, timestamp, source, chunk_text, chunk_index, content_hash,
+           embed_status, embed_error, embed_model, parent_id)
+        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
       `);
             rawLogStmt.bind([
                 rawLogId,
@@ -184,44 +208,95 @@ export class LocalStore {
                 chunkText,
                 0,
                 contentHash,
+                'no_embed', // Parent is not embedded (T-108)
+                null,
+                null,
+                null, // parent_id=NULL for parent rows
             ]);
             rawLogStmt.step();
             rawLogStmt.finalize();
-            // Insert embedding into vec_raw_log (auto-assigned id).
-            const vecStmt = this.#db.prepare(`INSERT INTO vec_raw_log(embedding) VALUES (?)`);
-            vecStmt.bind([embeddingJson]);
-            vecStmt.step();
-            vecStmt.finalize();
-            const vecRowid = this.#db.selectValue('SELECT last_insert_rowid()');
-            // Back-fill vec_rowid so raw-log-search can join without a mapping table.
-            const updateStmt = this.#db.prepare(`UPDATE raw_log SET vec_rowid = ? WHERE id = ?`);
-            updateStmt.bind([vecRowid, rawLogId]);
-            updateStmt.step();
-            updateStmt.finalize();
+            // T-108: Split parent into child chunks and embed each child.
+            // splitIntoChildren is a generator — iterate lazily to avoid OOM on large inputs.
+            let i = 0;
+            for (const childText of splitIntoChildren(chunkText)) {
+                const childId = crypto.randomUUID();
+                let childEmbedding = null;
+                let childEmbeddingJson = null;
+                let childEmbedStatus = 'pending';
+                let childEmbedError = null;
+                let childEmbedModel = null;
+                // Embed the child chunk
+                try {
+                    childEmbedding = await embed(childText);
+                    childEmbeddingJson = serializeEmbedding(childEmbedding);
+                    childEmbedStatus = 'done';
+                    childEmbedModel = 'Xenova/bge-small-en-v1.5';
+                }
+                catch (err) {
+                    childEmbedStatus = 'failed';
+                    childEmbedError = err instanceof Error ? err.message : String(err);
+                }
+                // Insert child row
+                const childStmt = this.#db.prepare(`
+          INSERT INTO raw_log
+            (id, user_id, session_id, session_label,
+             user_message, agent_response, timestamp, source, chunk_text, chunk_index, content_hash,
+             embed_status, embed_error, embed_model, parent_id)
+          VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+        `);
+                childStmt.bind([
+                    childId,
+                    this.#userId,
+                    exchange.sessionId,
+                    exchange.sessionLabel ?? null,
+                    exchange.userMessage,
+                    exchange.agentResponse,
+                    exchange.timestamp.toISOString(),
+                    exchange.source,
+                    childText,
+                    i, // chunk_index for ordering
+                    null, // No content_hash for children (they don't participate in dedup)
+                    childEmbedStatus,
+                    childEmbedError,
+                    childEmbedModel,
+                    rawLogId, // parent_id points to parent
+                ]);
+                childStmt.step();
+                childStmt.finalize();
+                // Insert child embedding into vec_raw_log if embedding succeeded
+                if (childEmbeddingJson !== null) {
+                    const vecStmt = this.#db.prepare(`INSERT INTO vec_raw_log(embedding) VALUES (?)`);
+                    vecStmt.bind([childEmbeddingJson]);
+                    vecStmt.step();
+                    vecStmt.finalize();
+                    const vecRowid = this.#db.selectValue('SELECT last_insert_rowid()');
+                    // Back-fill vec_rowid on child row
+                    const updateStmt = this.#db.prepare(`UPDATE raw_log SET vec_rowid = ? WHERE id = ?`);
+                    updateStmt.bind([vecRowid, childId]);
+                    updateStmt.step();
+                    updateStmt.finalize();
+                    // T-103: Append child embedding to in-memory cache
+                    this.#rawLogEmbeddingCache.push({ rowid: vecRowid, embedding: childEmbedding });
+                }
+                i++;
+            }
             this.#db.exec('COMMIT');
         }
         catch (err) {
             this.#db.exec('ROLLBACK');
             // Check for SQLite UNIQUE constraint error on content_hash.
             if (err instanceof Error && err.message.includes('UNIQUE constraint')) {
-                // Duplicate content — skip ingestion and fact extraction.
+                // Duplicate content — skip ingestion.
                 return {
                     rawLogId: '',
-                    factsExtracted: 0,
-                    factIds: [],
                     skipped: true,
                 };
             }
             // Re-throw other errors (e.g., real DB issues).
             throw err;
         }
-        // Layer 2: enqueue exchange for batched fact extraction (T-071).
-        // ExtractionQueue handles draining on interval or batch size threshold.
-        this.#extractionQueue.enqueue(exchange, this.#userId);
         return {
             rawLogId,
-            factsExtracted: 0,
-            factIds: [],
         };
     }
     /**
@@ -229,31 +304,16 @@ export class LocalStore {
      * See raw-log-search.ts for the full pipeline description.
      */
     async searchRawLog(query, limit = 10) {
-        return searchRawLog(this.#db, this.#userId, query, limit);
-    }
-    /**
-     * Wait for all queued fact extractions to complete.
-     * Call this before close() to ensure all async work is done.
-     * Delegates to ExtractionQueue.flush().
-     */
-    async drain() {
-        await this.#extractionQueue.flush();
+        // T-103: Pass in-memory embedding cache to searchRawLog (eliminates ~3,700ms SQLite load per query)
+        return searchRawLog(this.#db, this.#userId, query, limit, this.#rawLogEmbeddingCache);
     }
     /**
-     * Re-extract facts for orphaned raw_log chunks (chunks with no corresponding facts).
-     *
-     * This is useful when fact extraction failed during initial ingest (e.g., missing API key,
-     * rate limits, crashes). Re-running the normal seeder won't help because content-hash dedup
-     * skips already-ingested chunks before reaching the extraction phase.
-     *
-     * This method directly calls extractFacts() for each orphaned chunk, bypassing the dedup gate.
-     *
-     * @param throttleMs - Delay between extractions (default 1000ms) to stay under rate limits
-     * @returns Statistics: orphansFound, factsCreated
+     * Export all data for a user (for plumb export command).
+     * Returns raw database rows (no vector data).
      */
-    async reextractOrphans(throttleMs = 1000) {
-        // Query for raw_log entries with no corresponding facts.
-        const stmt = this.#db.prepare(`
+    exportAll(userId) {
+        // Export all raw_log entries (no vector data).
+        const rawLogStmt = this.#db.prepare(`
       SELECT
         id,
         user_id AS userId,
@@ -262,140 +322,171 @@ export class LocalStore {
         user_message AS userMessage,
         agent_response AS agentResponse,
         timestamp,
-        source
+        source,
+        chunk_text AS chunkText,
+        chunk_index AS chunkIndex,
+        content_hash AS contentHash,
+        embed_status AS embedStatus,
+        embed_error AS embedError,
+        embed_model AS embedModel
       FROM raw_log
       WHERE user_id = ?
-        AND NOT EXISTS (
-          SELECT 1 FROM facts
-          WHERE facts.source_session_id = raw_log.session_id
-        )
-      ORDER BY timestamp ASC
+      ORDER BY timestamp DESC
     `);
-        stmt.bind([this.#userId]);
-        const orphanRows = [];
-        while (stmt.step()) {
-            const row = stmt.get({});
-            orphanRows.push(row);
+        rawLogStmt.bind([userId]);
+        const rawLog = [];
+        while (rawLogStmt.step()) {
+            rawLog.push(rawLogStmt.get({}));
         }
-        stmt.finalize();
-        const orphansFound = orphanRows.length;
-        if (orphansFound === 0) {
-            return { orphansFound: 0, factsCreated: 0 };
+        rawLogStmt.finalize();
+        return { rawLog };
+    }
+    /**
+     * Start background backlog processor drain loop (T-095).
+     * Launches continuous async loop for embed backlog.
+     */
+    startBacklogProcessor() {
+        // Start embed drain loop
+        if (this.#embedDrainPromise === null) {
+            this.#embedDrainStopped = false;
+            this.#embedDrainPromise = this.#embedDrainLoop();
         }
-        let factsCreated = 0;
-        for (let i = 0; i < orphanRows.length; i++) {
-            const row = orphanRows[i];
-            if (!row)
-                continue;
-            // Reconstruct MessageExchange from raw_log data
-            const exchange = {
-                userMessage: row.userMessage,
-                agentResponse: row.agentResponse,
-                timestamp: new Date(row.timestamp),
-                source: row.source,
-                sessionId: row.sessionId,
-                ...(row.sessionLabel !== null ? { sessionLabel: row.sessionLabel } : {}),
-            };
-            // Extract facts directly (bypasses ingest dedup gate)
-            try {
-                const llmFn = this.#llmConfig
-                    ? (prompt) => callLLMWithConfig(prompt, this.#llmConfig)
-                    : undefined;
-                const facts = await extractFacts(exchange, this.#userId, this, llmFn);
-                factsCreated += facts.length;
-                console.log(`  ✅ [${i + 1}/${orphansFound}] Re-extracted ${facts.length} fact(s) from session ${row.sessionId}`);
-            }
-            catch (err) {
-                console.error(`  ❌ [${i + 1}/${orphansFound}] Failed to re-extract facts from session ${row.sessionId}:`, err);
+        // FIX 4: Health check - detect runaway loop that isn't processing or stopping
+        if (this.#healthCheckInterval === null) {
+            this.#healthCheckInterval = setInterval(() => {
+                const idleTime = Date.now() - this.#lastActivityTimestamp;
+                const MAX_IDLE_TIME = 300000; // 5 minutes of no activity
+                // If loop is running but idle for too long, force stop
+                if (idleTime > MAX_IDLE_TIME && !this.#embedDrainStopped) {
+                    console.warn(`[plumb] Drain loop idle for ${Math.round(idleTime / 1000)}s, forcing stop`);
+                    void this.stopBacklogProcessor();
+                }
+            }, 60000); // Check every minute
+        }
+    }
+    /**
+     * Stop background backlog processor drain loop (T-095).
+     * Signals loop to stop and awaits in-flight work.
+     */
+    async stopBacklogProcessor() {
+        // FIX 4: Clear health check interval
+        if (this.#healthCheckInterval !== null) {
+            clearInterval(this.#healthCheckInterval);
+            this.#healthCheckInterval = null;
+        }
+        // Signal loop to stop
+        this.#embedDrainStopped = true;
+        // Await drain loop Promise (waits for in-flight work to complete)
+        if (this.#embedDrainPromise !== null) {
+            await this.#embedDrainPromise;
+            this.#embedDrainPromise = null;
+        }
+    }
+    /**
+     * Continuous drain loop for embed backlog (T-095).
+     * Runs as fast as the Worker thread allows, with no artificial throttling.
+     * Only sleeps when the queue is empty.
+     */
+    async #embedDrainLoop() {
+        // FIX 2: Safety counter to detect infinite loops
+        let consecutiveEmptyBatches = 0;
+        const MAX_EMPTY_BATCHES = 1000; // Safety limit: stop after many empty iterations
+        while (!this.#embedDrainStopped) {
+            const processed = await this.#processEmbedBatch();
+            if (processed === 0) {
+                consecutiveEmptyBatches++;
+                // FIX 2: Safety check - if idle too long, verify stop flag
+                if (consecutiveEmptyBatches >= MAX_EMPTY_BATCHES) {
+                    console.warn('[plumb] Embed drain loop: hit safety limit, verifying stop flag');
+                    if (this.#embedDrainStopped)
+                        break;
+                    consecutiveEmptyBatches = 0; // Reset and continue
+                }
+                // Queue is empty — sleep before checking again
+                await new Promise(resolve => setTimeout(resolve, this.#embedIdleMs));
             }
-            // Throttle to stay under rate limits (skip delay after last item)
-            if (i < orphanRows.length - 1) {
-                await new Promise(resolve => setTimeout(resolve, throttleMs));
+            else {
+                consecutiveEmptyBatches = 0;
+                // FIX 4: Update activity timestamp
+                this.#lastActivityTimestamp = Date.now();
             }
+            // If processed > 0: immediately loop to grab the next batch
         }
-        return { orphansFound, factsCreated };
     }
     /**
-     * Get top subjects by fact count (for plumb status command).
-     * Returns subjects ordered by number of facts (non-deleted only).
+     * Process one batch of embed backlog rows (T-095).
+     * Uses Promise.all for parallelism across the batch (embed runs in Worker, no API limits).
+     * Returns count of rows processed.
      */
-    topSubjects(userId, limit = 5) {
+    async #processEmbedBatch() {
+        const BATCH_SIZE = 50; // Large batch — embed is CPU-bound, no rate limit
+        // T-108: Fetch pending child rows only (parent_id IS NOT NULL).
+        // Old parent rows (parent_id IS NULL, embed_status='pending') are left as-is for fallback search.
         const stmt = this.#db.prepare(`
-      SELECT subject, COUNT(*) as count
-      FROM facts
-      WHERE user_id = ? AND deleted_at IS NULL
-      GROUP BY subject
-      ORDER BY count DESC
+      SELECT id, chunk_text FROM raw_log
+      WHERE user_id = ? AND embed_status = 'pending' AND parent_id IS NOT NULL
+      ORDER BY rowid ASC
       LIMIT ?
     `);
-        stmt.bind([userId, limit]);
-        const results = [];
+        stmt.bind([this.#userId, BATCH_SIZE]);
+        const pendingRows = [];
         while (stmt.step()) {
-            results.push(stmt.get({}));
+            pendingRows.push(stmt.get({}));
         }
         stmt.finalize();
-        return results;
-    }
-    /**
-     * Export all data for a user (for plumb export command).
-     * Returns raw database rows (no vector data).
-     * Includes soft-deleted facts for transparency.
-     */
-    exportAll(userId) {
-        // Export all non-deleted facts only (soft-deleted facts are excluded).
-        const factStmt = this.#db.prepare(`
-      SELECT
-        id,
-        user_id AS userId,
-        subject,
-        predicate,
-        object,
-        confidence,
-        decay_rate AS decayRate,
-        timestamp,
-        source_session_id AS sourceSessionId,
-        source_session_label AS sourceSessionLabel,
-        context,
-        deleted_at AS deletedAt
-      FROM facts
-      WHERE user_id = ? AND deleted_at IS NULL
-      ORDER BY timestamp DESC
-    `);
-        factStmt.bind([userId]);
-        const factRows = [];
-        while (factStmt.step()) {
-            factRows.push(factStmt.get({}));
-        }
-        factStmt.finalize();
-        const facts = factRows.map((row) => ({
-            ...row,
-            deleted: false, // All exported facts are non-deleted
+        if (pendingRows.length === 0)
+            return 0;
+        // Process rows concurrently with Promise.all
+        await Promise.all(pendingRows.map(async (row) => {
+            try {
+                const embedding = await embed(row.chunk_text);
+                const embeddingJson = serializeEmbedding(embedding);
+                const embedModel = 'Xenova/bge-small-en-v1.5';
+                // Insert into vec_raw_log (transaction per row for isolation)
+                this.#db.exec('BEGIN');
+                const vecStmt = this.#db.prepare(`INSERT INTO vec_raw_log(embedding) VALUES (?)`);
+                vecStmt.bind([embeddingJson]);
+                vecStmt.step();
+                vecStmt.finalize();
+                const vecRowid = this.#db.selectValue('SELECT last_insert_rowid()');
+                // Update raw_log: embed_status='done', vec_rowid, embed_model
+                const updateStmt = this.#db.prepare(`
+            UPDATE raw_log
+            SET embed_status = 'done', embed_error = NULL, embed_model = ?, vec_rowid = ?
+            WHERE id = ?
+          `);
+                updateStmt.bind([embedModel, vecRowid, row.id]);
+                updateStmt.step();
+                updateStmt.finalize();
+                this.#db.exec('COMMIT');
+                // T-103: Append new embedding to in-memory cache
+                this.#rawLogEmbeddingCache.push({ rowid: vecRowid, embedding });
+            }
+            catch (err) {
+                // Embedding failed — update embed_status='failed' with error
+                const errorMsg = err instanceof Error ? err.message : String(err);
+                const updateStmt = this.#db.prepare(`
+            UPDATE raw_log
+            SET embed_status = 'failed', embed_error = ?
+            WHERE id = ?
+          `);
+                updateStmt.bind([errorMsg, row.id]);
+                updateStmt.step();
+                updateStmt.finalize();
+            }
         }));
-        // Export all raw_log entries (no vector data).
-        const rawLogStmt = this.#db.prepare(`
-      SELECT
-        id,
-        user_id AS userId,
-        session_id AS sessionId,
-        session_label AS sessionLabel,
-        user_message AS userMessage,
-        agent_response AS agentResponse,
-        timestamp,
-        source,
-        chunk_text AS chunkText,
-        chunk_index AS chunkIndex,
-        content_hash AS contentHash
-      FROM raw_log
-      WHERE user_id = ?
-      ORDER BY timestamp DESC
-    `);
-        rawLogStmt.bind([userId]);
-        const rawLog = [];
-        while (rawLogStmt.step()) {
-            rawLog.push(rawLogStmt.get({}));
+        // FIX 3: Periodic WAL checkpoint to prevent unbounded growth
+        const now = Date.now();
+        if (now - this.#lastCheckpoint > this.#checkpointIntervalMs) {
+            try {
+                this.#db.exec('PRAGMA wal_checkpoint(PASSIVE)');
+                this.#lastCheckpoint = now;
+            }
+            catch (e) {
+                console.warn('[plumb] WAL checkpoint failed:', e);
+            }
         }
-        rawLogStmt.finalize();
-        return { facts, rawLog };
+        return pendingRows.length;
     }
     /** Close the database connection. Call when done (e.g. in tests). */
     close() {