@getplumb/core 0.1.0

package/dist/local-store.js

@@ -0,0 +1,292 @@
+import Database from 'better-sqlite3';
+import * as sqliteVec from 'sqlite-vec';
+import { createHash } from 'node:crypto';
+import { homedir } from 'node:os';
+import { mkdirSync } from 'node:fs';
+import { join, dirname } from 'node:path';
+import { applySchema } from './schema.js';
+import { extractFacts } from './extractor.js';
+import { embed } from './embedder.js';
+import { formatExchange } from './chunker.js';
+import { searchRawLog } from './raw-log-search.js';
+import { searchFacts } from './fact-search.js';
+export class LocalStore {
+    #db;
+    #userId;
+    #inFlightExtractions = new Set();
+    /** Expose database for plugin use (e.g., NudgeManager) */
+    get db() {
+        return this.#db;
+    }
+    /** Expose userId for plugin use */
+    get userId() {
+        return this.#userId;
+    }
+    constructor(options = {}) {
+        const dbPath = options.dbPath ?? join(homedir(), '.plumb', 'memory.db');
+        this.#userId = options.userId ?? 'default';
+        mkdirSync(dirname(dbPath), { recursive: true });
+        this.#db = new Database(dbPath);
+        this.#db.pragma('journal_mode = WAL');
+        this.#db.pragma('foreign_keys = ON');
+        // Load sqlite-vec extension — vector operations implemented in T-004.
+        sqliteVec.load(this.#db);
+        applySchema(this.#db);
+    }
+    async store(fact) {
+        const id = crypto.randomUUID();
+        // Embed concatenated fact text for vector search.
+        const text = `${fact.subject} ${fact.predicate} ${fact.object} ${fact.context ?? ''}`.trim();
+        const embedding = await embed(text);
+        const vecBlob = Buffer.from(embedding.buffer);
+        const doInsert = this.#db.transaction(() => {
+            this.#db.prepare(`
+        INSERT INTO facts
+          (id, user_id, subject, predicate, object,
+           confidence, decay_rate, timestamp, source_session_id,
+           source_session_label, context)
+        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+      `).run(id, this.#userId, fact.subject, fact.predicate, fact.object, fact.confidence, fact.decayRate, fact.timestamp.toISOString(), fact.sourceSessionId, fact.sourceSessionLabel ?? null, fact.context ?? null);
+            // Insert embedding into vec_facts (auto-assigned rowid).
+            const vecInfo = this.#db.prepare(`INSERT INTO vec_facts(embedding) VALUES (?)`).run(vecBlob);
+            // Back-fill vec_rowid so fact-search can join without a mapping table.
+            this.#db.prepare(`UPDATE facts SET vec_rowid = ? WHERE id = ?`).run(vecInfo.lastInsertRowid, id);
+        });
+        doInsert();
+        return id;
+    }
+    async search(query, limit = 20) {
+        return searchFacts(this.#db, this.#userId, query, limit);
+    }
+    async delete(id) {
+        // Soft delete only — never hard delete.
+        this.#db.prepare(`
+      UPDATE facts SET deleted_at = ? WHERE id = ? AND user_id = ?
+    `).run(new Date().toISOString(), id, this.#userId);
+    }
+    async status() {
+        const factCount = this.#db.prepare(`SELECT COUNT(*) AS c FROM facts WHERE user_id = ? AND deleted_at IS NULL`).get(this.#userId).c;
+        const rawLogCount = this.#db.prepare(`SELECT COUNT(*) AS c FROM raw_log WHERE user_id = ?`).get(this.#userId).c;
+        const lastIngestionRow = this.#db.prepare(`SELECT MAX(timestamp) AS ts FROM raw_log WHERE user_id = ?`).get(this.#userId);
+        const pageCount = this.#db.pragma('page_count', { simple: true });
+        const pageSize = this.#db.pragma('page_size', { simple: true });
+        return {
+            factCount,
+            rawLogCount,
+            lastIngestion: lastIngestionRow.ts !== null ? new Date(lastIngestionRow.ts) : null,
+            storageBytes: pageCount * pageSize,
+        };
+    }
+    async ingest(exchange) {
+        const rawLogId = crypto.randomUUID();
+        const chunkText = formatExchange(exchange);
+        // Compute content hash for deduplication (scoped per userId).
+        const contentHash = createHash('sha256').update(chunkText).digest('hex');
+        // Embed before opening the synchronous DB transaction.
+        const embedding = await embed(chunkText);
+        const vecBlob = Buffer.from(embedding.buffer);
+        // Layer 1: write raw exchange to raw_log and store vector in vec_raw_log.
+        // vec_raw_log auto-assigns its own rowid; we store it back in raw_log.vec_rowid
+        // so raw-log-search can join the two tables without a separate mapping table.
+        const doInsert = this.#db.transaction(() => {
+            this.#db.prepare(`
+        INSERT INTO raw_log
+          (id, user_id, session_id, session_label,
+           user_message, agent_response, timestamp, source, chunk_text, chunk_index, content_hash)
+        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+      `).run(rawLogId, this.#userId, exchange.sessionId, exchange.sessionLabel ?? null, exchange.userMessage, exchange.agentResponse, exchange.timestamp.toISOString(), exchange.source, chunkText, 0, contentHash);
+            // Insert embedding into sqlite-vec (auto-assigned rowid).
+            const vecInfo = this.#db.prepare(`INSERT INTO vec_raw_log(embedding) VALUES (?)`).run(vecBlob);
+            // Back-fill vec_rowid so raw-log-search can join without a mapping table.
+            this.#db.prepare(`UPDATE raw_log SET vec_rowid = ? WHERE id = ?`).run(vecInfo.lastInsertRowid, rawLogId);
+        });
+        // Attempt insert — catch UNIQUE constraint violations (duplicate content_hash).
+        try {
+            doInsert();
+        }
+        catch (err) {
+            // Check for SQLite UNIQUE constraint error on content_hash.
+            if (err instanceof Error && 'code' in err && err.code === 'SQLITE_CONSTRAINT_UNIQUE') {
+                // Duplicate content — skip ingestion and fact extraction.
+                return {
+                    rawLogId: '',
+                    factsExtracted: 0,
+                    factIds: [],
+                    skipped: true,
+                };
+            }
+            // Re-throw other errors (e.g., real DB issues).
+            throw err;
+        }
+        // Layer 2: fire-and-forget fact extraction — never blocks ingest().
+        // Track the promise so drain() can wait for completion before close().
+        const extractionPromise = extractFacts(exchange, this.#userId, this)
+            .catch((err) => {
+            console.error('[plumb/local-store] Fact extraction failed:', err);
+            return []; // Return empty array on error
+        })
+            .finally(() => {
+            this.#inFlightExtractions.delete(extractionPromise);
+        });
+        this.#inFlightExtractions.add(extractionPromise);
+        return {
+            rawLogId,
+            factsExtracted: 0,
+            factIds: [],
+        };
+    }
+    /**
+     * Hybrid search over raw_log (Layer 1 retrieval).
+     * See raw-log-search.ts for the full pipeline description.
+     */
+    async searchRawLog(query, limit = 10) {
+        return searchRawLog(this.#db, this.#userId, query, limit);
+    }
+    /**
+     * Wait for all in-flight fact extractions to complete.
+     * Call this before close() to ensure all async work is done.
+     */
+    async drain() {
+        if (this.#inFlightExtractions.size === 0)
+            return;
+        await Promise.allSettled(Array.from(this.#inFlightExtractions));
+    }
+    /**
+     * Re-extract facts for orphaned raw_log chunks (chunks with no corresponding facts).
+     *
+     * This is useful when fact extraction failed during initial ingest (e.g., missing API key,
+     * rate limits, crashes). Re-running the normal seeder won't help because content-hash dedup
+     * skips already-ingested chunks before reaching the extraction phase.
+     *
+     * This method directly calls extractFacts() for each orphaned chunk, bypassing the dedup gate.
+     *
+     * @param throttleMs - Delay between extractions (default 1000ms) to stay under rate limits
+     * @returns Statistics: orphansFound, factsCreated
+     */
+    async reextractOrphans(throttleMs = 1000) {
+        // Query for raw_log entries with no corresponding facts.
+        // A session_id may have multiple raw_log chunks, but if ANY chunk has facts,
+        // we skip the entire session. This is conservative but prevents re-extracting
+        // sessions that already have partial facts.
+        const orphanRows = this.#db.prepare(`
+      SELECT
+        id,
+        user_id AS userId,
+        session_id AS sessionId,
+        session_label AS sessionLabel,
+        user_message AS userMessage,
+        agent_response AS agentResponse,
+        timestamp,
+        source
+      FROM raw_log
+      WHERE user_id = ?
+        AND NOT EXISTS (
+          SELECT 1 FROM facts
+          WHERE facts.source_session_id = raw_log.session_id
+        )
+      ORDER BY timestamp ASC
+    `).all(this.#userId);
+        const orphansFound = orphanRows.length;
+        if (orphansFound === 0) {
+            return { orphansFound: 0, factsCreated: 0 };
+        }
+        let factsCreated = 0;
+        for (let i = 0; i < orphanRows.length; i++) {
+            const row = orphanRows[i];
+            if (!row)
+                continue; // Type guard: skip if row is undefined (shouldn't happen)
+            // Reconstruct MessageExchange from raw_log data
+            const exchange = {
+                userMessage: row.userMessage,
+                agentResponse: row.agentResponse,
+                timestamp: new Date(row.timestamp),
+                source: row.source,
+                sessionId: row.sessionId,
+                // Conditionally include sessionLabel only if it's not null (exactOptionalPropertyTypes)
+                ...(row.sessionLabel !== null ? { sessionLabel: row.sessionLabel } : {}),
+            };
+            // Extract facts directly (bypasses ingest dedup gate)
+            try {
+                const facts = await extractFacts(exchange, this.#userId, this);
+                factsCreated += facts.length;
+                console.log(`  ✅ [${i + 1}/${orphansFound}] Re-extracted ${facts.length} fact(s) from session ${row.sessionId}`);
+            }
+            catch (err) {
+                console.error(`  ❌ [${i + 1}/${orphansFound}] Failed to re-extract facts from session ${row.sessionId}:`, err);
+            }
+            // Throttle to stay under rate limits (skip delay after last item)
+            if (i < orphanRows.length - 1) {
+                await new Promise(resolve => setTimeout(resolve, throttleMs));
+            }
+        }
+        return { orphansFound, factsCreated };
+    }
+    /**
+     * Get top subjects by fact count (for plumb status command).
+     * Returns subjects ordered by number of facts (non-deleted only).
+     */
+    topSubjects(userId, limit = 5) {
+        return this.#db.prepare(`
+      SELECT subject, COUNT(*) as count
+      FROM facts
+      WHERE user_id = ? AND deleted_at IS NULL
+      GROUP BY subject
+      ORDER BY count DESC
+      LIMIT ?
+    `).all(userId, limit);
+    }
+    /**
+     * Export all data for a user (for plumb export command).
+     * Returns raw database rows (no vector data).
+     * Includes soft-deleted facts for transparency.
+     */
+    exportAll(userId) {
+        // Export all non-deleted facts only (soft-deleted facts are excluded).
+        const factRows = this.#db.prepare(`
+      SELECT
+        id,
+        user_id AS userId,
+        subject,
+        predicate,
+        object,
+        confidence,
+        decay_rate AS decayRate,
+        timestamp,
+        source_session_id AS sourceSessionId,
+        source_session_label AS sourceSessionLabel,
+        context,
+        deleted_at AS deletedAt
+      FROM facts
+      WHERE user_id = ? AND deleted_at IS NULL
+      ORDER BY timestamp DESC
+    `).all(userId);
+        const facts = factRows.map((row) => ({
+            ...row,
+            deleted: false, // All exported facts are non-deleted
+        }));
+        // Export all raw_log entries (no vector data).
+        const rawLog = this.#db.prepare(`
+      SELECT
+        id,
+        user_id AS userId,
+        session_id AS sessionId,
+        session_label AS sessionLabel,
+        user_message AS userMessage,
+        agent_response AS agentResponse,
+        timestamp,
+        source,
+        chunk_text AS chunkText,
+        chunk_index AS chunkIndex,
+        content_hash AS contentHash
+      FROM raw_log
+      WHERE user_id = ?
+      ORDER BY timestamp DESC
+    `).all(userId);
+        return { facts, rawLog };
+    }
+    /** Close the database connection. Call when done (e.g. in tests). */
+    close() {
+        this.#db.close();
+    }
+}
+//# sourceMappingURL=local-store.js.map

package/dist/local-store.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"local-store.js","sourceRoot":"","sources":["../src/local-store.ts"],"names":[],"mappings":"AAAA,OAAO,QAAQ,MAAM,gBAAgB,CAAC;AACtC,OAAO,KAAK,SAAS,MAAM,YAAY,CAAC;AACxC,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAClC,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACpC,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAG1C,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AACtC,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC9C,OAAO,EAAE,YAAY,EAA2B,MAAM,qBAAqB,CAAC;AAC5E,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AA8C/C,MAAM,OAAO,UAAU;IACZ,GAAG,CAAoB;IACvB,OAAO,CAAS;IAChB,oBAAoB,GAAyB,IAAI,GAAG,EAAE,CAAC;IAEhE,0DAA0D;IAC1D,IAAI,EAAE;QACJ,OAAO,IAAI,CAAC,GAAG,CAAC;IAClB,CAAC;IAED,mCAAmC;IACnC,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAED,YAAY,UAA6B,EAAE;QACzC,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE,EAAE,QAAQ,EAAE,WAAW,CAAC,CAAC;QACxE,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,MAAM,IAAI,SAAS,CAAC;QAE3C,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAEhD,IAAI,CAAC,GAAG,GAAG,IAAI,QAAQ,CAAC,MAAM,CAAC,CAAC;QAChC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAC;QACtC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,mBAAmB,CAAC,CAAC;QAErC,sEAAsE;QACtE,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAEzB,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACxB,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAsB;QAChC,MAAM,EAAE,GAAG,MAAM,CAAC,UAAU,EAAE,CAAC;QAE/B,kDAAkD;QAClD,MAAM,IAAI,GAAG,GAAG,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,OAAO,IAAI,EAAE,EAAE,CAAC,IAAI,EAAE,CAAC;QAC7F,MAAM,SAAS,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC;QACpC,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QAE9C,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,GAAG,EAAE;YACzC,IAAI,CAAC,GAAG,CAAC,OAAO,CAIb;;;;;;OAMF,CAAC,CAAC,GAAG,CACJ,EAAE,EACF,IAAI,CAAC,OAAO,EACZ,IAAI,CAAC,OAAO,EACZ,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,UAAU,EACf,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,EAC5B,IAAI,CAAC,eAAe,EACpB,IAAI,CAAC,kBAAkB,IAAI,IAAI,EAC/B,IAAI,CAAC,OAAO,IAAI,IAAI,CACrB,CAAC;YAEF,yDAAyD;YACzD,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAC9B,6CAA6C,CAC9C,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YAEf,uEAAuE;YACvE,IAAI,CAAC,GAAG,CAAC,OAAO,CACd,6CAA6C,CAC9C,CAAC,GAAG,CAAC,OAAO,CAAC,eAAe,EAAE,EAAE,CAAC,CAAC;QACrC,CAAC,CAAC,CAAC;QAEH,QAAQ,EAAE,CAAC;QAEX,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,KAAK,GAAG,EAAE;QACpC,OAAO,WAAW,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;IAC3D,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,EAAU;QACrB,wCAAwC;QACxC,IAAI,CAAC,GAAG,CAAC,OAAO,CAA2B;;KAE1C,CAAC,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;IACrD,CAAC;IAED,KAAK,CAAC,MAAM;QACV,MAAM,SAAS,GAAI,IAAI,CAAC,GAAG,CAAC,OAAO,CACjC,0EAA0E,CAC3E,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAmB,CAAC,CAAC,CAAC;QAExC,MAAM,WAAW,GAAI,IAAI,CAAC,GAAG,CAAC,OAAO,CACnC,qDAAqD,CACtD,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAmB,CAAC,CAAC,CAAC;QAExC,MAAM,gBAAgB,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CACvC,4DAA4D,CAC7D,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAA0B,CAAC;QAE7C,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,YAAY,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAW,CAAC;QAC5E,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAW,CAAC;QAE1E,OAAO;YACL,SAAS;YACT,WAAW;YACX,aAAa,EAAE,gBAAgB,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,gBAAgB,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI;YAClF,YAAY,EAAE,SAAS,GAAG,QAAQ;SACnC,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,QAAyB;QACpC,MAAM,QAAQ,GAAG,MAAM,CAAC,UAAU,EAAE,CAAC;QACrC,MAAM,SAAS,GAAG,cAAc,CAAC,QAAQ,CAAC,CAAC;QAE3C,8DAA8D;QAC9D,MAAM,WAAW,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAEzE,uDAAuD;QACvD,MAAM,SAAS,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC,CAAC;QACzC,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QAE9C,0EAA0E;QAC1E,gFAAgF;QAChF,8EAA8E;QAC9E,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,GAAG,EAAE;YACzC,IAAI,CAAC,GAAG,CAAC,OAAO,CAGb;;;;;OAKF,CAAC,CAAC,GAAG,CACJ,QAAQ,EACR,IAAI,CAAC,OAAO,EACZ,QAAQ,CAAC,SAAS,EAClB,QAAQ,CAAC,YAAY,IAAI,IAAI,EAC7B,QAAQ,CAAC,WAAW,EACpB,QAAQ,CAAC,aAAa,EACtB,QAAQ,CAAC,SAAS,CAAC,WAAW,EAAE,EAChC,QAAQ,CAAC,MAAM,EACf,SAAS,EACT,CAAC,EACD,WAAW,CACZ,CAAC;YAEF,0DAA0D;YAC1D,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAC9B,+CAA+C,CAChD,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YAEf,0EAA0E;YAC1E,IAAI,CAAC,GAAG,CAAC,OAAO,CACd,+CAA+C,CAChD,CAAC,GAAG,CAAC,OAAO,CAAC,eAAe,EAAE,QAAQ,CAAC,CAAC;QAC3C,CAAC,CAAC,CAAC;QAEH,gFAAgF;QAChF,IAAI,CAAC;YACH,QAAQ,EAAE,CAAC;QACb,CAAC;QAAC,OAAO,GAAY,EAAE,CAAC;YACtB,4DAA4D;YAC5D,IAAI,GAAG,YAAY,KAAK,IAAI,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,IAAI,KAAK,0BAA0B,EAAE,CAAC;gBACrF,0DAA0D;gBAC1D,OAAO;oBACL,QAAQ,EAAE,EAAE;oBACZ,cAAc,EAAE,CAAC;oBACjB,OAAO,EAAE,EAAE;oBACX,OAAO,EAAE,IAAI;iBACd,CAAC;YACJ,CAAC;YACD,gDAAgD;YAChD,MAAM,GAAG,CAAC;QACZ,CAAC;QAED,oEAAoE;QACpE,uEAAuE;QACvE,MAAM,iBAAiB,GAAG,YAAY,CAAC,QAAQ,EAAE,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC;aACjE,KAAK,CAAC,CAAC,GAAY,EAAE,EAAE;YACtB,OAAO,CAAC,KAAK,CAAC,6CAA6C,EAAE,GAAG,CAAC,CAAC;YAClE,OAAO,EAAY,CAAC,CAAC,8BAA8B;QACrD,CAAC,CAAC;aACD,OAAO,CAAC,GAAG,EAAE;YACZ,IAAI,CAAC,oBAAoB,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC;QACtD,CAAC,CAAC,CAAC;QAEL,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC;QAEjD,OAAO;YACL,QAAQ;YACR,cAAc,EAAE,CAAC;YACjB,OAAO,EAAE,EAAE;SACZ,CAAC;IACJ,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,YAAY,CAAC,KAAa,EAAE,KAAK,GAAG,EAAE;QAC1C,OAAO,YAAY,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;IAC5D,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,oBAAoB,CAAC,IAAI,KAAK,CAAC;YAAE,OAAO;QACjD,MAAM,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC,CAAC;IAClE,CAAC;IAED;;;;;;;;;;;OAWG;IACH,KAAK,CAAC,gBAAgB,CAAC,UAAU,GAAG,IAAI;QACtC,yDAAyD;QACzD,6EAA6E;QAC7E,8EAA8E;QAC9E,4CAA4C;QAC5C,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAW;;;;;;;;;;;;;;;;;KAiB7C,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CASjB,CAAC;QAEH,MAAM,YAAY,GAAG,UAAU,CAAC,MAAM,CAAC;QAEvC,IAAI,YAAY,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO,EAAE,YAAY,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,CAAC;QAC9C,CAAC;QAED,IAAI,YAAY,GAAG,CAAC,CAAC;QAErB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3C,MAAM,GAAG,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;YAC1B,IAAI,CAAC,GAAG;gBAAE,SAAS,CAAC,0DAA0D;YAE9E,gDAAgD;YAChD,MAAM,QAAQ,GAAoB;gBAChC,WAAW,EAAE,GAAG,CAAC,WAAW;gBAC5B,aAAa,EAAE,GAAG,CAAC,aAAa;gBAChC,SAAS,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC;gBAClC,MAAM,EAAE,GAAG,CAAC,MAAuD;gBACnE,SAAS,EAAE,GAAG,CAAC,SAAS;gBACxB,wFAAwF;gBACxF,GAAG,CAAC,GAAG,CAAC,YAAY,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,GAAG,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACzE,CAAC;YAEF,sDAAsD;YACtD,IAAI,CAAC;gBACH,MAAM,KAAK,GAAG,MAAM,YAAY,CAAC,QAAQ,EAAE,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;gBAC/D,YAAY,IAAI,KAAK,CAAC,MAAM,CAAC;gBAE7B,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,YAAY,kBAAkB,KAAK,CAAC,MAAM,yBAAyB,GAAG,CAAC,SAAS,EAAE,CAAC,CAAC;YACnH,CAAC;YAAC,OAAO,GAAY,EAAE,CAAC;gBACtB,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,YAAY,6CAA6C,GAAG,CAAC,SAAS,GAAG,EAAE,GAAG,CAAC,CAAC;YACjH,CAAC;YAED,kEAAkE;YAClE,IAAI,CAAC,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC9B,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC,CAAC;YAChE,CAAC;QACH,CAAC;QAED,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,CAAC;IACxC,CAAC;IAED;;;OAGG;IACH,WAAW,CAAC,MAAc,EAAE,KAAK,GAAG,CAAC;QACnC,OAAO,IAAI,CAAC,GAAG,CAAC,OAAO,CAAmB;;;;;;;KAOzC,CAAC,CAAC,GAAG,CAAC,MAAM,EAAE,KAAK,CAA8C,CAAC;IACrE,CAAC;IAED;;;;OAIG;IACH,SAAS,CAAC,MAAc;QACtB,uEAAuE;QACvE,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAW;;;;;;;;;;;;;;;;;KAiB3C,CAAC,CAAC,GAAG,CAAC,MAAM,CAaX,CAAC;QAEH,MAAM,KAAK,GAAc,QAAQ,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YAC9C,GAAG,GAAG;YACN,OAAO,EAAE,KAAK,EAAE,qCAAqC;SACtD,CAAC,CAAC,CAAC;QAEJ,+CAA+C;QAC/C,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAW;;;;;;;;;;;;;;;;KAgBzC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAkB,CAAC;QAEhC,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;IAC3B,CAAC;IAED,qEAAqE;IACrE,KAAK;QACH,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC;IACnB,CAAC;CACF"}

package/dist/raw-log-search.d.ts

@@ -0,0 +1,33 @@
+/**
+ * Raw-log hybrid search (Layer 1 retrieval).
+ *
+ * Pipeline:
+ *   1. BM25 keyword search over all raw_log chunk_text (built at query time)
+ *   2. KNN vector search via sqlite-vec vec_raw_log virtual table
+ *   3. Reciprocal Rank Fusion (RRF, k=60) merges both ranked lists
+ *   4. Recency decay: score *= e^(-0.012 × age_in_days)
+ *   5. Cross-encoder reranker (top-20 candidates → Xenova/ms-marco-MiniLM-L-6-v2)
+ *   6. Return top-k by reranker score (falls back to RRF×decay if reranker fails)
+ *
+ * Search is cross-session: no session filter is applied.
+ * The caller (LocalStore) passes its internal db handle — no separate DB connection.
+ */
+import type Database from 'better-sqlite3';
+export interface RawLogSearchResult {
+    readonly chunk_text: string;
+    readonly session_id: string;
+    readonly session_label: string | null;
+    readonly timestamp: string;
+    /** Final score after RRF × recency_decay × reranker (or RRF × recency if reranker failed). */
+    readonly final_score: number;
+}
+/**
+ * Hybrid search over raw_log.
+ *
+ * @param db      The better-sqlite3 Database instance (with sqlite-vec loaded)
+ * @param userId  Scopes the search to this user's data
+ * @param query   Natural language query string
+ * @param limit   Number of results to return (default 10)
+ */
+export declare function searchRawLog(db: Database.Database, userId: string, query: string, limit?: number): Promise<readonly RawLogSearchResult[]>;
+//# sourceMappingURL=raw-log-search.d.ts.map

package/dist/raw-log-search.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"raw-log-search.d.ts","sourceRoot":"","sources":["../src/raw-log-search.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,KAAK,QAAQ,MAAM,gBAAgB,CAAC;AAe3C,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,8FAA8F;IAC9F,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAC9B;AAsDD;;;;;;;GAOG;AACH,wBAAsB,YAAY,CAChC,EAAE,EAAE,QAAQ,CAAC,QAAQ,EACrB,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,MAAM,EACb,KAAK,SAAK,GACT,OAAO,CAAC,SAAS,kBAAkB,EAAE,CAAC,CAoGxC"}

package/dist/raw-log-search.js

@@ -0,0 +1,137 @@
+/**
+ * Raw-log hybrid search (Layer 1 retrieval).
+ *
+ * Pipeline:
+ *   1. BM25 keyword search over all raw_log chunk_text (built at query time)
+ *   2. KNN vector search via sqlite-vec vec_raw_log virtual table
+ *   3. Reciprocal Rank Fusion (RRF, k=60) merges both ranked lists
+ *   4. Recency decay: score *= e^(-0.012 × age_in_days)
+ *   5. Cross-encoder reranker (top-20 candidates → Xenova/ms-marco-MiniLM-L-6-v2)
+ *   6. Return top-k by reranker score (falls back to RRF×decay if reranker fails)
+ *
+ * Search is cross-session: no session filter is applied.
+ * The caller (LocalStore) passes its internal db handle — no separate DB connection.
+ */
+import { Bm25 } from './bm25.js';
+import { embedQuery, rerankScores } from './embedder.js';
+// RRF constant (standard k=60; higher = less weight on top-1 rank).
+const RRF_K = 60;
+// Recency decay lambda for raw logs (medium — half-life ≈ 58 days).
+const RECENCY_LAMBDA = 0.012;
+// Number of candidates passed to the cross-encoder.
+const RERANK_TOP_K = 20;
+// ─── Helpers ──────────────────────────────────────────────────────────────────
+function ageInDays(timestamp) {
+    return (Date.now() - new Date(timestamp).getTime()) / (1_000 * 60 * 60 * 24);
+}
+function recencyDecay(timestamp) {
+    return Math.exp(-RECENCY_LAMBDA * ageInDays(timestamp));
+}
+/**
+ * Merge two ranked lists via Reciprocal Rank Fusion.
+ * Each list entry is [id, score] ordered by descending relevance (rank 1 = index 0).
+ * Returns a map of id → rrf_score.
+ */
+function rrf(vecRanked, bm25Ranked) {
+    const scores = new Map();
+    for (let rank = 0; rank < vecRanked.length; rank++) {
+        const id = vecRanked[rank]?.[0];
+        if (id === undefined)
+            continue;
+        scores.set(id, (scores.get(id) ?? 0) + 1 / (RRF_K + rank + 1));
+    }
+    for (let rank = 0; rank < bm25Ranked.length; rank++) {
+        const id = bm25Ranked[rank]?.[0];
+        if (id === undefined)
+            continue;
+        scores.set(id, (scores.get(id) ?? 0) + 1 / (RRF_K + rank + 1));
+    }
+    return scores;
+}
+// ─── Public API ───────────────────────────────────────────────────────────────
+/**
+ * Hybrid search over raw_log.
+ *
+ * @param db      The better-sqlite3 Database instance (with sqlite-vec loaded)
+ * @param userId  Scopes the search to this user's data
+ * @param query   Natural language query string
+ * @param limit   Number of results to return (default 10)
+ */
+export async function searchRawLog(db, userId, query, limit = 10) {
+    // ── 1. Fetch all raw_log rows for this user ──────────────────────────────
+    const allRows = db
+        .prepare(`SELECT id, session_id, session_label, timestamp, chunk_text
+       FROM raw_log
+       WHERE user_id = ?
+       ORDER BY timestamp DESC`)
+        .all(userId);
+    if (allRows.length === 0)
+        return [];
+    const idToRow = new Map(allRows.map((r) => [r.id, r]));
+    // ── 2. BM25 search ───────────────────────────────────────────────────────
+    const corpus = allRows.map((r) => r.chunk_text);
+    const bm25 = new Bm25(corpus);
+    const bm25RawScores = bm25.scores(query);
+    const bm25Ranked = allRows
+        .map((r, i) => [r.id, bm25RawScores[i] ?? 0])
+        .sort((a, b) => b[1] - a[1]);
+    // ── 3. Vector search via sqlite-vec ─────────────────────────────────────
+    const queryVec = await embedQuery(query);
+    const queryBlob = Buffer.from(queryVec.buffer);
+    // Fetch enough candidates for RRF merge (all docs for small corpora, top-N otherwise).
+    const vecFetchLimit = Math.min(allRows.length, Math.max(RERANK_TOP_K * 2, limit * 3, 50));
+    // Query vec_raw_log for nearest neighbours; resolve back to raw_log via vec_rowid column.
+    const vecRows = db
+        .prepare(`SELECT rowid, distance FROM vec_raw_log
+       WHERE embedding MATCH ?
+       ORDER BY distance
+       LIMIT ?`)
+        .all(queryBlob, vecFetchLimit);
+    const vecRanked = [];
+    for (const vecRow of vecRows) {
+        const logRow = db
+            .prepare(`SELECT id FROM raw_log WHERE vec_rowid = ? AND user_id = ?`)
+            .get(vecRow.rowid, userId);
+        if (logRow !== undefined) {
+            // sqlite-vec cosine distance is in [0,2] for normalized vectors.
+            // Similarity = 1 − distance gives [−1,1]; in practice ≥ 0 for meaningful matches.
+            vecRanked.push([logRow.id, 1 - vecRow.distance]);
+        }
+    }
+    // ── 4. RRF merge ────────────────────────────────────────────────────────
+    const rrfScores = rrf(vecRanked, bm25Ranked);
+    // ── 5. Recency decay ────────────────────────────────────────────────────
+    const decayedScores = [];
+    for (const [id, rrfScore] of rrfScores) {
+        const row = idToRow.get(id);
+        if (row === undefined)
+            continue;
+        const decay = recencyDecay(row.timestamp);
+        decayedScores.push([id, rrfScore * decay]);
+    }
+    decayedScores.sort((a, b) => b[1] - a[1]);
+    // ── 6. Take top candidates for reranking ────────────────────────────────
+    const candidates = decayedScores.slice(0, Math.max(RERANK_TOP_K, limit));
+    // ── 7. Cross-encoder reranking ──────────────────────────────────────────
+    const passages = candidates.map(([id]) => idToRow.get(id)?.chunk_text ?? '');
+    const rerankerScores = await rerankScores(query, passages);
+    // Detect all-zero fallback (reranker unavailable) → keep RRF×decay order.
+    const hasRerankerSignal = rerankerScores.some((s) => s !== 0);
+    const ranked = candidates.map(([id, rrfDecayScore], i) => ({
+        id,
+        finalScore: hasRerankerSignal ? (rerankerScores[i] ?? 0) : rrfDecayScore,
+    }));
+    ranked.sort((a, b) => b.finalScore - a.finalScore);
+    // ── 8. Build output ─────────────────────────────────────────────────────
+    return ranked.slice(0, limit).map(({ id, finalScore }) => {
+        const row = idToRow.get(id);
+        return {
+            chunk_text: row.chunk_text,
+            session_id: row.session_id,
+            session_label: row.session_label,
+            timestamp: row.timestamp,
+            final_score: finalScore,
+        };
+    });
+}
+//# sourceMappingURL=raw-log-search.js.map

package/dist/raw-log-search.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"raw-log-search.js","sourceRoot":"","sources":["../src/raw-log-search.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAGH,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAEzD,oEAAoE;AACpE,MAAM,KAAK,GAAG,EAAE,CAAC;AAEjB,oEAAoE;AACpE,MAAM,cAAc,GAAG,KAAK,CAAC;AAE7B,oDAAoD;AACpD,MAAM,YAAY,GAAG,EAAE,CAAC;AA4BxB,iFAAiF;AAEjF,SAAS,SAAS,CAAC,SAAiB;IAClC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,IAAI,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,CAAC,GAAG,CAAC,KAAK,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC;AAC/E,CAAC;AAED,SAAS,YAAY,CAAC,SAAiB;IACrC,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,cAAc,GAAG,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC;AAC1D,CAAC;AAED;;;;GAIG;AACH,SAAS,GAAG,CACV,SAAkC,EAClC,UAAmC;IAEnC,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IAEzC,KAAK,IAAI,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,EAAE,CAAC;QACnD,MAAM,EAAE,GAAG,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAChC,IAAI,EAAE,KAAK,SAAS;YAAE,SAAS;QAC/B,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC;IACjE,CAAC;IACD,KAAK,IAAI,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,UAAU,CAAC,MAAM,EAAE,IAAI,EAAE,EAAE,CAAC;QACpD,MAAM,EAAE,GAAG,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QACjC,IAAI,EAAE,KAAK,SAAS;YAAE,SAAS;QAC/B,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC;IACjE,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,iFAAiF;AAEjF;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,EAAqB,EACrB,MAAc,EACd,KAAa,EACb,KAAK,GAAG,EAAE;IAEV,4EAA4E;IAC5E,MAAM,OAAO,GAAG,EAAE;SACf,OAAO,CACN;;;+BAGyB,CAC1B;SACA,GAAG,CAAC,MAAM,CAAC,CAAC;IAEf,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAEpC,MAAM,OAAO,GAAG,IAAI,GAAG,CAAoB,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAE1E,4EAA4E;IAC5E,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;IAChD,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,CAAC;IAC9B,MAAM,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAEzC,MAAM,UAAU,GAA4B,OAAO;SAChD,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAoB,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;SAC9D,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAE/B,2EAA2E;IAC3E,MAAM,QAAQ,GAAG,MAAM,UAAU,CAAC,KAAK,CAAC,CAAC;IACzC,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IAE/C,uFAAuF;IACvF,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,YAAY,GAAG,CAAC,EAAE,KAAK,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;IAE1F,0FAA0F;IAC1F,MAAM,OAAO,GAAG,EAAE;SACf,OAAO,CACN;;;eAGS,CACV;SACA,GAAG,CAAC,SAAS,EAAE,aAAa,CAAC,CAAC;IAIjC,MAAM,SAAS,GAA4B,EAAE,CAAC;IAC9C,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,MAAM,MAAM,GAAG,EAAE;aACd,OAAO,CACN,4DAA4D,CAC7D;aACA,GAAG,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QAC7B,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACzB,iEAAiE;YACjE,kFAAkF;YAClF,SAAS,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC;QACnD,CAAC;IACH,CAAC;IAED,2EAA2E;IAC3E,MAAM,SAAS,GAAG,GAAG,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;IAE7C,2EAA2E;IAC3E,MAAM,aAAa,GAA4B,EAAE,CAAC;IAClD,KAAK,MAAM,CAAC,EAAE,EAAE,QAAQ,CAAC,IAAI,SAAS,EAAE,CAAC;QACvC,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC5B,IAAI,GAAG,KAAK,SAAS;YAAE,SAAS;QAChC,MAAM,KAAK,GAAG,YAAY,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAC1C,aAAa,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAC;IAC7C,CAAC;IACD,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAE1C,2EAA2E;IAC3E,MAAM,UAAU,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,YAAY,EAAE,KAAK,CAAC,CAAC,CAAC;IAEzE,2EAA2E;IAC3E,MAAM,QAAQ,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,UAAU,IAAI,EAAE,CAAC,CAAC;IAC7E,MAAM,cAAc,GAAG,MAAM,YAAY,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;IAE3D,0EAA0E;IAC1E,MAAM,iBAAiB,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;IAE9D,MAAM,MAAM,GAA8C,UAAU,CAAC,GAAG,CACtE,CAAC,CAAC,EAAE,EAAE,aAAa,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;QAC3B,EAAE;QACF,UAAU,EAAE,iBAAiB,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa;KACzE,CAAC,CACH,CAAC;IAEF,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC;IAEnD,2EAA2E;IAC3E,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE;QACvD,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,EAAE,CAAE,CAAC;QAC7B,OAAO;YACL,UAAU,EAAE,GAAG,CAAC,UAAU;YAC1B,UAAU,EAAE,GAAG,CAAC,UAAU;YAC1B,aAAa,EAAE,GAAG,CAAC,aAAa;YAChC,SAAS,EAAE,GAAG,CAAC,SAAS;YACxB,WAAW,EAAE,UAAU;SACxB,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC"}

package/dist/read-path.d.ts

@@ -0,0 +1,60 @@
+/**
+ * Read path — Layer 1 + Layer 2 retrieval, scoring, and tiering.
+ *
+ * buildMemoryContext() is called before every agent response. It queries both
+ * memory layers in parallel, re-scores facts with decay, tiers by confidence
+ * band, and returns a MemoryContext ready for formatContextBlock().
+ *
+ * Search is always cross-session — no session filter is applied.
+ */
+import type { SearchResult, Fact } from './types.js';
+import type { RawLogSearchResult } from './local-store.js';
+export interface ScoredFact {
+    readonly fact: Fact;
+    /** Decay-adjusted score in [0,1]. */
+    readonly score: number;
+    readonly ageInDays: number;
+}
+export interface RawChunk {
+    readonly chunkText: string;
+    readonly sessionId: string;
+    readonly sessionLabel: string | null;
+    readonly timestamp: Date;
+    /** Final score from hybrid search (RRF × decay × reranker). */
+    readonly score: number;
+}
+export interface MemoryContext {
+    readonly highConfidence: ScoredFact[];
+    readonly mediumConfidence: ScoredFact[];
+    readonly lowConfidence: ScoredFact[];
+    readonly relatedConversations: RawChunk[];
+}
+export interface ReadPathOptions {
+    /** Max facts returned per confidence tier. Default: 5. */
+    maxFactsPerTier?: number;
+    /** Max raw log chunks returned. Default: 3. */
+    maxRawChunks?: number;
+    /** Point-in-time for decay computation. Default: new Date(). */
+    now?: Date;
+}
+/**
+ * Minimal store interface required by the read path.
+ * LocalStore satisfies this; tests can pass a mock.
+ */
+export interface ReadPathStore {
+    search(query: string, limit?: number): Promise<readonly SearchResult[]>;
+    searchRawLog(query: string, limit?: number): Promise<readonly RawLogSearchResult[]>;
+}
+/**
+ * Build a structured MemoryContext for a given query.
+ *
+ * Queries Layer 1 (raw log hybrid search) and Layer 2 (fact store) in parallel.
+ * Facts are re-scored with scoreFact() and tiered by confidence band.
+ * Raw chunks are returned in hybrid-search score order.
+ *
+ * @param query   Natural language query (the incoming user message or context).
+ * @param store   Any store implementing ReadPathStore (typically LocalStore).
+ * @param options Optional limits and now-override.
+ */
+export declare function buildMemoryContext(query: string, store: ReadPathStore, options?: ReadPathOptions): Promise<MemoryContext>;
+//# sourceMappingURL=read-path.d.ts.map

package/dist/read-path.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"read-path.d.ts","sourceRoot":"","sources":["../src/read-path.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAGH,OAAO,KAAK,EAAE,YAAY,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AACrD,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AAI3D,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC;IACpB,qCAAqC;IACrC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B;AAED,MAAM,WAAW,QAAQ;IACvB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,QAAQ,CAAC,SAAS,EAAE,IAAI,CAAC;IACzB,+DAA+D;IAC/D,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,cAAc,EAAE,UAAU,EAAE,CAAC;IACtC,QAAQ,CAAC,gBAAgB,EAAE,UAAU,EAAE,CAAC;IACxC,QAAQ,CAAC,aAAa,EAAE,UAAU,EAAE,CAAC;IACrC,QAAQ,CAAC,oBAAoB,EAAE,QAAQ,EAAE,CAAC;CAC3C;AAED,MAAM,WAAW,eAAe;IAC9B,0DAA0D;IAC1D,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,+CAA+C;IAC/C,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,gEAAgE;IAChE,GAAG,CAAC,EAAE,IAAI,CAAC;CACZ;AAED;;;GAGG;AACH,MAAM,WAAW,aAAa;IAC5B,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,YAAY,EAAE,CAAC,CAAC;IACxE,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,kBAAkB,EAAE,CAAC,CAAC;CACrF;AASD;;;;;;;;;;GAUG;AACH,wBAAsB,kBAAkB,CACtC,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,aAAa,EACpB,OAAO,CAAC,EAAE,eAAe,GACxB,OAAO,CAAC,aAAa,CAAC,CAqDxB"}

package/dist/read-path.js

@@ -0,0 +1,76 @@
+/**
+ * Read path — Layer 1 + Layer 2 retrieval, scoring, and tiering.
+ *
+ * buildMemoryContext() is called before every agent response. It queries both
+ * memory layers in parallel, re-scores facts with decay, tiers by confidence
+ * band, and returns a MemoryContext ready for formatContextBlock().
+ *
+ * Search is always cross-session — no session filter is applied.
+ */
+import { scoreFact } from './scorer.js';
+// ─── Confidence band boundaries ───────────────────────────────────────────────
+const HIGH_THRESHOLD = 0.7;
+const LOW_THRESHOLD = 0.3;
+// ─── Public API ───────────────────────────────────────────────────────────────
+/**
+ * Build a structured MemoryContext for a given query.
+ *
+ * Queries Layer 1 (raw log hybrid search) and Layer 2 (fact store) in parallel.
+ * Facts are re-scored with scoreFact() and tiered by confidence band.
+ * Raw chunks are returned in hybrid-search score order.
+ *
+ * @param query   Natural language query (the incoming user message or context).
+ * @param store   Any store implementing ReadPathStore (typically LocalStore).
+ * @param options Optional limits and now-override.
+ */
+export async function buildMemoryContext(query, store, options) {
+    const maxFactsPerTier = options?.maxFactsPerTier ?? 5;
+    const maxRawChunks = options?.maxRawChunks ?? 3;
+    const now = options?.now ?? new Date();
+    // Fetch more candidates than needed so tiering has enough to fill each band.
+    const factCandidateLimit = maxFactsPerTier * 3 * 3; // 3 tiers × 3× headroom
+    const rawCandidateLimit = maxRawChunks * 3;
+    // ── Query Layer 1 and Layer 2 in parallel ─────────────────────────────────
+    const [searchResults, rawLogResults] = await Promise.all([
+        store.search(query, factCandidateLimit),
+        store.searchRawLog(query, rawCandidateLimit),
+    ]);
+    // ── Re-score facts with decay ──────────────────────────────────────────────
+    const scoredFacts = searchResults.map((result) => {
+        const { score } = scoreFact(result.fact, now);
+        const ageInDays = (now.getTime() - result.fact.timestamp.getTime()) / (1_000 * 60 * 60 * 24);
+        return { fact: result.fact, score, ageInDays };
+    });
+    // Sort by score descending.
+    scoredFacts.sort((a, b) => b.score - a.score);
+    // ── Tier facts into confidence bands ──────────────────────────────────────
+    const highConfidence = [];
+    const mediumConfidence = [];
+    const lowConfidence = [];
+    for (const sf of scoredFacts) {
+        if (sf.score > HIGH_THRESHOLD) {
+            if (highConfidence.length < maxFactsPerTier)
+                highConfidence.push(sf);
+        }
+        else if (sf.score >= LOW_THRESHOLD) {
+            if (mediumConfidence.length < maxFactsPerTier)
+                mediumConfidence.push(sf);
+        }
+        else {
+            if (lowConfidence.length < maxFactsPerTier)
+                lowConfidence.push(sf);
+        }
+    }
+    // ── Build raw chunks (already ranked by hybrid search score) ──────────────
+    const relatedConversations = rawLogResults
+        .slice(0, maxRawChunks)
+        .map((r) => ({
+        chunkText: r.chunk_text,
+        sessionId: r.session_id,
+        sessionLabel: r.session_label,
+        timestamp: new Date(r.timestamp),
+        score: r.final_score,
+    }));
+    return { highConfidence, mediumConfidence, lowConfidence, relatedConversations };
+}
+//# sourceMappingURL=read-path.js.map

package/dist/read-path.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"read-path.js","sourceRoot":"","sources":["../src/read-path.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AA+CxC,iFAAiF;AAEjF,MAAM,cAAc,GAAG,GAAG,CAAC;AAC3B,MAAM,aAAa,GAAG,GAAG,CAAC;AAE1B,iFAAiF;AAEjF;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,KAAa,EACb,KAAoB,EACpB,OAAyB;IAEzB,MAAM,eAAe,GAAG,OAAO,EAAE,eAAe,IAAI,CAAC,CAAC;IACtD,MAAM,YAAY,GAAG,OAAO,EAAE,YAAY,IAAI,CAAC,CAAC;IAChD,MAAM,GAAG,GAAG,OAAO,EAAE,GAAG,IAAI,IAAI,IAAI,EAAE,CAAC;IAEvC,6EAA6E;IAC7E,MAAM,kBAAkB,GAAG,eAAe,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,wBAAwB;IAC5E,MAAM,iBAAiB,GAAG,YAAY,GAAG,CAAC,CAAC;IAE3C,6EAA6E;IAC7E,MAAM,CAAC,aAAa,EAAE,aAAa,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QACvD,KAAK,CAAC,MAAM,CAAC,KAAK,EAAE,kBAAkB,CAAC;QACvC,KAAK,CAAC,YAAY,CAAC,KAAK,EAAE,iBAAiB,CAAC;KAC7C,CAAC,CAAC;IAEH,8EAA8E;IAC9E,MAAM,WAAW,GAAiB,aAAa,CAAC,GAAG,CAAC,CAAC,MAAoB,EAAE,EAAE;QAC3E,MAAM,EAAE,KAAK,EAAE,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAC9C,MAAM,SAAS,GACb,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC,GAAG,CAAC,KAAK,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC;QAC7E,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;IACjD,CAAC,CAAC,CAAC;IAEH,4BAA4B;IAC5B,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IAE9C,6EAA6E;IAC7E,MAAM,cAAc,GAAiB,EAAE,CAAC;IACxC,MAAM,gBAAgB,GAAiB,EAAE,CAAC;IAC1C,MAAM,aAAa,GAAiB,EAAE,CAAC;IAEvC,KAAK,MAAM,EAAE,IAAI,WAAW,EAAE,CAAC;QAC7B,IAAI,EAAE,CAAC,KAAK,GAAG,cAAc,EAAE,CAAC;YAC9B,IAAI,cAAc,CAAC,MAAM,GAAG,eAAe;gBAAE,cAAc,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACvE,CAAC;aAAM,IAAI,EAAE,CAAC,KAAK,IAAI,aAAa,EAAE,CAAC;YACrC,IAAI,gBAAgB,CAAC,MAAM,GAAG,eAAe;gBAAE,gBAAgB,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC3E,CAAC;aAAM,CAAC;YACN,IAAI,aAAa,CAAC,MAAM,GAAG,eAAe;gBAAE,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACrE,CAAC;IACH,CAAC;IAED,6EAA6E;IAC7E,MAAM,oBAAoB,GAAe,aAAa;SACnD,KAAK,CAAC,CAAC,EAAE,YAAY,CAAC;SACtB,GAAG,CAAC,CAAC,CAAqB,EAAE,EAAE,CAAC,CAAC;QAC/B,SAAS,EAAE,CAAC,CAAC,UAAU;QACvB,SAAS,EAAE,CAAC,CAAC,UAAU;QACvB,YAAY,EAAE,CAAC,CAAC,aAAa;QAC7B,SAAS,EAAE,IAAI,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC;QAChC,KAAK,EAAE,CAAC,CAAC,WAAW;KACrB,CAAC,CAAC,CAAC;IAEN,OAAO,EAAE,cAAc,EAAE,gBAAgB,EAAE,aAAa,EAAE,oBAAoB,EAAE,CAAC;AACnF,CAAC"}