conductor-remote 1.40.0 → 1.42.0

package/dist-node/src/search.js

@@ -0,0 +1,333 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { DatabaseSync } from 'node:sqlite';
+import { parseMessage } from "./transcript.js";
+/**
+ * Full-text search over the chat history, in a sidecar DB the relay owns.
+ *
+ * Two facts decide the whole shape of this file.
+ *
+ * **Prose is 1.2% of the transcript.** Measured over this Mac's 1.7M
+ * `session_messages` rows (3,106 MB of `content`): tool_result output is 799 MB,
+ * `type:"system"` frames 522 MB, tool_use arguments 162 MB, thinking 77 MB — and
+ * what a person would ever search for, the assistant's own words plus the prompts
+ * they typed, is **38 MB**. So this indexes prose only. A grep of the raw column
+ * would be a 3 GB scan to search 38 MB, and it would rank a file dump the agent
+ * happened to `cat` above the sentence that explained the decision.
+ *
+ * **`node:sqlite` ships FTS5.** Porter stemming, `bm25()`, `snippet()`, `NEAR()`
+ * all work on the bundled SQLite (3.51.2), so the index costs no runtime
+ * dependency — which the tarball rule requires (see CLAUDE.md ▸ Traps). Measured
+ * on the full history: 7.6s to build, 111,079 chunks, queries in 1–7ms.
+ *
+ * The index is **never** written into `conductor.db`. That handle is read-only and
+ * stays that way; this opens its own file under the relay's state dir, and it is
+ * disposable — delete it and the next start rebuilds it.
+ */
+/** Bump to force a rebuild: a tokenizer or extraction change makes every stored chunk wrong. */
+const SCHEMA_VERSION = 1;
+/**
+ * Source rows advanced per tick. The cursor moves by *scanned* rowid rather than
+ * matched rowid, so a caught-up index re-scans nothing — get that wrong and every
+ * idle poll re-reads the 3 GB tail looking for rows it already rejected.
+ */
+const WINDOW_ROWS = 4000;
+/** Yield to the event loop between batches: the backfill is ~19ms of blocking work per window. */
+const BACKFILL_PAUSE_MS = 5;
+/** Once caught up, look for new messages at about the rate a chat produces them. */
+const IDLE_POLL_MS = 15_000;
+/** A pathological single message can't be allowed to dominate the index. */
+const MAX_CHUNK_CHARS = 64_000;
+/** How many chunks a query ranks before they are folded into workspaces. */
+const CHUNK_LIMIT = 300;
+/**
+ * Snippet highlight markers. Control characters rather than brackets: they survive
+ * JSON, they need no escaping on the way to the phone, and no transcript contains
+ * them, so `web/src/lib/format.ts` can split on them without a parser. The client
+ * must not render them literally — see `splitSnippet`.
+ */
+export const HIT_OPEN = '\u0001';
+export const HIT_CLOSE = '\u0002';
+/**
+ * Turn a phone query into an FTS5 MATCH expression.
+ *
+ * Every token is quoted, because FTS5 reads `-`, `*`, `:`, `(`, `AND`, `NEAR` and
+ * friends as syntax: an unquoted apostrophe or hyphen is a *parse error*, not a
+ * poor result, so a raw query would fail rather than under-match. Tokens are OR'd
+ * because someone reaching for a workspace on a phone is recalling it, not
+ * filtering it — BM25 is what puts the message matching four of four words above
+ * the one matching one, and requiring all four would return nothing whenever a
+ * single word is misremembered.
+ *
+ * The last token gets a prefix `*` so search-as-you-type matches mid-word, but only
+ * from three characters: `"a"*` matches a large fraction of the index and would
+ * spend the whole query budget on a keystroke that means nothing yet.
+ */
+export function matchQuery(raw) {
+    const tokens = raw.toLowerCase().match(/[\p{L}\p{N}_]+/gu);
+    if (!tokens?.length)
+        return null;
+    const terms = tokens.map(t => `"${t}"`);
+    const last = tokens[tokens.length - 1];
+    if (!/\s$/.test(raw) && last.length >= 3)
+        terms[terms.length - 1] = `"${last}"*`;
+    return terms.join(' OR ');
+}
+/** The tokens `matchQuery` will search for — what a caller matches names against. */
+export function queryTokens(raw) {
+    return raw.toLowerCase().match(/[\p{L}\p{N}_]+/gu) ?? [];
+}
+export class SearchIndex {
+    source;
+    file;
+    db = null;
+    openError = null;
+    cursor = 0;
+    caughtUp = false;
+    timer = null;
+    sourceMax = 0;
+    constructor(source, file) {
+        this.source = source;
+        this.file = file;
+    }
+    /** Open (or rebuild) the sidecar and start indexing in the background. */
+    start() {
+        try {
+            this.open();
+        }
+        catch (err) {
+            // A search index is a convenience; failing to open one must never stop the relay
+            // serving state, transcripts or sends. Report it on /api/search instead.
+            this.openError = err instanceof Error ? err.message : String(err);
+            console.warn(`⚠ search index unavailable (${this.openError}) — /api/search will report it`);
+            return;
+        }
+        this.schedule(0);
+    }
+    stop() {
+        if (this.timer)
+            clearTimeout(this.timer);
+        this.timer = null;
+    }
+    open() {
+        fs.mkdirSync(path.dirname(this.file), { recursive: true });
+        const db = new DatabaseSync(this.file);
+        db.exec('PRAGMA journal_mode = WAL');
+        db.exec('PRAGMA synchronous = NORMAL');
+        // A dev relay on another port shares this file with the LaunchAgent's. WAL lets them
+        // both read; the writer that loses waits rather than throwing away its batch, and a
+        // tick that still fails is retried by the scheduler with nothing lost (the cursor
+        // only advances on commit).
+        db.exec('PRAGMA busy_timeout = 5000');
+        db.exec('CREATE TABLE IF NOT EXISTS meta (k TEXT PRIMARY KEY, v TEXT NOT NULL)');
+        const version = Number(this.readMeta(db, 'version') ?? 0);
+        if (version !== SCHEMA_VERSION) {
+            db.exec('DROP TABLE IF EXISTS chunks');
+            db.exec(`
+				CREATE VIRTUAL TABLE chunks USING fts5(
+					body,
+					session_id UNINDEXED,
+					src_rowid UNINDEXED,
+					role UNINDEXED,
+					at UNINDEXED,
+					tokenize='porter unicode61'
+				)
+			`);
+            db.prepare('INSERT OR REPLACE INTO meta(k, v) VALUES (?, ?)').run('version', String(SCHEMA_VERSION));
+            db.prepare('INSERT OR REPLACE INTO meta(k, v) VALUES (?, ?)').run('cursor', '0');
+            if (version)
+                console.log(`search index schema ${version} → ${SCHEMA_VERSION}, rebuilding`);
+        }
+        this.db = db;
+        this.cursor = Number(this.readMeta(db, 'cursor') ?? 0);
+    }
+    readMeta(db, key) {
+        const row = db.prepare('SELECT v FROM meta WHERE k = ?').get(key);
+        return row?.v ?? null;
+    }
+    schedule(ms) {
+        if (this.timer)
+            clearTimeout(this.timer);
+        this.timer = setTimeout(() => {
+            let more = false;
+            try {
+                more = this.tick();
+            }
+            catch (err) {
+                console.warn(`⚠ search index tick failed: ${err instanceof Error ? err.message : err}`);
+            }
+            this.schedule(more ? BACKFILL_PAUSE_MS : IDLE_POLL_MS);
+        }, ms);
+        this.timer.unref?.();
+    }
+    /**
+     * Index one window of source rows. Returns true while there is more to do.
+     *
+     * The window is picked by rowid *before* the prose filter runs, so the cursor
+     * advances past rows that hold nothing worth indexing. Filtering first and
+     * advancing to the last match instead would leave a caught-up index re-scanning
+     * every tool_result between the last prose row and the end of the table, every
+     * poll, forever.
+     */
+    tick() {
+        const db = this.db;
+        if (!db)
+            return false;
+        const window = this.source.query('SELECT rowid FROM session_messages WHERE rowid > ? ORDER BY rowid LIMIT ?', [this.cursor, WINDOW_ROWS]);
+        if (!window.length) {
+            this.caughtUp = true;
+            return false;
+        }
+        const end = window[window.length - 1].rowid;
+        // Only rows that can hold prose: a plain-text prompt, or a frame carrying a text
+        // block. Everything else is tool plumbing and 98.8% of the bytes.
+        const rows = this.source.query(`SELECT rowid, id, session_id, role, content, full_message, created_at, sent_at, queue_order
+			 FROM session_messages
+			 WHERE rowid > ? AND rowid <= ? AND session_id IS NOT NULL
+			   AND (role = 'user' OR content LIKE '%"type":"text"%')
+			 ORDER BY rowid`, [this.cursor, end]);
+        const insert = db.prepare('INSERT INTO chunks(body, session_id, src_rowid, role, at) VALUES (?, ?, ?, ?, ?)');
+        db.exec('BEGIN');
+        try {
+            for (const row of rows) {
+                // Reuse the transcript parser rather than a second JSON walk: it already knows
+                // that text inside a `type:"user"` frame is injected context and not the user's
+                // words, and indexing something the chat view would never show is how a search
+                // result becomes impossible to find once you open it.
+                for (const entry of parseMessage(row, null)) {
+                    if (entry.role !== 'user' && entry.role !== 'assistant')
+                        continue;
+                    const body = entry.text.trim();
+                    if (!body)
+                        continue;
+                    insert.run(body.slice(0, MAX_CHUNK_CHARS), row.session_id, row.rowid, entry.role, entry.ts);
+                }
+            }
+            db.prepare('INSERT OR REPLACE INTO meta(k, v) VALUES (?, ?)').run('cursor', String(end));
+            db.exec('COMMIT');
+        }
+        catch (err) {
+            db.exec('ROLLBACK');
+            throw err;
+        }
+        this.cursor = end;
+        return true;
+    }
+    status() {
+        if (this.openError)
+            return { chunks: 0, ready: false, progress: 0, error: this.openError };
+        const db = this.db;
+        if (!db)
+            return { chunks: 0, ready: false, progress: 0 };
+        const chunks = Number(db.prepare('SELECT COUNT(*) c FROM chunks').get().c);
+        if (this.caughtUp)
+            return { chunks, ready: true, progress: 1 };
+        // Only re-read the source's high-water mark while backfilling; it costs a query
+        // and the answer only matters for the progress bar.
+        if (!this.sourceMax) {
+            const max = this.source.query('SELECT MAX(rowid) m FROM session_messages')[0]?.m;
+            this.sourceMax = max ?? 0;
+        }
+        const progress = this.sourceMax ? Math.min(1, this.cursor / this.sourceMax) : 0;
+        return { chunks, ready: false, progress };
+    }
+    /** Top matching chunks, best first. Empty when the query has no searchable tokens. */
+    search(raw, limit = CHUNK_LIMIT) {
+        const db = this.db;
+        if (!db)
+            return [];
+        const match = matchQuery(raw);
+        if (!match)
+            return [];
+        let rows;
+        try {
+            rows = db
+                .prepare(`SELECT session_id, src_rowid, role, at, -bm25(chunks) AS score,
+					        snippet(chunks, 0, ?, ?, '…', 24) AS snippet
+					 FROM chunks WHERE chunks MATCH ? ORDER BY bm25(chunks) LIMIT ?`)
+                .all(HIT_OPEN, HIT_CLOSE, match, limit);
+        }
+        catch (err) {
+            // A MATCH that still fails to parse is a bug in matchQuery, not user error —
+            // report it rather than showing an empty result that looks like "no matches".
+            throw new Error(`search failed for ${JSON.stringify(match)}: ${err instanceof Error ? err.message : err}`);
+        }
+        return rows.map(r => ({
+            sessionId: r.session_id,
+            srcRowid: Number(r.src_rowid),
+            role: r.role === 'user' ? 'user' : 'assistant',
+            at: r.at,
+            score: Number(r.score),
+            snippet: r.snippet
+        }));
+    }
+}
+const SNIPPETS_PER_RESULT = 3;
+/**
+ * Fold chunk hits up into workspaces.
+ *
+ * Chunk-level results are the wrong unit here: one long conversation produces a
+ * dozen and buries every other workspace. What to do with those dozen is the whole
+ * ranking decision, and it was measured rather than guessed — searching this Mac's
+ * history for "removing adding lamp manual", where the right answer is a chat that
+ * says "Add by name is gone. Removed the form":
+ *
+ *   summing every hit   → right answer ranks 9th
+ *   best single hit     → 5th
+ *   sum of the top 3    → 5th, and steadier across other queries
+ *
+ * Summing everything ranks by *volume*: a 32-message conversation about lamps beat
+ * the four messages that actually removed the feature. So only the best
+ * `SNIPPETS_PER_RESULT` hits score, which caps what repetition can buy and makes
+ * the number mean something the user can check — the score is exactly the strength
+ * of the snippets shown under the row. `hits` still counts them all.
+ *
+ * Hits whose session no longer resolves to a workspace are dropped; a result nobody
+ * can open is worse than one fewer result.
+ */
+export function foldHits(hits, resolve) {
+    const byWorkspace = new Map();
+    for (const hit of hits) {
+        const workspace = resolve(hit.sessionId);
+        if (!workspace)
+            continue;
+        let entry = byWorkspace.get(workspace.id);
+        if (!entry) {
+            entry = {
+                workspace,
+                sessionId: null,
+                hits: 0,
+                score: 0,
+                at: null,
+                snippets: [],
+                byName: false,
+                bestBySession: new Map()
+            };
+            byWorkspace.set(workspace.id, entry);
+        }
+        entry.hits++;
+        if (!entry.at || hit.at > entry.at)
+            entry.at = hit.at;
+        entry.bestBySession.set(hit.sessionId, Math.max(entry.bestBySession.get(hit.sessionId) ?? 0, hit.score));
+        // `hits` arrives in BM25 order, so the first few of a workspace are its best few:
+        // scoring and snippeting the same slice needs no second sort.
+        if (entry.snippets.length < SNIPPETS_PER_RESULT) {
+            entry.score += hit.score;
+            entry.snippets.push({ sessionId: hit.sessionId, role: hit.role, at: hit.at, text: hit.snippet });
+        }
+    }
+    const results = [];
+    for (const entry of byWorkspace.values()) {
+        const { bestBySession, ...rest } = entry;
+        let sessionId = null;
+        let best = -Infinity;
+        for (const [id, score] of bestBySession) {
+            if (score <= best)
+                continue;
+            best = score;
+            sessionId = id;
+        }
+        results.push({ ...rest, sessionId });
+    }
+    return results.sort((a, b) => b.score - a.score);
+}