conductor-remote 1.92.1 → 1.92.2

package/dist-node/src/search.js

@@ -1,9 +1,5 @@
-import fs from 'node:fs';
-import path from 'node:path';
-import { DatabaseSync } from 'node:sqlite';
+import { Worker } from 'node:worker_threads';
 import { chatCursor } from "./chat-cursor.js";
-import { HIT_CLOSE, HIT_OPEN } from "./shared.js";
-import { parseMessage } from "./transcript.js";
 /**
  * Full-text search over the chat history, in a sidecar DB the relay owns.
  *
@@ -22,7 +18,7 @@ import { parseMessage } from "./transcript.js";
  * together. Skipping it was the original cut and it was wrong: the chat view
  * *renders* thinking, so a hit there opens to something you can read, and the
  * reasoning is where a decision gets explained before the reply summarises it.
- * Note the shape trap behind the source query below: **every thinking block sits
+ * Note the shape trap behind the worker's source query: **every thinking block sits
  * in a row with no text block beside it** (0 of 102,773 rows carry both), so a
  * prefilter written for `"type":"text"` excludes 100% of thinking rather than
  * some of it — the bug that made this cut invisible.
@@ -43,20 +39,6 @@ import { parseMessage } from "./transcript.js";
  * stays that way; this opens its own file under the relay's state dir, and it is
  * disposable — delete it and the next start rebuilds it.
  */
-/** Bump to force a rebuild: a tokenizer or extraction change makes every stored chunk wrong. */
-const SCHEMA_VERSION = 2;
-/**
- * Source rows advanced per tick. The cursor moves by *scanned* rowid rather than
- * matched rowid, so a caught-up index re-scans nothing — get that wrong and every
- * idle poll re-reads the 3 GB tail looking for rows it already rejected.
- */
-const WINDOW_ROWS = 4000;
-/** Yield to the event loop between batches: the backfill is ~19ms of blocking work per window. */
-const BACKFILL_PAUSE_MS = 5;
-/** Once caught up, look for new messages at about the rate a chat produces them. */
-const IDLE_POLL_MS = 15_000;
-/** A pathological single message can't be allowed to dominate the index. */
-const MAX_CHUNK_CHARS = 64_000;
 /** How many chunks a query ranks before they are folded into workspaces. */
 const CHUNK_LIMIT = 300;
 /**
@@ -67,8 +49,6 @@ const CHUNK_LIMIT = 300;
  * must not render them literally.
  */
 export { HIT_CLOSE, HIT_OPEN } from "./shared.js";
-/** The `TranscriptEntry` roles this index keeps, and the set `search()` maps a stored role back through. */
-const INDEXED_ROLES = new Set(['user', 'assistant', 'thinking']);
 /**
  * Turn a phone query into an FTS5 MATCH expression.
  *
@@ -135,213 +115,126 @@ export function matchQuery(raw) {
 }
 /** The tokens `matchQuery` will search for — what a caller matches names against. */
 export { queryTokens } from "./shared.js";
+/**
+ * Main-thread facade for the disposable full-text index.
+ *
+ * `node:sqlite` is synchronous. Keeping its connection here meant a contended
+ * sidecar write, a backfill batch, or an expensive FTS rank stopped every HTTP
+ * route even though Conductor's AppleScript process itself is asynchronous. The
+ * worker owns both SQLite handles now; only small structured-clone messages cross
+ * back to the server thread.
+ */
 export class SearchIndex {
-    source;
+    sourceDbPath;
     file;
-    db = null;
-    openError = null;
-    cursor = 0;
-    caughtUp = false;
-    timer = null;
-    sourceMax = 0;
-    constructor(source, file) {
-        this.source = source;
+    worker = null;
+    indexStatus = { chunks: 0, ready: false, progress: 0 };
+    nextId = 1;
+    pending = new Map();
+    stopping = false;
+    constructor(sourceDbPath, file) {
+        this.sourceDbPath = sourceDbPath;
         this.file = file;
     }
-    /** Open (or rebuild) the sidecar and start indexing in the background. */
+    /** Spawn the index worker. Search remains a convenience: startup failure is non-fatal. */
     start() {
+        if (this.worker)
+            return;
+        this.stopping = false;
+        this.indexStatus = { chunks: 0, ready: false, progress: 0 };
+        const module = import.meta.url.endsWith('.ts') ? './search-worker.ts' : './search-worker.js';
         try {
-            this.open();
+            const worker = new Worker(new URL(module, import.meta.url), {
+                workerData: { sourceDbPath: this.sourceDbPath, file: this.file },
+                // The CLI suppresses node:sqlite's still-experimental warning in the main
+                // isolate; warning state does not cross into a worker.
+                execArgv: [...process.execArgv, '--disable-warning=ExperimentalWarning']
+            });
+            this.worker = worker;
+            worker.on('message', message => this.onMessage(message));
+            worker.on('error', error => {
+                if (this.worker === worker)
+                    this.workerFailed(error.message);
+            });
+            worker.on('exit', code => {
+                if (this.worker !== worker)
+                    return;
+                this.worker = null;
+                if (!this.stopping && !this.indexStatus.error)
+                    this.workerFailed(`worker exited with code ${code}`);
+            });
         }
         catch (err) {
-            // A search index is a convenience; failing to open one must never stop the relay
-            // serving state, transcripts or sends. Report it on /api/search instead.
-            this.openError = err instanceof Error ? err.message : String(err);
-            console.warn(`⚠ search index unavailable (${this.openError}) — /api/search will report it`);
-            return;
+            this.workerFailed(err instanceof Error ? err.message : String(err));
         }
-        this.schedule(0);
-    }
-    stop() {
-        if (this.timer)
-            clearTimeout(this.timer);
-        this.timer = null;
     }
-    open() {
-        fs.mkdirSync(path.dirname(this.file), { recursive: true });
-        const db = new DatabaseSync(this.file);
-        db.exec('PRAGMA journal_mode = WAL');
-        db.exec('PRAGMA synchronous = NORMAL');
-        // A dev relay on another port shares this file with the LaunchAgent's. WAL lets them
-        // both read; the writer that loses waits rather than throwing away its batch, and a
-        // tick that still fails is retried by the scheduler with nothing lost (the cursor
-        // only advances on commit).
-        db.exec('PRAGMA busy_timeout = 5000');
-        db.exec('CREATE TABLE IF NOT EXISTS meta (k TEXT PRIMARY KEY, v TEXT NOT NULL)');
-        const version = Number(this.readMeta(db, 'version') ?? 0);
-        if (version !== SCHEMA_VERSION) {
-            db.exec('DROP TABLE IF EXISTS chunks');
-            db.exec(`
-				CREATE VIRTUAL TABLE chunks USING fts5(
-					body,
-					session_id UNINDEXED,
-					src_rowid UNINDEXED,
-					role UNINDEXED,
-					at UNINDEXED,
-					tokenize='porter unicode61'
-				)
-			`);
-            db.prepare('INSERT OR REPLACE INTO meta(k, v) VALUES (?, ?)').run('version', String(SCHEMA_VERSION));
-            db.prepare('INSERT OR REPLACE INTO meta(k, v) VALUES (?, ?)').run('cursor', '0');
-            if (version)
-                console.log(`search index schema ${version} → ${SCHEMA_VERSION}, rebuilding`);
-        }
-        this.db = db;
-        this.cursor = Number(this.readMeta(db, 'cursor') ?? 0);
+    async stop() {
+        this.stopping = true;
+        const worker = this.worker;
+        this.worker = null;
+        this.rejectPending('search index stopped');
+        if (worker)
+            await worker.terminate();
     }
-    readMeta(db, key) {
-        const row = db.prepare('SELECT v FROM meta WHERE k = ?').get(key);
-        return row?.v ?? null;
+    status() {
+        return { ...this.indexStatus };
     }
-    schedule(ms) {
-        if (this.timer)
-            clearTimeout(this.timer);
-        this.timer = setTimeout(() => {
-            let more = false;
+    /**
+     * Top matching chunks, best first. The SQLite work happens entirely in the
+     * worker, so awaiting a slow rank does not stop unrelated API requests.
+     */
+    search(raw, options = {}) {
+        if (!matchQuery(raw) || (options.sessionIds && !options.sessionIds.length))
+            return Promise.resolve([]);
+        const worker = this.worker;
+        if (!worker || this.indexStatus.error)
+            return Promise.resolve([]);
+        const id = this.nextId++;
+        return new Promise((resolve, reject) => {
+            this.pending.set(id, { resolve, reject });
             try {
-                more = this.tick();
+                worker.postMessage({
+                    id,
+                    type: 'search',
+                    raw,
+                    options: { limit: CHUNK_LIMIT, ...options }
+                });
             }
-            catch (err) {
-                console.warn(`⚠ search index tick failed: ${err instanceof Error ? err.message : err}`);
+            catch (error) {
+                this.pending.delete(id);
+                reject(error instanceof Error ? error : new Error(String(error)));
             }
-            this.schedule(more ? BACKFILL_PAUSE_MS : IDLE_POLL_MS);
-        }, ms);
-        this.timer.unref?.();
+        });
     }
-    /**
-     * Index one window of source rows. Returns true while there is more to do.
-     *
-     * The window is picked by rowid *before* the prose filter runs, so the cursor
-     * advances past rows that hold nothing worth indexing. Filtering first and
-     * advancing to the last match instead would leave a caught-up index re-scanning
-     * every tool_result between the last prose row and the end of the table, every
-     * poll, forever.
-     */
-    tick() {
-        const db = this.db;
-        if (!db)
-            return false;
-        const window = this.source.query('SELECT rowid FROM session_messages WHERE rowid > ? ORDER BY rowid LIMIT ?', [this.cursor, WINDOW_ROWS]);
-        if (!window.length) {
-            this.caughtUp = true;
-            return false;
-        }
-        const end = window[window.length - 1].rowid;
-        // Only rows that can hold prose: a plain-text prompt, or a frame carrying a text or
-        // thinking block. Everything else is tool plumbing and the bulk of the bytes.
-        // The thinking clause is not redundant with the text one — the two block types never
-        // share a row (see this file's header), so dropping it drops thinking entirely.
-        const rows = this.source.query(`SELECT rowid, id, session_id, role, content, full_message, created_at, sent_at, queue_order
-			 FROM session_messages
-			 WHERE rowid > ? AND rowid <= ? AND session_id IS NOT NULL
-			   AND (role = 'user' OR content LIKE '%"type":"text"%' OR content LIKE '%"type":"thinking"%')
-			 ORDER BY rowid`, [this.cursor, end]);
-        const insert = db.prepare('INSERT INTO chunks(body, session_id, src_rowid, role, at) VALUES (?, ?, ?, ?, ?)');
-        db.exec('BEGIN');
-        try {
-            for (const row of rows) {
-                // Reuse the transcript parser rather than a second JSON walk: it already knows
-                // that text inside a `type:"user"` frame is injected context and not the user's
-                // words, and indexing something the chat view would never show is how a search
-                // result becomes impossible to find once you open it.
-                for (const entry of parseMessage(row, null)) {
-                    if (!INDEXED_ROLES.has(entry.role))
-                        continue;
-                    const body = entry.text.trim();
-                    if (!body)
-                        continue;
-                    insert.run(body.slice(0, MAX_CHUNK_CHARS), row.session_id, row.rowid, entry.role, entry.ts);
-                }
-            }
-            db.prepare('INSERT OR REPLACE INTO meta(k, v) VALUES (?, ?)').run('cursor', String(end));
-            db.exec('COMMIT');
+    onMessage(message) {
+        if (message.type === 'status') {
+            this.indexStatus = message.status;
+            return;
         }
-        catch (err) {
-            db.exec('ROLLBACK');
-            throw err;
+        if (message.type === 'log') {
+            console[message.level](message.message);
+            return;
         }
-        this.cursor = end;
-        return true;
+        const pending = this.pending.get(message.id);
+        if (!pending)
+            return;
+        this.pending.delete(message.id);
+        if (message.type === 'result')
+            pending.resolve(message.hits);
+        else
+            pending.reject(new Error(message.error));
     }
-    status() {
-        if (this.openError)
-            return { chunks: 0, ready: false, progress: 0, error: this.openError };
-        const db = this.db;
-        if (!db)
-            return { chunks: 0, ready: false, progress: 0 };
-        const chunks = Number(db.prepare('SELECT COUNT(*) c FROM chunks').get().c);
-        if (this.caughtUp)
-            return { chunks, ready: true, progress: 1 };
-        // Only re-read the source's high-water mark while backfilling; it costs a query
-        // and the answer only matters for the progress bar.
-        if (!this.sourceMax) {
-            const max = this.source.query('SELECT MAX(rowid) m FROM session_messages')[0]?.m;
-            this.sourceMax = max ?? 0;
-        }
-        const progress = this.sourceMax ? Math.min(1, this.cursor / this.sourceMax) : 0;
-        return { chunks, ready: false, progress };
+    workerFailed(error) {
+        if (this.indexStatus.error === error)
+            return;
+        this.indexStatus = { chunks: 0, ready: false, progress: 0, error };
+        this.rejectPending(error);
+        console.warn(`⚠ search index unavailable (${error}) — /api/search will report it`);
     }
-    /**
-     * Top matching chunks, best first. Empty when the query has no searchable tokens.
-     *
-     * `sessionIds` scopes the ranking, not just the result: it is the repo filter,
-     * resolved to chat ids by the caller because this index knows nothing about
-     * workspaces. It has to sit inside the query rather than be applied to the
-     * chunks it returns, because the `limit` is spent before any post-filter runs
-     * — a common word fills all 300 slots from the busiest repo and a smaller one
-     * folds up to nothing. Measured on this Mac's 205k chunks with the largest repo's
-     * 1,005 chats as the list: +0.3ms on a rare word, +60ms on the worst common-word
-     * query (165ms → 220ms), still under the phone's 250ms debounce. The list rides
-     * in as one JSON parameter through `json_each`, so its length never meets
-     * SQLite's bound-variable limit. An empty list matches nothing, which is the
-     * right answer for a repo with no chats and never "everything".
-     */
-    search(raw, { limit = CHUNK_LIMIT, sessionIds } = {}) {
-        const db = this.db;
-        if (!db)
-            return [];
-        const match = matchQuery(raw);
-        if (!match)
-            return [];
-        if (sessionIds && !sessionIds.length)
-            return [];
-        const scope = sessionIds ? 'AND session_id IN (SELECT value FROM json_each(?))' : '';
-        const params = sessionIds
-            ? [HIT_OPEN, HIT_CLOSE, match, JSON.stringify(sessionIds), limit]
-            : [HIT_OPEN, HIT_CLOSE, match, limit];
-        let rows;
-        try {
-            rows = db
-                .prepare(`SELECT session_id, src_rowid, role, at, -bm25(chunks) AS score,
-					        snippet(chunks, 0, ?, ?, '…', 24) AS snippet
-					 FROM chunks WHERE chunks MATCH ? ${scope} ORDER BY bm25(chunks) LIMIT ?`)
-                .all(...params);
-        }
-        catch (err) {
-            // A MATCH that still fails to parse is a bug in matchQuery, not user error —
-            // report it rather than showing an empty result that looks like "no matches".
-            throw new Error(`search failed for ${JSON.stringify(match)}: ${err instanceof Error ? err.message : err}`);
-        }
-        return rows.map(r => ({
-            sessionId: r.session_id,
-            srcRowid: Number(r.src_rowid),
-            // An index written before SCHEMA_VERSION 2 is dropped on open, so an unknown role
-            // here is a bug rather than an old row — fall back to the neutral one.
-            role: INDEXED_ROLES.has(r.role) ? r.role : 'assistant',
-            at: r.at,
-            score: Number(r.score),
-            snippet: r.snippet
-        }));
+    rejectPending(message) {
+        for (const pending of this.pending.values())
+            pending.reject(new Error(message));
+        this.pending.clear();
     }
 }
 const SNIPPETS_PER_RESULT = 3;

package/dist-node/src/server.js

@@ -53,7 +53,7 @@ const planUsage = new PlanUsageService();
 // Full-text index over the chat prose, in the relay's own sidecar DB — never in
 // Conductor's (see src/search.ts). It backfills in the background and is disposable:
 // deleting the file rebuilds it on the next start.
-const search = new SearchIndex(db, path.join(stateDir(), 'search.db'));
+const search = new SearchIndex(cfg.dbPath, path.join(stateDir(), 'search.db'));
 search.start();
 /**
  * The MCP tools, bound to this relay over loopback.
@@ -879,7 +879,7 @@ const server = http.createServer(async (req, res) => {
                 const scope = scoped
                     ? { sessionIds: reads.searchSessionIds(repos.length ? repos : undefined, includeArchived) }
                     : {};
-                const hits = search.search(q, scope);
+                const hits = await search.search(q, scope);
                 const targets = reads.searchTargets([...new Set(hits.map(h => h.sessionId))]);
                 const fromChats = foldHits(hits, sid => {
                     const workspace = targets.get(sid)?.workspace ?? null;

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "conductor-remote",
-	"version": "1.92.1",
+	"version": "1.92.2",
 	"type": "module",
 	"packageManager": "yarn@4.15.0",
 	"description": "Phone control panel for local Conductor agents. Reads ride SQLite + git; prompts ride Conductor's own dispatch path.",