@clawchatsai/connector 0.2.4 → 0.2.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,367 @@
1
+ // SearchStore: the connector's full-text index of ClawChats chats (data/search.db, node:sqlite FTS5).
2
+ //
3
+ // Derived data only. The gateway's transcripts are the source of truth; this file can be deleted at
4
+ // any time and the indexer (indexer.js) rebuilds it. A schema/tokenizer change bumps SCHEMA, which
5
+ // drops and rebuilds the index instead of migrating it.
6
+ //
7
+ // chats one row per indexed chat: gateway stamps (sessionId, change time, delta cursor),
8
+ // title/archived for matching and filtering, sync bookkeeping and the last error
9
+ // docs searchable documents: message text (kind 'message') and selection comments
10
+ // (kind 'comment', ord = attachment position), keyed to the gateway entry id
11
+ // docs_fts FTS5 over docs (text, quote), porter stemming + prefix index: "deplo" finds deploy
12
+ // chats_fts FTS5 over chat titles
13
+ // comment_files selection-comment bodies by media URL (immutable files: fetched once)
14
+ // chat_meta a chat's tags, summary and keywords, copied from the tag store (server/tags/) so
15
+ // search matches them; meta_fts over them. Dropped with the chat; the tagger refills it.
16
+
17
+ import fs from 'node:fs';
18
+ import path from 'node:path';
19
+ import { Database } from '../bootstrap/native.js';
20
+ import { normalizeTag } from '../tags/model.js';
21
+
22
+ const SCHEMA = 2;
23
+ const TOKENIZE = "tokenize = 'porter unicode61 remove_diacritics 2', prefix = '2 3'";
24
+ const MAX_QUERY_CHARS = 500;
25
+ const MAX_QUERY_TERMS = 16;
26
+ const HIT_SAMPLE = 5000; // best-ranked hits read per query, to pick each chat's top snippets
27
+ export const SNIPPET_OPEN = '\u0002';
28
+ export const SNIPPET_CLOSE = '\u0003';
29
+ const MISSING_FILE_RETRY_MS = 24 * 60 * 60 * 1000;
30
+
31
+ /**
32
+ * User input → FTS5 query. Every word must appear (AND), each as a prefix so results follow typing;
33
+ * "double quotes" make an exact phrase. Words are passed as FTS strings, so operators and
34
+ * punctuation in the input are plain text. Returns null when nothing searchable is left.
35
+ */
36
+ export function toFtsQuery(input) {
37
+ const s = String(input || '').slice(0, MAX_QUERY_CHARS);
38
+ const terms = [];
39
+ const re = /"([^"]*)"|(\S+)/g;
40
+ let m;
41
+ while ((m = re.exec(s)) && terms.length < MAX_QUERY_TERMS) {
42
+ const phrase = m[1] !== undefined;
43
+ const raw = (phrase ? m[1] : m[2]).trim();
44
+ // A term FTS5 would tokenize to nothing (only punctuation) can't match: drop it.
45
+ if (!raw || !/[\p{L}\p{N}]/u.test(raw)) continue;
46
+ const quoted = `"${raw.replace(/"/g, '""')}"`;
47
+ terms.push(phrase ? quoted : `${quoted}*`);
48
+ }
49
+ return terms.length ? terms.join(' AND ') : null;
50
+ }
51
+
52
+ /**
53
+ * `#tag` words of a query (a filter: only chats carrying every one) and the rest (searched as text).
54
+ * "#laser-show deploy" = chats tagged laser-show that mention deploy; "#laser-show" alone lists them.
55
+ */
56
+ export function parseQuery(input) {
57
+ const tags = [];
58
+ const text = String(input || '').slice(0, MAX_QUERY_CHARS).replace(/(^|\s)#([^\s#"]+)/g, (all, pre, raw) => {
59
+ const t = normalizeTag(raw);
60
+ if (t && !tags.includes(t)) tags.push(t);
61
+ return pre;
62
+ }).trim();
63
+ return { text, tags };
64
+ }
65
+
66
+ // bm25 is negative, lower = better. A title, tag or summary match and many hits lift a chat; recency breaks ties.
67
+ const BOOST = { title: 4, tag: 3, about: 2 };
68
+
69
+ function rankChats(groups, matched) {
70
+ for (const g of groups) {
71
+ g.score = g.best - Math.log1p(g.hits) * 0.5;
72
+ for (const [kind, keys] of Object.entries(matched)) if (keys.has(g.key)) g.score -= BOOST[kind];
73
+ }
74
+ return groups.sort((a, b) => a.score - b.score || (b.lastTs || 0) - (a.lastTs || 0));
75
+ }
76
+
77
+ const parseList = v => { try { const a = JSON.parse(v || '[]'); return Array.isArray(a) ? a : []; } catch { return []; } };
78
+
79
+ export class SearchStore {
80
+ /** @param {string|null} file database path; null = in memory (tests) */
81
+ constructor(file) {
82
+ if (file) fs.mkdirSync(path.dirname(file), { recursive: true });
83
+ this.db = new Database(file || ':memory:');
84
+ if (file) { try { fs.chmodSync(file, 0o600); } catch { /* best effort: copies of transcripts */ } }
85
+ this.db.exec('PRAGMA journal_mode = WAL; PRAGMA foreign_keys = ON; PRAGMA busy_timeout = 2000');
86
+ this._open();
87
+ }
88
+
89
+ _open() {
90
+ const v = this._meta('schema');
91
+ if (v !== null && Number(v) !== SCHEMA) this._dropAll();
92
+ this.db.exec(`
93
+ CREATE TABLE IF NOT EXISTS meta (k TEXT PRIMARY KEY, v TEXT);
94
+ CREATE TABLE IF NOT EXISTS chats (
95
+ key TEXT PRIMARY KEY, agent_id TEXT, session_id TEXT, title TEXT NOT NULL DEFAULT '',
96
+ archived INTEGER NOT NULL DEFAULT 0, stamp INTEGER NOT NULL DEFAULT 0, synced_stamp INTEGER NOT NULL DEFAULT -1,
97
+ cursor TEXT, total INTEGER, last_id TEXT, synced_at INTEGER, full_at INTEGER, docs INTEGER NOT NULL DEFAULT 0,
98
+ error TEXT, error_at INTEGER, attempts INTEGER NOT NULL DEFAULT 0);
99
+ CREATE TABLE IF NOT EXISTS docs (
100
+ id INTEGER PRIMARY KEY, key TEXT NOT NULL REFERENCES chats(key) ON DELETE CASCADE,
101
+ message_id TEXT NOT NULL, kind TEXT NOT NULL, ord INTEGER NOT NULL DEFAULT 0,
102
+ role TEXT NOT NULL, ts INTEGER, text TEXT NOT NULL, quote TEXT NOT NULL DEFAULT '',
103
+ UNIQUE (key, message_id, kind, ord));
104
+ CREATE INDEX IF NOT EXISTS docs_key ON docs(key);
105
+ CREATE VIRTUAL TABLE IF NOT EXISTS docs_fts USING fts5(text, quote, content = 'docs', content_rowid = 'id', ${TOKENIZE});
106
+ CREATE TRIGGER IF NOT EXISTS docs_ai AFTER INSERT ON docs BEGIN
107
+ INSERT INTO docs_fts(rowid, text, quote) VALUES (new.id, new.text, new.quote); END;
108
+ CREATE TRIGGER IF NOT EXISTS docs_ad AFTER DELETE ON docs BEGIN
109
+ INSERT INTO docs_fts(docs_fts, rowid, text, quote) VALUES ('delete', old.id, old.text, old.quote); END;
110
+ CREATE TRIGGER IF NOT EXISTS docs_au AFTER UPDATE OF text, quote ON docs BEGIN
111
+ INSERT INTO docs_fts(docs_fts, rowid, text, quote) VALUES ('delete', old.id, old.text, old.quote);
112
+ INSERT INTO docs_fts(rowid, text, quote) VALUES (new.id, new.text, new.quote); END;
113
+ CREATE VIRTUAL TABLE IF NOT EXISTS chats_fts USING fts5(title, content = 'chats', content_rowid = 'rowid', ${TOKENIZE});
114
+ CREATE TRIGGER IF NOT EXISTS chats_ai AFTER INSERT ON chats BEGIN
115
+ INSERT INTO chats_fts(rowid, title) VALUES (new.rowid, new.title); END;
116
+ CREATE TRIGGER IF NOT EXISTS chats_ad AFTER DELETE ON chats BEGIN
117
+ INSERT INTO chats_fts(chats_fts, rowid, title) VALUES ('delete', old.rowid, old.title); END;
118
+ CREATE TRIGGER IF NOT EXISTS chats_au AFTER UPDATE OF title ON chats BEGIN
119
+ INSERT INTO chats_fts(chats_fts, rowid, title) VALUES ('delete', old.rowid, old.title);
120
+ INSERT INTO chats_fts(rowid, title) VALUES (new.rowid, new.title); END;
121
+ CREATE TABLE IF NOT EXISTS comment_files (url TEXT PRIMARY KEY, body TEXT, status TEXT NOT NULL, fetched_at INTEGER NOT NULL);
122
+ CREATE TABLE IF NOT EXISTS chat_meta (
123
+ key TEXT PRIMARY KEY REFERENCES chats(key) ON DELETE CASCADE, tag_list TEXT NOT NULL DEFAULT '[]',
124
+ tags TEXT NOT NULL DEFAULT '', summary TEXT NOT NULL DEFAULT '', keywords TEXT NOT NULL DEFAULT '');
125
+ CREATE VIRTUAL TABLE IF NOT EXISTS meta_fts USING fts5(tags, summary, keywords, content = 'chat_meta', content_rowid = 'rowid', ${TOKENIZE});
126
+ CREATE TRIGGER IF NOT EXISTS meta_ai AFTER INSERT ON chat_meta BEGIN
127
+ INSERT INTO meta_fts(rowid, tags, summary, keywords) VALUES (new.rowid, new.tags, new.summary, new.keywords); END;
128
+ CREATE TRIGGER IF NOT EXISTS meta_ad AFTER DELETE ON chat_meta BEGIN
129
+ INSERT INTO meta_fts(meta_fts, rowid, tags, summary, keywords) VALUES ('delete', old.rowid, old.tags, old.summary, old.keywords); END;
130
+ CREATE TRIGGER IF NOT EXISTS meta_au AFTER UPDATE ON chat_meta BEGIN
131
+ INSERT INTO meta_fts(meta_fts, rowid, tags, summary, keywords) VALUES ('delete', old.rowid, old.tags, old.summary, old.keywords);
132
+ INSERT INTO meta_fts(rowid, tags, summary, keywords) VALUES (new.rowid, new.tags, new.summary, new.keywords); END;
133
+ `);
134
+ this._setMeta('schema', String(SCHEMA));
135
+ }
136
+
137
+ _dropAll() {
138
+ this.db.exec(`
139
+ DROP TABLE IF EXISTS meta_fts; DROP TABLE IF EXISTS chat_meta;
140
+ DROP TABLE IF EXISTS docs_fts; DROP TABLE IF EXISTS chats_fts; DROP TABLE IF EXISTS docs;
141
+ DROP TABLE IF EXISTS chats; DROP TABLE IF EXISTS comment_files; DROP TABLE IF EXISTS meta;`);
142
+ }
143
+
144
+ _meta(k) {
145
+ try { return this.db.prepare('SELECT v FROM meta WHERE k = ?').get(k)?.v ?? null; } catch { return null; }
146
+ }
147
+
148
+ _setMeta(k, v) {
149
+ if (v === null) this.db.prepare('DELETE FROM meta WHERE k = ?').run(k);
150
+ else this.db.prepare('INSERT INTO meta (k, v) VALUES (?, ?) ON CONFLICT(k) DO UPDATE SET v = excluded.v').run(k, v);
151
+ }
152
+
153
+ _tx(fn) {
154
+ this.db.exec('BEGIN IMMEDIATE');
155
+ try { const r = fn(); this.db.exec('COMMIT'); return r; } catch (e) { this.db.exec('ROLLBACK'); throw e; }
156
+ }
157
+
158
+ close() { try { this.db.close(); } catch { /* closed */ } }
159
+
160
+ /** Throw everything away (Rebuild). The indexer refills it. */
161
+ reset() {
162
+ this._tx(() => { this.db.exec('DELETE FROM docs; DELETE FROM chats; DELETE FROM comment_files; DELETE FROM meta'); this._setMeta('schema', String(SCHEMA)); });
163
+ }
164
+
165
+ /**
166
+ * The indexer's extraction rules changed (a connector update fixed or widened what gets indexed):
167
+ * every chat is re-read whole on the next pass. Documents stay searchable meanwhile.
168
+ */
169
+ requireIndexerVersion(version) {
170
+ if (this._meta('indexer') === String(version)) return false;
171
+ this._tx(() => {
172
+ this.db.exec('UPDATE chats SET synced_stamp = -1, cursor = NULL, total = NULL, last_id = NULL');
173
+ this._setMeta('indexer', String(version));
174
+ });
175
+ return true;
176
+ }
177
+
178
+ /** When the index last finished a pass over every chat (null until the first backfill is done). */
179
+ get completedAt() { const v = this._meta('completed_at'); return v ? Number(v) : null; }
180
+ set completedAt(ts) { this._setMeta('completed_at', ts ? String(ts) : null); }
181
+
182
+ // ── Chats ─────────────────────────────────────────────────────────
183
+
184
+ getChat(key) { return this.db.prepare('SELECT * FROM chats WHERE key = ?').get(key) || null; }
185
+
186
+ chatKeys() { return this.db.prepare('SELECT key FROM chats').all().map(r => r.key); }
187
+
188
+ /** Title, archive state, agent and change stamp from a gateway row; creates the chat if new. */
189
+ upsertChat({ key, agentId = null, title = '', archived = false, stamp = 0 }) {
190
+ this.db.prepare(`INSERT INTO chats (key, agent_id, title, archived, stamp) VALUES (?, ?, ?, ?, ?)
191
+ ON CONFLICT(key) DO UPDATE SET agent_id = excluded.agent_id, title = excluded.title, archived = excluded.archived,
192
+ stamp = MAX(chats.stamp, excluded.stamp)`).run(key, agentId, title, archived ? 1 : 0, stamp);
193
+ }
194
+
195
+ deleteChat(key) { return this.db.prepare('DELETE FROM chats WHERE key = ?').run(key).changes > 0; }
196
+
197
+ /** Chats whose transcript changed since they were last indexed (or that failed), oldest change first. */
198
+ staleKeys() {
199
+ return this.db.prepare('SELECT key FROM chats WHERE synced_stamp < stamp OR error IS NOT NULL ORDER BY stamp').all().map(r => r.key);
200
+ }
201
+
202
+ /** Chats due a full re-read (self-check against drift), least recently verified first. */
203
+ dueForFullRead(olderThan, limit) {
204
+ return this.db.prepare('SELECT key FROM chats WHERE full_at IS NULL OR full_at < ? ORDER BY COALESCE(full_at, 0) LIMIT ?').all(olderThan, limit).map(r => r.key);
205
+ }
206
+
207
+ markFailed(key, message, now) {
208
+ this.db.prepare('UPDATE chats SET error = ?, error_at = ?, attempts = attempts + 1 WHERE key = ?').run(String(message).slice(0, 500), now, key);
209
+ }
210
+
211
+ // ── Documents ─────────────────────────────────────────────────────
212
+
213
+ /**
214
+ * Store a chat's documents after a sync. `full` replaces the chat's whole set (rows no longer in the
215
+ * gateway's history go); otherwise `docs` are upserted (delta: appended entries only). Stamps the
216
+ * chat as synced up to `stamp` in the same transaction, so a crash never leaves it marked current.
217
+ */
218
+ writeDocs(key, docs, { full, sessionId, cursor, total = null, lastId = null, stamp, now }) {
219
+ return this._tx(() => {
220
+ if (full) this.db.prepare('DELETE FROM docs WHERE key = ?').run(key);
221
+ const put = this.db.prepare(`INSERT INTO docs (key, message_id, kind, ord, role, ts, text, quote) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
222
+ ON CONFLICT(key, message_id, kind, ord) DO UPDATE SET role = excluded.role, ts = excluded.ts, text = excluded.text, quote = excluded.quote
223
+ WHERE docs.text IS NOT excluded.text OR docs.quote IS NOT excluded.quote OR docs.role IS NOT excluded.role OR docs.ts IS NOT excluded.ts`);
224
+ for (const d of docs) put.run(key, d.messageId, d.kind, d.ord || 0, d.role, d.ts ?? null, d.text, d.quote || '');
225
+ const count = this.db.prepare('SELECT COUNT(*) n FROM docs WHERE key = ?').get(key).n;
226
+ this.db.prepare(`UPDATE chats SET session_id = ?, cursor = ?, total = ?, last_id = ?, synced_stamp = MAX(synced_stamp, ?), synced_at = ?,
227
+ full_at = CASE WHEN ? THEN ? ELSE full_at END, docs = ?, error = NULL, error_at = NULL, attempts = 0 WHERE key = ?`)
228
+ .run(sessionId ?? null, cursor ?? null, total, lastId, stamp, now, full ? 1 : 0, now, count, key);
229
+ return count;
230
+ });
231
+ }
232
+
233
+ /** The chat was checked and nothing changed: current up to `stamp`. */
234
+ markSynced(key, stamp, now) {
235
+ this.db.prepare('UPDATE chats SET synced_stamp = MAX(synced_stamp, ?), synced_at = ?, error = NULL, error_at = NULL, attempts = 0 WHERE key = ?').run(stamp, now, key);
236
+ }
237
+
238
+ // ── Comment files ─────────────────────────────────────────────────
239
+
240
+ /** A cached comment body: { body } when fetched, { missing: true } inside the retry window, else null. */
241
+ commentFile(url, now) {
242
+ const r = this.db.prepare('SELECT body, status, fetched_at FROM comment_files WHERE url = ?').get(url);
243
+ if (!r) return null;
244
+ if (r.status === 'ok') return { body: r.body };
245
+ return now - r.fetched_at < MISSING_FILE_RETRY_MS ? { missing: true } : null;
246
+ }
247
+
248
+ putCommentFile(url, body, now) {
249
+ this.db.prepare(`INSERT INTO comment_files (url, body, status, fetched_at) VALUES (?, ?, ?, ?)
250
+ ON CONFLICT(url) DO UPDATE SET body = excluded.body, status = excluded.status, fetched_at = excluded.fetched_at`)
251
+ .run(url, body ?? null, body == null ? 'missing' : 'ok', now);
252
+ }
253
+
254
+ // ── Tags and summaries (copied from the tag store) ─────────────────
255
+
256
+ /** A chat's tags, summary and keywords; nothing happens for a chat the index doesn't have (yet). */
257
+ setMeta(key, { tags = [], summary = '', keywords = [] }) {
258
+ if (!this.getChat(key)) return false;
259
+ this.db.prepare(`INSERT INTO chat_meta (key, tag_list, tags, summary, keywords) VALUES (?, ?, ?, ?, ?)
260
+ ON CONFLICT(key) DO UPDATE SET tag_list = excluded.tag_list, tags = excluded.tags, summary = excluded.summary, keywords = excluded.keywords
261
+ WHERE chat_meta.tag_list IS NOT excluded.tag_list OR chat_meta.summary IS NOT excluded.summary OR chat_meta.keywords IS NOT excluded.keywords`)
262
+ .run(key, JSON.stringify(tags), tags.join(' '), summary, keywords.join(' · '));
263
+ return true;
264
+ }
265
+
266
+ hasMeta(key) { return !!this.db.prepare('SELECT 1 FROM chat_meta WHERE key = ?').get(key); }
267
+
268
+ /** A chat's messages and comments, oldest first (what the tagger reads), and their count. */
269
+ messages(key) {
270
+ return this.db.prepare('SELECT role, kind, text, quote, ts FROM docs WHERE key = ? ORDER BY ts, id').all(key);
271
+ }
272
+
273
+ messageCount(key) { return this.db.prepare("SELECT COUNT(*) n FROM docs WHERE key = ? AND kind = 'message'").get(key).n; }
274
+
275
+ /** Chats with at least `minMessages` messages: key, title, change stamp, message count, text size. */
276
+ taggable(minMessages) {
277
+ return this.db.prepare(`SELECT c.key, c.title, c.stamp, COUNT(*) messages, SUM(LENGTH(d.text)) chars FROM chats c
278
+ JOIN docs d ON d.key = c.key AND d.kind = 'message' GROUP BY c.key HAVING COUNT(*) >= ?`).all(minMessages);
279
+ }
280
+
281
+ // ── Query ─────────────────────────────────────────────────────────
282
+
283
+ /**
284
+ * One row per chat, best first: { key, title, archived, hits, titleMatch, tagMatch, summaryMatch, lastTs,
285
+ * tags, summary, snippets: [{ messageId, kind, role, ts, snippet }] }. Snippet matches are wrapped in
286
+ * SNIPPET_OPEN/SNIPPET_CLOSE. `#tag` words in `q` filter (parseQuery; a tag starting with it counts);
287
+ * a query of only tags lists those chats, latest first.
288
+ * @param {object} o
289
+ * @param {string} o.q
290
+ * @param {'user'|'assistant'|null} [o.role] only documents by this author (title, tag and summary matches still count)
291
+ * @param {boolean} [o.archived=true] include archived chats
292
+ * @param {number} [o.limit=50] chats
293
+ * @param {number} [o.perChat=3] snippets per chat
294
+ */
295
+ search({ q, role = null, archived = true, limit = 50, perChat = 3 }) {
296
+ const { text, tags } = parseQuery(q);
297
+ const match = toFtsQuery(text);
298
+ if (!match && !tags.length) return { results: [], truncated: false };
299
+ const arch = archived ? '' : 'AND c.archived = 0';
300
+ const only = tags.length ? new Set(this._keysTagged(tags, arch)) : null;
301
+ const allowed = key => !only || only.has(key);
302
+ const matched = { title: new Set(), tag: new Set(), about: new Set() };
303
+ const groups = new Map();
304
+ const group = key => groups.get(key) || groups.set(key, { key, hits: 0, best: 0, lastTs: null, snippets: [] }).get(key);
305
+ const roleSql = role ? 'AND d.role = ?' : '';
306
+ const args = role ? [match, role] : [match];
307
+ if (!match) for (const key of only) group(key);
308
+ else {
309
+ if (!role) {
310
+ const keys = (sql, m) => this.db.prepare(sql).all(m).map(r => r.key).filter(allowed);
311
+ matched.title = new Set(keys(`SELECT c.key FROM chats_fts JOIN chats c ON c.rowid = chats_fts.rowid WHERE chats_fts MATCH ? ${arch}`, match));
312
+ const meta = `SELECT c.key FROM meta_fts JOIN chat_meta m ON m.rowid = meta_fts.rowid JOIN chats c ON c.key = m.key WHERE meta_fts MATCH ? ${arch}`;
313
+ matched.tag = new Set(keys(meta, `{tags} : (${match})`));
314
+ matched.about = new Set(keys(meta, `{summary keywords} : (${match})`));
315
+ }
316
+ // bm25() can't sit inside an aggregate: rank in a materialized CTE, then group.
317
+ const counts = this.db.prepare(`WITH h AS MATERIALIZED (SELECT d.key, d.ts, bm25(docs_fts, 1.0, 0.6) score
318
+ FROM docs_fts JOIN docs d ON d.id = docs_fts.rowid JOIN chats c ON c.key = d.key
319
+ WHERE docs_fts MATCH ? ${roleSql} ${arch})
320
+ SELECT key, COUNT(*) n, MIN(score) best, MAX(ts) last_ts FROM h GROUP BY key`).all(...args);
321
+ for (const r of counts) if (allowed(r.key)) Object.assign(group(r.key), { hits: r.n, best: r.best, lastTs: r.last_ts });
322
+ for (const keys of Object.values(matched)) for (const key of keys) group(key);
323
+ }
324
+ const info = this.db.prepare('SELECT c.title, c.archived, c.stamp, m.tag_list, m.summary FROM chats c LEFT JOIN chat_meta m ON m.key = c.key WHERE c.key = ?');
325
+ for (const g of groups.values()) { g.info = info.get(g.key); g.lastTs ??= g.info?.stamp || null; }
326
+ const ranked = match ? rankChats([...groups.values()], matched) : [...groups.values()].sort((a, b) => (b.lastTs || 0) - (a.lastTs || 0));
327
+ const page = ranked.slice(0, limit);
328
+ if (match) {
329
+ const wanted = new Set(page.map(g => g.key));
330
+ const hits = this.db.prepare(`SELECT d.key, d.message_id, d.kind, d.role, d.ts,
331
+ snippet(docs_fts, -1, '${SNIPPET_OPEN}', '${SNIPPET_CLOSE}', '…', 24) snip
332
+ FROM docs_fts JOIN docs d ON d.id = docs_fts.rowid JOIN chats c ON c.key = d.key
333
+ WHERE docs_fts MATCH ? ${roleSql} ${arch} ORDER BY bm25(docs_fts, 1.0, 0.6) LIMIT ${HIT_SAMPLE}`).all(...args);
334
+ for (const h of hits) {
335
+ const g = wanted.has(h.key) && groups.get(h.key);
336
+ if (g && g.snippets.length < perChat) g.snippets.push({ messageId: h.message_id, kind: h.kind, role: h.role, ts: h.ts, snippet: h.snip });
337
+ }
338
+ }
339
+ const results = page.map(g => ({
340
+ key: g.key, title: g.info?.title || '', archived: !!g.info?.archived, hits: g.hits,
341
+ titleMatch: matched.title.has(g.key), tagMatch: matched.tag.has(g.key), summaryMatch: matched.about.has(g.key),
342
+ lastTs: g.lastTs, tags: parseList(g.info?.tag_list), summary: g.info?.summary || '', snippets: g.snippets,
343
+ }));
344
+ return { results, truncated: ranked.length > limit };
345
+ }
346
+
347
+ /** Chats carrying, for each of `tags`, a tag that starts with it (results follow typing, like words do). */
348
+ _keysTagged(tags, arch) {
349
+ const each = tags.map(() => "EXISTS (SELECT 1 FROM json_each(m.tag_list) WHERE value LIKE ? ESCAPE '\\')").join(' AND ');
350
+ const prefixes = tags.map(t => `${t.replace(/[\\%_]/g, c => `\\${c}`)}%`);
351
+ return this.db.prepare(`SELECT m.key FROM chat_meta m JOIN chats c ON c.key = m.key WHERE ${each} ${arch}`).all(...prefixes).map(r => r.key);
352
+ }
353
+
354
+ /** Coverage numbers for the status line and the Settings check. */
355
+ stats() {
356
+ const c = this.db.prepare(`SELECT COUNT(*) chats, SUM(synced_stamp >= stamp AND error IS NULL) current,
357
+ SUM(error IS NOT NULL) failed, SUM(synced_at IS NULL) never FROM chats`).get();
358
+ const d = this.db.prepare("SELECT COUNT(*) docs, SUM(kind = 'comment') comments FROM docs").get();
359
+ return { chats: c.chats || 0, current: c.current || 0, failed: c.failed || 0, neverIndexed: c.never || 0, docs: d.docs || 0, comments: d.comments || 0 };
360
+ }
361
+
362
+ /** Per-chat coverage rows (Settings): key, docs, last sync, error. */
363
+ coverage() {
364
+ return this.db.prepare('SELECT key, title, docs, synced_at, full_at, error, attempts, synced_stamp >= stamp AS current FROM chats ORDER BY title').all()
365
+ .map(r => ({ key: r.key, title: r.title, docs: r.docs, syncedAt: r.synced_at, fullAt: r.full_at, current: !!r.current && !r.error, error: r.error, attempts: r.attempts }));
366
+ }
367
+ }
@@ -0,0 +1,141 @@
1
+ import { test } from 'node:test';
2
+ import assert from 'node:assert/strict';
3
+ import fs from 'node:fs';
4
+ import os from 'node:os';
5
+ import path from 'node:path';
6
+ import { SearchStore, toFtsQuery, SNIPPET_OPEN, SNIPPET_CLOSE } from './store.js';
7
+
8
+ const A = 'agent:main:dashboard:a', B = 'agent:main:dashboard:b', C = 'agent:main:dashboard:c';
9
+ const now = 1000;
10
+
11
+ function seeded() {
12
+ const s = new SearchStore(null);
13
+ s.upsertChat({ key: A, title: 'Connector deploy notes', stamp: 1 });
14
+ s.upsertChat({ key: B, title: 'Laser show', stamp: 1 });
15
+ s.upsertChat({ key: C, title: 'Old stuff', archived: true, stamp: 1 });
16
+ s.writeDocs(A, [
17
+ { messageId: 'a1', kind: 'message', role: 'user', ts: 10, text: 'how do we deploy the connector?' },
18
+ { messageId: 'a2', kind: 'message', role: 'assistant', ts: 11, text: 'Run deploy-local.sh, then reload.' },
19
+ ], { full: true, sessionId: 's', stamp: 1, now });
20
+ s.writeDocs(B, [
21
+ ...Array.from({ length: 30 }, (_, i) => ({ messageId: `b${i}`, kind: 'message', role: 'assistant', ts: 100 + i, text: `laser frame ${i} deployed` })),
22
+ { messageId: 'b-c', kind: 'comment', ord: 0, role: 'user', ts: 200, text: 'make the galvo faster', quote: 'scan rate is 20k' },
23
+ ], { full: true, sessionId: 's', stamp: 1, now });
24
+ s.writeDocs(C, [{ messageId: 'c1', kind: 'message', role: 'user', ts: 5, text: 'archived deploy talk' }], { full: true, sessionId: 's', stamp: 1, now });
25
+ return s;
26
+ }
27
+
28
+ test('query parsing: AND of prefixes, phrases, punctuation is text, caps', () => {
29
+ assert.equal(toFtsQuery('deplo conn'), '"deplo"* AND "conn"*');
30
+ assert.equal(toFtsQuery('"exact phrase" x'), '"exact phrase" AND "x"*');
31
+ assert.equal(toFtsQuery('x OR y'), '"x"* AND "OR"* AND "y"*', 'operators are plain words');
32
+ assert.equal(toFtsQuery('NEAR( -- ** )'), '"NEAR("*', 'punctuation-only terms dropped');
33
+ assert.equal(toFtsQuery('-- ** ()'), null);
34
+ assert.equal(toFtsQuery(' '), null);
35
+ assert.equal(toFtsQuery('w '.repeat(40)).split(' AND ').length, 16);
36
+ });
37
+
38
+ test('awkward input never breaks a query', () => {
39
+ const s = seeded();
40
+ for (const q of ['a"b', '"unclosed', 'x*', 'col:umn', 'NOT', '^start', 'C++', 'sessions.search', '日本語', "it's", '\u0000', '""'])
41
+ assert.doesNotThrow(() => s.search({ q }), q);
42
+ assert.equal(s.search({ q: 'deploy-local' }).results[0].key, A);
43
+ });
44
+
45
+ test('partial words and inflections match; one row per chat with hit counts', () => {
46
+ const s = seeded();
47
+ const { results } = s.search({ q: 'deplo' });
48
+ assert.deepEqual(results.map(r => r.key).sort(), [A, B, C].sort());
49
+ const b = results.find(r => r.key === B);
50
+ assert.equal(b.hits, 30);
51
+ assert.equal(b.snippets.length, 3, 'top snippets per chat');
52
+ assert.ok(b.snippets[0].snippet.includes(`${SNIPPET_OPEN}deployed${SNIPPET_CLOSE}`));
53
+ assert.ok(s.search({ q: 'deploying' }).results.length === 3, 'porter stemming');
54
+ });
55
+
56
+ test('a long chat no longer crowds out the others', () => {
57
+ const s = seeded();
58
+ const { results } = s.search({ q: 'deploy', limit: 2 });
59
+ assert.equal(results.length, 2);
60
+ assert.ok(results.some(r => r.key === A), 'chat A has 2 hits and a title match: it ranks first-page');
61
+ });
62
+
63
+ test('title matches count, even without message hits; role filter and archived filter', () => {
64
+ const s = seeded();
65
+ const laser = s.search({ q: 'laser show' }).results;
66
+ assert.equal(laser[0].key, B);
67
+ assert.equal(laser[0].titleMatch, true);
68
+ const t = s.search({ q: 'notes' }).results;
69
+ assert.deepEqual(t.map(r => [r.key, r.hits, r.titleMatch]), [[A, 0, true]]);
70
+ assert.deepEqual(s.search({ q: 'deploy', role: 'user' }).results.map(r => r.key).sort(), [A, C].sort());
71
+ assert.ok(!s.search({ q: 'deploy', archived: false }).results.some(r => r.key === C));
72
+ assert.equal(s.search({ q: 'deploy' }).results.find(r => r.key === C).archived, true);
73
+ });
74
+
75
+ test('comments are searchable by comment and by quoted passage', () => {
76
+ const s = seeded();
77
+ const byComment = s.search({ q: 'galvo' }).results;
78
+ assert.equal(byComment[0].key, B);
79
+ assert.equal(byComment[0].snippets[0].kind, 'comment');
80
+ assert.equal(s.search({ q: 'scan rate' }).results[0].key, B);
81
+ });
82
+
83
+ test('full writes replace, delta writes upsert; chat delete and reset clear everything', () => {
84
+ const s = seeded();
85
+ s.writeDocs(A, [{ messageId: 'a9', kind: 'message', role: 'user', ts: 20, text: 'brand new words' }], { full: false, sessionId: 's', stamp: 2, now });
86
+ assert.equal(s.search({ q: 'brand' }).results.length, 1);
87
+ assert.equal(s.search({ q: 'reload' }).results.length, 1, 'delta keeps older docs');
88
+ s.writeDocs(A, [{ messageId: 'a1', kind: 'message', role: 'user', ts: 10, text: 'edited text' }], { full: true, sessionId: 's2', stamp: 3, now });
89
+ assert.equal(s.search({ q: 'brand' }).results.length, 0, 'full read drops docs no longer in history');
90
+ assert.equal(s.search({ q: 'edited' }).results[0].key, A);
91
+ assert.equal(s.getChat(A).docs, 1);
92
+ assert.ok(s.deleteChat(B));
93
+ assert.equal(s.search({ q: 'galvo' }).results.length, 0, 'cascade removes FTS rows');
94
+ s.upsertChat({ key: A, title: 'Renamed chat', stamp: 3 });
95
+ assert.equal(s.search({ q: 'renamed' }).results[0].key, A);
96
+ assert.equal(s.search({ q: 'connector notes' }).results.length, 0, 'old title gone from the title index');
97
+ s.reset();
98
+ assert.equal(s.stats().chats, 0);
99
+ assert.equal(s.search({ q: 'edited' }).results.length, 0);
100
+ });
101
+
102
+ test('stale tracking, failures, and the synced stamp', () => {
103
+ const s = new SearchStore(null);
104
+ s.upsertChat({ key: A, stamp: 5 });
105
+ assert.deepEqual(s.staleKeys(), [A]);
106
+ assert.equal(s.stats().neverIndexed, 1);
107
+ s.writeDocs(A, [], { full: true, stamp: 5, now });
108
+ assert.deepEqual(s.staleKeys(), []);
109
+ s.upsertChat({ key: A, stamp: 4 });
110
+ assert.equal(s.getChat(A).stamp, 5, 'stamps never go back');
111
+ s.upsertChat({ key: A, stamp: 9 });
112
+ s.markFailed(A, 'boom', now);
113
+ assert.equal(s.getChat(A).attempts, 1);
114
+ assert.deepEqual(s.staleKeys(), [A]);
115
+ s.markSynced(A, 9, now);
116
+ assert.equal(s.getChat(A).error, null);
117
+ assert.deepEqual(s.staleKeys(), []);
118
+ });
119
+
120
+ test('comment file cache: bodies kept, missing files retried after a day', () => {
121
+ const s = new SearchStore(null);
122
+ s.putCommentFile('media://inbound/a.txt', 'body', now);
123
+ assert.deepEqual(s.commentFile('media://inbound/a.txt', now), { body: 'body' });
124
+ s.putCommentFile('media://inbound/b.txt', null, now);
125
+ assert.deepEqual(s.commentFile('media://inbound/b.txt', now + 1000), { missing: true });
126
+ assert.equal(s.commentFile('media://inbound/b.txt', now + 25 * 3600_000), null);
127
+ });
128
+
129
+ test('a schema change rebuilds the file instead of migrating it', () => {
130
+ const file = path.join(fs.mkdtempSync(path.join(os.tmpdir(), 'search-')), 'search.db');
131
+ const s = seeded.call(null);
132
+ s.close();
133
+ const a = new SearchStore(file);
134
+ a.upsertChat({ key: A, title: 'x', stamp: 1 });
135
+ a.db.prepare("UPDATE meta SET v = '0' WHERE k = 'schema'").run();
136
+ a.close();
137
+ const b = new SearchStore(file);
138
+ assert.equal(b.stats().chats, 0);
139
+ assert.equal(fs.statSync(file).mode & 0o777, 0o600);
140
+ b.close();
141
+ });
@@ -0,0 +1,111 @@
1
+ // Search documents from gateway history entries (chat.history / chat.message.get), and the
2
+ // selection-comment attachment format. Pure functions; the indexer (indexer.js) does the I/O.
3
+ //
4
+ // What is searchable is what the chat shows as text: user and assistant text blocks. Tool calls,
5
+ // tool results and thinking stay out, as in the gateway's own transcript index. Selection comments
6
+ // travel as `selection-comment.txt` attachments on a (usually empty) user message, so their text is
7
+ // only in the inbound media store; each comment becomes its own document on that message.
8
+
9
+ export const MAX_DOC_CHARS = 200_000;
10
+ // Room transcripts of team chats prefix every entry with its author (server/team.js).
11
+ const TEAM_LABEL_PREFIX = /^\[([^\]\n]{1,100})\]\n\n/;
12
+ // Reply directives the gateway strips before showing a reply (Control UI / ClawChats alike).
13
+ const DIRECTIVES = /\[\[\s*(?:reply_to_current|reply_to\s*:[^\]]*|audio_as_voice)\s*\]\]/g;
14
+ const COMMENT_FILE = /^selection-comment\.txt$/i;
15
+ const INBOUND_URL = /^media:\/\/inbound\/[^/?#\\]+$/;
16
+
17
+ /** The gateway's transcript metadata on an entry (`__<platform>`, found by shape, as the browser does). */
18
+ export function entryMeta(m) {
19
+ if (!m || typeof m !== 'object') return null;
20
+ for (const k of Object.keys(m)) {
21
+ if (k.startsWith('__') && m[k] && typeof m[k] === 'object' && !Array.isArray(m[k]) && typeof m[k].id === 'string') return { field: k, meta: m[k] };
22
+ }
23
+ return null;
24
+ }
25
+
26
+ export function textOf(m) {
27
+ const c = m?.content;
28
+ if (typeof c === 'string') return c;
29
+ if (Array.isArray(c)) return c.filter(p => p?.type === 'text' && typeof p.text === 'string').map(p => p.text).join('\n');
30
+ return '';
31
+ }
32
+
33
+ function clean(text) {
34
+ return String(text || '').replace(DIRECTIVES, ' ').replace(/[ \t]+\n/g, '\n').trim().slice(0, MAX_DOC_CHARS);
35
+ }
36
+
37
+ function timestampOf(m) {
38
+ const t = typeof m?.timestamp === 'number' ? m.timestamp : Date.parse(m?.timestamp);
39
+ return Number.isFinite(t) ? t : null;
40
+ }
41
+
42
+ /** Inbound `selection-comment.txt` attachments of an entry: [{ url }] in attachment order. */
43
+ export function commentAttachments(m) {
44
+ const media = entryMeta(m)?.meta?.media;
45
+ if (!Array.isArray(media)) return [];
46
+ return media.filter(x => x && typeof x.url === 'string' && INBOUND_URL.test(x.url) && COMMENT_FILE.test(x.fileName || '')).map(x => ({ url: x.url }));
47
+ }
48
+
49
+ /**
50
+ * Parse a selection-comment body (frontend/chat/selection-comments.js formatSelectionComment, the
51
+ * Control UI's format). The quoted passage is cut by its declared UTF-16 length, so a passage that
52
+ * itself contains "User comment:" can't be misread. Returns { quote, comment } or null.
53
+ */
54
+ export function parseSelectionComment(body) {
55
+ if (typeof body !== 'string' || !body.startsWith('Selected text:\n')) return null;
56
+ const head = 'Selected text:\n';
57
+ const tailAt = body.lastIndexOf('\n\nSource session: ');
58
+ if (tailAt < 0) return null;
59
+ const len = /\nSelected text UTF-16 length: (\d+)/.exec(body.slice(tailAt));
60
+ let quote, rest;
61
+ if (len && head.length + Number(len[1]) <= tailAt) {
62
+ quote = body.slice(head.length, head.length + Number(len[1]));
63
+ rest = body.slice(head.length + quote.length, tailAt);
64
+ } else {
65
+ // No usable length: split at the first comment marker.
66
+ const at = body.indexOf('\n\nUser comment:\n', head.length);
67
+ quote = body.slice(head.length, at >= 0 && at < tailAt ? at : tailAt);
68
+ rest = body.slice(head.length + quote.length, tailAt);
69
+ }
70
+ const comment = rest.startsWith('\n\nUser comment:\n') ? rest.slice('\n\nUser comment:\n'.length) : '';
71
+ return { quote: clean(quote), comment: clean(comment) };
72
+ }
73
+
74
+ /**
75
+ * Who wrote a room entry, from the room's author map (team store): 'user' (the owner), 'person'
76
+ * (someone else in the room), 'assistant' (an agent) or null for timeline lines ("Kamil joined").
77
+ */
78
+ function teamRole(author, label) {
79
+ if (author?.type === 'system' || (!author && label === 'System')) return null;
80
+ if (author?.type === 'user') return 'user';
81
+ if (author?.type === 'person') return 'person';
82
+ return 'assistant';
83
+ }
84
+
85
+ /**
86
+ * Documents for one history entry. `team` (room author map) switches to room parsing: entries are
87
+ * assistant-role injections prefixed with "[Label]\n\n" and attributed through the map.
88
+ * Returns { id, ts, truncated, docs: [{ kind: 'message', role, text }], comments: [{ url }] }, or
89
+ * null for entries with nothing searchable and no id.
90
+ */
91
+ export function documentsOf(m, { team = null } = {}) {
92
+ const em = entryMeta(m);
93
+ const id = em?.meta.id;
94
+ if (!id || (m.role !== 'user' && m.role !== 'assistant')) return null;
95
+ const ts = timestampOf(m);
96
+ const truncated = em.meta.truncated === true || em.meta.truncated === 'true';
97
+ let role = m.role;
98
+ let text = textOf(m);
99
+ if (team) {
100
+ if (m.role !== 'assistant') return null;
101
+ const match = TEAM_LABEL_PREFIX.exec(text);
102
+ role = teamRole(team[id], match?.[1]);
103
+ if (!role) return null;
104
+ if (match) text = text.slice(match[0].length);
105
+ }
106
+ text = clean(text);
107
+ const docs = text ? [{ kind: 'message', role, text }] : [];
108
+ const comments = m.role === 'user' ? commentAttachments(m) : [];
109
+ if (!docs.length && !comments.length && !truncated) return null;
110
+ return { id, ts, truncated, docs, comments };
111
+ }