@clawchatsai/connector 0.2.4 → 0.2.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/server/controllers/search.js +66 -0
- package/server/controllers/tags.js +80 -0
- package/server/controllers/title.js +13 -8
- package/server/gateway.js +1 -1
- package/server/index.js +65 -4
- package/server/search/indexer.js +383 -0
- package/server/search/indexer.test.js +267 -0
- package/server/search/store.js +367 -0
- package/server/search/store.test.js +141 -0
- package/server/search/text.js +111 -0
- package/server/search/text.test.js +57 -0
- package/server/store/legacy-threads.js +35 -8
- package/server/tags/model.js +132 -0
- package/server/tags/model.test.js +73 -0
- package/server/tags/store.js +143 -0
- package/server/tags/tagger.js +328 -0
- package/server/tags/tagger.test.js +309 -0
- package/server/team.js +11 -4
- package/server/team.test.js +12 -0
|
@@ -0,0 +1,383 @@
|
|
|
1
|
+
// SearchIndexer: keeps the SearchStore (store.js) in step with the gateway's chats.
|
|
2
|
+
//
|
|
3
|
+
// Self-healing by construction: every chat carries the gateway's change stamp (its latest activity
|
|
4
|
+
// time) and the stamp it was last indexed at. Anything that can make the index fall behind only
|
|
5
|
+
// leaves a chat "stale", and stale chats are re-read until they're current:
|
|
6
|
+
// - reconcile(): a full roster pass (on connect, then every RECONCILE_MS) adds new chats, drops
|
|
7
|
+
// deleted/incognito/hidden ones, refreshes titles and archive state, and marks changed chats stale.
|
|
8
|
+
// On an empty store this is the backfill.
|
|
9
|
+
// - sessions.changed: the changed chat is marked stale and synced after a short quiet period.
|
|
10
|
+
// - a weekly full re-read of every chat (a few per pass) catches drift a stamp can't show.
|
|
11
|
+
// A sync reads the chat through chat.history: the delta cursor from the last read when the
|
|
12
|
+
// gateway still accepts it (appended entries only), else every page (full read, replacing the
|
|
13
|
+
// chat's documents). Long messages the display cap truncated are fetched whole (chat.message.get);
|
|
14
|
+
// selection comments are read from the inbound media store once and cached.
|
|
15
|
+
//
|
|
16
|
+
// Events (the tagger, server/tags/, follows them): 'synced' (key) a chat's documents are current,
|
|
17
|
+
// 'removed' (key) a chat left the index (deleted, incognito, hidden), 'ready' the first full pass
|
|
18
|
+
// after a build finished.
|
|
19
|
+
|
|
20
|
+
import { EventEmitter } from 'node:events';
|
|
21
|
+
import { documentsOf, entryMeta, parseSelectionComment } from './text.js';
|
|
22
|
+
|
|
23
|
+
const idOf = m => entryMeta(m)?.meta.id ?? null;
|
|
24
|
+
|
|
25
|
+
// Bump when what a chat's documents are made of changes (text.js rules, delta handling): every chat
|
|
26
|
+
// indexed by an older connector is then read again whole.
|
|
27
|
+
export const INDEXER_VERSION = 2;
|
|
28
|
+
const RECONCILE_MS = 10 * 60 * 1000;
|
|
29
|
+
const QUIET_MS = 3000; // a chat syncs this long after its last change...
|
|
30
|
+
const MAX_WAIT_MS = 20_000; // ...but no later than this after the first one
|
|
31
|
+
const FULL_READ_EVERY_MS = 7 * 24 * 60 * 60 * 1000;
|
|
32
|
+
const FULL_READS_PER_PASS = 10;
|
|
33
|
+
const PAGE_LIMIT = 200;
|
|
34
|
+
const PAGE_BYTES = 4 * 1024 * 1024;
|
|
35
|
+
const MAX_CHARS = 200_000; // chat.history text cap per entry (gateway default 8000)
|
|
36
|
+
const MAX_PAGES = 200;
|
|
37
|
+
const MAX_DELTA_ROUNDS = 25;
|
|
38
|
+
const MAX_FULL_FETCHES = 100; // truncated entries fetched whole per sync
|
|
39
|
+
const BACKOFF_MS = [30_000, 2 * 60_000, 10 * 60_000, 60 * 60_000];
|
|
40
|
+
const LIST_PAGE = 1000;
|
|
41
|
+
const backoffMs = attempts => BACKOFF_MS[Math.min(Math.max(attempts, 1) - 1, BACKOFF_MS.length - 1)];
|
|
42
|
+
|
|
43
|
+
/** The gateway's change stamp for a session row: its latest activity. */
|
|
44
|
+
export function stampOf(row) {
|
|
45
|
+
return Math.max(0, ...['updatedAt', 'lastActivityAt', 'lastInteractionAt'].map(k => Number(row?.[k]) || 0));
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/** The chat title the browser shows (ChatStore.threadTitle), for title matching. */
|
|
49
|
+
export function titleOf(row) {
|
|
50
|
+
const t = row && (row.label || row.displayName || row.derivedTitle || row.autoLabel);
|
|
51
|
+
return typeof t === 'string' ? t.trim() : '';
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
export class SearchIndexer extends EventEmitter {
|
|
55
|
+
/**
|
|
56
|
+
* @param {object} o
|
|
57
|
+
* @param {import('./store.js').SearchStore} o.store
|
|
58
|
+
* @param {(method: string, params: object, timeoutMs?: number) => Promise<any>} o.request gateway RPC
|
|
59
|
+
* @param {(url: string, meta: { sessionKey: string, field: string }) => Promise<string|null>} o.fetchComment
|
|
60
|
+
* body of an inbound media file, null when the gateway no longer has it
|
|
61
|
+
* @param {{ observe(key, row): boolean, isVisible(key): boolean }} o.lens which sessions are ClawChats chats
|
|
62
|
+
* @param {{ room(key): object|null }} [o.team] team chats: room author maps
|
|
63
|
+
* @param {(frame: object) => void} [o.notify] status changes for browsers
|
|
64
|
+
*/
|
|
65
|
+
constructor({ store, request, fetchComment, lens, team = null, notify = () => {}, logger = console, now = () => Date.now(), quietMs = QUIET_MS, maxWaitMs = MAX_WAIT_MS }) {
|
|
66
|
+
super();
|
|
67
|
+
Object.assign(this, { store, request, fetchComment, lens, team, notify, log: logger, now, quietMs, maxWaitMs });
|
|
68
|
+
this.timers = new Map(); // key -> { timer, first }
|
|
69
|
+
this.queue = new Set(); // keys waiting for a sync, in order
|
|
70
|
+
this.running = false;
|
|
71
|
+
this.connected = false;
|
|
72
|
+
this.reconciling = null;
|
|
73
|
+
this.lastReconcileAt = null;
|
|
74
|
+
this.interval = null;
|
|
75
|
+
this.closed = false;
|
|
76
|
+
if (store.requireIndexerVersion(INDEXER_VERSION)) this.log.info?.(`[search] indexer v${INDEXER_VERSION}: every chat will be re-read`);
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
start() {
|
|
80
|
+
this.interval = setInterval(() => { if (this.connected) this.reconcile().catch(() => {}); }, RECONCILE_MS);
|
|
81
|
+
this.interval.unref?.();
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
close() {
|
|
85
|
+
this.closed = true;
|
|
86
|
+
clearInterval(this.interval);
|
|
87
|
+
for (const t of this.timers.values()) clearTimeout(t.timer);
|
|
88
|
+
this.timers.clear();
|
|
89
|
+
this.queue.clear();
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
onConnected() {
|
|
93
|
+
this.connected = true;
|
|
94
|
+
this.reconcile().catch(e => this.log.warn?.(`[search] reconcile: ${e.message}`));
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
onDisconnected() { this.connected = false; }
|
|
98
|
+
|
|
99
|
+
/** Status for the browser: whether results are complete yet, and coverage numbers. */
|
|
100
|
+
status() {
|
|
101
|
+
const s = this.store.stats();
|
|
102
|
+
const pending = s.chats - s.current;
|
|
103
|
+
return {
|
|
104
|
+
ready: this.store.completedAt != null,
|
|
105
|
+
building: this.running || !!this.reconciling || pending > 0,
|
|
106
|
+
pending, ...s,
|
|
107
|
+
completedAt: this.store.completedAt, lastReconcileAt: this.lastReconcileAt,
|
|
108
|
+
};
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
_notify() {
|
|
112
|
+
clearTimeout(this._notifyTimer);
|
|
113
|
+
this._notifyTimer = setTimeout(() => { try { this.notify({ type: 'clawchats', event: 'search-index', status: this.status() }); } catch { /* no browsers */ } }, 500);
|
|
114
|
+
this._notifyTimer.unref?.();
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
/** Throw the index away and build it again from the gateway. */
|
|
118
|
+
async rebuild() {
|
|
119
|
+
this.queue.clear();
|
|
120
|
+
this.store.reset();
|
|
121
|
+
this._notify();
|
|
122
|
+
if (this.connected) await this.reconcile();
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
// ── Roster ────────────────────────────────────────────────────────
|
|
126
|
+
|
|
127
|
+
/** Whether a session's transcript belongs in the index. */
|
|
128
|
+
_indexable(key, row) {
|
|
129
|
+
if (!key || row?.incognito) return false;
|
|
130
|
+
const visible = row && typeof row === 'object' ? this.lens.observe(key, row) : this.lens.isVisible(key);
|
|
131
|
+
return visible && !this.lens.parentOf?.(key);
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
reconcile() {
|
|
135
|
+
if (this.reconciling) return this.reconciling;
|
|
136
|
+
this.reconciling = this._reconcile().finally(() => { this.reconciling = null; });
|
|
137
|
+
return this.reconciling;
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
async _reconcile() {
|
|
141
|
+
const rows = [];
|
|
142
|
+
let offset = 0, complete = false;
|
|
143
|
+
for (let page = 0; page < 50; page++) {
|
|
144
|
+
const res = await this.request('sessions.list', { archived: 'all', limit: LIST_PAGE, offset, includeDerivedTitles: true, excludeCron: true, excludeSystem: true }, 60_000);
|
|
145
|
+
// A reply without a roster is a failure, never "the gateway has no chats" (that would drop the index).
|
|
146
|
+
if (!Array.isArray(res?.sessions)) throw new Error('sessions.list returned no session list');
|
|
147
|
+
rows.push(...res.sessions);
|
|
148
|
+
if (!res?.hasMore || res.nextOffset == null || res.nextOffset <= offset) { complete = !res?.hasMore; break; }
|
|
149
|
+
offset = res.nextOffset;
|
|
150
|
+
}
|
|
151
|
+
// Learn the whole roster before judging rows: a child can be listed before its parent.
|
|
152
|
+
for (const row of rows) if (row?.key) this.lens.observe(row.key, row);
|
|
153
|
+
const live = new Set();
|
|
154
|
+
for (const row of rows) {
|
|
155
|
+
if (!row?.key || !this._indexable(row.key, row)) continue;
|
|
156
|
+
live.add(row.key);
|
|
157
|
+
this.store.upsertChat({ key: row.key, agentId: row.agentId || row.key.split(':')[1] || null, title: titleOf(row), archived: !!row.archived, stamp: stampOf(row) });
|
|
158
|
+
}
|
|
159
|
+
// Only a complete roster proves a chat is gone.
|
|
160
|
+
if (complete) for (const key of this.store.chatKeys()) if (!live.has(key)) this._remove(key);
|
|
161
|
+
for (const key of this.store.staleKeys()) this._enqueue(key);
|
|
162
|
+
for (const key of this.store.dueForFullRead(this.now() - FULL_READ_EVERY_MS, FULL_READS_PER_PASS)) {
|
|
163
|
+
if (this.store.getChat(key)?.full_at != null) this._enqueue(key, { full: true });
|
|
164
|
+
}
|
|
165
|
+
this.lastReconcileAt = this.now();
|
|
166
|
+
this._notify();
|
|
167
|
+
this._drain();
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
/** `sessions.changed` from the connector's subscription (gateway.js). */
|
|
171
|
+
onSessionsChanged(payload) {
|
|
172
|
+
const key = payload?.sessionKey;
|
|
173
|
+
if (!key || this.closed) return;
|
|
174
|
+
if (payload.reason === 'delete') { if (this._remove(key)) this._notify(); return; }
|
|
175
|
+
const row = payload.session && typeof payload.session === 'object' ? { key, ...payload.session } : { key, ...payload };
|
|
176
|
+
if (!this._indexable(key, row)) {
|
|
177
|
+
// Became incognito/hidden/nested: its documents go. Undecided partial rows change nothing.
|
|
178
|
+
if (row.incognito || this.lens.isVisible(key) === false) { if (this._remove(key)) this._notify(); }
|
|
179
|
+
return;
|
|
180
|
+
}
|
|
181
|
+
const prev = this.store.getChat(key);
|
|
182
|
+
const title = 'label' in row || 'displayName' in row || 'derivedTitle' in row || 'autoLabel' in row ? titleOf(row) : prev?.title || '';
|
|
183
|
+
const archived = 'archived' in row ? !!row.archived : !!prev?.archived;
|
|
184
|
+
const stamp = Math.max(stampOf(row), prev?.stamp || 0, this.now());
|
|
185
|
+
this.store.upsertChat({ key, agentId: row.agentId || prev?.agent_id || key.split(':')[1] || null, title, archived, stamp });
|
|
186
|
+
this._schedule(key);
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
_remove(key) {
|
|
190
|
+
const gone = this.store.deleteChat(key);
|
|
191
|
+
if (gone) this.emit('removed', key);
|
|
192
|
+
return gone;
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
// ── Queue ─────────────────────────────────────────────────────────
|
|
196
|
+
|
|
197
|
+
/** Debounced sync: quiet period after the last change, bounded by maxWaitMs; `delayMs` = a retry. */
|
|
198
|
+
_schedule(key, delayMs = null) {
|
|
199
|
+
const t = this.timers.get(key);
|
|
200
|
+
const first = t?.first ?? this.now();
|
|
201
|
+
if (t) clearTimeout(t.timer);
|
|
202
|
+
const wait = delayMs ?? Math.max(0, Math.min(this.quietMs, first + this.maxWaitMs - this.now()));
|
|
203
|
+
const timer = setTimeout(() => { this.timers.delete(key); this._enqueue(key); this._drain(); }, wait);
|
|
204
|
+
timer.unref?.();
|
|
205
|
+
this.timers.set(key, { timer, first });
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
_enqueue(key, { full = false } = {}) {
|
|
209
|
+
if (full) (this._fullNext ??= new Set()).add(key);
|
|
210
|
+
this.queue.add(key);
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
_drain() {
|
|
214
|
+
if (!this.running && !this.closed) this.draining = this._drainLoop();
|
|
215
|
+
return this.draining;
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
/** Resolves once no reconcile, sync or debounced change is outstanding (tests, rebuild). */
|
|
219
|
+
async idle() {
|
|
220
|
+
for (let i = 0; i < 100; i++) {
|
|
221
|
+
if (this.reconciling) await this.reconciling.catch(() => {});
|
|
222
|
+
else if (this.running) await this.draining;
|
|
223
|
+
else if (this.timers.size) await new Promise(r => setTimeout(r, Math.min(this.quietMs, 50) + 1));
|
|
224
|
+
else if (this.queue.size && this.connected) await this._drain();
|
|
225
|
+
else return;
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
async _drainLoop() {
|
|
230
|
+
this.running = true;
|
|
231
|
+
try {
|
|
232
|
+
while (this.queue.size && this.connected && !this.closed) {
|
|
233
|
+
const key = this.queue.values().next().value;
|
|
234
|
+
this.queue.delete(key);
|
|
235
|
+
const chat = this.store.getChat(key);
|
|
236
|
+
if (!chat) continue;
|
|
237
|
+
const full = !!this._fullNext?.delete(key);
|
|
238
|
+
if (!full && chat.synced_stamp >= chat.stamp && !chat.error) continue;
|
|
239
|
+
if (chat.error) {
|
|
240
|
+
// Failing chats wait out their backoff (a later change can't shorten it).
|
|
241
|
+
const left = chat.error_at + backoffMs(chat.attempts) - this.now();
|
|
242
|
+
if (left > 0) { this._schedule(key, left); continue; }
|
|
243
|
+
}
|
|
244
|
+
try { await this.syncChat(key, { full }); this.emit('synced', key); }
|
|
245
|
+
catch (e) {
|
|
246
|
+
if (!this.connected) { this.queue.add(key); break; } // the gateway went away: not this chat's fault
|
|
247
|
+
if (this.store.getChat(key)) {
|
|
248
|
+
this.store.markFailed(key, e.message, this.now());
|
|
249
|
+
this._schedule(key, backoffMs(this.store.getChat(key).attempts));
|
|
250
|
+
}
|
|
251
|
+
this.log.warn?.(`[search] ${key}: ${e.message}`);
|
|
252
|
+
}
|
|
253
|
+
this._notify();
|
|
254
|
+
}
|
|
255
|
+
if (!this.queue.size && this.store.stats().neverIndexed === 0 && this.store.completedAt == null && this.lastReconcileAt) {
|
|
256
|
+
this.store.completedAt = this.now();
|
|
257
|
+
this.emit('ready');
|
|
258
|
+
this._notify();
|
|
259
|
+
}
|
|
260
|
+
} finally {
|
|
261
|
+
this.running = false;
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
// ── Sync one chat ─────────────────────────────────────────────────
|
|
266
|
+
|
|
267
|
+
/**
|
|
268
|
+
* Bring one chat up to date. Cheapest first: the delta since the last read; else a one-entry probe
|
|
269
|
+
* (same window, same entry count, same last entry = unchanged); else a full read. `full` skips
|
|
270
|
+
* both shortcuts (the weekly drift check).
|
|
271
|
+
*/
|
|
272
|
+
async syncChat(key, { full = false } = {}) {
|
|
273
|
+
const chat = this.store.getChat(key);
|
|
274
|
+
if (!chat) return;
|
|
275
|
+
const stamp = chat.stamp; // indexed up to here; a change during the read re-marks it stale
|
|
276
|
+
const team = this.team?.room?.(key)?.authors || null;
|
|
277
|
+
const indexed = chat.synced_at != null;
|
|
278
|
+
if (!full && indexed && chat.cursor) {
|
|
279
|
+
const delta = await this._readDelta(key, chat.cursor);
|
|
280
|
+
if (delta) {
|
|
281
|
+
const docs = await this._documents(key, delta.messages, team);
|
|
282
|
+
this.store.writeDocs(key, docs, { full: false, sessionId: chat.session_id, cursor: delta.cursor, stamp, now: this.now() });
|
|
283
|
+
return;
|
|
284
|
+
}
|
|
285
|
+
}
|
|
286
|
+
if (!full && indexed && !chat.cursor && chat.total != null) {
|
|
287
|
+
const probe = await this.request('chat.history', { sessionKey: key, limit: 1, maxBytes: 256 * 1024, maxChars: 1000 }, 30_000);
|
|
288
|
+
const last = (probe?.messages || []).at(-1);
|
|
289
|
+
if (probe?.sessionId === chat.session_id && probe?.totalMessages === chat.total && idOf(last) === chat.last_id) {
|
|
290
|
+
this.store.markSynced(key, stamp, this.now());
|
|
291
|
+
return;
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
const read = await this._readAll(key);
|
|
295
|
+
const docs = await this._documents(key, read.messages, team);
|
|
296
|
+
this.store.writeDocs(key, docs, { full: true, sessionId: read.sessionId, cursor: read.cursor, total: read.total, lastId: read.lastId, stamp, now: this.now() });
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
/** Entries appended since `cursor`, or null when the gateway wants a full read (branch changed, window reset, unknown cursor). */
|
|
300
|
+
async _readDelta(key, cursor) {
|
|
301
|
+
const messages = [];
|
|
302
|
+
let cur = cursor;
|
|
303
|
+
for (let round = 0; round < MAX_DELTA_ROUNDS; round++) {
|
|
304
|
+
let res;
|
|
305
|
+
try { res = await this.request('chat.history', { sessionKey: key, cursor: cur, maxBytes: PAGE_BYTES, maxChars: MAX_CHARS }, 60_000); }
|
|
306
|
+
catch (e) { if (/cursor|invalid|reset/i.test(e.message)) return null; throw e; }
|
|
307
|
+
if (res?.kind !== 'delta' || typeof res.deltaCursor !== 'string') return null;
|
|
308
|
+
// Delta entries are session.message payloads ({ sessionKey, message, messageId, … }), not bare entries.
|
|
309
|
+
const batch = (Array.isArray(res.messages) ? res.messages : []).map(e => (e?.message && typeof e.message === 'object' ? e.message : e));
|
|
310
|
+
messages.push(...batch);
|
|
311
|
+
if (!batch.length || res.deltaCursor === cur) return { messages, cursor: res.deltaCursor };
|
|
312
|
+
cur = res.deltaCursor;
|
|
313
|
+
}
|
|
314
|
+
return null; // a delta this long is cheaper read whole
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
/** Every page of a chat's history, newest first from the gateway, returned oldest first. */
|
|
318
|
+
async _readAll(key) {
|
|
319
|
+
const pages = [];
|
|
320
|
+
let offset = 0, sessionId = null, cursor = null, total = null, lastId = null;
|
|
321
|
+
for (let page = 0; page < MAX_PAGES; page++) {
|
|
322
|
+
const res = await this.request('chat.history', { sessionKey: key, limit: PAGE_LIMIT, offset, maxBytes: PAGE_BYTES, maxChars: MAX_CHARS }, 60_000);
|
|
323
|
+
if (page === 0) {
|
|
324
|
+
sessionId = res?.sessionId || null;
|
|
325
|
+
cursor = typeof res?.deltaCursor === 'string' ? res.deltaCursor : null;
|
|
326
|
+
total = Number.isInteger(res?.totalMessages) ? res.totalMessages : null;
|
|
327
|
+
lastId = idOf((res?.messages || []).at(-1));
|
|
328
|
+
}
|
|
329
|
+
else if (res?.sessionId && sessionId && res.sessionId !== sessionId) throw new Error('chat was reset while it was being read');
|
|
330
|
+
pages.unshift(Array.isArray(res?.messages) ? res.messages : []);
|
|
331
|
+
const next = res?.nextOffset;
|
|
332
|
+
if (!res?.hasMore || !Number.isFinite(next) || next <= offset) break;
|
|
333
|
+
offset = next;
|
|
334
|
+
}
|
|
335
|
+
const seen = new Set();
|
|
336
|
+
const messages = [];
|
|
337
|
+
for (const m of pages.flat()) {
|
|
338
|
+
const id = idOf(m);
|
|
339
|
+
if (id && seen.has(id)) continue; // pages can overlap while the chat grows
|
|
340
|
+
if (id) seen.add(id);
|
|
341
|
+
messages.push(m);
|
|
342
|
+
}
|
|
343
|
+
return { messages, sessionId, cursor, total, lastId };
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
/** History entries → store documents (whole text for truncated entries, comment bodies resolved). */
|
|
347
|
+
async _documents(key, messages, team) {
|
|
348
|
+
const out = [];
|
|
349
|
+
let fetches = 0;
|
|
350
|
+
for (let m of messages) {
|
|
351
|
+
let entry = documentsOf(m, { team });
|
|
352
|
+
if (!entry) continue;
|
|
353
|
+
if (entry.truncated && fetches < MAX_FULL_FETCHES) {
|
|
354
|
+
fetches++;
|
|
355
|
+
const whole = await this._wholeMessage(key, entry.id);
|
|
356
|
+
if (whole) { m = whole; entry = documentsOf(m, { team }) || entry; }
|
|
357
|
+
}
|
|
358
|
+
for (const d of entry.docs) out.push({ messageId: entry.id, kind: d.kind, role: d.role, ts: entry.ts, text: d.text });
|
|
359
|
+
for (let i = 0; i < entry.comments.length; i++) {
|
|
360
|
+
const parsed = parseSelectionComment(await this._commentBody(key, m, entry.comments[i].url));
|
|
361
|
+
if (parsed && (parsed.comment || parsed.quote)) out.push({ messageId: entry.id, kind: 'comment', ord: i, role: 'user', ts: entry.ts, text: parsed.comment, quote: parsed.quote });
|
|
362
|
+
}
|
|
363
|
+
}
|
|
364
|
+
return out;
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
async _wholeMessage(key, messageId) {
|
|
368
|
+
try {
|
|
369
|
+
const res = await this.request('chat.message.get', { sessionKey: key, messageId, maxChars: 500_000 }, 30_000);
|
|
370
|
+
return res?.ok && res.message && typeof res.message === 'object' ? res.message : null;
|
|
371
|
+
} catch { return null; } // keep the capped text
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
async _commentBody(key, m, url) {
|
|
375
|
+
const cached = this.store.commentFile(url, this.now());
|
|
376
|
+
if (cached?.body != null) return cached.body;
|
|
377
|
+
if (cached?.missing) return null;
|
|
378
|
+
const field = Object.keys(m).find(k => k.startsWith('__') && m[k]?.media) || '__openclaw';
|
|
379
|
+
const body = await this.fetchComment(url, { sessionKey: key, field });
|
|
380
|
+
this.store.putCommentFile(url, typeof body === 'string' ? body.slice(0, 256 * 1024) : null, this.now());
|
|
381
|
+
return body;
|
|
382
|
+
}
|
|
383
|
+
}
|
|
@@ -0,0 +1,267 @@
|
|
|
1
|
+
import { test } from 'node:test';
|
|
2
|
+
import assert from 'node:assert/strict';
|
|
3
|
+
import { SearchStore } from './store.js';
|
|
4
|
+
import { SearchIndexer } from './indexer.js';
|
|
5
|
+
import { SessionLens } from '../session-lens.js';
|
|
6
|
+
|
|
7
|
+
const A = 'agent:main:dashboard:a', B = 'agent:main:dashboard:b';
|
|
8
|
+
const SECRET = 'agent:main:dashboard:incog', CRON = 'agent:main:cron:job';
|
|
9
|
+
const quiet = { info() {}, warn() {} };
|
|
10
|
+
|
|
11
|
+
const msg = (id, role, text, extra = {}) => ({ role, content: [{ type: 'text', text }], timestamp: Number(id.replace(/\D/g, '')) || 1, __openclaw: { id, ...extra } });
|
|
12
|
+
const comment = (id, url) => ({ role: 'user', content: '', timestamp: 1, __openclaw: { id, media: [{ url, fileName: 'selection-comment.txt', contentType: 'text/plain' }] } });
|
|
13
|
+
const commentBody = (text, c) => `Selected text:\n${text}\n\nUser comment:\n${c}\n\nSource session: ${A}\nSelected text UTF-16 length: ${text.length}\nDOM text UTF-16 range: [0, 1)`;
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* A gateway with sessions and transcripts. chat.history pages newest-first by offset, hands out a
|
|
17
|
+
* delta cursor `<sessionId>:<count>` on the first page, and answers a cursor with the appended
|
|
18
|
+
* entries, or `reset` after a branch change / window reset.
|
|
19
|
+
*/
|
|
20
|
+
function fakeGateway({ delta = true } = {}) {
|
|
21
|
+
const chats = new Map(); // key -> { row, sessionId, messages, branchChanged }
|
|
22
|
+
const calls = [];
|
|
23
|
+
const failing = new Set();
|
|
24
|
+
const full = new Map(); // messageId -> whole message (chat.message.get)
|
|
25
|
+
const add = (key, messages = [], row = {}) => chats.set(key, { row: { key, kind: 'direct', createdVia: 'operator', updatedAt: 1, ...row }, sessionId: 's1', messages, branchChanged: false });
|
|
26
|
+
async function request(method, p) {
|
|
27
|
+
calls.push({ method, ...p });
|
|
28
|
+
if (method === 'sessions.list') return { sessions: [...chats.values()].map(c => c.row), hasMore: false };
|
|
29
|
+
const c = chats.get(p.sessionKey);
|
|
30
|
+
if (failing.has(p.sessionKey)) throw new Error('boom');
|
|
31
|
+
if (method === 'chat.message.get') return full.has(p.messageId) ? { ok: true, message: full.get(p.messageId) } : { ok: false, unavailableReason: 'not_found' };
|
|
32
|
+
if (method !== 'chat.history') throw new Error(`unexpected ${method}`);
|
|
33
|
+
if (!c) throw new Error('session not found');
|
|
34
|
+
if (p.cursor) {
|
|
35
|
+
const [sid, n] = p.cursor.split(':');
|
|
36
|
+
if (sid !== c.sessionId || c.branchChanged || Number(n) > c.messages.length) return { kind: 'reset' };
|
|
37
|
+
// Real gateways wrap delta entries as session.message payloads.
|
|
38
|
+
const appended = c.messages.slice(Number(n)).map(m => ({ sessionKey: p.sessionKey, agentId: 'main', message: m, messageId: m.__openclaw.id }));
|
|
39
|
+
return { kind: 'delta', messages: appended, deltaCursor: `${c.sessionId}:${c.messages.length}` };
|
|
40
|
+
}
|
|
41
|
+
const n = c.messages.length, offset = p.offset || 0, limit = p.limit || 200;
|
|
42
|
+
const start = Math.max(0, n - offset - limit);
|
|
43
|
+
const page = c.messages.slice(start, n - offset);
|
|
44
|
+
return { sessionId: c.sessionId, messages: page, totalMessages: n, hasMore: start > 0, nextOffset: offset + page.length,
|
|
45
|
+
...(delta && offset === 0 ? { deltaCursor: `${c.sessionId}:${n}` } : {}) };
|
|
46
|
+
}
|
|
47
|
+
return { chats, calls, failing, full, add, request };
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function setup(gw, { files = {}, team = null } = {}) {
|
|
51
|
+
const store = new SearchStore(null);
|
|
52
|
+
const lens = new SessionLens({ broadcast() {}, logger: quiet });
|
|
53
|
+
const fetched = [];
|
|
54
|
+
let clock = 1_000_000;
|
|
55
|
+
const indexer = new SearchIndexer({
|
|
56
|
+
store, request: gw.request, lens, team, logger: quiet, quietMs: 5, maxWaitMs: 20, now: () => clock,
|
|
57
|
+
fetchComment: async url => { fetched.push(url); return url in files ? files[url] : null; },
|
|
58
|
+
});
|
|
59
|
+
return { store, indexer, fetched, tick: ms => { clock += ms; } };
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
const keys = r => r.results.map(x => x.key).sort();
|
|
63
|
+
|
|
64
|
+
test('backfill: every visible chat indexed; incognito, cron and children stay out; then ready', async () => {
|
|
65
|
+
const gw = fakeGateway();
|
|
66
|
+
gw.add(A, [msg('m1', 'user', 'deploy the connector'), msg('m2', 'assistant', 'done, reload the plugin')]);
|
|
67
|
+
gw.add(B, Array.from({ length: 450 }, (_, i) => msg(`b${i}`, 'assistant', `laser frame ${i}`)));
|
|
68
|
+
gw.add(SECRET, [msg('s1', 'user', 'deploy secret')], { incognito: true });
|
|
69
|
+
gw.add(CRON, [msg('c1', 'user', 'deploy cron')]);
|
|
70
|
+
gw.add('agent:main:dashboard:child', [msg('k1', 'user', 'deploy child')], { spawnedBy: A, parentSessionKey: A });
|
|
71
|
+
const { store, indexer } = setup(gw);
|
|
72
|
+
assert.equal(indexer.status().ready, false);
|
|
73
|
+
indexer.onConnected();
|
|
74
|
+
await indexer.idle();
|
|
75
|
+
assert.deepEqual(keys(store.search({ q: 'deplo' })), [A]);
|
|
76
|
+
assert.equal(store.search({ q: 'frame' }).results[0].hits, 450, 'every page of a long chat');
|
|
77
|
+
const st = indexer.status();
|
|
78
|
+
assert.equal(st.ready, true);
|
|
79
|
+
assert.equal(st.chats, 2);
|
|
80
|
+
assert.equal(st.pending, 0);
|
|
81
|
+
});
|
|
82
|
+
|
|
83
|
+
test('live changes: appended entries arrive through the delta cursor, no full re-read', async () => {
|
|
84
|
+
const gw = fakeGateway();
|
|
85
|
+
gw.add(A, [msg('m1', 'user', 'first words')]);
|
|
86
|
+
const { store, indexer, tick } = setup(gw);
|
|
87
|
+
indexer.onConnected();
|
|
88
|
+
await indexer.idle();
|
|
89
|
+
gw.chats.get(A).messages.push(msg('m2', 'assistant', 'galvanometer tuning'));
|
|
90
|
+
gw.calls.length = 0;
|
|
91
|
+
tick(10);
|
|
92
|
+
indexer.onSessionsChanged({ sessionKey: A, reason: 'message', session: { kind: 'direct', createdVia: 'operator', updatedAt: 2 } });
|
|
93
|
+
await indexer.idle();
|
|
94
|
+
assert.equal(store.search({ q: 'galvan' }).results[0].key, A);
|
|
95
|
+
assert.ok(gw.calls.every(c => c.method !== 'chat.history' || c.cursor), 'delta only');
|
|
96
|
+
// A change with nothing new is a single delta round.
|
|
97
|
+
gw.calls.length = 0;
|
|
98
|
+
tick(10);
|
|
99
|
+
indexer.onSessionsChanged({ sessionKey: A, reason: 'patch' });
|
|
100
|
+
await indexer.idle();
|
|
101
|
+
assert.equal(gw.calls.length, 1);
|
|
102
|
+
});
|
|
103
|
+
|
|
104
|
+
test('branch change or window reset: the gateway refuses the cursor and the chat is re-read whole', async () => {
|
|
105
|
+
const gw = fakeGateway();
|
|
106
|
+
gw.add(A, [msg('m1', 'user', 'keep this'), msg('m2', 'assistant', 'rewound away')]);
|
|
107
|
+
const { store, indexer, tick } = setup(gw);
|
|
108
|
+
indexer.onConnected();
|
|
109
|
+
await indexer.idle();
|
|
110
|
+
gw.chats.get(A).messages = [msg('m1', 'user', 'keep this'), msg('m3', 'assistant', 'new branch reply')];
|
|
111
|
+
gw.chats.get(A).branchChanged = true;
|
|
112
|
+
tick(10);
|
|
113
|
+
indexer.onSessionsChanged({ sessionKey: A, reason: 'rewind' });
|
|
114
|
+
await indexer.idle();
|
|
115
|
+
assert.equal(store.search({ q: 'rewound' }).results.length, 0);
|
|
116
|
+
assert.equal(store.search({ q: 'branch' }).results[0].key, A);
|
|
117
|
+
assert.equal(store.search({ q: 'keep' }).results.length, 1);
|
|
118
|
+
});
|
|
119
|
+
|
|
120
|
+
test('without a delta cursor: an unchanged chat costs one probe, a changed one a full read', async () => {
|
|
121
|
+
const gw = fakeGateway({ delta: false });
|
|
122
|
+
gw.add(A, [msg('m1', 'user', 'alpha')]);
|
|
123
|
+
const { store, indexer, tick } = setup(gw);
|
|
124
|
+
indexer.onConnected();
|
|
125
|
+
await indexer.idle();
|
|
126
|
+
gw.calls.length = 0;
|
|
127
|
+
tick(10);
|
|
128
|
+
indexer.onSessionsChanged({ sessionKey: A, reason: 'patch' });
|
|
129
|
+
await indexer.idle();
|
|
130
|
+
assert.deepEqual(gw.calls.map(c => [c.method, c.limit]), [['chat.history', 1]]);
|
|
131
|
+
gw.chats.get(A).messages.push(msg('m2', 'assistant', 'bravo'));
|
|
132
|
+
tick(10);
|
|
133
|
+
indexer.onSessionsChanged({ sessionKey: A, reason: 'message' });
|
|
134
|
+
await indexer.idle();
|
|
135
|
+
assert.equal(store.search({ q: 'bravo' }).results[0].key, A);
|
|
136
|
+
});
|
|
137
|
+
|
|
138
|
+
test('deletes, incognito and reconcile after a missed event', async () => {
|
|
139
|
+
const gw = fakeGateway();
|
|
140
|
+
gw.add(A, [msg('m1', 'user', 'apple')]);
|
|
141
|
+
gw.add(B, [msg('m2', 'user', 'apple pie')]);
|
|
142
|
+
const { store, indexer } = setup(gw);
|
|
143
|
+
indexer.onConnected();
|
|
144
|
+
await indexer.idle();
|
|
145
|
+
gw.chats.delete(A);
|
|
146
|
+
indexer.onSessionsChanged({ sessionKey: A, reason: 'delete' });
|
|
147
|
+
assert.deepEqual(keys(store.search({ q: 'apple' })), [B]);
|
|
148
|
+
gw.chats.get(B).row.incognito = true;
|
|
149
|
+
indexer.onSessionsChanged({ sessionKey: B, reason: 'patch', session: { kind: 'direct', createdVia: 'operator', incognito: true } });
|
|
150
|
+
assert.equal(store.search({ q: 'apple' }).results.length, 0);
|
|
151
|
+
// Deleted while the connector wasn't listening: the next roster pass drops it.
|
|
152
|
+
gw.chats.get(B).row.incognito = false;
|
|
153
|
+
indexer.store.upsertChat({ key: B, stamp: 1 });
|
|
154
|
+
gw.chats.delete(B);
|
|
155
|
+
gw.add('agent:main:dashboard:c', [msg('m3', 'user', 'cherry')]);
|
|
156
|
+
await indexer.reconcile();
|
|
157
|
+
await indexer.idle();
|
|
158
|
+
assert.deepEqual(store.chatKeys().sort(), ['agent:main:dashboard:c']);
|
|
159
|
+
});
|
|
160
|
+
|
|
161
|
+
test('a roster reply without a session list fails the pass and keeps the index', async () => {
|
|
162
|
+
const gw = fakeGateway();
|
|
163
|
+
gw.add(A, [msg('m1', 'user', 'apple')]);
|
|
164
|
+
const { store, indexer } = setup(gw);
|
|
165
|
+
indexer.onConnected();
|
|
166
|
+
await indexer.idle();
|
|
167
|
+
const real = gw.request;
|
|
168
|
+
indexer.request = async (method, p) => method === 'sessions.list' ? { ok: false, error: { code: 'UNAVAILABLE' } } : real(method, p);
|
|
169
|
+
await assert.rejects(indexer.reconcile(), /no session list/);
|
|
170
|
+
assert.deepEqual(store.chatKeys(), [A]);
|
|
171
|
+
});
|
|
172
|
+
|
|
173
|
+
test('titles and archive state follow the gateway without re-reading transcripts', async () => {
|
|
174
|
+
const gw = fakeGateway();
|
|
175
|
+
gw.add(A, [msg('m1', 'user', 'x')], { label: 'Proxmox restore', archived: false });
|
|
176
|
+
const { store, indexer, tick } = setup(gw);
|
|
177
|
+
indexer.onConnected();
|
|
178
|
+
await indexer.idle();
|
|
179
|
+
assert.equal(store.search({ q: 'proxm' }).results[0].titleMatch, true);
|
|
180
|
+
tick(10);
|
|
181
|
+
indexer.onSessionsChanged({ sessionKey: A, reason: 'patch', session: { kind: 'direct', createdVia: 'operator', label: 'Homiox backups', archived: true } });
|
|
182
|
+
assert.equal(store.search({ q: 'proxm' }).results.length, 0);
|
|
183
|
+
assert.equal(store.search({ q: 'homiox' }).results[0].archived, true);
|
|
184
|
+
});
|
|
185
|
+
|
|
186
|
+
test('selection comments: fetched once, cached, missing files skipped', async () => {
|
|
187
|
+
const gw = fakeGateway();
|
|
188
|
+
const url = 'media://inbound/selection-comment---1.txt';
|
|
189
|
+
gw.add(A, [comment('u1', url), comment('u2', 'media://inbound/selection-comment---gone.txt')]);
|
|
190
|
+
const { store, indexer, fetched } = setup(gw, { files: { [url]: commentBody('scan rate is 20k', 'make the galvo faster') } });
|
|
191
|
+
indexer.onConnected();
|
|
192
|
+
await indexer.idle();
|
|
193
|
+
const hit = store.search({ q: 'galvo' }).results[0];
|
|
194
|
+
assert.equal(hit.snippets[0].kind, 'comment');
|
|
195
|
+
assert.equal(hit.snippets[0].messageId, 'u1');
|
|
196
|
+
assert.equal(store.search({ q: 'scan rate' }).results.length, 1);
|
|
197
|
+
await indexer.syncChat(A, { full: true });
|
|
198
|
+
assert.equal(fetched.filter(u => u === url).length, 1, 'cached after the first read');
|
|
199
|
+
});
|
|
200
|
+
|
|
201
|
+
test('display-capped entries are fetched whole', async () => {
|
|
202
|
+
const gw = fakeGateway();
|
|
203
|
+
gw.add(A, [msg('m1', 'assistant', 'start of a long reply…', { truncated: true, reason: 'display-cap' })]);
|
|
204
|
+
gw.full.set('m1', msg('m1', 'assistant', 'start of a long reply and the needle at the end'));
|
|
205
|
+
const { store, indexer } = setup(gw);
|
|
206
|
+
indexer.onConnected();
|
|
207
|
+
await indexer.idle();
|
|
208
|
+
assert.equal(store.search({ q: 'needle' }).results[0].key, A);
|
|
209
|
+
});
|
|
210
|
+
|
|
211
|
+
test('a failing chat is recorded, retried after its backoff, and does not block the others', async () => {
|
|
212
|
+
const gw = fakeGateway();
|
|
213
|
+
gw.add(A, [msg('m1', 'user', 'alpha')]);
|
|
214
|
+
gw.add(B, [msg('m2', 'user', 'beta')]);
|
|
215
|
+
gw.failing.add(A);
|
|
216
|
+
const { store, indexer, tick } = setup(gw);
|
|
217
|
+
indexer.onConnected();
|
|
218
|
+
await indexer.idle();
|
|
219
|
+
assert.equal(store.search({ q: 'beta' }).results.length, 1);
|
|
220
|
+
assert.equal(store.getChat(A).error, 'boom');
|
|
221
|
+
assert.equal(indexer.status().failed, 1);
|
|
222
|
+
assert.equal(indexer.status().ready, false, 'not complete while a chat has never been indexed');
|
|
223
|
+
gw.failing.delete(A);
|
|
224
|
+
tick(31_000);
|
|
225
|
+
await indexer.syncChat(A); // what the retry timer runs
|
|
226
|
+
assert.equal(store.search({ q: 'alpha' }).results.length, 1);
|
|
227
|
+
assert.equal(store.getChat(A).error, null);
|
|
228
|
+
});
|
|
229
|
+
|
|
230
|
+
test('team rooms: entries attributed through the room author map', async () => {
|
|
231
|
+
const gw = fakeGateway();
|
|
232
|
+
gw.add(A, [msg('e1', 'assistant', '[Houman]\n\nship the laser'), msg('e2', 'assistant', '[homiabot]\n\nshipping the laser'), msg('e3', 'assistant', '[System]\n\nKamil joined')]);
|
|
233
|
+
const team = { room: key => key === A ? { authors: { e1: { type: 'user' }, e2: { type: 'agent', agentId: 'main' }, e3: { type: 'system' } } } : null };
|
|
234
|
+
const { store, indexer } = setup(gw, { team });
|
|
235
|
+
indexer.onConnected();
|
|
236
|
+
await indexer.idle();
|
|
237
|
+
assert.deepEqual(store.search({ q: 'laser', role: 'user' }).results[0].snippets.map(s => s.messageId), ['e1']);
|
|
238
|
+
assert.equal(store.search({ q: 'joined' }).results.length, 0);
|
|
239
|
+
});
|
|
240
|
+
|
|
241
|
+
test('rebuild empties the index and fills it again', async () => {
|
|
242
|
+
const gw = fakeGateway();
|
|
243
|
+
gw.add(A, [msg('m1', 'user', 'alpha')]);
|
|
244
|
+
const { store, indexer } = setup(gw);
|
|
245
|
+
indexer.onConnected();
|
|
246
|
+
await indexer.idle();
|
|
247
|
+
await indexer.rebuild();
|
|
248
|
+
await indexer.idle();
|
|
249
|
+
assert.equal(store.search({ q: 'alpha' }).results.length, 1);
|
|
250
|
+
assert.equal(indexer.status().ready, true);
|
|
251
|
+
});
|
|
252
|
+
|
|
253
|
+
test('a connector update that changes indexing re-reads every chat', async () => {
|
|
254
|
+
const gw = fakeGateway();
|
|
255
|
+
gw.add(A, [msg('m1', 'user', 'alpha')]);
|
|
256
|
+
const { store, indexer } = setup(gw);
|
|
257
|
+
indexer.onConnected();
|
|
258
|
+
await indexer.idle();
|
|
259
|
+
store.db.prepare("UPDATE meta SET v = '1' WHERE k = 'indexer'").run();
|
|
260
|
+
gw.calls.length = 0;
|
|
261
|
+
const next = new SearchIndexer({ store, request: gw.request, lens: new SessionLens({ broadcast() {}, logger: quiet }), logger: quiet, fetchComment: async () => null, quietMs: 5 });
|
|
262
|
+
assert.equal(store.search({ q: 'alpha' }).results.length, 1, 'still searchable while it re-reads');
|
|
263
|
+
next.onConnected();
|
|
264
|
+
await next.idle();
|
|
265
|
+
assert.ok(gw.calls.some(c => c.method === 'chat.history' && !c.cursor && c.limit === 200), 'full read');
|
|
266
|
+
assert.deepEqual(store.staleKeys(), []);
|
|
267
|
+
});
|