@clawchatsai/connector 0.1.16 → 0.1.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -2
- package/dist/index.js +49 -133
- package/package.json +2 -1
- package/server/bootstrap/native.js +0 -4
- package/server/config.js +9 -17
- package/server/controllers/filesystem.js +2 -2
- package/server/controllers/gateway-media.js +42 -0
- package/server/controllers/gateway-media.test.js +23 -0
- package/server/debug.js +14 -1
- package/server/debug.test.js +22 -0
- package/server/gateway.js +55 -563
- package/server/gateway.test.js +27 -0
- package/server/history-export.js +10 -1
- package/server/index.js +34 -205
- package/server/session-lens.js +291 -0
- package/server/session-lens.test.js +179 -0
- package/server/store/extras-store.js +92 -0
- package/server/store/legacy-threads.js +42 -0
- package/server/store/legacy-threads.test.js +50 -0
- package/server/util/helpers.js +2 -105
- package/server/controllers/files.js +0 -64
- package/server/controllers/memory.js +0 -86
- package/server/controllers/messages.js +0 -118
- package/server/controllers/threads.js +0 -119
- package/server/controllers/workspaces.js +0 -116
- package/server/gateway-cleanup.js +0 -31
- package/server/gateway-sync.js +0 -571
- package/server/providers/memory-config.js +0 -52
- package/server/providers/memory.js +0 -108
- package/server/thread-move.js +0 -47
package/server/gateway-sync.js
DELETED
|
@@ -1,571 +0,0 @@
|
|
|
1
|
-
// Keeps ClawChats projects/threads and the gateway's session organization in step.
|
|
2
|
-
//
|
|
3
|
-
// ClawChats project <-> gateway session group (catalog entry + each session's `category`)
|
|
4
|
-
// ClawChats title <-> session `label` (custom name, unique per agent store)
|
|
5
|
-
// ClawChats pinned <-> session `pinned`
|
|
6
|
-
// ClawChats delete <-> `sessions.delete`
|
|
7
|
-
//
|
|
8
|
-
// The gateway owns organization. ClawChats actions are written to the gateway first
|
|
9
|
-
// (controllers call the push methods and fail if the gateway rejects); gateway-side
|
|
10
|
-
// changes arrive as `sessions.changed` events (plus a full reconcile on connect, on
|
|
11
|
-
// unkeyed/group events and every few minutes, since slow clients can miss events).
|
|
12
|
-
// Every write compares first, so the connector's own changes echo back as no-ops.
|
|
13
|
-
//
|
|
14
|
-
// A thread is only pulled from the gateway (moved, renamed, deleted) once the connector
|
|
15
|
-
// has seen its session at least once (`threads.gateway_seen_at`). Until then the
|
|
16
|
-
// connector pushes ClawChats' state to the gateway instead. That makes the first run
|
|
17
|
-
// after upgrade/import a seed, and keeps never-imported threads safe.
|
|
18
|
-
|
|
19
|
-
import fs from 'node:fs';
|
|
20
|
-
import path from 'node:path';
|
|
21
|
-
import { parseSessionKey } from './util/helpers.js';
|
|
22
|
-
import { moveThread as moveThreadRows } from './thread-move.js';
|
|
23
|
-
|
|
24
|
-
export const DEFAULT_PROJECT_LABEL = 'Default';
|
|
25
|
-
const PLACEHOLDER_TITLES = new Set(['New chat', '']);
|
|
26
|
-
const LABEL_MAX = 512;
|
|
27
|
-
const MAX_SUFFIX = 1000;
|
|
28
|
-
const PERIODIC_RECONCILE_MS = 5 * 60 * 1000;
|
|
29
|
-
const RECONCILE_DEBOUNCE_MS = 400;
|
|
30
|
-
// Refuse to mirror mass deletions found during a reconcile (missed events are normal;
|
|
31
|
-
// dozens of sessions vanishing at once is far more likely a gateway-side problem).
|
|
32
|
-
const MAX_RECONCILE_DELETES = 5;
|
|
33
|
-
|
|
34
|
-
export function withSuffix(base, n) {
|
|
35
|
-
return n <= 1 ? base : `${base} (${n})`;
|
|
36
|
-
}
|
|
37
|
-
|
|
38
|
-
export function isPushableTitle(title) {
|
|
39
|
-
return typeof title === 'string' && !PLACEHOLDER_TITLES.has(title.trim());
|
|
40
|
-
}
|
|
41
|
-
|
|
42
|
-
function isNotFound(err) {
|
|
43
|
-
return /not found|no session|unknown session/i.test(err?.message || '');
|
|
44
|
-
}
|
|
45
|
-
|
|
46
|
-
function isLabelTaken(err) {
|
|
47
|
-
return /label already in use/i.test(err?.message || '');
|
|
48
|
-
}
|
|
49
|
-
|
|
50
|
-
function slugify(label, taken) {
|
|
51
|
-
const base = (label.toLowerCase().replace(/[^a-z0-9-]+/g, '-').replace(/^-+|-+$/g, '') || 'project').slice(0, 28);
|
|
52
|
-
let name = base, n = 2;
|
|
53
|
-
while (taken.has(name)) name = `${base}-${n++}`;
|
|
54
|
-
return name;
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
export class GatewaySync {
|
|
58
|
-
constructor({ gateway, getDb, closeDb, getWorkspaces, setWorkspaces, dataDir, uploadsDir, broadcast, logger = console }) {
|
|
59
|
-
this.gateway = gateway;
|
|
60
|
-
this.getDb = getDb;
|
|
61
|
-
this.closeDb = closeDb;
|
|
62
|
-
this.getWorkspaces = getWorkspaces;
|
|
63
|
-
this.setWorkspaces = setWorkspaces;
|
|
64
|
-
this.dataDir = dataDir;
|
|
65
|
-
this.uploadsDir = uploadsDir;
|
|
66
|
-
this._changed = { projects: false, threads: new Set() };
|
|
67
|
-
this.broadcast = broadcast;
|
|
68
|
-
this.log = logger;
|
|
69
|
-
this.statePath = path.join(dataDir, 'gateway-sync.json');
|
|
70
|
-
this.state = this._loadState();
|
|
71
|
-
this.locations = new Map(); // threadId -> project name, for threads that moved away from their key's project
|
|
72
|
-
this.labels = new Map(); // label -> session key, from the last full list (pre-check only; gateway is authoritative)
|
|
73
|
-
this._queue = Promise.resolve();
|
|
74
|
-
// key -> newest gateway `updatedAt` applied or written. Event rows are captured
|
|
75
|
-
// snapshots that can arrive after a newer write; anything older is ignored
|
|
76
|
-
// (same nondecreasing-recency rule the gateway's Control UI uses).
|
|
77
|
-
this.recency = new Map();
|
|
78
|
-
this._reconcileQueued = false;
|
|
79
|
-
this._reconcileTimer = null;
|
|
80
|
-
this._periodic = null;
|
|
81
|
-
this._indexLocations();
|
|
82
|
-
}
|
|
83
|
-
|
|
84
|
-
// ── State ────────────────────────────────────────────────────────────
|
|
85
|
-
|
|
86
|
-
_loadState() {
|
|
87
|
-
try {
|
|
88
|
-
const s = JSON.parse(fs.readFileSync(this.statePath, 'utf8'));
|
|
89
|
-
if (s && s.version === 1) return s;
|
|
90
|
-
} catch { /* first run */ }
|
|
91
|
-
return { version: 1, catalog: null };
|
|
92
|
-
}
|
|
93
|
-
|
|
94
|
-
_saveState() {
|
|
95
|
-
fs.writeFileSync(this.statePath, JSON.stringify(this.state, null, 2));
|
|
96
|
-
}
|
|
97
|
-
|
|
98
|
-
/**
|
|
99
|
-
* Gateway-first write for a ClawChats action: runs in the sync queue and tags failures
|
|
100
|
-
* so the HTTP layer answers 502 and the caller skips its local change.
|
|
101
|
-
*/
|
|
102
|
-
async push(fn) {
|
|
103
|
-
try { return await this.run(fn); }
|
|
104
|
-
catch (e) { e.gatewayError = true; throw e; }
|
|
105
|
-
}
|
|
106
|
-
|
|
107
|
-
/** sessions.patch that records the resulting `updatedAt`, so older event snapshots are ignored. */
|
|
108
|
-
async _patch(params) {
|
|
109
|
-
const res = await this.gateway.request('sessions.patch', params);
|
|
110
|
-
this._seen(params.key, res?.entry?.updatedAt);
|
|
111
|
-
return res;
|
|
112
|
-
}
|
|
113
|
-
|
|
114
|
-
/** Record a gateway `updatedAt`; returns false if it is older than one already seen. */
|
|
115
|
-
_seen(key, updatedAt) {
|
|
116
|
-
if (typeof updatedAt !== 'number') return true;
|
|
117
|
-
const prev = this.recency.get(key);
|
|
118
|
-
if (prev !== undefined && updatedAt < prev) return false;
|
|
119
|
-
this.recency.set(key, updatedAt);
|
|
120
|
-
return true;
|
|
121
|
-
}
|
|
122
|
-
|
|
123
|
-
/** Serialize all sync work (pushes, event handling, reconciles). */
|
|
124
|
-
run(fn) {
|
|
125
|
-
const next = this._queue.then(fn);
|
|
126
|
-
this._queue = next.catch(() => {});
|
|
127
|
-
return next;
|
|
128
|
-
}
|
|
129
|
-
|
|
130
|
-
// ── Thread location (key → project) ─────────────────────────────────
|
|
131
|
-
|
|
132
|
-
_projects() {
|
|
133
|
-
return Object.values(this.getWorkspaces().workspaces).sort((a, b) => (a.order ?? 999) - (b.order ?? 999));
|
|
134
|
-
}
|
|
135
|
-
|
|
136
|
-
_indexLocations() {
|
|
137
|
-
this.locations.clear();
|
|
138
|
-
for (const p of this._projects()) {
|
|
139
|
-
let rows = [];
|
|
140
|
-
try { rows = this.getDb(p.name).prepare('SELECT id, session_key FROM threads').all(); } catch { continue; }
|
|
141
|
-
for (const r of rows) {
|
|
142
|
-
const parsed = parseSessionKey(r.session_key);
|
|
143
|
-
if (parsed && parsed.workspace !== p.name) this.locations.set(r.id, p.name);
|
|
144
|
-
}
|
|
145
|
-
}
|
|
146
|
-
}
|
|
147
|
-
|
|
148
|
-
/** parseSessionKey + current project of threads that moved. Same shape as parseSessionKey. */
|
|
149
|
-
locate(sessionKey) {
|
|
150
|
-
const parsed = parseSessionKey(sessionKey);
|
|
151
|
-
if (!parsed) return null;
|
|
152
|
-
const moved = this.locations.get(parsed.threadId);
|
|
153
|
-
return moved ? { ...parsed, workspace: moved } : parsed;
|
|
154
|
-
}
|
|
155
|
-
|
|
156
|
-
// ── Lifecycle ────────────────────────────────────────────────────────
|
|
157
|
-
|
|
158
|
-
async onConnected() {
|
|
159
|
-
try {
|
|
160
|
-
await this.gateway.request('sessions.subscribe', {});
|
|
161
|
-
} catch (e) {
|
|
162
|
-
this.log.warn?.(`[sync] sessions.subscribe failed: ${e.message}`);
|
|
163
|
-
}
|
|
164
|
-
this.scheduleReconcile(0);
|
|
165
|
-
if (!this._periodic) {
|
|
166
|
-
this._periodic = setInterval(() => this.scheduleReconcile(0), PERIODIC_RECONCILE_MS);
|
|
167
|
-
this._periodic.unref?.();
|
|
168
|
-
}
|
|
169
|
-
}
|
|
170
|
-
|
|
171
|
-
stop() {
|
|
172
|
-
if (this._periodic) clearInterval(this._periodic);
|
|
173
|
-
if (this._reconcileTimer) clearTimeout(this._reconcileTimer);
|
|
174
|
-
this._periodic = this._reconcileTimer = null;
|
|
175
|
-
}
|
|
176
|
-
|
|
177
|
-
scheduleReconcile(delay = RECONCILE_DEBOUNCE_MS) {
|
|
178
|
-
if (this._reconcileTimer) clearTimeout(this._reconcileTimer);
|
|
179
|
-
this._reconcileTimer = setTimeout(() => {
|
|
180
|
-
this._reconcileTimer = null;
|
|
181
|
-
if (this._reconcileQueued) return; // one waiting pass covers everything that happened before it starts
|
|
182
|
-
this._reconcileQueued = true;
|
|
183
|
-
this.run(() => { this._reconcileQueued = false; return this.reconcile(); })
|
|
184
|
-
.catch(e => this.log.error?.(`[sync] reconcile failed: ${e.message}`));
|
|
185
|
-
}, delay);
|
|
186
|
-
this._reconcileTimer.unref?.();
|
|
187
|
-
}
|
|
188
|
-
|
|
189
|
-
/** `sessions.changed` event from the gateway. */
|
|
190
|
-
onSessionsChanged(payload) {
|
|
191
|
-
if (!payload) return;
|
|
192
|
-
const key = payload.sessionKey;
|
|
193
|
-
if (!key || payload.reason === 'groups') { this.scheduleReconcile(); return; }
|
|
194
|
-
if (key.includes('__clawchats_title_')) return;
|
|
195
|
-
if (!parseSessionKey(key)) return; // not a ClawChats thread
|
|
196
|
-
this.run(async () => {
|
|
197
|
-
if (payload.reason === 'delete') return this._onSessionDeleted(key);
|
|
198
|
-
if (payload.session) return this._applyEventRow(payload.session, payload.reason);
|
|
199
|
-
}).catch(e => this.log.error?.(`[sync] ${payload.reason} ${key}: ${e.message}`));
|
|
200
|
-
}
|
|
201
|
-
|
|
202
|
-
// ── Push: ClawChats → gateway (called by controllers inside run()) ───
|
|
203
|
-
|
|
204
|
-
async _catalogNames() {
|
|
205
|
-
const res = await this.gateway.request('sessions.groups.list', {});
|
|
206
|
-
return (res?.groups || []).slice().sort((a, b) => a.position - b.position).map(g => g.name);
|
|
207
|
-
}
|
|
208
|
-
|
|
209
|
-
_uniqueProjectLabel(label, exceptName = null) {
|
|
210
|
-
const used = new Set(this._projects().filter(p => p.name !== exceptName).map(p => p.label));
|
|
211
|
-
let n = 1;
|
|
212
|
-
while (used.has(withSuffix(label, n))) n++;
|
|
213
|
-
return withSuffix(label, n);
|
|
214
|
-
}
|
|
215
|
-
|
|
216
|
-
/** New project. Returns the (possibly suffixed) label to store. */
|
|
217
|
-
async projectCreated(label) {
|
|
218
|
-
const unique = this._uniqueProjectLabel(label);
|
|
219
|
-
const names = await this._catalogNames();
|
|
220
|
-
if (!names.includes(unique)) await this.gateway.request('sessions.groups.put', { names: [...names, unique] });
|
|
221
|
-
this.state.catalog = [...new Set([...names, unique])];
|
|
222
|
-
this._saveState();
|
|
223
|
-
return unique;
|
|
224
|
-
}
|
|
225
|
-
|
|
226
|
-
/** Project renamed. Returns the (possibly suffixed) label to store. */
|
|
227
|
-
async projectRenamed(name, fromLabel, toLabel) {
|
|
228
|
-
const unique = this._uniqueProjectLabel(toLabel, name);
|
|
229
|
-
if (unique === fromLabel) return unique;
|
|
230
|
-
const names = await this._catalogNames();
|
|
231
|
-
if (names.includes(fromLabel)) {
|
|
232
|
-
await this.gateway.request('sessions.groups.rename', { name: fromLabel, to: unique });
|
|
233
|
-
} else if (!names.includes(unique)) {
|
|
234
|
-
await this.gateway.request('sessions.groups.put', { names: [...names, unique] });
|
|
235
|
-
}
|
|
236
|
-
this.state.catalog = await this._catalogNames();
|
|
237
|
-
this._saveState();
|
|
238
|
-
return unique;
|
|
239
|
-
}
|
|
240
|
-
|
|
241
|
-
/** Project deleted (its sessions were already deleted). */
|
|
242
|
-
async projectDeleted(label) {
|
|
243
|
-
const names = await this._catalogNames();
|
|
244
|
-
if (names.includes(label)) {
|
|
245
|
-
try { await this.gateway.request('sessions.groups.delete', { name: label }); }
|
|
246
|
-
catch (e) { if (!/unknown session group/i.test(e.message)) throw e; }
|
|
247
|
-
}
|
|
248
|
-
this.state.catalog = (this.state.catalog || names).filter(n => n !== label);
|
|
249
|
-
this._saveState();
|
|
250
|
-
}
|
|
251
|
-
|
|
252
|
-
/** Projects reordered; `labels` in the new order. */
|
|
253
|
-
async projectsReordered(labels) {
|
|
254
|
-
const names = await this._catalogNames();
|
|
255
|
-
const ordered = [...labels.filter(l => names.includes(l)), ...names.filter(n => !labels.includes(n))];
|
|
256
|
-
if (ordered.join('\u0000') !== names.join('\u0000')) await this.gateway.request('sessions.groups.put', { names: ordered });
|
|
257
|
-
this.state.catalog = ordered;
|
|
258
|
-
this._saveState();
|
|
259
|
-
}
|
|
260
|
-
|
|
261
|
-
/**
|
|
262
|
-
* Set a thread's title as the session's custom name, adding " (n)" until the gateway
|
|
263
|
-
* accepts it. Returns the accepted title, or the input unchanged when the session
|
|
264
|
-
* doesn't exist yet (it gets pushed when first seen) or the title is a placeholder.
|
|
265
|
-
*/
|
|
266
|
-
async pushTitle(sessionKey, title) {
|
|
267
|
-
if (!isPushableTitle(title)) return title;
|
|
268
|
-
const base = title.trim().slice(0, LABEL_MAX - 8);
|
|
269
|
-
for (let n = 1; n <= MAX_SUFFIX; n++) {
|
|
270
|
-
const candidate = withSuffix(base, n);
|
|
271
|
-
const owner = this.labels.get(candidate);
|
|
272
|
-
if (owner && owner !== sessionKey) continue;
|
|
273
|
-
try {
|
|
274
|
-
await this._patch({ key: sessionKey, label: candidate });
|
|
275
|
-
this._rememberLabel(sessionKey, candidate);
|
|
276
|
-
return candidate;
|
|
277
|
-
} catch (e) {
|
|
278
|
-
if (isNotFound(e)) return title;
|
|
279
|
-
if (isLabelTaken(e)) { this.labels.set(candidate, '?'); continue; }
|
|
280
|
-
throw e;
|
|
281
|
-
}
|
|
282
|
-
}
|
|
283
|
-
throw new Error(`no free title for "${base}"`);
|
|
284
|
-
}
|
|
285
|
-
|
|
286
|
-
_rememberLabel(sessionKey, label) {
|
|
287
|
-
for (const [l, k] of this.labels) if (k === sessionKey) this.labels.delete(l);
|
|
288
|
-
this.labels.set(label, sessionKey);
|
|
289
|
-
}
|
|
290
|
-
|
|
291
|
-
async pushPinned(sessionKey, pinned) {
|
|
292
|
-
try { await this._patch({ key: sessionKey, pinned: !!pinned }); }
|
|
293
|
-
catch (e) { if (!isNotFound(e)) throw e; }
|
|
294
|
-
}
|
|
295
|
-
|
|
296
|
-
/** Gateway-first thread delete. Throws if the gateway refuses (not-found is fine). */
|
|
297
|
-
async deleteSession(sessionKey) {
|
|
298
|
-
try { await this.gateway.request('sessions.delete', { key: sessionKey, deleteTranscript: true }); }
|
|
299
|
-
catch (e) { if (!isNotFound(e)) throw e; }
|
|
300
|
-
}
|
|
301
|
-
|
|
302
|
-
/** Title generated server-side (not user-initiated): push, adopt a suffix if one was needed. */
|
|
303
|
-
titleGenerated(projectName, threadId, sessionKey, title) {
|
|
304
|
-
this.run(async () => {
|
|
305
|
-
const accepted = await this.pushTitle(sessionKey, title);
|
|
306
|
-
if (accepted !== title) this._setLocalTitle(projectName, threadId, accepted);
|
|
307
|
-
}).catch(e => this.log.warn?.(`[sync] title push failed for ${threadId}: ${e.message}`));
|
|
308
|
-
}
|
|
309
|
-
|
|
310
|
-
// ── Pull: gateway → ClawChats ────────────────────────────────────────
|
|
311
|
-
|
|
312
|
-
_projectByLabel(label) {
|
|
313
|
-
return this._projects().find(p => p.label === label) || null;
|
|
314
|
-
}
|
|
315
|
-
|
|
316
|
-
_ensureProject(label) {
|
|
317
|
-
const existing = this._projectByLabel(label);
|
|
318
|
-
if (existing) return existing;
|
|
319
|
-
const ws = this.getWorkspaces();
|
|
320
|
-
const name = slugify(label, new Set(Object.keys(ws.workspaces)));
|
|
321
|
-
const maxOrder = Math.max(-1, ...Object.values(ws.workspaces).map(p => p.order ?? -1));
|
|
322
|
-
ws.workspaces[name] = { name, label, color: null, icon: null, agent: 'main', createdAt: Date.now(), order: maxOrder + 1 };
|
|
323
|
-
this.setWorkspaces(ws);
|
|
324
|
-
this.getDb(name);
|
|
325
|
-
this.log.info?.(`[sync] created project "${label}" (${name}) from gateway group`);
|
|
326
|
-
this._changed.projects = true;
|
|
327
|
-
return ws.workspaces[name];
|
|
328
|
-
}
|
|
329
|
-
|
|
330
|
-
_findThread(sessionKey) {
|
|
331
|
-
const loc = this.locate(sessionKey);
|
|
332
|
-
if (!loc || !this.getWorkspaces().workspaces[loc.workspace]) return null;
|
|
333
|
-
const row = this.getDb(loc.workspace).prepare('SELECT id, session_key, title, pinned, gateway_seen_at FROM threads WHERE id = ?').get(loc.threadId);
|
|
334
|
-
return row && row.session_key === sessionKey ? { ...row, project: loc.workspace } : null;
|
|
335
|
-
}
|
|
336
|
-
|
|
337
|
-
_setLocalTitle(project, threadId, title) {
|
|
338
|
-
this.getDb(project).prepare('UPDATE threads SET title = ? WHERE id = ?').run(title, threadId);
|
|
339
|
-
this.broadcast(JSON.stringify({ type: 'clawchats', event: 'thread-title-updated', threadId, workspace: project, title }));
|
|
340
|
-
}
|
|
341
|
-
|
|
342
|
-
_moveThread(threadId, from, to) {
|
|
343
|
-
if (!moveThreadRows(this.getDb, threadId, from, to)) return;
|
|
344
|
-
this.locations.set(threadId, to);
|
|
345
|
-
this.log.info?.(`[sync] moved thread ${threadId} ${from} → ${to}`);
|
|
346
|
-
this._changed.threads.add(from).add(to);
|
|
347
|
-
}
|
|
348
|
-
|
|
349
|
-
/** First sight of a session for a ClawChats thread: push ClawChats' state, mark seen. */
|
|
350
|
-
async _seed(t, row) {
|
|
351
|
-
const project = this.getWorkspaces().workspaces[t.project];
|
|
352
|
-
const patch = {};
|
|
353
|
-
if (row.category !== project.label) patch.category = project.label;
|
|
354
|
-
if (!!row.pinned !== !!t.pinned) patch.pinned = !!t.pinned;
|
|
355
|
-
if (Object.keys(patch).length) await this._patch({ key: row.key, ...patch });
|
|
356
|
-
if (isPushableTitle(t.title) && row.label !== t.title) {
|
|
357
|
-
const accepted = await this.pushTitle(row.key, t.title);
|
|
358
|
-
if (accepted !== t.title) this._setLocalTitle(t.project, t.id, accepted);
|
|
359
|
-
}
|
|
360
|
-
this.getDb(t.project).prepare('UPDATE threads SET gateway_seen_at = ? WHERE id = ?').run(Date.now(), t.id);
|
|
361
|
-
}
|
|
362
|
-
|
|
363
|
-
/** Gateway → ClawChats for one field set. `row` may be partial: absent fields mean "unchanged". */
|
|
364
|
-
_pullFields(t, row, source) {
|
|
365
|
-
if (typeof row.category === 'string' && row.category) {
|
|
366
|
-
const target = this._projectByLabel(row.category);
|
|
367
|
-
if (!target) return 'unknown-group';
|
|
368
|
-
if (target.name !== t.project) {
|
|
369
|
-
this.log.info?.(`[sync] ${source}: "${row.category}" for ${t.id} (updatedAt ${row.updatedAt})`);
|
|
370
|
-
this._moveThread(t.id, t.project, target.name);
|
|
371
|
-
t.project = target.name;
|
|
372
|
-
}
|
|
373
|
-
}
|
|
374
|
-
if (typeof row.label === 'string' && row.label && row.label !== t.title) this._setLocalTitle(t.project, t.id, row.label);
|
|
375
|
-
if (typeof row.pinned === 'boolean' && row.pinned !== !!t.pinned) {
|
|
376
|
-
this.getDb(t.project).prepare('UPDATE threads SET pinned = ? WHERE id = ?').run(row.pinned ? 1 : 0, t.id);
|
|
377
|
-
this._changed.threads.add(t.project);
|
|
378
|
-
}
|
|
379
|
-
return 'ok';
|
|
380
|
-
}
|
|
381
|
-
|
|
382
|
-
/**
|
|
383
|
-
* `sessions.changed` row. Never writes to the gateway (event rows are partial captured
|
|
384
|
-
* snapshots); anything needing a write or a complete row is left to a full reconcile.
|
|
385
|
-
*/
|
|
386
|
-
async _applyEventRow(row, reason) {
|
|
387
|
-
const t = this._findThread(row.key);
|
|
388
|
-
if (!t) return;
|
|
389
|
-
if (!t.gateway_seen_at) { this.scheduleReconcile(); return; } // seeding happens in a full pass
|
|
390
|
-
if (!this._seen(row.key, row.updatedAt)) return; // older than what we already applied/wrote
|
|
391
|
-
if (this._pullFields(t, row, `event ${reason}`) === 'unknown-group') { this.scheduleReconcile(); return; }
|
|
392
|
-
if ('category' in row && !row.category) this.scheduleReconcile(); // explicit ungroup: reconcile decides
|
|
393
|
-
this._flushChanged();
|
|
394
|
-
}
|
|
395
|
-
|
|
396
|
-
/** Full `sessions.list` row during reconcile: complete, so absence means unset. */
|
|
397
|
-
async _applyListRow(row) {
|
|
398
|
-
const t = this._findThread(row.key);
|
|
399
|
-
if (!t) return;
|
|
400
|
-
if (!this._seen(row.key, row.updatedAt)) return;
|
|
401
|
-
if (!t.gateway_seen_at) return this._seed(t, row);
|
|
402
|
-
if (!row.category) {
|
|
403
|
-
// Ungrouped in the gateway → Default (both sides).
|
|
404
|
-
await this._patch({ key: row.key, category: DEFAULT_PROJECT_LABEL });
|
|
405
|
-
row = { ...row, category: DEFAULT_PROJECT_LABEL };
|
|
406
|
-
}
|
|
407
|
-
this._ensureProject(row.category);
|
|
408
|
-
this._pullFields(t, row, 'reconcile');
|
|
409
|
-
if (!row.label && isPushableTitle(t.title)) {
|
|
410
|
-
const accepted = await this.pushTitle(row.key, t.title);
|
|
411
|
-
if (accepted !== t.title) this._setLocalTitle(t.project, t.id, accepted);
|
|
412
|
-
}
|
|
413
|
-
}
|
|
414
|
-
|
|
415
|
-
_onSessionDeleted(sessionKey) {
|
|
416
|
-
const t = this._findThread(sessionKey);
|
|
417
|
-
if (!t || !t.gateway_seen_at) return;
|
|
418
|
-
this._deleteLocalThread(t);
|
|
419
|
-
this._flushChanged();
|
|
420
|
-
}
|
|
421
|
-
|
|
422
|
-
_deleteLocalThread(t) {
|
|
423
|
-
this.getDb(t.project).prepare('DELETE FROM threads WHERE id = ?').run(t.id);
|
|
424
|
-
this.locations.delete(t.id);
|
|
425
|
-
try { fs.rmSync(path.join(this.uploadsDir, t.id), { recursive: true }); } catch { /* none */ }
|
|
426
|
-
this.log.info?.(`[sync] deleted thread ${t.id} (session deleted in gateway)`);
|
|
427
|
-
this._changed.threads.add(t.project);
|
|
428
|
-
}
|
|
429
|
-
|
|
430
|
-
_flushChanged() {
|
|
431
|
-
const c = this._changed;
|
|
432
|
-
this._changed = { projects: false, threads: new Set() };
|
|
433
|
-
if (c.projects) this.broadcast(JSON.stringify({ type: 'clawchats', event: 'workspaces-changed' }));
|
|
434
|
-
if (c.threads.size) this.broadcast(JSON.stringify({ type: 'clawchats', event: 'threads-changed', workspaces: [...c.threads] }));
|
|
435
|
-
}
|
|
436
|
-
|
|
437
|
-
async _listAllSessions() {
|
|
438
|
-
const out = [];
|
|
439
|
-
let offset = 0;
|
|
440
|
-
for (let page = 0; page < 50; page++) {
|
|
441
|
-
const res = await this.gateway.request('sessions.list', { archived: 'all', limit: 1000, offset }, 60000);
|
|
442
|
-
out.push(...(res?.sessions || []));
|
|
443
|
-
if (!res?.hasMore || res.nextOffset == null) break;
|
|
444
|
-
offset = res.nextOffset;
|
|
445
|
-
}
|
|
446
|
-
return out;
|
|
447
|
-
}
|
|
448
|
-
|
|
449
|
-
/** Full two-way pass. Must run inside run(). */
|
|
450
|
-
async reconcile() {
|
|
451
|
-
if (!this.gateway.connected) return;
|
|
452
|
-
const [names, sessions] = await Promise.all([this._catalogNames(), this._listAllSessions()]);
|
|
453
|
-
this.labels = new Map(sessions.filter(s => s.label).map(s => [s.label, s.key]));
|
|
454
|
-
const firstRun = !Array.isArray(this.state.catalog);
|
|
455
|
-
const last = this.state.catalog || [];
|
|
456
|
-
|
|
457
|
-
// 1. Duplicate project labels (exact, like the gateway) → suffix in ClawChats.
|
|
458
|
-
const seen = new Set();
|
|
459
|
-
const ws = this.getWorkspaces();
|
|
460
|
-
for (const p of this._projects()) {
|
|
461
|
-
if (seen.has(p.label)) { p.label = this._uniqueProjectLabel(p.label, p.name); this._changed.projects = true; }
|
|
462
|
-
seen.add(p.label);
|
|
463
|
-
}
|
|
464
|
-
if (this._changed.projects) this.setWorkspaces(ws);
|
|
465
|
-
|
|
466
|
-
// 2. Group-level changes made in the gateway since the last pass.
|
|
467
|
-
const projectsToDelete = new Set();
|
|
468
|
-
if (!firstRun) {
|
|
469
|
-
const labels = new Set(this._projects().map(p => p.label));
|
|
470
|
-
const vanished = last.filter(n => !names.includes(n) && labels.has(n));
|
|
471
|
-
const appeared = names.filter(n => !last.includes(n) && !labels.has(n));
|
|
472
|
-
if (vanished.length === 1 && appeared.length === 1) {
|
|
473
|
-
const p = this._projectByLabel(vanished[0]);
|
|
474
|
-
p.label = appeared[0];
|
|
475
|
-
this.setWorkspaces(this.getWorkspaces());
|
|
476
|
-
this.log.info?.(`[sync] project "${vanished[0]}" renamed to "${appeared[0]}" in gateway`);
|
|
477
|
-
this._changed.projects = true;
|
|
478
|
-
} else {
|
|
479
|
-
for (const n of appeared) this._ensureProject(n);
|
|
480
|
-
for (const n of vanished) projectsToDelete.add(this._projectByLabel(n).name);
|
|
481
|
-
}
|
|
482
|
-
}
|
|
483
|
-
this._ensureProject(DEFAULT_PROJECT_LABEL);
|
|
484
|
-
|
|
485
|
-
// 3. Sessions ↔ threads.
|
|
486
|
-
const present = new Set();
|
|
487
|
-
for (const row of sessions) {
|
|
488
|
-
present.add(row.key);
|
|
489
|
-
if (!parseSessionKey(row.key)) continue;
|
|
490
|
-
try { await this._applyListRow(row); }
|
|
491
|
-
catch (e) { this.log.warn?.(`[sync] ${row.key}: ${e.message}`); }
|
|
492
|
-
}
|
|
493
|
-
|
|
494
|
-
// 4. Seen threads whose session is gone were deleted in the gateway.
|
|
495
|
-
const gone = [];
|
|
496
|
-
for (const p of this._projects()) {
|
|
497
|
-
for (const t of this.getDb(p.name).prepare('SELECT id, session_key, gateway_seen_at FROM threads WHERE gateway_seen_at IS NOT NULL').all()) {
|
|
498
|
-
if (!present.has(t.session_key)) gone.push({ ...t, project: p.name });
|
|
499
|
-
}
|
|
500
|
-
}
|
|
501
|
-
if (gone.length > MAX_RECONCILE_DELETES) {
|
|
502
|
-
this.log.warn?.(`[sync] ${gone.length} synced threads have no gateway session — NOT deleting (limit ${MAX_RECONCILE_DELETES}). Check the gateway.`);
|
|
503
|
-
} else {
|
|
504
|
-
for (const t of gone) this._deleteLocalThread(t);
|
|
505
|
-
}
|
|
506
|
-
|
|
507
|
-
// 5. Projects whose group was deleted in the gateway: chats moved to Default above.
|
|
508
|
-
const def = this._projectByLabel(DEFAULT_PROJECT_LABEL);
|
|
509
|
-
for (const name of projectsToDelete) {
|
|
510
|
-
if (name === def.name) continue;
|
|
511
|
-
const db = this.getDb(name);
|
|
512
|
-
for (const t of db.prepare('SELECT id FROM threads').all()) this._moveThread(t.id, name, def.name);
|
|
513
|
-
this._deleteLocalProject(name);
|
|
514
|
-
}
|
|
515
|
-
|
|
516
|
-
// 6. Catalog ⇄ projects.
|
|
517
|
-
const projects = this._projects();
|
|
518
|
-
let finalNames;
|
|
519
|
-
if (firstRun) {
|
|
520
|
-
// Seed: ClawChats order; keep non-ClawChats groups that still have members.
|
|
521
|
-
const members = new Map();
|
|
522
|
-
for (const s of sessions) if (s.category) members.set(s.category, (members.get(s.category) || 0) + 1);
|
|
523
|
-
const claw = new Set();
|
|
524
|
-
for (const s of sessions) if (parseSessionKey(s.key) && this._findThread(s.key)) claw.add(s.key);
|
|
525
|
-
const foreignMembers = new Map();
|
|
526
|
-
for (const s of sessions) if (s.category && !claw.has(s.key)) foreignMembers.set(s.category, (foreignMembers.get(s.category) || 0) + 1);
|
|
527
|
-
for (const n of names) if (!this._projectByLabel(n) && foreignMembers.get(n)) this._ensureProject(n);
|
|
528
|
-
finalNames = this._projects().map(p => p.label);
|
|
529
|
-
} else {
|
|
530
|
-
// Gateway order wins; projects missing from the catalog are appended.
|
|
531
|
-
finalNames = [...names.filter(n => this._projectByLabel(n)), ...projects.map(p => p.label).filter(l => !names.includes(l))];
|
|
532
|
-
const wsNow = this.getWorkspaces();
|
|
533
|
-
finalNames.forEach((label, i) => {
|
|
534
|
-
const p = this._projectByLabel(label);
|
|
535
|
-
if (p && p.order !== i) { wsNow.workspaces[p.name].order = i; this._changed.projects = true; }
|
|
536
|
-
});
|
|
537
|
-
if (this._changed.projects) this.setWorkspaces(wsNow);
|
|
538
|
-
}
|
|
539
|
-
const current = await this._catalogNames();
|
|
540
|
-
if (finalNames.join('\u0000') !== current.join('\u0000')) {
|
|
541
|
-
// Groups not in finalNames must be empty by now; put rejects dropping non-empty ones.
|
|
542
|
-
await this.gateway.request('sessions.groups.put', { names: finalNames });
|
|
543
|
-
}
|
|
544
|
-
this.state.catalog = finalNames;
|
|
545
|
-
this._saveState();
|
|
546
|
-
|
|
547
|
-
// 7. Leftover title-generation sessions.
|
|
548
|
-
for (const s of sessions) {
|
|
549
|
-
if (s.key.includes('__clawchats_title_')) {
|
|
550
|
-
try { await this.deleteSession(s.key); } catch (e) { this.log.warn?.(`[sync] title session cleanup: ${e.message}`); }
|
|
551
|
-
}
|
|
552
|
-
}
|
|
553
|
-
|
|
554
|
-
const unseen = this._projects().reduce((n, p) => n + this.getDb(p.name).prepare('SELECT COUNT(*) c FROM threads WHERE gateway_seen_at IS NULL').get().c, 0);
|
|
555
|
-
this.log.info?.(`[sync] reconcile done: ${sessions.length} sessions, ${finalNames.length} groups, ${unseen} ClawChats threads not in the gateway`);
|
|
556
|
-
this._flushChanged();
|
|
557
|
-
}
|
|
558
|
-
|
|
559
|
-
_deleteLocalProject(name) {
|
|
560
|
-
const ws = this.getWorkspaces();
|
|
561
|
-
if (!ws.workspaces[name] || Object.keys(ws.workspaces).length <= 1) return;
|
|
562
|
-
this.closeDb(name);
|
|
563
|
-
const dbPath = path.join(this.dataDir, `${name}.db`);
|
|
564
|
-
for (const suffix of ['', '-wal', '-shm']) { try { fs.unlinkSync(dbPath + suffix); } catch { /* ok */ } }
|
|
565
|
-
delete ws.workspaces[name];
|
|
566
|
-
if (ws.active === name) ws.active = this._projectByLabel(DEFAULT_PROJECT_LABEL)?.name || Object.keys(ws.workspaces)[0];
|
|
567
|
-
this.setWorkspaces(ws);
|
|
568
|
-
this.log.info?.(`[sync] deleted project ${name} (group deleted in gateway)`);
|
|
569
|
-
this._changed.projects = true;
|
|
570
|
-
}
|
|
571
|
-
}
|
|
@@ -1,52 +0,0 @@
|
|
|
1
|
-
import fs from 'node:fs';
|
|
2
|
-
import path from 'node:path';
|
|
3
|
-
import os from 'node:os';
|
|
4
|
-
|
|
5
|
-
/**
|
|
6
|
-
* Discover memory backend configuration from OpenClaw config + optional env overrides.
|
|
7
|
-
* env vars (MEMORY_PROVIDER, QDRANT_HOST, etc.) are read by the plugin host (src/index.ts)
|
|
8
|
-
* and passed in via envOverrides to keep env vars out of the server bundle.
|
|
9
|
-
*
|
|
10
|
-
* This file is intentionally network-free: it only reads local config files.
|
|
11
|
-
* The actual provider implementations (Qdrant, Postgres) live in memory.js.
|
|
12
|
-
*/
|
|
13
|
-
export function discoverMemoryConfig(envOverrides = {}) {
|
|
14
|
-
const defaults = { provider: 'qdrant', host: 'localhost', port: 6333, collection: null };
|
|
15
|
-
let oc = null;
|
|
16
|
-
for (const cfgPath of [path.join(os.homedir(), '.openclaw', 'openclaw.json'), '/etc/openclaw/openclaw.json']) {
|
|
17
|
-
try { oc = JSON.parse(fs.readFileSync(cfgPath, 'utf8')); break; } catch { /* try next */ }
|
|
18
|
-
}
|
|
19
|
-
|
|
20
|
-
let cfg = { ...defaults };
|
|
21
|
-
if (oc) {
|
|
22
|
-
const vs = oc.plugins?.slots?.memory
|
|
23
|
-
? oc.plugins?.entries?.[oc.plugins.slots.memory]?.config?.oss?.vectorStore
|
|
24
|
-
: null;
|
|
25
|
-
if (vs) {
|
|
26
|
-
if (vs.provider) cfg.provider = vs.provider;
|
|
27
|
-
if (vs.config?.host) cfg.host = vs.config.host;
|
|
28
|
-
if (vs.config?.port) cfg.port = vs.config.port;
|
|
29
|
-
if (vs.config?.collectionName) cfg.collection = vs.config.collectionName;
|
|
30
|
-
if (vs.config?.user) cfg.pgUser = vs.config.user;
|
|
31
|
-
if (vs.config?.password) cfg.pgPassword = vs.config.password;
|
|
32
|
-
if (vs.config?.dbname) cfg.pgDbName = vs.config.dbname;
|
|
33
|
-
}
|
|
34
|
-
const wsDir = oc.agents?.defaults?.workspace;
|
|
35
|
-
if (wsDir) cfg.workspaceDir = wsDir;
|
|
36
|
-
}
|
|
37
|
-
|
|
38
|
-
if (envOverrides.provider) cfg.provider = envOverrides.provider;
|
|
39
|
-
if (envOverrides.host) cfg.host = envOverrides.host;
|
|
40
|
-
if (envOverrides.port) cfg.port = parseInt(envOverrides.port, 10);
|
|
41
|
-
if (envOverrides.collection) cfg.collection = envOverrides.collection;
|
|
42
|
-
if (envOverrides.pgUrl) cfg.pgUrl = envOverrides.pgUrl;
|
|
43
|
-
if (envOverrides.qdrantUrl && !envOverrides.host) {
|
|
44
|
-
try {
|
|
45
|
-
const u = new URL(envOverrides.qdrantUrl);
|
|
46
|
-
cfg.host = u.hostname;
|
|
47
|
-
if (u.port) cfg.port = parseInt(u.port, 10);
|
|
48
|
-
} catch { /* ignore */ }
|
|
49
|
-
}
|
|
50
|
-
if (!cfg.workspaceDir) cfg.workspaceDir = path.join(os.homedir(), '.openclaw', 'workspace');
|
|
51
|
-
return cfg;
|
|
52
|
-
}
|