agent-loft 0.2.125 → 0.2.126
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/package.json +1 -1
- package/src/channels/web/api.ts +4 -0
- package/src/channels/web/memory-sync-api.ts +44 -0
- package/src/core/chat-history.ts +40 -13
- package/src/core/db.ts +12 -0
- package/src/core/memory-curator.ts +51 -0
- package/src/core/memory-summaries.ts +4 -3
- package/src/core/memory-sync-schema.ts +53 -0
- package/src/core/memory-sync-store.ts +434 -0
- package/src/core/memory-sync.ts +200 -0
- package/src/core/personal.ts +2 -0
- package/src/main.ts +3 -0
- package/src/mcp/memory-server.ts +19 -6
- package/web/settings/memory-sync.js +119 -0
- package/web/settings/memory.js +2 -0
|
@@ -0,0 +1,434 @@
|
|
|
1
|
+
// The relay orders committed versions. Each sender includes the values it edited from, so
|
|
2
|
+
// clock skew never decides whose text survives a concurrent edit.
|
|
3
|
+
import { randomBytes } from "node:crypto";
|
|
4
|
+
import { sql, transaction } from "./db.js";
|
|
5
|
+
import { ftsIndexText } from "./fts.js";
|
|
6
|
+
import { readMemory, setMemoryOptions } from "./memory.js";
|
|
7
|
+
import { syncKinds, tables, type SyncKind } from "./memory-sync-schema.js";
|
|
8
|
+
|
|
9
|
+
type Body = Record<string, string | number | null>;
|
|
10
|
+
export type Document = { gid: string; kind: SyncKind; body: Body | null; revision: string | null; conflicts: string[] };
|
|
11
|
+
export type Change = {
|
|
12
|
+
gid: string;
|
|
13
|
+
kind: SyncKind;
|
|
14
|
+
op: string;
|
|
15
|
+
baseRevision: string | null;
|
|
16
|
+
base: Body | null;
|
|
17
|
+
body: Body | null;
|
|
18
|
+
resolve?: string[];
|
|
19
|
+
actor?: string;
|
|
20
|
+
};
|
|
21
|
+
type Mapping = { kind: SyncKind; local_key: string | null; gid: string };
|
|
22
|
+
type DocRow = { seq: number; kind: SyncKind; gid: string; body: string | null; revision: string | null; conflicts: string };
|
|
23
|
+
export type Account = {
|
|
24
|
+
user_id: string;
|
|
25
|
+
device: string;
|
|
26
|
+
relay: string | null;
|
|
27
|
+
url: string | null;
|
|
28
|
+
token: string | null;
|
|
29
|
+
cursor: number;
|
|
30
|
+
last_sync: number | null;
|
|
31
|
+
error: string | null;
|
|
32
|
+
};
|
|
33
|
+
export const randomId = () => randomBytes(16).toString("hex");
|
|
34
|
+
export const idPattern = /^[0-9a-f]{32}$/;
|
|
35
|
+
export const account = (u: string) => sql("SELECT * FROM sync_account WHERE user_id=?").get(u) as Account | undefined;
|
|
36
|
+
const mapping = (u: string, gid: string) => sql("SELECT * FROM sync_map WHERE user_id=? AND gid=?").get(u, gid) as Mapping | undefined;
|
|
37
|
+
const stored = (u: string, gid: string) => sql("SELECT * FROM sync_doc WHERE user_id=? AND gid=?").get(u, gid) as DocRow | undefined;
|
|
38
|
+
const decode = (r: DocRow): Document => ({
|
|
39
|
+
gid: r.gid,
|
|
40
|
+
kind: r.kind,
|
|
41
|
+
body: r.body ? JSON.parse(r.body) : null,
|
|
42
|
+
revision: r.revision,
|
|
43
|
+
conflicts: JSON.parse(r.conflicts),
|
|
44
|
+
});
|
|
45
|
+
const columns = {
|
|
46
|
+
about: ["text"],
|
|
47
|
+
item: ["text", "source", "tier", "locked", "created_at", "updated_at", "observed_at", "valid_until", "source_conv", "deleted_at"],
|
|
48
|
+
message: ["conv_key", "agent", "role", "text", "ts", "msg_id", "source_order"],
|
|
49
|
+
summary: ["kind", "scope", "title", "text", "project", "from_ts", "to_ts", "updated_at"],
|
|
50
|
+
};
|
|
51
|
+
|
|
52
|
+
export function activate(u: string, device = randomId(), url: string | null = null, token: string | null = null) {
|
|
53
|
+
transaction(() => {
|
|
54
|
+
if (account(u)) throw new Error("该账号已配置同步");
|
|
55
|
+
readMemory(u); // finish any legacy item migration before taking the initial inventory
|
|
56
|
+
sql("INSERT INTO sync_account(user_id,device,url,token) VALUES(?,?,?,?)").run(u, device, url, token);
|
|
57
|
+
for (const kind of syncKinds) {
|
|
58
|
+
const col = kind === "about" ? "data" : "id";
|
|
59
|
+
sql(`UPDATE ${tables[kind]} SET ${col}=${col} WHERE user_id=?`).run(u);
|
|
60
|
+
}
|
|
61
|
+
});
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
const prefix = (device: string) => `sync/${device}/`;
|
|
65
|
+
function qualify(value: unknown, device: string) {
|
|
66
|
+
return typeof value === "string" && !value.startsWith("sync/") ? prefix(device) + value : value;
|
|
67
|
+
}
|
|
68
|
+
function localize(value: unknown, device: string) {
|
|
69
|
+
return typeof value === "string" && value.startsWith(prefix(device)) ? value.slice(prefix(device).length) : value;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
function snapshot(u: string, m: Mapping, device: string): Body | null {
|
|
73
|
+
if (m.kind === "about") return { text: readMemory(u).about };
|
|
74
|
+
if (!m.local_key) return null;
|
|
75
|
+
const body = sql(`SELECT ${columns[m.kind].join(",")} FROM ${tables[m.kind]} WHERE user_id=? AND id=?`).get(u, m.local_key) as
|
|
76
|
+
| Body
|
|
77
|
+
| undefined;
|
|
78
|
+
if (!body) return null;
|
|
79
|
+
if (m.kind === "message") body.source_order ??= Number(m.local_key);
|
|
80
|
+
for (const k of ["conv_key", "source_conv", "scope"]) if (typeof body[k] === "string") body[k] = qualify(body[k], device) as string;
|
|
81
|
+
return body;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
function saveDoc(u: string, d: Document) {
|
|
85
|
+
sql("INSERT OR REPLACE INTO sync_doc(user_id,gid,kind,body,revision,conflicts) VALUES(?,?,?,?,?,?)").run(
|
|
86
|
+
u,
|
|
87
|
+
d.gid,
|
|
88
|
+
d.kind,
|
|
89
|
+
d.body ? JSON.stringify(d.body) : null,
|
|
90
|
+
d.revision,
|
|
91
|
+
JSON.stringify(d.conflicts),
|
|
92
|
+
);
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
export function staged(u: string, limit = 48, gid?: string): Change[] {
|
|
96
|
+
const a = account(u)!;
|
|
97
|
+
const rows = sql(`SELECT m.*,q.op,q.resolution FROM sync_dirty q JOIN sync_map m USING(user_id,gid)
|
|
98
|
+
WHERE q.user_id=? ${gid ? "AND q.gid=?" : ""} ORDER BY q.seq LIMIT ?`).iterate(u, ...(gid ? [gid] : []), limit) as Iterable<
|
|
99
|
+
Mapping & { op: string; resolution: string | null }
|
|
100
|
+
>;
|
|
101
|
+
const out: Change[] = [];
|
|
102
|
+
let bytes = 0;
|
|
103
|
+
for (const r of rows) {
|
|
104
|
+
const base = stored(u, r.gid);
|
|
105
|
+
const c: Change = {
|
|
106
|
+
gid: r.gid,
|
|
107
|
+
kind: r.kind,
|
|
108
|
+
op: r.op,
|
|
109
|
+
baseRevision: base?.revision ?? null,
|
|
110
|
+
base: (r.kind === "about" || r.kind === "item") && base?.body ? JSON.parse(base.body) : null,
|
|
111
|
+
body: snapshot(u, r, a.device),
|
|
112
|
+
};
|
|
113
|
+
if (r.kind === "item" && r.local_key) {
|
|
114
|
+
const log = sql("SELECT actor FROM memory_log WHERE user_id=? AND item_id=? ORDER BY id DESC LIMIT 1").get(u, r.local_key);
|
|
115
|
+
if (log?.actor) c.actor = String(log.actor);
|
|
116
|
+
}
|
|
117
|
+
if (r.resolution) c.resolve = JSON.parse(r.resolution);
|
|
118
|
+
out.push(c);
|
|
119
|
+
bytes += Buffer.byteLength(JSON.stringify(c));
|
|
120
|
+
if (bytes >= 512 * 1024) break;
|
|
121
|
+
}
|
|
122
|
+
return out;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
// Freeze the exact batch across process restarts and lost acknowledgements. Later edits stay
|
|
126
|
+
// dirty until the acknowledged snapshot becomes their base.
|
|
127
|
+
export function preparePush(u: string): Change[] {
|
|
128
|
+
return transaction(() => {
|
|
129
|
+
const flight = sql("SELECT change FROM sync_flight WHERE user_id=? ORDER BY rowid").all(u);
|
|
130
|
+
if (flight.length) return flight.map((r) => JSON.parse(String(r.change)) as Change);
|
|
131
|
+
const batch = staged(u);
|
|
132
|
+
for (const c of batch) sql("INSERT INTO sync_flight VALUES(?,?,?)").run(u, c.gid, JSON.stringify(c));
|
|
133
|
+
return batch;
|
|
134
|
+
});
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
function merge(old: Document | undefined, change: Change): Document {
|
|
138
|
+
const { gid, kind, body, base } = change;
|
|
139
|
+
if (!body || old?.body === null) return { gid, kind, body: null, revision: null, conflicts: [] };
|
|
140
|
+
const previous = old?.body;
|
|
141
|
+
if ((!previous && kind !== "about") || kind === "message" || kind === "summary")
|
|
142
|
+
return { gid, kind, body, revision: change.op, conflicts: [] };
|
|
143
|
+
const current = previous ?? { text: "" };
|
|
144
|
+
const original = base ?? (kind === "about" ? { text: "" } : {});
|
|
145
|
+
// An automatic change made offline before another instance pinned the item must
|
|
146
|
+
// pass the same protection check as an automatic change made on that instance.
|
|
147
|
+
const protectedItem = (b: Body) => b.locked === 1 || b.source === "user" || b.source === "asked";
|
|
148
|
+
if (kind === "item" && old && change.actor !== "user" && protectedItem(current) && !protectedItem(original)) return old;
|
|
149
|
+
// Keep the version that was put into the trash. A restore must name a base that was already trashed.
|
|
150
|
+
if (kind === "item" && old && current.deleted_at && !original.deleted_at && !body.deleted_at) return old;
|
|
151
|
+
const result = { ...current };
|
|
152
|
+
let revision = old?.revision ?? null;
|
|
153
|
+
let choices = [...(old?.conflicts ?? [])];
|
|
154
|
+
for (const field of columns[kind]) {
|
|
155
|
+
const next = body[field]!;
|
|
156
|
+
if (
|
|
157
|
+
field === "text" &&
|
|
158
|
+
change.resolve &&
|
|
159
|
+
old?.revision === change.baseRevision &&
|
|
160
|
+
JSON.stringify(choices) === JSON.stringify(change.resolve)
|
|
161
|
+
) {
|
|
162
|
+
revision = change.op;
|
|
163
|
+
result.text = next;
|
|
164
|
+
choices = [];
|
|
165
|
+
} else if (next !== original[field]) {
|
|
166
|
+
if (field === "text") revision = change.op;
|
|
167
|
+
if (
|
|
168
|
+
field === "text" &&
|
|
169
|
+
(choices.length || (current.text !== next && (current.text !== original.text || (old?.revision ?? null) !== change.baseRevision)))
|
|
170
|
+
) {
|
|
171
|
+
choices = [...new Set([...(choices.length ? choices : [String(current.text)]), String(next)])].sort();
|
|
172
|
+
result.text = old?.conflicts.length ? current.text! : (original.text ?? "");
|
|
173
|
+
} else if (field === "source" && ["user", "asked"].includes(String(current.source)) && !["user", "asked"].includes(String(next))) {
|
|
174
|
+
continue;
|
|
175
|
+
} else if (field === "updated_at") result[field] = Math.max(Number(current[field]), Number(next));
|
|
176
|
+
else result[field] = next;
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
if (result.deleted_at) {
|
|
180
|
+
choices = [];
|
|
181
|
+
if (!original.deleted_at && body.deleted_at) {
|
|
182
|
+
result.text = body.text!;
|
|
183
|
+
revision = change.op;
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
return { gid, kind, body: result, revision, conflicts: choices };
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
function apply(u: string, d: Document) {
|
|
190
|
+
const a = account(u)!;
|
|
191
|
+
const m = mapping(u, d.gid);
|
|
192
|
+
let key = m?.local_key ?? null;
|
|
193
|
+
const b = d.body && { ...d.body };
|
|
194
|
+
sql("UPDATE sync_account SET applying=1 WHERE user_id=?").run(u);
|
|
195
|
+
if (d.kind === "about") {
|
|
196
|
+
const value = String(b?.text ?? "");
|
|
197
|
+
if (readMemory(u).about !== value) setMemoryOptions(u, { about: value });
|
|
198
|
+
} else if (b) {
|
|
199
|
+
for (const k of ["conv_key", "source_conv", "scope"]) if (typeof b[k] === "string") b[k] = localize(b[k], a.device) as string;
|
|
200
|
+
if (d.kind === "item") key ??= "m_" + randomId();
|
|
201
|
+
if (d.kind === "summary" && !key) {
|
|
202
|
+
// A newer incarnation can arrive before the old tombstone on a compacted feed.
|
|
203
|
+
// Reuse its local row and detach the old identity so a late purge can't delete the new summary.
|
|
204
|
+
const row = sql("SELECT id FROM memory_summaries WHERE user_id=? AND kind=? AND scope=?").get(u, b.kind!, b.scope!);
|
|
205
|
+
if (row) {
|
|
206
|
+
key = String(row.id);
|
|
207
|
+
sql("UPDATE sync_map SET local_key=NULL WHERE user_id=? AND kind='summary' AND local_key=?").run(u, key);
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
const fields = columns[d.kind];
|
|
211
|
+
const vals = fields.map((k) => b[k]!);
|
|
212
|
+
if (key && sql(`SELECT 1 FROM ${tables[d.kind]} WHERE user_id=? AND id=?`).get(u, key)) {
|
|
213
|
+
sql(`UPDATE ${tables[d.kind]} SET ${fields.map((k) => k + "=?").join(",")} WHERE user_id=? AND id=?`).run(...vals, u, key);
|
|
214
|
+
} else {
|
|
215
|
+
const insert = sql(`INSERT INTO ${tables[d.kind]}(user_id,id,${fields.join(",")}) VALUES(?,?,${fields.map(() => "?").join(",")})`);
|
|
216
|
+
const saved = insert.run(u, key, ...vals);
|
|
217
|
+
key ??= String(saved.lastInsertRowid);
|
|
218
|
+
}
|
|
219
|
+
if (d.kind === "message" || d.kind === "summary") {
|
|
220
|
+
const fts = d.kind === "message" ? "history_fts" : "memory_summaries_fts";
|
|
221
|
+
sql(`DELETE FROM ${fts} WHERE rowid=?`).run(key);
|
|
222
|
+
if (d.kind === "summary" || sql("SELECT 1 FROM index_state WHERE name='history_fts' AND upto>=?").get(Number(key))) {
|
|
223
|
+
sql(`INSERT INTO ${fts}(rowid,text) VALUES(?,?)`).run(key, ftsIndexText((b.title ? b.title + "\n" : "") + b.text));
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
} else if (key) {
|
|
227
|
+
if (d.kind === "item") sql("DELETE FROM memory_log WHERE user_id=? AND item_id=?").run(u, key);
|
|
228
|
+
if (d.kind === "message" || d.kind === "summary") {
|
|
229
|
+
const fts = d.kind === "message" ? "history_fts" : "memory_summaries_fts";
|
|
230
|
+
sql(`DELETE FROM ${fts} WHERE rowid=?`).run(key);
|
|
231
|
+
}
|
|
232
|
+
sql(`DELETE FROM ${tables[d.kind]} WHERE user_id=? AND id=?`).run(u, key);
|
|
233
|
+
key = null;
|
|
234
|
+
}
|
|
235
|
+
sql(
|
|
236
|
+
"INSERT INTO sync_map(user_id,kind,local_key,gid) VALUES(?,?,?,?) ON CONFLICT(user_id,gid) DO UPDATE SET local_key=excluded.local_key",
|
|
237
|
+
).run(u, d.kind, d.kind === "about" ? "about" : key, d.gid);
|
|
238
|
+
sql("UPDATE sync_account SET applying=0 WHERE user_id=?").run(u);
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
function commit(u: string, c: Change, local: boolean) {
|
|
242
|
+
const old = stored(u, c.gid);
|
|
243
|
+
const d = merge(old && decode(old), c);
|
|
244
|
+
if (!local || JSON.stringify(d.body) !== JSON.stringify(c.body)) apply(u, d);
|
|
245
|
+
saveDoc(u, d);
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
export function flushRelay(u: string, gid?: string) {
|
|
249
|
+
return transaction(() => {
|
|
250
|
+
const batch = staged(u, 48, gid);
|
|
251
|
+
for (const c of batch) {
|
|
252
|
+
commit(u, c, true);
|
|
253
|
+
sql("DELETE FROM sync_dirty WHERE user_id=? AND op=?").run(u, c.op);
|
|
254
|
+
}
|
|
255
|
+
return batch.length;
|
|
256
|
+
});
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
export function acceptPush(u: string, device: string, changes: Change[]) {
|
|
260
|
+
transaction(() => {
|
|
261
|
+
for (const c of changes) {
|
|
262
|
+
validateChange(c);
|
|
263
|
+
if (sql("SELECT 1 FROM sync_receipt WHERE user_id=? AND device=? AND op=?").get(u, device, c.op)) continue;
|
|
264
|
+
flushRelay(u, c.gid);
|
|
265
|
+
const old = stored(u, c.gid);
|
|
266
|
+
if ((old && old.kind !== c.kind) || (!old && c.kind !== "about" && !c.gid.startsWith(device + ":"))) throw new Error("对象来源无效");
|
|
267
|
+
if ((c.kind === "message" || c.kind === "summary") && c.body && !c.gid.startsWith(device + ":"))
|
|
268
|
+
throw new Error("只能修改来源实例的聊天");
|
|
269
|
+
commit(u, c, false);
|
|
270
|
+
sql("INSERT INTO sync_receipt VALUES(?,?,?)").run(u, device, c.op);
|
|
271
|
+
}
|
|
272
|
+
});
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
export function acknowledge(u: string, batch: Change[]) {
|
|
276
|
+
transaction(() => {
|
|
277
|
+
for (const c of batch) {
|
|
278
|
+
const m = mapping(u, c.gid);
|
|
279
|
+
sql("DELETE FROM sync_flight WHERE user_id=? AND gid=?").run(u, c.gid);
|
|
280
|
+
// If purged while this request was in flight, never write the old content back to disk.
|
|
281
|
+
if (m?.local_key || c.kind === "about")
|
|
282
|
+
saveDoc(u, {
|
|
283
|
+
...c,
|
|
284
|
+
revision:
|
|
285
|
+
(c.base?.text ?? (c.kind === "about" ? "" : undefined)) !== c.body?.text ||
|
|
286
|
+
c.resolve ||
|
|
287
|
+
(!c.base?.deleted_at && c.body?.deleted_at)
|
|
288
|
+
? c.op
|
|
289
|
+
: c.baseRevision,
|
|
290
|
+
conflicts: c.resolve ? [] : JSON.parse(stored(u, c.gid)?.conflicts ?? "[]"),
|
|
291
|
+
});
|
|
292
|
+
sql("DELETE FROM sync_dirty WHERE user_id=? AND op=?").run(u, c.op);
|
|
293
|
+
}
|
|
294
|
+
});
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
export function feed(u: string, cursor: number) {
|
|
298
|
+
return transaction(() => {
|
|
299
|
+
flushRelay(u);
|
|
300
|
+
const docs: Document[] = [];
|
|
301
|
+
let bytes = 0;
|
|
302
|
+
for (const r of sql("SELECT * FROM sync_doc WHERE user_id=? AND seq>? ORDER BY seq LIMIT 48").iterate(u, cursor) as Iterable<DocRow>) {
|
|
303
|
+
docs.push(decode(r));
|
|
304
|
+
cursor = r.seq;
|
|
305
|
+
bytes += (r.body?.length ?? 0) * 3;
|
|
306
|
+
if (bytes >= 512 * 1024) break;
|
|
307
|
+
}
|
|
308
|
+
return { docs, cursor };
|
|
309
|
+
});
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
export function receive(u: string, docs: Document[], cursor: number) {
|
|
313
|
+
return transaction(() => {
|
|
314
|
+
if (docs.some((d) => sql("SELECT 1 FROM sync_dirty WHERE user_id=? AND gid=?").get(u, d.gid))) return false;
|
|
315
|
+
for (const d of docs) {
|
|
316
|
+
validateDocument(d);
|
|
317
|
+
apply(u, d);
|
|
318
|
+
saveDoc(u, d);
|
|
319
|
+
}
|
|
320
|
+
sql("UPDATE sync_account SET cursor=? WHERE user_id=?").run(cursor, u);
|
|
321
|
+
return true;
|
|
322
|
+
});
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
export function resolveConflict(u: string, gid: string, choices: string[], text: string) {
|
|
326
|
+
transaction(() => {
|
|
327
|
+
const old = stored(u, gid);
|
|
328
|
+
if (!old?.body || !choices?.length || old.conflicts !== JSON.stringify(choices)) throw new Error("冲突版本已变化,请刷新");
|
|
329
|
+
const d = decode(old);
|
|
330
|
+
if (
|
|
331
|
+
(d.kind !== "item" && d.kind !== "about") ||
|
|
332
|
+
typeof text !== "string" ||
|
|
333
|
+
(d.kind === "item" && !text.trim()) ||
|
|
334
|
+
text.length > (d.kind === "item" ? 500 : 2000)
|
|
335
|
+
)
|
|
336
|
+
throw new Error("解决内容无效");
|
|
337
|
+
const m = mapping(u, gid);
|
|
338
|
+
d.body = m ? snapshot(u, m, account(u)!.device) : null;
|
|
339
|
+
if (!d.body || d.body.deleted_at) throw new Error("这条记忆已删除");
|
|
340
|
+
d.body.text = text.trim();
|
|
341
|
+
apply(u, d);
|
|
342
|
+
sql(
|
|
343
|
+
"INSERT INTO sync_dirty(user_id,gid,op,resolution) VALUES(?,?,?,?) ON CONFLICT(user_id,gid) DO UPDATE SET op=excluded.op,resolution=excluded.resolution",
|
|
344
|
+
).run(u, gid, randomId(), JSON.stringify(choices));
|
|
345
|
+
});
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
export function syncConflicts(u: string) {
|
|
349
|
+
return (sql("SELECT * FROM sync_doc WHERE user_id=? AND conflicts!='[]'").all(u) as DocRow[]).map((r) => {
|
|
350
|
+
const d = decode(r);
|
|
351
|
+
return { gid: d.gid, kind: d.kind, text: d.body?.text, choices: d.conflicts };
|
|
352
|
+
});
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
function validateBody(kind: SyncKind, b: Body | null, origin: string) {
|
|
356
|
+
if (b === null) return;
|
|
357
|
+
if (!b || typeof b !== "object" || Array.isArray(b) || Object.keys(b).length !== columns[kind].length) throw new Error("同步字段不完整");
|
|
358
|
+
const string = (k: string, max: number) => typeof b[k] === "string" && (b[k] as string).length <= max;
|
|
359
|
+
const number = (k: string) => typeof b[k] === "number" && Number.isSafeInteger(b[k]) && Number(b[k]) >= 0;
|
|
360
|
+
const qualified = (k: string) => string(k, 2048) && /^sync\/[0-9a-f]{32}\/.+$/s.test(String(b[k]));
|
|
361
|
+
const fromOrigin = (k: string) => qualified(k) && String(b[k]).startsWith(prefix(origin));
|
|
362
|
+
let valid = columns[kind].every((k) => Object.hasOwn(b, k));
|
|
363
|
+
switch (kind) {
|
|
364
|
+
case "about":
|
|
365
|
+
valid &&= string("text", 2000);
|
|
366
|
+
break;
|
|
367
|
+
case "item":
|
|
368
|
+
valid &&=
|
|
369
|
+
string("text", 500) &&
|
|
370
|
+
!!b.text &&
|
|
371
|
+
["asked", "agent", "user", "auto"].includes(String(b.source)) &&
|
|
372
|
+
["active", "core", "archive"].includes(String(b.tier)) &&
|
|
373
|
+
(b.locked === 0 || b.locked === 1) &&
|
|
374
|
+
["created_at", "updated_at"].every(number) &&
|
|
375
|
+
["observed_at", "valid_until", "deleted_at"].every((k) => b[k] === null || number(k)) &&
|
|
376
|
+
(b.source_conv === null || qualified("source_conv"));
|
|
377
|
+
break;
|
|
378
|
+
case "message":
|
|
379
|
+
valid &&=
|
|
380
|
+
string("text", Number.MAX_SAFE_INTEGER) &&
|
|
381
|
+
fromOrigin("conv_key") &&
|
|
382
|
+
string("agent", 256) &&
|
|
383
|
+
(b.role === "u" || b.role === "a") &&
|
|
384
|
+
number("ts") &&
|
|
385
|
+
number("source_order") &&
|
|
386
|
+
(b.msg_id === null || string("msg_id", 2048));
|
|
387
|
+
break;
|
|
388
|
+
case "summary":
|
|
389
|
+
valid &&=
|
|
390
|
+
fromOrigin("scope") &&
|
|
391
|
+
(b.kind === "month" || b.kind === "conv") &&
|
|
392
|
+
string("text", 1500) &&
|
|
393
|
+
string("title", 80) &&
|
|
394
|
+
(b.project === null || string("project", 8192)) &&
|
|
395
|
+
["from_ts", "to_ts", "updated_at"].every(number);
|
|
396
|
+
break;
|
|
397
|
+
}
|
|
398
|
+
if (!valid) throw new Error("同步数据格式无效");
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
function validateKey(d: { kind: SyncKind; gid: string }) {
|
|
402
|
+
if (
|
|
403
|
+
!d ||
|
|
404
|
+
!syncKinds.includes(d.kind) ||
|
|
405
|
+
typeof d.gid !== "string" ||
|
|
406
|
+
!(d.kind === "about" ? d.gid === "about" : /^[0-9a-f]{32}:[0-9a-f]{32}$/.test(d.gid))
|
|
407
|
+
)
|
|
408
|
+
throw new Error("同步对象标识无效");
|
|
409
|
+
}
|
|
410
|
+
|
|
411
|
+
function validChoices(v: unknown, kind: SyncKind) {
|
|
412
|
+
return Array.isArray(v) && v.every((s) => typeof s === "string" && s.length <= (kind === "about" ? 2000 : 500));
|
|
413
|
+
}
|
|
414
|
+
function validateChange(c: Change) {
|
|
415
|
+
validateKey(c);
|
|
416
|
+
if (c.baseRevision !== null && !idPattern.test(c.baseRevision)) throw new Error("基础版本无效");
|
|
417
|
+
if (typeof c.op !== "string" || !idPattern.test(c.op) || (c.resolve && !validChoices(c.resolve, c.kind))) throw new Error("同步变更无效");
|
|
418
|
+
validateBody(c.kind, c.base, c.gid.split(":")[0]!);
|
|
419
|
+
validateBody(c.kind, c.body, c.gid.split(":")[0]!);
|
|
420
|
+
if (c.kind === "about" && !c.body) throw new Error("个人档案格式无效");
|
|
421
|
+
}
|
|
422
|
+
function validateDocument(d: Document) {
|
|
423
|
+
validateKey(d);
|
|
424
|
+
if (d.revision !== null && !idPattern.test(d.revision)) throw new Error("同步版本无效");
|
|
425
|
+
if (!validChoices(d.conflicts, d.kind)) throw new Error("同步版本无效");
|
|
426
|
+
validateBody(d.kind, d.body, d.gid.split(":")[0]!);
|
|
427
|
+
}
|
|
428
|
+
|
|
429
|
+
export function removeSyncAccount(u: string) {
|
|
430
|
+
transaction(() => {
|
|
431
|
+
for (const t of ["sync_account", "sync_invite", "sync_peer", "sync_dirty", "sync_flight", "sync_map", "sync_doc", "sync_receipt"])
|
|
432
|
+
sql(`DELETE FROM ${t} WHERE user_id=?`).run(u);
|
|
433
|
+
});
|
|
434
|
+
}
|
|
@@ -0,0 +1,200 @@
|
|
|
1
|
+
import { createHash, randomBytes } from "node:crypto";
|
|
2
|
+
import { sql, transaction } from "./db.js";
|
|
3
|
+
import { getUser } from "./users.js";
|
|
4
|
+
import {
|
|
5
|
+
acceptPush,
|
|
6
|
+
account,
|
|
7
|
+
acknowledge,
|
|
8
|
+
activate,
|
|
9
|
+
feed,
|
|
10
|
+
flushRelay,
|
|
11
|
+
idPattern,
|
|
12
|
+
randomId,
|
|
13
|
+
receive,
|
|
14
|
+
preparePush,
|
|
15
|
+
syncConflicts,
|
|
16
|
+
type Change,
|
|
17
|
+
type Document,
|
|
18
|
+
} from "./memory-sync-store.js";
|
|
19
|
+
|
|
20
|
+
const hash = (s: string) => createHash("sha256").update(s).digest("hex");
|
|
21
|
+
const active = (u: string) => {
|
|
22
|
+
const user = getUser(u);
|
|
23
|
+
return !!user && !user.disabled;
|
|
24
|
+
};
|
|
25
|
+
export function issueInvite(u: string) {
|
|
26
|
+
return transaction(() => {
|
|
27
|
+
if (account(u)?.url) throw new Error("此账号已连接中转实例");
|
|
28
|
+
if (!account(u)) activate(u);
|
|
29
|
+
const code = randomBytes(16).toString("hex");
|
|
30
|
+
const expires = Date.now() + 10 * 60_000;
|
|
31
|
+
sql("INSERT OR REPLACE INTO sync_invite VALUES(?,?,?)").run(u, hash(code), expires);
|
|
32
|
+
return { code, expires };
|
|
33
|
+
});
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export function redeem(code: string, device: string) {
|
|
37
|
+
return transaction(() => {
|
|
38
|
+
if (!idPattern.test(code) || !idPattern.test(device)) throw new Error("无效配对码或实例");
|
|
39
|
+
const invite = sql("DELETE FROM sync_invite WHERE hash=? AND expires>? RETURNING user_id").get(hash(code), Date.now()) as
|
|
40
|
+
| { user_id: string }
|
|
41
|
+
| undefined;
|
|
42
|
+
if (!invite || !active(invite.user_id)) throw new Error("配对码无效或已过期");
|
|
43
|
+
if (account(invite.user_id)?.device === device) throw new Error("不能与当前实例自身配对");
|
|
44
|
+
const token = randomBytes(32).toString("hex");
|
|
45
|
+
sql("INSERT INTO sync_peer(user_id,device,hash) VALUES(?,?,?) ON CONFLICT(user_id,device) DO UPDATE SET hash=excluded.hash").run(
|
|
46
|
+
invite.user_id,
|
|
47
|
+
device,
|
|
48
|
+
hash(token),
|
|
49
|
+
);
|
|
50
|
+
return { token, relay: account(invite.user_id)!.device, version: 1 };
|
|
51
|
+
});
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
export function authenticate(token: string) {
|
|
55
|
+
if (!/^[a-f0-9]{64}$/.test(token)) return;
|
|
56
|
+
const peer = sql("SELECT user_id,device FROM sync_peer WHERE hash=?").get(hash(token)) as { user_id: string; device: string } | undefined;
|
|
57
|
+
if (peer && active(peer.user_id)) return peer;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
export function revoke(u: string, device: string) {
|
|
61
|
+
sql("DELETE FROM sync_peer WHERE user_id=? AND device=?").run(u, device);
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function endpoint(raw: string) {
|
|
65
|
+
const url = new URL(raw);
|
|
66
|
+
if (url.protocol !== "https:" || url.username || url.password || url.search || url.hash) throw new Error("请输入中转实例的 HTTPS 地址");
|
|
67
|
+
return url.origin + url.pathname.replace(/\/$/, "");
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
const maxBytes = 64 * 1024 * 1024;
|
|
71
|
+
async function post(url: string, path: string, body: unknown, token?: string) {
|
|
72
|
+
const response = await fetch(url + "/api/memory-sync/v1/" + path, {
|
|
73
|
+
method: "POST",
|
|
74
|
+
redirect: "error",
|
|
75
|
+
headers: { "content-type": "application/json", ...(token ? { authorization: "Bearer " + token } : {}) },
|
|
76
|
+
body: JSON.stringify(body),
|
|
77
|
+
signal: AbortSignal.timeout(45_000),
|
|
78
|
+
});
|
|
79
|
+
if (!response.ok) {
|
|
80
|
+
await response.body?.cancel();
|
|
81
|
+
throw new Error(`中转返回 HTTP ${response.status}`);
|
|
82
|
+
}
|
|
83
|
+
const data: Buffer[] = [];
|
|
84
|
+
let size = 0;
|
|
85
|
+
for await (const chunk of response.body!) {
|
|
86
|
+
size += chunk.byteLength;
|
|
87
|
+
if (size > maxBytes) throw new Error("同步响应过大");
|
|
88
|
+
data.push(Buffer.from(chunk));
|
|
89
|
+
}
|
|
90
|
+
return JSON.parse(Buffer.concat(data).toString()) as Record<string, unknown>;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
const joining = new Set<string>();
|
|
94
|
+
export async function join(u: string, address: string, code: string) {
|
|
95
|
+
const url = endpoint(address);
|
|
96
|
+
const old = account(u);
|
|
97
|
+
if ((old && old.url !== url) || joining.has(u)) throw new Error("此账号已经配置同步");
|
|
98
|
+
const device = old?.device ?? randomId();
|
|
99
|
+
joining.add(u);
|
|
100
|
+
try {
|
|
101
|
+
const result = await post(url, "pair", { code: code.trim(), device });
|
|
102
|
+
if (result.version !== 1 || typeof result.token !== "string" || !/^[a-f0-9]{64}$/.test(result.token)) throw new Error("配对响应无效");
|
|
103
|
+
if (!active(u)) throw new Error("账号不可用");
|
|
104
|
+
if (typeof result.relay !== "string" || !idPattern.test(result.relay)) throw new Error("无效中转标识");
|
|
105
|
+
if (old && old.relay !== result.relay) throw new Error("配对码属于其他账号或中转实例");
|
|
106
|
+
transaction(() => {
|
|
107
|
+
if (!old) activate(u, device, url, result.token as string);
|
|
108
|
+
sql("UPDATE sync_account SET relay=?,token=?,error=NULL WHERE user_id=?").run(result.relay as string, result.token as string, u);
|
|
109
|
+
});
|
|
110
|
+
} finally {
|
|
111
|
+
joining.delete(u);
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
export function status(u: string) {
|
|
116
|
+
const a = account(u);
|
|
117
|
+
return {
|
|
118
|
+
device: a?.device,
|
|
119
|
+
mode: a ? (a.url ? "client" : "relay") : "off",
|
|
120
|
+
url: a?.url,
|
|
121
|
+
lastSync: a?.last_sync,
|
|
122
|
+
error: a?.error,
|
|
123
|
+
pending: Number(sql("SELECT count(*) AS n FROM sync_dirty WHERE user_id=?").get(u)?.n),
|
|
124
|
+
peers: sql("SELECT device,last_sync FROM sync_peer WHERE user_id=? ORDER BY device").all(u),
|
|
125
|
+
conflicts: syncConflicts(u),
|
|
126
|
+
};
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
export function servePush(peer: { user_id: string; device: string }, changes: Change[]) {
|
|
130
|
+
acceptPush(peer.user_id, peer.device, changes);
|
|
131
|
+
touchPeer(peer);
|
|
132
|
+
}
|
|
133
|
+
export function servePull(peer: { user_id: string; device: string }, cursor: number) {
|
|
134
|
+
const page = feed(peer.user_id, cursor);
|
|
135
|
+
touchPeer(peer);
|
|
136
|
+
return page;
|
|
137
|
+
}
|
|
138
|
+
function touchPeer(peer: { user_id: string; device: string }) {
|
|
139
|
+
const now = Date.now();
|
|
140
|
+
sql("UPDATE sync_peer SET last_sync=? WHERE user_id=? AND device=?").run(now, peer.user_id, peer.device);
|
|
141
|
+
sql("UPDATE sync_account SET last_sync=? WHERE user_id=?").run(now, peer.user_id);
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
const running = new Map<string, Promise<void>>();
|
|
145
|
+
let changed: (u: string) => void = () => {};
|
|
146
|
+
async function exchange(u: string) {
|
|
147
|
+
const a = account(u);
|
|
148
|
+
if (!a || !active(u)) return;
|
|
149
|
+
const current = () => active(u) && account(u)?.device === a.device;
|
|
150
|
+
if (!a.url || !a.token) {
|
|
151
|
+
while (current() && flushRelay(u)) await new Promise((r) => setImmediate(r));
|
|
152
|
+
return;
|
|
153
|
+
}
|
|
154
|
+
let cursor = a.cursor;
|
|
155
|
+
do {
|
|
156
|
+
const batch = preparePush(u);
|
|
157
|
+
if (batch.length) {
|
|
158
|
+
const ack = await post(a.url, "push", { changes: batch }, a.token);
|
|
159
|
+
if (!current()) return;
|
|
160
|
+
if (ack.ok !== true) throw new Error("中转未确认写入");
|
|
161
|
+
acknowledge(u, batch);
|
|
162
|
+
}
|
|
163
|
+
const page = await post(a.url, "pull", { cursor }, a.token);
|
|
164
|
+
if (!current()) return;
|
|
165
|
+
if (
|
|
166
|
+
!Array.isArray(page.docs) ||
|
|
167
|
+
page.docs.length > 48 ||
|
|
168
|
+
typeof page.cursor !== "number" ||
|
|
169
|
+
!Number.isSafeInteger(page.cursor) ||
|
|
170
|
+
(page.docs.length ? page.cursor <= cursor : page.cursor !== cursor)
|
|
171
|
+
)
|
|
172
|
+
throw new Error("中转游标无效");
|
|
173
|
+
if (receive(u, page.docs as Document[], page.cursor)) {
|
|
174
|
+
cursor = page.cursor;
|
|
175
|
+
if ((page.docs as Document[]).some((d) => d.kind === "about" || d.kind === "item")) changed(u);
|
|
176
|
+
}
|
|
177
|
+
if (!batch.length && !page.docs.length) break;
|
|
178
|
+
} while (current());
|
|
179
|
+
sql("UPDATE sync_account SET last_sync=?,error=NULL WHERE user_id=? AND device=?").run(Date.now(), u, a.device);
|
|
180
|
+
}
|
|
181
|
+
export function syncNow(u: string) {
|
|
182
|
+
const ongoing = running.get(u);
|
|
183
|
+
if (ongoing) return ongoing;
|
|
184
|
+
const work = exchange(u)
|
|
185
|
+
.catch((err) => {
|
|
186
|
+
sql("UPDATE sync_account SET error=? WHERE user_id=?").run(err instanceof Error ? err.message : "同步失败", u);
|
|
187
|
+
throw err;
|
|
188
|
+
})
|
|
189
|
+
.finally(() => running.delete(u));
|
|
190
|
+
running.set(u, work);
|
|
191
|
+
return work;
|
|
192
|
+
}
|
|
193
|
+
export function startMemorySync(onChanged: (u: string) => void) {
|
|
194
|
+
changed = onChanged;
|
|
195
|
+
const tick = () => {
|
|
196
|
+
for (const row of sql("SELECT user_id FROM sync_account").all()) void syncNow(String(row.user_id)).catch(() => {});
|
|
197
|
+
};
|
|
198
|
+
tick();
|
|
199
|
+
return setInterval(tick, 15_000).unref();
|
|
200
|
+
}
|
package/src/core/personal.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
// 「这条对话是谁的」在个人数据(记忆、飞书绑定)上的落点。
|
|
2
|
+
import { removeSyncAccount } from "./memory-sync-store.js";
|
|
2
3
|
import { deleteMemory, memoryPrompt, readMemory } from "./memory.js";
|
|
3
4
|
import { purgeConversationHistory, purgeHistory } from "./chat-history.js";
|
|
4
5
|
import { deleteConvSummary } from "./memory-summaries.js";
|
|
@@ -76,6 +77,7 @@ export function purgeHiddenConversationsOnce(): number {
|
|
|
76
77
|
|
|
77
78
|
/** 删用户时一起删掉的个人数据:记忆、聊天记录索引、飞书和微信绑定、定时任务、技能(他建的技能,装了的人也就没了)、专家和 MCP */
|
|
78
79
|
export function purgeUserData(userId: string): void {
|
|
80
|
+
removeSyncAccount(userId);
|
|
79
81
|
deleteMemory(userId);
|
|
80
82
|
purgeHistory(userId);
|
|
81
83
|
unlinkUser(userId);
|
package/src/main.ts
CHANGED
|
@@ -13,6 +13,7 @@ import { startFeishu, stopFeishu } from "./channels/feishu/index.js";
|
|
|
13
13
|
import { startWeixin, stopWeixin } from "./channels/weixin/index.js";
|
|
14
14
|
import { startWeb, stopWeb } from "./channels/web/server.js";
|
|
15
15
|
import { home } from "./home.js";
|
|
16
|
+
import { startMemorySync } from "./core/memory-sync.js";
|
|
16
17
|
import { startMemoryCurator } from "./core/memory-curator.js";
|
|
17
18
|
import { backupBeforeMigration, startBackups } from "./core/backup.js";
|
|
18
19
|
import { purgeHiddenConversationsOnce } from "./core/personal.js";
|
|
@@ -87,6 +88,7 @@ async function main(): Promise<void> {
|
|
|
87
88
|
.then((n) => n && console.log(`[记忆] 全文索引补上了 ${n} 句聊天`))
|
|
88
89
|
.catch((err) => console.error("[记忆] 建全文索引失败:", err));
|
|
89
90
|
startMemoryCurator();
|
|
91
|
+
const syncTimer = startMemorySync(u => hub.memoryChanged(u));
|
|
90
92
|
startBackups();
|
|
91
93
|
|
|
92
94
|
startFeishu();
|
|
@@ -115,6 +117,7 @@ async function main(): Promise<void> {
|
|
|
115
117
|
|
|
116
118
|
let stopping = false;
|
|
117
119
|
const shutdown = async (signal: string) => {
|
|
120
|
+
clearInterval(syncTimer);
|
|
118
121
|
if (stopping) return;
|
|
119
122
|
stopping = true;
|
|
120
123
|
console.log(`[退出] 收到 ${signal},正在收尾…`);
|