agent-loft 0.2.88 → 0.2.89

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -6,9 +6,12 @@
6
6
  // - 实时:hub 的事件流(见 startHistoryCapture),从现在起说的话;
7
7
  // - 回填:第一次用时把已有对话从 agent 的记录里读一遍(见 memory-curator.ts),只做一次。
8
8
  //
9
- // 搜索不用 SQLite 的 FTS5:trigram 分词搜不到两个字的中文词(「豆豆」)。先用 instr 在库里筛出
10
- // 含有某个词的句子,再在这里按命中情况打分,不用把整个人的记录都读出来。
9
+ // 搜索用 FTS5(history_fts,切词方式见 fts.ts)在库里筛出候选,再在这里按命中情况打分。
10
+ // 索引不在写入时维护,而是按 index_state 里的进度补(见 syncHistoryIndex):写入路径不用多一步,
11
+ // 导入的、老版本写的也都能补上。英文词中间的片段(lowercase 写的 openclash 搜 clash)FTS 查不到,
12
+ // 最近的一段记录再用 instr 扫一遍兜底,扫描量有上限。
11
13
  import { sql, transaction } from "./db.js";
14
+ import { bigrams, ftsIndexText, ftsMatch, isCjk, needles, terms } from "./fts.js";
12
15
 
13
16
  export type HistoryRecord = {
14
17
  /** 实时用户消息的编号,撤销排队消息时用于精确移除 */
@@ -33,6 +36,8 @@ export type CurationLog = {
33
36
  /** 用的哪个 agent / 模型 */
34
37
  by?: string;
35
38
  error?: string;
39
+ /** 这次写了几份对话摘要 */
40
+ summaries?: number;
36
41
  };
37
42
 
38
43
  export type Meta = {
@@ -44,10 +49,24 @@ export type Meta = {
44
49
  lastDaily?: string;
45
50
  /** 上次整理的结果,设置页显示 */
46
51
  lastCuration?: CurationLog;
52
+ /** 整轮都失败时的退避:连着失败了几轮、下次什么时候再试 */
53
+ curateRetry?: { failures: number; nextAt: number };
54
+ /** 每段对话连着整理失败了几次。到上限的先搁置(不丢),用户点「现在整理」时再试 */
55
+ curateAttempts?: Record<string, number>;
56
+ /** 月度摘要上次检查的日期(本地日期 YYYY-MM-DD),一天查一次 */
57
+ lastRollup?: string;
47
58
  };
48
59
 
49
60
  /** 一句话最多存这么多字:长篇代码、整页网页没必要全存 */
50
- const TEXT_MAX = 4000;
61
+ export const TEXT_MAX = 8000;
62
+ /** 太长时头尾都留:长回复的结论通常在最后 */
63
+ const TEXT_TAIL = 2000;
64
+
65
+ function capText(text: string): string {
66
+ if (text.length <= TEXT_MAX) return text;
67
+ const gap = "\n…(中间省略)…\n";
68
+ return `${text.slice(0, TEXT_MAX - TEXT_TAIL - gap.length)}${gap}${text.slice(-TEXT_TAIL)}`;
69
+ }
51
70
 
52
71
  function checkUser(userId: string): string {
53
72
  if (!/^[A-Za-z0-9_]{1,64}$/.test(userId)) throw new Error("用户 id 不合法");
@@ -64,7 +83,7 @@ function toRecord(row: Row): HistoryRecord {
64
83
 
65
84
  function insertRecords(userId: string, records: HistoryRecord[]): void {
66
85
  const insert = sql("INSERT INTO history (user_id, conv_key, agent, role, text, ts, msg_id) VALUES (?, ?, ?, ?, ?, ?, ?)");
67
- for (const r of records) insert.run(userId, r.k, r.a, r.r, r.t.slice(0, TEXT_MAX), r.ts, r.id ?? null);
86
+ for (const r of records) insert.run(userId, r.k, r.a, r.r, capText(r.t), r.ts, r.id ?? null);
68
87
  }
69
88
 
70
89
  export function appendRecords(userId: string, records: HistoryRecord[]): void {
@@ -88,14 +107,34 @@ export function countRecords(userId: string): { count: number; conversations: nu
88
107
 
89
108
  export function removeHistoryRecord(userId: string, key: string, id: string): void {
90
109
  checkUser(userId);
91
- sql("DELETE FROM history WHERE user_id = ? AND conv_key = ? AND msg_id = ?").run(userId, key, id);
110
+ transaction(() => {
111
+ sql("DELETE FROM history_fts WHERE rowid IN (SELECT id FROM history WHERE user_id = ? AND conv_key = ? AND msg_id = ?)").run(userId, key, id);
112
+ sql("DELETE FROM history WHERE user_id = ? AND conv_key = ? AND msg_id = ?").run(userId, key, id);
113
+ });
114
+ }
115
+
116
+ function deleteConversationRows(userId: string, key: string): void {
117
+ sql("DELETE FROM history_fts WHERE rowid IN (SELECT id FROM history WHERE user_id = ? AND conv_key = ?)").run(userId, key);
118
+ sql("DELETE FROM history WHERE user_id = ? AND conv_key = ?").run(userId, key);
119
+ }
120
+
121
+ /** 对话被删了:这段的聊天记录和整理进度一起删,不只是搜不到 */
122
+ export function purgeConversationHistory(userId: string, key: string): void {
123
+ checkUser(userId);
124
+ transaction(() => {
125
+ deleteConversationRows(userId, key);
126
+ const meta = readMeta(userId);
127
+ delete meta.extracted[key];
128
+ if (meta.curateAttempts) delete meta.curateAttempts[key];
129
+ writeMeta(userId, meta);
130
+ });
92
131
  }
93
132
 
94
133
  /** 回退后只保留当前对话剩下的内容,避免搜索再搜出已经移除的消息 */
95
134
  export function replaceConversationRecords(userId: string, key: string, records: HistoryRecord[]): void {
96
135
  checkUser(userId);
97
136
  transaction(() => {
98
- sql("DELETE FROM history WHERE user_id = ? AND conv_key = ?").run(userId, key);
137
+ deleteConversationRows(userId, key);
99
138
  insertRecords(userId, records);
100
139
  });
101
140
  }
@@ -112,7 +151,7 @@ export function readMeta(userId: string): Meta {
112
151
  return { ...m, extracted: m.extracted && typeof m.extracted === "object" ? m.extracted : {} };
113
152
  }
114
153
 
115
- export function patchMeta(userId: string, patch: Partial<Meta>): void {
154
+ export function patchMeta(userId: string, patch: { [K in keyof Meta]?: Meta[K] | undefined }): void {
116
155
  writeMeta(userId, { ...readMeta(userId), ...patch });
117
156
  }
118
157
 
@@ -133,8 +172,10 @@ export function markExtracted(userId: string, key: string, upTo: number): void {
133
172
  export function purgeHistory(userId: string): void {
134
173
  checkUser(userId);
135
174
  transaction(() => {
175
+ sql("DELETE FROM history_fts WHERE rowid IN (SELECT id FROM history WHERE user_id = ?)").run(userId);
136
176
  sql("DELETE FROM history WHERE user_id = ?").run(userId);
137
177
  sql("DELETE FROM history_meta WHERE user_id = ?").run(userId);
178
+ sql("DELETE FROM search_stats WHERE user_id = ?").run(userId);
138
179
  });
139
180
  }
140
181
 
@@ -148,44 +189,21 @@ export function pendingConversations(
148
189
  now = Date.now(),
149
190
  ): { key: string; records: HistoryRecord[]; last: number }[] {
150
191
  const meta = readMeta(userId);
151
- // 每条对话整理到的时刻在 meta 里,交给 json_each 在库里比,只取还没整理过的
152
- const rows = sql(
153
- `SELECT h.id, h.conv_key, h.agent, h.role, h.text, h.ts, h.msg_id FROM history h LEFT JOIN json_each(?) e ON e.key = h.conv_key
154
- WHERE h.user_id = ? AND h.ts > coalesce(e.value, 0) ORDER BY h.id`,
155
- ).all(JSON.stringify(meta.extracted), userId) as Row[];
156
- const byKey = new Map<string, HistoryRecord[]>();
157
- for (const r of rows.map(toRecord)) {
158
- let list = byKey.get(r.k);
159
- if (!list) byKey.set(r.k, (list = []));
160
- list.push(r);
161
- }
192
+ // 先只看每段对话的最后一句(走覆盖索引,不读正文),再去读真正要整理的那几段。
193
+ // 不能把 extracted 交给库去 JOIN:json_each 没有索引,句数 × 对话数,几年后要跑几分钟还卡住主进程
194
+ const convs = sql("SELECT conv_key, max(ts) AS last FROM history WHERE user_id = ? GROUP BY conv_key").all(userId) as { conv_key: string; last: number }[];
162
195
  const out: { key: string; records: HistoryRecord[]; last: number }[] = [];
163
- for (const [key, records] of byKey) {
164
- const last = Math.max(...records.map((r) => r.ts));
165
- if (now - last >= idleMs) out.push({ key, records, last });
196
+ const rows = sql(`SELECT ${COLUMNS} FROM history WHERE user_id = ? AND conv_key = ? AND ts > ? ORDER BY id`);
197
+ for (const { conv_key: key, last } of convs.sort((a, b) => a.last - b.last)) {
198
+ const done = meta.extracted[key] ?? 0;
199
+ if (last <= done || now - last < idleMs) continue;
200
+ out.push({ key, records: (rows.all(userId, key, done) as Row[]).map(toRecord), last });
166
201
  }
167
202
  return out;
168
203
  }
169
204
 
170
205
  // ---------- 搜索 ----------
171
206
 
172
- /** 把查询拆成词:按空白和标点切开。中文不切词,长的中文词另外拆成两字一组做模糊匹配 */
173
- function terms(query: string): string[] {
174
- return query
175
- .toLowerCase()
176
- .split(/[\s,,。.!!??、;;::"'“”‘’()()[\]【】<>《》/\\|]+/)
177
- .map((t) => t.trim())
178
- .filter(Boolean);
179
- }
180
-
181
- const isCjk = (s: string) => /[㐀-鿿]/.test(s);
182
-
183
- function bigrams(s: string): string[] {
184
- const out: string[] = [];
185
- for (let i = 0; i + 2 <= s.length; i++) out.push(s.slice(i, i + 2));
186
- return out;
187
- }
188
-
189
207
  function occurrences(hay: string, needle: string, cap = 5): number {
190
208
  let n = 0;
191
209
  for (let i = hay.indexOf(needle); i >= 0 && n < cap; i = hay.indexOf(needle, i + needle.length)) n++;
@@ -234,27 +252,124 @@ function snippet(text: string, query: string, max: number): string {
234
252
  return `${start > 0 ? "…" : ""}${flat.slice(start, start + max)}${start + max < flat.length ? "…" : ""}`;
235
253
  }
236
254
 
237
- export type SearchHit = { key: string; ts: number; user?: string; assistant?: string; score: number };
255
+ // ---------- 全文索引的进度 ----------
256
+
257
+ const INDEX_BATCH = 2000;
258
+ const INDEX_NAME = "history_fts";
259
+
260
+ function indexedUpTo(): number {
261
+ return (sql("SELECT upto FROM index_state WHERE name = ?").get(INDEX_NAME) as { upto: number } | undefined)?.upto ?? 0;
262
+ }
263
+
264
+ /** 补一批,一批一个事务:锁拿得短,别的进程不用等太久。返回补了几句 */
265
+ function indexBatch(): number {
266
+ const top = (sql("SELECT max(id) AS m FROM history").get() as { m: number | null }).m ?? 0;
267
+ if (top <= indexedUpTo()) return 0;
268
+ return transaction(() => {
269
+ const rows = sql("SELECT id, text FROM history WHERE id > ? ORDER BY id LIMIT ?").all(indexedUpTo(), INDEX_BATCH) as { id: number; text: string }[];
270
+ if (rows.length === 0) return 0;
271
+ const insert = sql("INSERT INTO history_fts (rowid, text) VALUES (?, ?)");
272
+ for (const r of rows) insert.run(r.id, ftsIndexText(r.text));
273
+ sql("INSERT OR REPLACE INTO index_state (name, upto) VALUES (?, ?)").run(INDEX_NAME, rows[rows.length - 1]!.id);
274
+ return rows.length;
275
+ });
276
+ }
277
+
278
+ /** 把还没进索引的句子补进去。搜索前都会调一次,平时只多一次查询 */
279
+ export function syncHistoryIndex(): number {
280
+ let total = 0;
281
+ for (let n = indexBatch(); n > 0; n = indexBatch()) total += n;
282
+ return total;
283
+ }
284
+
285
+ /** 启动时用:分批补,每批之间让出事件循环。第一次升级要给全部老记录建索引,不能卡住服务 */
286
+ export async function buildHistoryIndex(): Promise<number> {
287
+ let total = 0;
288
+ for (let n = indexBatch(); n > 0; n = indexBatch()) {
289
+ total += n;
290
+ await new Promise((resolve) => setImmediate(resolve));
291
+ }
292
+ return total;
293
+ }
294
+
295
+ export type SearchHit = { key: string; ts: number; user?: string; assistant?: string; score: number; rowId: number };
296
+
297
+ export type SearchOptions = {
298
+ limit?: number;
299
+ skip?: (key: string) => boolean;
300
+ now?: number;
301
+ /** 只要这个时间之后的(含) */
302
+ after?: number;
303
+ /** 只要这个时间之前的(不含) */
304
+ before?: number;
305
+ /** 同一段对话最多给几条,免得结果全挤在一段里 */
306
+ perConv?: number;
307
+ };
308
+
309
+ /** FTS 交给打分的候选上限:最相关的这么多句,加最新的这么多句 */
310
+ const CANDIDATES = 2000;
311
+ const CANDIDATES_RECENT = 500;
312
+ /** 跳过的太多时最多往后翻这么多句,防止一个词命中全库时翻个没完 */
313
+ const SCAN_MAX = 100_000;
314
+ /** 兜底扫描最多扫最近这么多句 */
315
+ const SCAN_RECENT = 20_000;
316
+
317
+ /** 英文词中间的片段 FTS 查不到:最近的记录再扫一遍。中文已经按字索引,不用 */
318
+ function scanRecent(userId: string, query: string, timeSql: string, timeArgs: number[]): Row[] {
319
+ const latin = needles(query).filter((n) => !isCjk(n));
320
+ if (latin.length === 0) return [];
321
+ const floor = (sql("SELECT id FROM history WHERE user_id = ? ORDER BY id DESC LIMIT 1 OFFSET ?").get(userId, SCAN_RECENT - 1) as { id: number } | undefined)?.id ?? 0;
322
+ return sql(
323
+ `SELECT ${COLUMNS} FROM history WHERE user_id = ? AND id >= ?${timeSql} AND (${latin.map(() => "instr(lower(text), ?) > 0").join(" OR ")})`,
324
+ ).all(userId, floor, ...timeArgs, ...latin) as Row[];
325
+ }
238
326
 
239
327
  /**
240
328
  * 在这个用户的聊天记录里搜。结果是一问一答为单位,相关的在前(同样相关的新的在前)。
241
329
  * skip:不要的对话(被删掉的、当前这条)。
242
330
  */
243
- export function searchHistory(
244
- userId: string,
245
- query: string,
246
- opts: { limit?: number; skip?: (key: string) => boolean; now?: number } = {},
247
- ): SearchHit[] {
331
+ export function searchHistory(userId: string, query: string, opts: SearchOptions = {}): SearchHit[] {
248
332
  checkUser(userId);
249
- // 能得分的句子至少含有一个词(长的中文词是至少含有它的一个两字组合),先在库里筛出来
250
- const needles = [...new Set(terms(query).flatMap((t) => (isCjk(t) && t.length >= 4 ? [t, ...bigrams(t)] : [t])))].slice(0, 200);
251
- if (needles.length === 0) return [];
252
- const rows = sql(
253
- `SELECT ${COLUMNS} FROM history WHERE user_id = ? AND (${needles.map(() => "instr(lower(text), ?) > 0").join(" OR ")})`,
254
- ).all(userId, ...needles) as Row[];
333
+ const match = ftsMatch(query);
334
+ if (!match && needles(query).length === 0) return [];
335
+ const limit = opts.limit ?? 6;
336
+ const time: string[] = [];
337
+ const timeArgs: number[] = [];
338
+ if (opts.after !== undefined) {
339
+ time.push(" AND ts >= ?");
340
+ timeArgs.push(opts.after);
341
+ }
342
+ if (opts.before !== undefined) {
343
+ time.push(" AND ts < ?");
344
+ timeArgs.push(opts.before);
345
+ }
346
+ const timeSql = time.join("");
347
+ syncHistoryIndex();
348
+ const byId = new Map<number, Row>();
349
+ if (match) {
350
+ // CROSS JOIN 固定先查索引:普通 JOIN 时规划器会选「扫这个人的全部记录,每行跑一次 FTS」,慢上千倍
351
+ const from = `SELECT h.id, h.conv_key, h.agent, h.role, h.text, h.ts, h.msg_id FROM history_fts CROSS JOIN history h ON h.id = history_fts.rowid
352
+ WHERE history_fts MATCH ? AND h.user_id = ?${timeSql.replace(/ ts /g, " h.ts ")}`;
353
+ // 边读边过滤,名额只算 skip 留下的:不然常见词的候选可能全是要跳过的对话(按专家筛选时、当前对话很长时),结果就空了
354
+ const take = (order: string, quota: number) => {
355
+ let kept = 0;
356
+ let seen = 0;
357
+ for (const r of sql(`${from} ORDER BY ${order}`).iterate(match, userId, ...timeArgs) as Iterable<Row>) {
358
+ if (++seen > SCAN_MAX || kept >= quota) break;
359
+ if (opts.skip?.(r.conv_key)) continue;
360
+ byId.set(r.id, r);
361
+ kept++;
362
+ }
363
+ };
364
+ // 最相关的一批,再加最新的一批:相关度差不多时新的该靠前,不能因为排不进前面就连候选都不是
365
+ take("history_fts.rank", CANDIDATES);
366
+ take("history_fts.rowid DESC", CANDIDATES_RECENT);
367
+ }
368
+ // 候选不够用才扫:常见词候选早就够了,不多花这一遍
369
+ if (byId.size < Math.max(limit, 50)) for (const r of scanRecent(userId, query, timeSql, timeArgs)) byId.set(r.id, r);
255
370
  const now = opts.now ?? Date.now();
256
371
  const scored: { row: Row; score: number }[] = [];
257
- for (const row of rows) {
372
+ for (const row of byId.values()) {
258
373
  if (opts.skip?.(row.conv_key)) continue;
259
374
  const s = scoreText(row.text, query);
260
375
  if (s === 0) continue;
@@ -265,17 +380,20 @@ export function searchHistory(
265
380
  scored.sort((a, b) => b.score - a.score || b.row.ts - a.row.ts || a.row.id - b.row.id);
266
381
  const out: SearchHit[] = [];
267
382
  const used = new Set<number>();
268
- const limit = opts.limit ?? 6;
269
- const next = sql(`SELECT ${COLUMNS} FROM history WHERE user_id = ? AND id > ? ORDER BY id LIMIT 1`);
270
- const prev = sql(`SELECT ${COLUMNS} FROM history WHERE user_id = ? AND id < ? ORDER BY id DESC LIMIT 1`);
383
+ const perConv = new Map<string, number>();
384
+ // 配对在同一段对话里找:同时开着几段对话时,用户记录里紧挨着的下一句可能是别的对话的
385
+ const next = sql(`SELECT ${COLUMNS} FROM history WHERE user_id = ? AND conv_key = ? AND id > ? ORDER BY id LIMIT 1`);
386
+ const prev = sql(`SELECT ${COLUMNS} FROM history WHERE user_id = ? AND conv_key = ? AND id < ? ORDER BY id DESC LIMIT 1`);
271
387
  for (const { row, score } of scored) {
272
388
  if (out.length >= limit) break;
273
389
  if (used.has(row.id)) continue;
390
+ if (opts.perConv && (perConv.get(row.conv_key) ?? 0) >= opts.perConv) continue;
274
391
  // 配上同一条对话里紧挨着的另一半:问配答,答配问
275
- const pair = (row.role === "u" ? next : prev).get(userId, row.id) as Row | undefined;
276
- const partner = pair && pair.conv_key === row.conv_key && pair.role !== row.role ? pair : undefined;
392
+ const pair = (row.role === "u" ? next : prev).get(userId, row.conv_key, row.id) as Row | undefined;
393
+ const partner = pair && pair.role !== row.role ? pair : undefined;
277
394
  used.add(row.id);
278
395
  if (partner) used.add(partner.id);
396
+ perConv.set(row.conv_key, (perConv.get(row.conv_key) ?? 0) + 1);
279
397
  const user = row.role === "u" ? row : partner;
280
398
  const assistant = row.role === "a" ? row : partner;
281
399
  out.push({
@@ -284,7 +402,57 @@ export function searchHistory(
284
402
  ...(user ? { user: snippet(user.text, query, 300) } : {}),
285
403
  ...(assistant ? { assistant: snippet(assistant.text, query, 600) } : {}),
286
404
  score,
405
+ rowId: row.id,
287
406
  });
288
407
  }
289
408
  return out;
290
409
  }
410
+
411
+ /** 读一段对话的原话。around:以这句为中心;否则从 offset 开始(从头数)。total 是这段一共几句 */
412
+ export function readConversationRecords(
413
+ userId: string,
414
+ key: string,
415
+ opts: { around?: number; offset?: number; limit?: number } = {},
416
+ ): { rows: (HistoryRecord & { rowId: number })[]; total: number; offset: number } {
417
+ checkUser(userId);
418
+ const limit = Math.max(1, Math.min(100, Math.round(Number(opts.limit ?? 20)) || 20));
419
+ const total = (sql("SELECT count(*) AS n FROM history WHERE user_id = ? AND conv_key = ?").get(userId, key) as { n: number }).n;
420
+ let offset = Math.max(0, Math.round(Number(opts.offset ?? 0)) || 0);
421
+ if (opts.around !== undefined && Number.isFinite(opts.around)) {
422
+ const before = (sql("SELECT count(*) AS n FROM history WHERE user_id = ? AND conv_key = ? AND id < ?").get(userId, key, opts.around) as { n: number }).n;
423
+ offset = Math.max(0, before - Math.floor(limit / 2));
424
+ }
425
+ const rows = sql(`SELECT ${COLUMNS} FROM history WHERE user_id = ? AND conv_key = ? ORDER BY id LIMIT ? OFFSET ?`).all(userId, key, limit, offset) as Row[];
426
+ return { rows: rows.map((r) => ({ ...toRecord(r), rowId: r.id })), total, offset };
427
+ }
428
+
429
+ /** 按顺序一批批读出全部记录(导出用),不用一次读进内存 */
430
+ export function* iterateRecords(userId: string, batch = 1000): Generator<HistoryRecord[]> {
431
+ checkUser(userId);
432
+ const stmt = sql(`SELECT ${COLUMNS} FROM history WHERE user_id = ? AND id > ? ORDER BY id LIMIT ?`);
433
+ let after = 0;
434
+ for (;;) {
435
+ const rows = stmt.all(userId, after, batch) as Row[];
436
+ if (rows.length === 0) return;
437
+ after = rows[rows.length - 1]!.id;
438
+ yield rows.map(toRecord);
439
+ }
440
+ }
441
+
442
+ // ---------- 搜索统计 ----------
443
+ //
444
+ // agent 搜了多少次、多少次没结果。用来判断关键词加摘要够不够用,要不要上语义检索。
445
+
446
+ export function recordSearch(userId: string, empty: boolean, day: string): void {
447
+ checkUser(userId);
448
+ sql(
449
+ `INSERT INTO search_stats (user_id, day, searches, empty) VALUES (?, ?, 1, ?)
450
+ ON CONFLICT (user_id, day) DO UPDATE SET searches = searches + 1, empty = empty + excluded.empty`,
451
+ ).run(userId, day, empty ? 1 : 0);
452
+ }
453
+
454
+ export function searchStats(userId: string, sinceDay: string): { searches: number; empty: number } {
455
+ checkUser(userId);
456
+ const row = sql("SELECT coalesce(sum(searches), 0) AS s, coalesce(sum(empty), 0) AS e FROM search_stats WHERE user_id = ? AND day >= ?").get(userId, sinceDay) as { s: number; e: number };
457
+ return { searches: row.s, empty: row.e };
458
+ }
package/src/core/db.ts CHANGED
@@ -2,7 +2,8 @@
2
2
  //
3
3
  // 放在这里的是会越用越多、或者好几个进程都要写的:会话(conversations)、会话归属(session_owners)、
4
4
  // 聊天记录索引(history / history_meta)、每轮回复的脚注(footers)、用量(usage_sessions / usage_daily)、
5
- // 每个人的记忆(memory)、资料库里的文件索引(files,文件本身还在上传目录里,见 library.ts)。
5
+ // 每个人的记忆(memory 存设置和「关于我」,memory_items 存条目,memory_log 存改动记录,
6
+ // memory_summaries 存摘要)、资料库里的文件索引(files,文件本身还在上传目录里,见 library.ts)。
6
7
  // 设置、用户、专家这类小配置还是 JSON 文件,方便直接打开看、手改。
7
8
  //
8
9
  // 主进程和每个会话的记忆工具子进程(mcp/memory-server.ts)会同时打开它:WAL 模式下读不挡写,
@@ -62,6 +63,21 @@ CREATE TABLE IF NOT EXISTS history (
62
63
  );
63
64
  CREATE INDEX IF NOT EXISTS history_user ON history (user_id, id);
64
65
  CREATE INDEX IF NOT EXISTS history_user_conv ON history (user_id, conv_key);
66
+ -- 查「每段对话最后一句是什么时候」只走这个索引,不用读正文(见 chat-history.ts 的 pendingConversations)
67
+ CREATE INDEX IF NOT EXISTS history_user_conv_ts ON history (user_id, conv_key, ts);
68
+
69
+ -- 聊天记录的全文索引:只存索引不存原文(原文在 history 里),rowid 就是 history.id。
70
+ -- 写入时不碰它,搜索和启动时按 index_state 里的进度补上(见 chat-history.ts 的 syncHistoryIndex)
71
+ CREATE VIRTUAL TABLE IF NOT EXISTS history_fts USING fts5(text, content='', contentless_delete=1, tokenize='unicode61');
72
+ CREATE TABLE IF NOT EXISTS index_state (name TEXT PRIMARY KEY, upto INTEGER NOT NULL);
73
+
74
+ CREATE TABLE IF NOT EXISTS search_stats (
75
+ user_id TEXT NOT NULL,
76
+ day TEXT NOT NULL,
77
+ searches INTEGER NOT NULL,
78
+ empty INTEGER NOT NULL,
79
+ PRIMARY KEY (user_id, day)
80
+ );
65
81
 
66
82
  CREATE TABLE IF NOT EXISTS history_meta (
67
83
  user_id TEXT PRIMARY KEY,
@@ -96,6 +112,58 @@ CREATE TABLE IF NOT EXISTS memory (
96
112
  data TEXT NOT NULL
97
113
  );
98
114
 
115
+ CREATE TABLE IF NOT EXISTS memory_items (
116
+ user_id TEXT NOT NULL,
117
+ id TEXT NOT NULL,
118
+ text TEXT NOT NULL,
119
+ source TEXT NOT NULL,
120
+ tier TEXT NOT NULL,
121
+ locked INTEGER NOT NULL DEFAULT 0,
122
+ created_at INTEGER NOT NULL,
123
+ updated_at INTEGER NOT NULL,
124
+ last_used_at INTEGER,
125
+ observed_at INTEGER,
126
+ valid_until INTEGER,
127
+ source_conv TEXT,
128
+ deleted_at INTEGER,
129
+ PRIMARY KEY (user_id, id)
130
+ );
131
+ CREATE INDEX IF NOT EXISTS memory_items_live ON memory_items (user_id, deleted_at, tier);
132
+
133
+ CREATE TABLE IF NOT EXISTS memory_log (
134
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
135
+ user_id TEXT NOT NULL,
136
+ item_id TEXT NOT NULL,
137
+ action TEXT NOT NULL,
138
+ actor TEXT NOT NULL,
139
+ old_text TEXT,
140
+ new_text TEXT,
141
+ old_tier TEXT,
142
+ new_tier TEXT,
143
+ conv_key TEXT,
144
+ note TEXT,
145
+ at INTEGER NOT NULL,
146
+ undone_at INTEGER
147
+ );
148
+ CREATE INDEX IF NOT EXISTS memory_log_user ON memory_log (user_id, id);
149
+ CREATE INDEX IF NOT EXISTS memory_log_item ON memory_log (user_id, item_id);
150
+
151
+ CREATE TABLE IF NOT EXISTS memory_summaries (
152
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
153
+ user_id TEXT NOT NULL,
154
+ kind TEXT NOT NULL,
155
+ scope TEXT NOT NULL,
156
+ title TEXT NOT NULL,
157
+ text TEXT NOT NULL,
158
+ project TEXT,
159
+ from_ts INTEGER NOT NULL,
160
+ to_ts INTEGER NOT NULL,
161
+ updated_at INTEGER NOT NULL,
162
+ UNIQUE (user_id, kind, scope)
163
+ );
164
+ CREATE INDEX IF NOT EXISTS memory_summaries_time ON memory_summaries (user_id, kind, to_ts);
165
+ CREATE VIRTUAL TABLE IF NOT EXISTS memory_summaries_fts USING fts5(text, content='', contentless_delete=1, tokenize='unicode61');
166
+
99
167
  CREATE TABLE IF NOT EXISTS files (
100
168
  id TEXT PRIMARY KEY,
101
169
  owner TEXT NOT NULL,
@@ -131,6 +199,9 @@ export function database(): DatabaseSync {
131
199
  return opened;
132
200
  }
133
201
 
202
+ /** memory 表里哪些行还带着老格式的条目。坏掉的 JSON 不能让 json_type 抛错:一行坏数据不该让服务起不来 */
203
+ export const LEGACY_MEMORY_ITEMS = "CASE WHEN json_valid(data) THEN json_type(data, '$.items') END = 'array'";
204
+
134
205
  /** 「记忆改成默认关」的升级迁移跑过了(见 memory.ts 的 migrateMemoryDefault) */
135
206
  export const MEMORY_DEFAULT_OFF = "memory-default-off";
136
207
 
@@ -0,0 +1,53 @@
1
+ // 全文索引(SQLite FTS5)用的文字处理:写进索引的文字怎么切,搜索词怎么变成 MATCH 表达式。
2
+ //
3
+ // FTS5 自带的 trigram 分词搜不到两个字的中文词(「豆豆」),unicode61 又把一整串中文当成一个词。
4
+ // 所以写进索引之前把中日韩文字一个字一个字隔开,搜的时候用短语查询:「软路由」→ "软 路 由"。
5
+ // 英文词整个保留,驼峰词另外拆开补一份(搜 clash 能搜到 OpenClash)。
6
+
7
+ const CJK = "\\u3040-\\u30ff\\u3400-\\u9fff\\uac00-\\ud7af\\uf900-\\ufaff";
8
+ const CJK_CHAR = new RegExp(`([${CJK}])`, "g");
9
+ const HAS_CJK = new RegExp(`[${CJK}]`);
10
+
11
+ export const isCjk = (s: string) => HAS_CJK.test(s);
12
+
13
+ const spaced = (text: string) => text.replace(CJK_CHAR, " $1 ");
14
+
15
+ /** 写进索引的文字 */
16
+ export function ftsIndexText(text: string): string {
17
+ // 先按词切再看里面有没有大小写变化:一个正则写到底的话,很长的一串字母(贴进来的 hash、base64)会回溯到平方级
18
+ const camel = text.replace(/[A-Za-z0-9]+/g, (w) => (w.length <= 64 && /[a-z0-9][A-Z]/.test(w) ? `${w} ${w.replace(/([a-z0-9])([A-Z])/g, "$1 $2")}` : w));
19
+ return spaced(camel);
20
+ }
21
+
22
+ /** 把查询拆成词:按空白和标点切开。中文不切词 */
23
+ export function terms(query: string): string[] {
24
+ return query
25
+ .toLowerCase()
26
+ .split(/[\s,,。.!!??、;;::"'“”‘’()()[\]【】<>《》/\\|]+/)
27
+ .map((t) => t.trim())
28
+ .filter(Boolean);
29
+ }
30
+
31
+ export function bigrams(s: string): string[] {
32
+ const out: string[] = [];
33
+ for (let i = 0; i + 2 <= s.length; i++) out.push(s.slice(i, i + 2));
34
+ return out;
35
+ }
36
+
37
+ /** 要找的东西:每个词本身,长的中文词再加上它的两字组合(「软路由防污染」也能找到「软路由怎么防 DNS 污染」) */
38
+ export function needles(query: string): string[] {
39
+ return [...new Set(terms(query).flatMap((t) => (isCjk(t) && t.length >= 4 ? [t, ...bigrams(t)] : [t])))].slice(0, 64);
40
+ }
41
+
42
+ /** 查询变成 MATCH 表达式,任何一个词命中都算候选(排序在外面做)。没有能搜的词(全是符号)就是 undefined */
43
+ export function ftsMatch(query: string): string | undefined {
44
+ const phrases: string[] = [];
45
+ for (const needle of needles(query)) {
46
+ if (!/[\p{L}\p{N}]/u.test(needle)) continue;
47
+ const phrase = spaced(needle).replace(/"/g, "").replace(/\s+/g, " ").trim();
48
+ if (!phrase) continue;
49
+ // 纯英文数字按前缀搜:clash 也能搜到 clashx
50
+ phrases.push(/^[a-z0-9]+$/.test(phrase) ? `"${phrase}"*` : `"${phrase}"`);
51
+ }
52
+ return phrases.length ? phrases.join(" OR ") : undefined;
53
+ }
package/src/core/hub.ts CHANGED
@@ -59,7 +59,7 @@ import { mcpEvents } from "./mcp.js";
59
59
  import { isMachineCommand, sessionSkills } from "./skill-runtime.js";
60
60
  import { skillEvents } from "./skills.js";
61
61
  import { isSilentReply } from "./task-prompt.js";
62
- import { findTask } from "./tasks.js";
62
+ import { findTask, listTasks, taskEvents } from "./tasks.js";
63
63
  import { patchWeixinAccount } from "./weixin-accounts.js";
64
64
 
65
65
  // ---------- 事件 ----------
@@ -193,6 +193,8 @@ export type ConversationInfo = {
193
193
  scheduled?: { id: string; text: string; at: number }[];
194
194
  /** 定时任务的专用对话 */
195
195
  taskId?: string;
196
+ /** 有定时任务把结果发回这条对话(含暂停的任务) */
197
+ hasTasks?: boolean;
196
198
  };
197
199
 
198
200
  /** 顶栏、列表里显示的专家 */
@@ -292,6 +294,12 @@ class Hub extends EventEmitter<{ event: [EventEnvelope]; task: [TaskTurnEnd]; "p
292
294
  super();
293
295
  this.setMaxListeners(100);
294
296
  interactions.on("created", (i) => this.emitEvent(i.convKey, { type: "ix", id: i.id }));
297
+ taskEvents.on("changed", (owner, keys) => {
298
+ for (const key of new Set(keys)) {
299
+ const conv = getConversation(key);
300
+ if (conv && memoryOwnerOf(conv) === owner) this.emitEvent(key, { type: "meta" });
301
+ }
302
+ });
295
303
  // 技能装了、改了、停用了、变量改了:受影响的人空闲的会话换上新的
296
304
  skillEvents.on("changed", (users) => this.personalChanged(new Set(users)));
297
305
  // 专家修改只影响自定义专家;MCP 和内置默认配置修改也更新该账号的内置会话。
@@ -637,6 +645,10 @@ class Hub extends EventEmitter<{ event: [EventEnvelope]; task: [TaskTurnEnd]; "p
637
645
  expert: this.expertBadge(conv),
638
646
  ...(memoryOwnerOf(conv) ? { personal: true } : {}),
639
647
  ...(conv.taskId ? { taskId: conv.taskId } : {}),
648
+ ...(() => {
649
+ const owner = memoryOwnerOf(conv);
650
+ return owner && listTasks(owner).some((task) => task.target.key === key) ? { hasTasks: true } : {};
651
+ })(),
640
652
  ...(() => {
641
653
  const list = listScheduled(key);
642
654
  return list.length ? { scheduled: list.map(({ id, text, at }) => ({ id, text, at })) } : {};
@@ -715,7 +727,8 @@ class Hub extends EventEmitter<{ event: [EventEnvelope]; task: [TaskTurnEnd]; "p
715
727
  // 专家:自定义的按它的配置来;被删了就在这里抛错,这条对话不能再用
716
728
  const expert = this.expertFor(conv);
717
729
  // 记忆在拉起会话这一刻读一次:会话里 agent 自己记下的,上下文里本来就有;
718
- // 用户在设置里改了,memoryChanged() 会让空闲的会话下次说话时带着新的重新拉起。专家可以不带记忆
730
+ // 用户在设置里改了、后台整理改了、别的对话里记了,memoryChanged() 会让空闲的会话下次说话时
731
+ // 带着新的重新拉起。专家可以不带记忆
719
732
  const memory = expert && !expert.memory ? undefined : sessionMemory(conv);
720
733
  // 定时任务工具:和记忆一样只给有主人的对话(飞书群聊里谁都能让机器人建任务,算不清是谁的)
721
734
  const taskOwner = memoryOwnerOf(conv);
@@ -1544,16 +1557,18 @@ class Hub extends EventEmitter<{ event: [EventEnvelope]; task: [TaskTurnEnd]; "p
1544
1557
  * 用户在设置里改了记忆(关于我、增删条目、开关):他名下活着的会话都要换上新的。
1545
1558
  * agent 自己用 remember / forget 改的不走这里 —— 那些本来就在上下文里,没必要重启进程、白白丢掉提示缓存。
1546
1559
  */
1547
- memoryChanged(userId: string): void {
1548
- this.personalChanged(new Set([userId]));
1560
+ /** except:这几条对话不用换(改动就是它们自己做的,上下文里本来就有) */
1561
+ memoryChanged(userId: string, opts: { except?: Set<string> } = {}): void {
1562
+ this.personalChanged(new Set([userId]), false, opts.except);
1549
1563
  }
1550
1564
 
1551
1565
  /**
1552
1566
  * 这些人的记忆、技能、专家或 MCP 变了:他们名下活着的会话空闲下来就重新拉起。
1553
1567
  * onlyExperts:只管用自定义专家的会话(修改自定义专家配置时使用)
1554
1568
  */
1555
- private personalChanged(users: Set<string>, onlyExperts = false): void {
1569
+ private personalChanged(users: Set<string>, onlyExperts = false, except?: Set<string>): void {
1556
1570
  for (const [key, live] of this.live) {
1571
+ if (except?.has(key)) continue;
1557
1572
  const conv = getConversation(key);
1558
1573
  const owner = conv ? memoryOwnerOf(conv) : undefined;
1559
1574
  if (!owner || !users.has(owner) || (onlyExperts && !conv?.expert)) continue;