agent-loft 0.2.59 → 0.2.60
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -7
- package/package.json +2 -2
- package/src/channels/web/api.ts +26 -3
- package/src/channels/web/library-api.ts +59 -0
- package/src/core/chat-history.ts +92 -71
- package/src/core/db.ts +367 -0
- package/src/core/experts.ts +60 -2
- package/src/core/footers.ts +21 -43
- package/src/core/hub.ts +3 -16
- package/src/core/library.ts +401 -0
- package/src/core/memory.ts +34 -60
- package/src/core/store.ts +47 -19
- package/src/core/uploads.ts +3 -46
- package/src/core/usage.ts +62 -72
- package/src/main.ts +7 -6
- package/src/mcp/memory-server.ts +4 -7
- package/web/css/library.css +266 -0
- package/web/index.html +1 -0
- package/web/lib/dom.js +2 -0
- package/web/state.js +3 -3
- package/web/views/composer.js +36 -1
- package/web/views/layout.js +11 -0
- package/web/views/library.js +309 -0
- package/web/views/messages.js +3 -3
- package/web/views/remote.js +2 -1
- package/web/views/sidebar.js +1 -0
- package/web/views/topbar.js +3 -2
|
@@ -0,0 +1,401 @@
|
|
|
1
|
+
// 资料库:用户发过的附件、agent 生成的文件,按人汇总在一处,可以浏览、搜索、删掉,或者拿到别的对话里再用。
|
|
2
|
+
//
|
|
3
|
+
// 文件本身不搬家,还在上传目录里(data/uploads/<对话>/…,见 uploads.ts):历史消息里写着的路径照样能预览,
|
|
4
|
+
// agent 读文件的范围也还是「这条对话的上传目录」。这里只在数据库的 files 表里记一份索引:谁的、哪条对话、
|
|
5
|
+
// 什么时候、从哪来(upload:用户发的;agent:agent 回复里给出的新文件,复制了一份进来)。
|
|
6
|
+
//
|
|
7
|
+
// 收录跟着 hub 的事件流走(见 startLibraryCapture):
|
|
8
|
+
// - 用户消息带的附件:发出去时收进来。上传了没发的不收,一天后清掉(见 tidyLibrary);
|
|
9
|
+
// - agent 的回复里提到的、这一轮新写出来的图片 / 文档 / 音视频:复制一份进这条对话的上传目录再收进来,
|
|
10
|
+
// 之后 agent 改了、删了原文件,资料库里的不受影响。
|
|
11
|
+
// 从资料库拿到别的对话里用:硬链接一份进那条对话的上传目录(见 attachToConversation),不占双份空间。
|
|
12
|
+
// 这份也记一行(source = copy,origin 指向原来那条),列表里不显示;删原件时连它一起删,空间才真的腾出来。
|
|
13
|
+
//
|
|
14
|
+
// 归属跟着对话的主人走:没有主人的对话(飞书群聊、没绑定账号的私聊)的文件归管理员,规则同 users.ts 的 ownedBy。
|
|
15
|
+
import crypto from "node:crypto";
|
|
16
|
+
import fs from "node:fs";
|
|
17
|
+
import path from "node:path";
|
|
18
|
+
import { sql, transaction } from "./db.js";
|
|
19
|
+
import { hub } from "./hub.js";
|
|
20
|
+
import { replyFileType, resolveReplyFile } from "./reply-files.js";
|
|
21
|
+
import { getConversation } from "./store.js";
|
|
22
|
+
import { ensureUploadsDir, kindOf, safeName, sniff, uploadsRoot } from "./uploads.js";
|
|
23
|
+
import { listUsers, ownedBy, type User } from "./users.js";
|
|
24
|
+
|
|
25
|
+
type Viewer = Pick<User, "id" | "role">;
|
|
26
|
+
import type { AttachmentKind, SavedAttachment } from "./types.js";
|
|
27
|
+
|
|
28
|
+
/** upload:用户发的;agent:agent 生成的;copy:从资料库放进别的对话的那份(列表里不显示) */
|
|
29
|
+
export type LibrarySource = "upload" | "agent" | "copy";
|
|
30
|
+
|
|
31
|
+
export type LibraryFile = {
|
|
32
|
+
id: string;
|
|
33
|
+
/** 用户 id;空串 = 没有主人的对话里的,归管理员 */
|
|
34
|
+
owner: string;
|
|
35
|
+
convKey: string;
|
|
36
|
+
path: string;
|
|
37
|
+
name: string;
|
|
38
|
+
kind: AttachmentKind;
|
|
39
|
+
size: number;
|
|
40
|
+
source: LibrarySource;
|
|
41
|
+
createdAt: number;
|
|
42
|
+
};
|
|
43
|
+
|
|
44
|
+
/** 筛选:图片 / 文档(其余的文件)/ 音视频 */
|
|
45
|
+
export type LibraryKind = "image" | "file" | "media";
|
|
46
|
+
|
|
47
|
+
type Row = { id: string; owner: string; conv_key: string; path: string; name: string; kind: AttachmentKind; size: number; source: LibrarySource; origin: string | null; created_at: number };
|
|
48
|
+
|
|
49
|
+
const COLUMNS = "id, owner, conv_key, path, name, kind, size, source, origin, created_at";
|
|
50
|
+
|
|
51
|
+
function toFile(r: Row): LibraryFile {
|
|
52
|
+
return { id: r.id, owner: r.owner, convKey: r.conv_key, path: r.path, name: r.name, kind: r.kind, size: r.size, source: r.source, createdAt: r.created_at };
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/** agent 生成的文件最多收这么大的(复制一份要占地方) */
|
|
56
|
+
const AGENT_FILE_MAX = 50 * 1024 * 1024;
|
|
57
|
+
/** 一轮最多收这么多个 agent 生成的文件 */
|
|
58
|
+
const AGENT_FILES_PER_TURN = 10;
|
|
59
|
+
/** 上传了一直没发出去的,过这么久清掉 */
|
|
60
|
+
const UNSENT_TTL = 24 * 60 * 60 * 1000;
|
|
61
|
+
|
|
62
|
+
export function recordFile(params: {
|
|
63
|
+
owner?: string;
|
|
64
|
+
convKey: string;
|
|
65
|
+
path: string;
|
|
66
|
+
name: string;
|
|
67
|
+
kind: AttachmentKind;
|
|
68
|
+
size: number;
|
|
69
|
+
source: LibrarySource;
|
|
70
|
+
origin?: string;
|
|
71
|
+
createdAt?: number;
|
|
72
|
+
}): boolean {
|
|
73
|
+
const res = sql(`INSERT OR IGNORE INTO files (${COLUMNS}) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(
|
|
74
|
+
`f_${crypto.randomBytes(8).toString("hex")}`,
|
|
75
|
+
params.owner ?? "",
|
|
76
|
+
params.convKey,
|
|
77
|
+
path.resolve(params.path),
|
|
78
|
+
params.name,
|
|
79
|
+
params.kind,
|
|
80
|
+
params.size,
|
|
81
|
+
params.source,
|
|
82
|
+
params.origin ?? null,
|
|
83
|
+
params.createdAt ?? Date.now(),
|
|
84
|
+
);
|
|
85
|
+
return Number(res.changes) > 0;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
export function getLibraryFile(id: string): LibraryFile | undefined {
|
|
89
|
+
const row = sql(`SELECT ${COLUMNS} FROM files WHERE id = ?`).get(id) as Row | undefined;
|
|
90
|
+
return row ? toFile(row) : undefined;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
export function canSeeFile(user: Viewer, file: LibraryFile): boolean {
|
|
94
|
+
return ownedBy(user, file.owner || undefined);
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* 这个用户看得到的那部分,拼成 SQL 条件:自己的;管理员还有没主人的、主人已经被删掉的
|
|
99
|
+
* (和 ownedBy 一样:别的管理员的不算)
|
|
100
|
+
*/
|
|
101
|
+
function visibleWhere(user: Viewer): { where: string; args: string[] } {
|
|
102
|
+
if (user.role !== "admin") return { where: "owner = ?", args: [user.id] };
|
|
103
|
+
const others = listUsers()
|
|
104
|
+
.map((u) => u.id)
|
|
105
|
+
.filter((id) => id !== user.id);
|
|
106
|
+
return { where: `(owner = ? OR owner NOT IN (${others.map(() => "?").join(", ") || "''"}))`, args: [user.id, ...others] };
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
const KIND_WHERE: Record<LibraryKind, string> = {
|
|
110
|
+
image: "kind = 'image'",
|
|
111
|
+
file: "kind = 'file'",
|
|
112
|
+
media: "kind IN ('audio', 'video')",
|
|
113
|
+
};
|
|
114
|
+
|
|
115
|
+
/** 按时间从新到旧一页一页地列。before:上一页最后一个的 { createdAt, id } */
|
|
116
|
+
export function listLibrary(
|
|
117
|
+
user: Viewer,
|
|
118
|
+
opts: { kind?: LibraryKind; q?: string; before?: { at: number; id: string }; limit?: number } = {},
|
|
119
|
+
): { files: LibraryFile[]; more: boolean } {
|
|
120
|
+
const { where, args } = visibleWhere(user);
|
|
121
|
+
const conds = [where, "source != 'copy'"];
|
|
122
|
+
const params: (string | number)[] = [...args];
|
|
123
|
+
if (opts.kind) conds.push(KIND_WHERE[opts.kind]);
|
|
124
|
+
const q = opts.q?.trim();
|
|
125
|
+
if (q) {
|
|
126
|
+
conds.push("instr(lower(name), ?) > 0");
|
|
127
|
+
params.push(q.toLowerCase());
|
|
128
|
+
}
|
|
129
|
+
if (opts.before) {
|
|
130
|
+
conds.push("(created_at < ? OR (created_at = ? AND id < ?))");
|
|
131
|
+
params.push(opts.before.at, opts.before.at, opts.before.id);
|
|
132
|
+
}
|
|
133
|
+
const limit = Math.max(1, Math.min(200, opts.limit ?? 60));
|
|
134
|
+
const rows = sql(`SELECT ${COLUMNS} FROM files WHERE ${conds.join(" AND ")} ORDER BY created_at DESC, id DESC LIMIT ?`).all(
|
|
135
|
+
...params,
|
|
136
|
+
limit + 1,
|
|
137
|
+
) as Row[];
|
|
138
|
+
return { files: rows.slice(0, limit).map(toFile), more: rows.length > limit };
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
/** 一共多少个、多大 */
|
|
142
|
+
export function librarySummary(user: Viewer): { count: number; size: number } {
|
|
143
|
+
const { where, args } = visibleWhere(user);
|
|
144
|
+
const row = sql(`SELECT count(*) AS n, coalesce(sum(size), 0) AS s FROM files WHERE ${where} AND source != 'copy'`).get(...args) as { n: number; s: number };
|
|
145
|
+
return { count: row.n, size: row.s };
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/**
|
|
149
|
+
* 从资料库删掉:文件和索引都删,放进别的对话的那几份也一起删(硬链接不全删掉空间腾不出来)。
|
|
150
|
+
* 放它们的小目录(每次上传一个)空了也一并删掉
|
|
151
|
+
*/
|
|
152
|
+
export function deleteLibraryFile(file: LibraryFile): void {
|
|
153
|
+
const all = sql("SELECT id, path FROM files WHERE id = ? OR origin = ?").all(file.id, file.id) as { id: string; path: string }[];
|
|
154
|
+
for (const f of all) {
|
|
155
|
+
fs.rmSync(f.path, { force: true });
|
|
156
|
+
removeEmptyDirs(path.dirname(f.path));
|
|
157
|
+
}
|
|
158
|
+
sql("DELETE FROM files WHERE id = ? OR origin = ?").run(file.id, file.id);
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
/** 往上删空目录,到上传目录的根为止 */
|
|
162
|
+
function removeEmptyDirs(dir: string): void {
|
|
163
|
+
const root = path.resolve(uploadsRoot);
|
|
164
|
+
for (let d = path.resolve(dir); d.startsWith(root + path.sep); d = path.dirname(d)) {
|
|
165
|
+
try {
|
|
166
|
+
fs.rmdirSync(d);
|
|
167
|
+
} catch {
|
|
168
|
+
return; // 不空、或者已经没了
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
// ---------- 拿到对话里用 ----------
|
|
174
|
+
|
|
175
|
+
/**
|
|
176
|
+
* 资料库里的文件放进这条对话:硬链接进它的上传目录(跨不了文件系统就复制),agent 照常能读。
|
|
177
|
+
* 返回和刚上传的附件一样的东西,交给发消息时认领
|
|
178
|
+
*/
|
|
179
|
+
export function attachToConversation(file: LibraryFile, key: string): SavedAttachment {
|
|
180
|
+
const dir = path.join(ensureUploadsDir(key), `lib-${crypto.randomBytes(6).toString("hex")}`);
|
|
181
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
182
|
+
const target = path.join(dir, safeName(file.name));
|
|
183
|
+
try {
|
|
184
|
+
fs.linkSync(file.path, target);
|
|
185
|
+
} catch {
|
|
186
|
+
fs.copyFileSync(file.path, target);
|
|
187
|
+
}
|
|
188
|
+
// 记下这一份:发出去时不会再当成新上传的收一遍,也不会被当成「上传了没发」清掉
|
|
189
|
+
const conv = getConversation(key);
|
|
190
|
+
recordFile({ owner: conv?.owner, convKey: key, path: target, name: path.basename(target), kind: file.kind, size: file.size, source: "copy", origin: file.id });
|
|
191
|
+
const mediaType = sniffFile(target);
|
|
192
|
+
return { kind: file.kind, path: target, name: path.basename(target), size: fs.statSync(target).size, ...(mediaType ? { mediaType } : {}) };
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
function sniffFile(file: string): string | undefined {
|
|
196
|
+
const head = Buffer.alloc(12);
|
|
197
|
+
let fd: number | undefined;
|
|
198
|
+
try {
|
|
199
|
+
fd = fs.openSync(file, "r");
|
|
200
|
+
fs.readSync(fd, head, 0, head.length, 0);
|
|
201
|
+
return sniff(head)?.mediaType;
|
|
202
|
+
} catch {
|
|
203
|
+
return undefined;
|
|
204
|
+
} finally {
|
|
205
|
+
if (fd !== undefined) fs.closeSync(fd);
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
/** 按文件头,认不出再按后缀 */
|
|
210
|
+
function kindOfFile(file: string): AttachmentKind {
|
|
211
|
+
return kindOf(sniffFile(file) ?? replyFileType(file));
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
// ---------- 收录 ----------
|
|
215
|
+
|
|
216
|
+
/** 回复里提到的这些后缀才当成「交给用户的东西」收进来:代码、配置之类的不算 */
|
|
217
|
+
const DELIVERABLE = "png|jpe?g|gif|webp|svg|pdf|docx?|xlsx?|pptx?|csv|zip|mp3|m4a|wav|mp4|mov|webm";
|
|
218
|
+
const CANDIDATE = new RegExp(`[^\\s<>"'\`()()\\[\\]{},。;:!?、“”‘’「」|*]+\\.(?:${DELIVERABLE})(?![\\p{L}\\p{N}_/-])`, "giu");
|
|
219
|
+
|
|
220
|
+
/** 回复里像是文件路径、后缀是图片 / 文档 / 音视频的 */
|
|
221
|
+
export function deliverablePaths(text: string): string[] {
|
|
222
|
+
return [...new Set(text.match(CANDIDATE) ?? [])];
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
function within(file: string, dir: string): boolean {
|
|
226
|
+
return file.startsWith(path.resolve(dir) + path.sep);
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
/**
|
|
230
|
+
* 这一轮 agent 新写出来的文件(修改时间不早于这一轮开始)复制进对话的上传目录,收进资料库。
|
|
231
|
+
* 用户自己传的(本来就在上传目录里)、太大的、敏感的(resolveReplyFile 挡掉)不收
|
|
232
|
+
*/
|
|
233
|
+
export async function captureAgentFiles(key: string, text: string, since: number): Promise<number> {
|
|
234
|
+
const conv = getConversation(key);
|
|
235
|
+
if (!conv) return 0;
|
|
236
|
+
const seen = new Set<string>();
|
|
237
|
+
let n = 0;
|
|
238
|
+
for (const p of deliverablePaths(text)) {
|
|
239
|
+
if (n >= AGENT_FILES_PER_TURN) break;
|
|
240
|
+
const real = resolveReplyFile(p, conv.cwd);
|
|
241
|
+
if (!real || seen.has(real) || within(real, uploadsRoot)) continue;
|
|
242
|
+
seen.add(real);
|
|
243
|
+
let stat: fs.Stats;
|
|
244
|
+
try {
|
|
245
|
+
stat = fs.statSync(real);
|
|
246
|
+
} catch {
|
|
247
|
+
continue;
|
|
248
|
+
}
|
|
249
|
+
// 修改时间留两秒余量:有的文件系统只精确到秒
|
|
250
|
+
if (stat.mtimeMs < since - 2000 || stat.size === 0 || stat.size > AGENT_FILE_MAX) continue;
|
|
251
|
+
const dir = path.join(ensureUploadsDir(key), `agent-${crypto.randomBytes(6).toString("hex")}`);
|
|
252
|
+
const target = path.join(dir, safeName(path.basename(real)));
|
|
253
|
+
try {
|
|
254
|
+
await fs.promises.mkdir(dir, { recursive: true });
|
|
255
|
+
await fs.promises.copyFile(real, target);
|
|
256
|
+
} catch (err) {
|
|
257
|
+
console.warn(`[资料库] 收录 ${real} 失败:`, err instanceof Error ? err.message : err);
|
|
258
|
+
continue;
|
|
259
|
+
}
|
|
260
|
+
recordFile({ owner: conv.owner, convKey: key, path: target, name: path.basename(target), kind: kindOfFile(target), size: stat.size, source: "agent" });
|
|
261
|
+
n++;
|
|
262
|
+
}
|
|
263
|
+
return n;
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
let capturing = false;
|
|
267
|
+
|
|
268
|
+
export function startLibraryCapture(): void {
|
|
269
|
+
if (capturing) return;
|
|
270
|
+
capturing = true;
|
|
271
|
+
/** 每条对话这一轮从什么时候开始(agent 生成的文件按修改时间认) */
|
|
272
|
+
const turnStart = new Map<string, number>();
|
|
273
|
+
hub.on("event", (env) => {
|
|
274
|
+
const ev = env.ev;
|
|
275
|
+
try {
|
|
276
|
+
if (ev.type === "turn_start") {
|
|
277
|
+
turnStart.set(env.key, env.ts);
|
|
278
|
+
return;
|
|
279
|
+
}
|
|
280
|
+
if (ev.type === "user") {
|
|
281
|
+
if (!ev.files?.length) return;
|
|
282
|
+
const conv = getConversation(env.key);
|
|
283
|
+
for (const f of ev.files) {
|
|
284
|
+
const file = path.resolve(f.path);
|
|
285
|
+
if (!within(file, uploadsRoot) || !fs.existsSync(file)) continue;
|
|
286
|
+
recordFile({ owner: conv?.owner, convKey: env.key, path: file, name: f.name, kind: f.kind, size: f.size ?? fs.statSync(file).size, source: "upload", createdAt: env.ts });
|
|
287
|
+
}
|
|
288
|
+
return;
|
|
289
|
+
}
|
|
290
|
+
if (ev.type === "result" && ev.text) {
|
|
291
|
+
const since = turnStart.get(env.key) ?? env.ts - 10 * 60_000;
|
|
292
|
+
turnStart.delete(env.key);
|
|
293
|
+
void captureAgentFiles(env.key, ev.text, since).catch((err) => console.warn("[资料库] 收录 agent 文件失败:", err));
|
|
294
|
+
}
|
|
295
|
+
} catch (err) {
|
|
296
|
+
console.warn("[资料库] 收录失败:", err instanceof Error ? err.message : err);
|
|
297
|
+
}
|
|
298
|
+
});
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
// ---------- 启动时 ----------
|
|
302
|
+
|
|
303
|
+
function walk(dir: string, out: string[] = []): string[] {
|
|
304
|
+
let entries: fs.Dirent[];
|
|
305
|
+
try {
|
|
306
|
+
entries = fs.readdirSync(dir, { withFileTypes: true });
|
|
307
|
+
} catch {
|
|
308
|
+
return out;
|
|
309
|
+
}
|
|
310
|
+
for (const e of entries) {
|
|
311
|
+
const full = path.join(dir, e.name);
|
|
312
|
+
if (e.isDirectory()) walk(full, out);
|
|
313
|
+
else if (e.isFile()) out.push(full);
|
|
314
|
+
}
|
|
315
|
+
return out;
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
/** 上传目录里的文件属于哪条对话(第一级目录名) */
|
|
319
|
+
function convKeyOf(file: string): string | undefined {
|
|
320
|
+
const first = path.relative(uploadsRoot, file).split(path.sep)[0];
|
|
321
|
+
if (!first) return undefined;
|
|
322
|
+
try {
|
|
323
|
+
return decodeURIComponent(first);
|
|
324
|
+
} catch {
|
|
325
|
+
return undefined;
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
/**
|
|
330
|
+
* 升级时把上传目录里已有的文件收进资料库,只做一次。以前切换会话时会清掉旧附件,留下来的都是还在用的那些
|
|
331
|
+
*/
|
|
332
|
+
export function backfillLibrary(): number {
|
|
333
|
+
return transaction(() => {
|
|
334
|
+
if (sql("SELECT 1 FROM migrations WHERE name = 'library-backfill'").get()) return 0;
|
|
335
|
+
let n = 0;
|
|
336
|
+
for (const file of walk(uploadsRoot)) {
|
|
337
|
+
const key = convKeyOf(file);
|
|
338
|
+
const conv = key ? getConversation(key) : undefined;
|
|
339
|
+
if (!key || !conv) continue;
|
|
340
|
+
const stat = fs.statSync(file);
|
|
341
|
+
const agentMade = path.relative(uploadsRoot, file).split(path.sep).some((s) => s.startsWith("agent-"));
|
|
342
|
+
const added = recordFile({
|
|
343
|
+
owner: conv.owner,
|
|
344
|
+
convKey: key,
|
|
345
|
+
path: file,
|
|
346
|
+
name: path.basename(file),
|
|
347
|
+
kind: kindOfFile(file),
|
|
348
|
+
size: stat.size,
|
|
349
|
+
source: agentMade ? "agent" : "upload",
|
|
350
|
+
createdAt: Math.round(stat.mtimeMs),
|
|
351
|
+
});
|
|
352
|
+
if (added) n++;
|
|
353
|
+
}
|
|
354
|
+
sql("INSERT INTO migrations (name, at) VALUES ('library-backfill', ?)").run(Date.now());
|
|
355
|
+
return n;
|
|
356
|
+
});
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
/**
|
|
360
|
+
* 启动时收拾:对话已经不在了的,上传目录和索引一起删;文件已经没了的索引删掉;
|
|
361
|
+
* 上传了一直没发出去(不在索引里)、超过一天的文件删掉
|
|
362
|
+
*/
|
|
363
|
+
export function tidyLibrary(now = Date.now()): { removed: number } {
|
|
364
|
+
let removed = 0;
|
|
365
|
+
let entries: string[] = [];
|
|
366
|
+
try {
|
|
367
|
+
entries = fs.readdirSync(uploadsRoot);
|
|
368
|
+
} catch {
|
|
369
|
+
// 还没有上传目录
|
|
370
|
+
}
|
|
371
|
+
for (const entry of entries) {
|
|
372
|
+
let key: string;
|
|
373
|
+
try {
|
|
374
|
+
key = decodeURIComponent(entry);
|
|
375
|
+
} catch {
|
|
376
|
+
continue;
|
|
377
|
+
}
|
|
378
|
+
if (getConversation(key)) continue;
|
|
379
|
+
fs.rmSync(path.join(uploadsRoot, entry), { recursive: true, force: true });
|
|
380
|
+
removed += Number(sql("DELETE FROM files WHERE conv_key = ?").run(key).changes);
|
|
381
|
+
}
|
|
382
|
+
const indexed = new Set<string>();
|
|
383
|
+
for (const row of sql("SELECT id, path FROM files").all() as { id: string; path: string }[]) {
|
|
384
|
+
if (fs.existsSync(row.path)) indexed.add(row.path);
|
|
385
|
+
else {
|
|
386
|
+
sql("DELETE FROM files WHERE id = ?").run(row.id);
|
|
387
|
+
removed++;
|
|
388
|
+
}
|
|
389
|
+
}
|
|
390
|
+
for (const file of walk(uploadsRoot)) {
|
|
391
|
+
if (indexed.has(file)) continue;
|
|
392
|
+
try {
|
|
393
|
+
if (now - fs.statSync(file).mtimeMs < UNSENT_TTL) continue;
|
|
394
|
+
fs.rmSync(file, { force: true });
|
|
395
|
+
removeEmptyDirs(path.dirname(file));
|
|
396
|
+
} catch {
|
|
397
|
+
// 刚好被删了
|
|
398
|
+
}
|
|
399
|
+
}
|
|
400
|
+
return { removed };
|
|
401
|
+
}
|
package/src/core/memory.ts
CHANGED
|
@@ -1,19 +1,17 @@
|
|
|
1
1
|
// 记忆:agent 对「这个人」的长期了解,像 ChatGPT 的记忆那样。
|
|
2
2
|
//
|
|
3
|
-
//
|
|
3
|
+
// 每个用户一份,存在数据库的 memory 表里(见 db.ts),一人一行:
|
|
4
4
|
// - about:用户自己写的「关于我」;
|
|
5
5
|
// - items:一条条记住的事,用户在设置里加的,或者 agent 在对话里用 remember 工具记下的。
|
|
6
6
|
// 开会话时拼成一段文字放进系统提示(见 memoryPrompt),Claude / Codex、Web / 飞书共用同一份。
|
|
7
7
|
//
|
|
8
8
|
// 归属跟着对话的主人走(见 memoryOwnerOf):别人的对话里绝不注入你的记忆,也不往你的记忆里写。
|
|
9
|
-
// 记忆隔离和多用户一样是「信任型」的:agent
|
|
9
|
+
// 记忆隔离和多用户一样是「信任型」的:agent 都以同一个系统用户运行,工作模式下能读到数据库文件。
|
|
10
10
|
//
|
|
11
|
-
//
|
|
12
|
-
//
|
|
11
|
+
// 不做内存缓存,每次都查库:agent 的记忆工具跑在单独的进程里(见 mcp/memory-server.ts),
|
|
12
|
+
// 两边都会写。读—改—写放在一个事务里,两边同时改也不会互相冲掉。
|
|
13
13
|
import crypto from "node:crypto";
|
|
14
|
-
import
|
|
15
|
-
import path from "node:path";
|
|
16
|
-
import { dataDir, readJson, writeJsonAtomic } from "../home.js";
|
|
14
|
+
import { MEMORY_DEFAULT_OFF, sql, transaction } from "./db.js";
|
|
17
15
|
|
|
18
16
|
export type MemoryItem = {
|
|
19
17
|
id: string;
|
|
@@ -94,12 +92,10 @@ export const MEMORY_LIMITS = {
|
|
|
94
92
|
promptItems: 4000,
|
|
95
93
|
};
|
|
96
94
|
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
/** 用户 id 只会是 users.ts 生成的那种;校验一下,免得拼出目录外的路径 */
|
|
100
|
-
function fileOf(userId: string): string {
|
|
95
|
+
/** 用户 id 只会是 users.ts 生成的那种 */
|
|
96
|
+
function checkUser(userId: string): string {
|
|
101
97
|
if (!/^[A-Za-z0-9_]{1,64}$/.test(userId)) throw new Error("用户 id 不合法");
|
|
102
|
-
return
|
|
98
|
+
return userId;
|
|
103
99
|
}
|
|
104
100
|
|
|
105
101
|
function empty(): UserMemory {
|
|
@@ -107,30 +103,37 @@ function empty(): UserMemory {
|
|
|
107
103
|
}
|
|
108
104
|
|
|
109
105
|
/**
|
|
110
|
-
* 记忆从默认开改成默认关(0.2.46
|
|
106
|
+
* 记忆从默认开改成默认关(0.2.46):升级前就有、但还没存过记忆的用户,补一份开着的,
|
|
111
107
|
* 让老实例里的人保持原样。只跑一次,跑过留个标记;新装的第一次启动时还没有用户,什么都不补。
|
|
112
108
|
*/
|
|
113
109
|
export function migrateMemoryDefault(userIds: string[]): void {
|
|
114
|
-
const
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
110
|
+
const n = transaction(() => {
|
|
111
|
+
if (sql("SELECT 1 FROM migrations WHERE name = ?").get(MEMORY_DEFAULT_OFF)) return 0;
|
|
112
|
+
let added = 0;
|
|
113
|
+
for (const id of userIds) {
|
|
114
|
+
if (sql("SELECT 1 FROM memory WHERE user_id = ?").get(checkUser(id))) continue;
|
|
115
|
+
write(id, { ...empty(), enabled: true });
|
|
116
|
+
added++;
|
|
117
|
+
}
|
|
118
|
+
sql("INSERT INTO migrations (name, at) VALUES (?, ?)").run(MEMORY_DEFAULT_OFF, Date.now());
|
|
119
|
+
return added;
|
|
120
|
+
});
|
|
124
121
|
if (n) console.log(`[记忆] 默认改成关闭:升级前的 ${n} 个用户保持开启`);
|
|
125
122
|
}
|
|
126
123
|
|
|
127
124
|
export function readMemory(userId: string): UserMemory {
|
|
128
|
-
const
|
|
125
|
+
const row = sql("SELECT data FROM memory WHERE user_id = ?").get(checkUser(userId)) as { data: string } | undefined;
|
|
126
|
+
let raw: Partial<UserMemory> & { autoExtract?: boolean } = {};
|
|
127
|
+
try {
|
|
128
|
+
raw = row ? JSON.parse(row.data) : {};
|
|
129
|
+
} catch {
|
|
130
|
+
// 坏了就当没有
|
|
131
|
+
}
|
|
129
132
|
const items = Array.isArray(raw.items)
|
|
130
133
|
? raw.items.filter((i): i is MemoryItem => !!i && typeof i.id === "string" && typeof i.text === "string")
|
|
131
134
|
: [];
|
|
132
135
|
return {
|
|
133
|
-
// 默认关:后台整理会占订阅额度,夜里还会单独开一个 5
|
|
136
|
+
// 默认关:后台整理会占订阅额度,夜里还会单独开一个 5 小时窗口。存过的都带着 enabled,老用户不受影响
|
|
134
137
|
enabled: raw.enabled === true,
|
|
135
138
|
// 早先的版本只有一个「自动整理」开关:关着的就是不整理
|
|
136
139
|
curate: cleanCurate(raw.curate, raw.autoExtract === false ? { ...DEFAULT_CURATE, when: "off" } : DEFAULT_CURATE),
|
|
@@ -147,44 +150,15 @@ export function readMemory(userId: string): UserMemory {
|
|
|
147
150
|
}
|
|
148
151
|
|
|
149
152
|
function write(userId: string, memory: UserMemory): void {
|
|
150
|
-
|
|
151
|
-
writeJsonAtomic(fileOf(userId), memory);
|
|
153
|
+
sql("INSERT OR REPLACE INTO memory (user_id, data) VALUES (?, ?)").run(checkUser(userId), JSON.stringify(memory));
|
|
152
154
|
}
|
|
153
155
|
|
|
154
|
-
const sleepSync = (ms: number) => Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
|
|
155
|
-
|
|
156
156
|
/**
|
|
157
|
-
*
|
|
158
|
-
*
|
|
159
|
-
* 用 O_EXCL 建锁文件;持锁的进程被杀了留下的锁,5 秒后当作废。改动都是毫秒级的,等不到 3 秒就报错。
|
|
157
|
+
* 读—改—写放进一个事务:主进程(设置页、自动整理)和每个会话的记忆工具子进程都会写同一个人的记忆,
|
|
158
|
+
* 不这样的话两边同时改,后写的那个会把先写的冲掉(新记的一条、刚保存的「关于我」悄悄没了)。
|
|
160
159
|
*/
|
|
161
160
|
function locked<T>(userId: string, fn: (memory: UserMemory) => T): T {
|
|
162
|
-
|
|
163
|
-
fs.mkdirSync(memoryDir, { recursive: true, mode: 0o700 });
|
|
164
|
-
const deadline = Date.now() + 3000;
|
|
165
|
-
for (;;) {
|
|
166
|
-
try {
|
|
167
|
-
fs.writeFileSync(lock, String(process.pid), { flag: "wx", mode: 0o600 });
|
|
168
|
-
break;
|
|
169
|
-
} catch (err) {
|
|
170
|
-
if ((err as NodeJS.ErrnoException).code !== "EEXIST") throw err;
|
|
171
|
-
try {
|
|
172
|
-
if (Date.now() - fs.statSync(lock).mtimeMs > 5000) {
|
|
173
|
-
fs.rmSync(lock, { force: true });
|
|
174
|
-
continue;
|
|
175
|
-
}
|
|
176
|
-
} catch {
|
|
177
|
-
continue; // 刚好被释放了
|
|
178
|
-
}
|
|
179
|
-
if (Date.now() > deadline) throw new Error("记忆文件正被别处修改,稍后再试");
|
|
180
|
-
sleepSync(15);
|
|
181
|
-
}
|
|
182
|
-
}
|
|
183
|
-
try {
|
|
184
|
-
return fn(readMemory(userId));
|
|
185
|
-
} finally {
|
|
186
|
-
fs.rmSync(lock, { force: true });
|
|
187
|
-
}
|
|
161
|
+
return transaction(() => fn(readMemory(userId)));
|
|
188
162
|
}
|
|
189
163
|
|
|
190
164
|
function cleanText(text: unknown, max: number, what: string): string {
|
|
@@ -283,9 +257,9 @@ export function clearMemory(userId: string): void {
|
|
|
283
257
|
locked(userId, (memory) => write(userId, { ...memory, about: "", items: [] }));
|
|
284
258
|
}
|
|
285
259
|
|
|
286
|
-
/**
|
|
260
|
+
/** 删用户时:整个删掉 */
|
|
287
261
|
export function deleteMemory(userId: string): void {
|
|
288
|
-
|
|
262
|
+
sql("DELETE FROM memory WHERE user_id = ?").run(checkUser(userId));
|
|
289
263
|
}
|
|
290
264
|
|
|
291
265
|
// ---------- 注入系统提示 ----------
|
package/src/core/store.ts
CHANGED
|
@@ -4,8 +4,8 @@
|
|
|
4
4
|
// 的一条对话。它记着用哪个专家(core/experts.ts)和它的引擎(agent)、在哪个目录,以及 agent 的会话 id
|
|
5
5
|
// 和模型。专家定了就不换:飞书里换专家等于开新会话(见 hub.setExpert)。
|
|
6
6
|
import path from "node:path";
|
|
7
|
-
import { dataDir, readJson, writeJsonAtomic } from "../home.js";
|
|
8
7
|
import { config } from "./config.js";
|
|
8
|
+
import { sql, transaction } from "./db.js";
|
|
9
9
|
|
|
10
10
|
export type AgentBinding = {
|
|
11
11
|
/** agent 侧的会话 id(Claude 的 sessionId / Codex 的 threadId) */
|
|
@@ -61,13 +61,24 @@ export type Conversation = {
|
|
|
61
61
|
|
|
62
62
|
type StoreShape = Record<string, Conversation>;
|
|
63
63
|
|
|
64
|
-
|
|
64
|
+
// 存在数据库的 conversations 表里(见 db.ts),一条对话一行。读进内存后都改内存里的对象,
|
|
65
|
+
// 落盘时和上次写下去的比,只写变了的、删掉没了的。
|
|
65
66
|
let cache: StoreShape | null = null;
|
|
67
|
+
/** 每条对话上次写进数据库的内容 */
|
|
68
|
+
const written = new Map<string, string>();
|
|
66
69
|
let persistTimer: NodeJS.Timeout | null = null;
|
|
67
70
|
|
|
68
71
|
function load(): StoreShape {
|
|
69
72
|
if (cache) return cache;
|
|
70
|
-
cache =
|
|
73
|
+
cache = {};
|
|
74
|
+
for (const row of sql("SELECT key, data FROM conversations").all() as { key: string; data: string }[]) {
|
|
75
|
+
try {
|
|
76
|
+
cache[row.key] = JSON.parse(row.data) as Conversation;
|
|
77
|
+
written.set(row.key, row.data);
|
|
78
|
+
} catch {
|
|
79
|
+
console.warn(`[store] 会话 ${row.key} 的数据坏了,跳过`);
|
|
80
|
+
}
|
|
81
|
+
}
|
|
71
82
|
// 聊天模式取消了(换成了专家):以前聊天模式的对话直接丢掉,不迁移
|
|
72
83
|
let dropped = 0;
|
|
73
84
|
for (const [key, conv] of Object.entries(cache)) {
|
|
@@ -79,11 +90,28 @@ function load(): StoreShape {
|
|
|
79
90
|
}
|
|
80
91
|
if (dropped) {
|
|
81
92
|
console.log(`[store] 聊天模式已取消,丢掉了 ${dropped} 条聊天模式的对话`);
|
|
82
|
-
|
|
93
|
+
writeChanges();
|
|
83
94
|
}
|
|
84
95
|
return cache;
|
|
85
96
|
}
|
|
86
97
|
|
|
98
|
+
function writeChanges(): void {
|
|
99
|
+
const all = cache ?? {};
|
|
100
|
+
const changed: [string, string][] = [];
|
|
101
|
+
for (const [key, conv] of Object.entries(all)) {
|
|
102
|
+
const data = JSON.stringify(conv);
|
|
103
|
+
if (written.get(key) !== data) changed.push([key, data]);
|
|
104
|
+
}
|
|
105
|
+
const removed = [...written.keys()].filter((key) => !(key in all));
|
|
106
|
+
if (changed.length === 0 && removed.length === 0) return;
|
|
107
|
+
transaction(() => {
|
|
108
|
+
for (const [key, data] of changed) sql("INSERT OR REPLACE INTO conversations (key, data) VALUES (?, ?)").run(key, data);
|
|
109
|
+
for (const key of removed) sql("DELETE FROM conversations WHERE key = ?").run(key);
|
|
110
|
+
});
|
|
111
|
+
for (const [key, data] of changed) written.set(key, data);
|
|
112
|
+
for (const key of removed) written.delete(key);
|
|
113
|
+
}
|
|
114
|
+
|
|
87
115
|
/** 这条对话的专家 id:自定义专家的 id,或者内置专家(就是 agent 本身) */
|
|
88
116
|
export function expertIdOf(conv: Pick<Conversation, "agent" | "expert">): string {
|
|
89
117
|
return conv.expert ?? conv.agent;
|
|
@@ -94,12 +122,12 @@ function persist(immediate = false): void {
|
|
|
94
122
|
if (immediate) {
|
|
95
123
|
if (persistTimer) clearTimeout(persistTimer);
|
|
96
124
|
persistTimer = null;
|
|
97
|
-
|
|
125
|
+
writeChanges();
|
|
98
126
|
return;
|
|
99
127
|
}
|
|
100
128
|
persistTimer ??= setTimeout(() => {
|
|
101
129
|
persistTimer = null;
|
|
102
|
-
|
|
130
|
+
writeChanges();
|
|
103
131
|
}, 500);
|
|
104
132
|
}
|
|
105
133
|
|
|
@@ -222,23 +250,23 @@ export function putConversation(conv: Conversation): void {
|
|
|
222
250
|
// /sessions、/resume 列的是 agent 自己存的全部历史(磁盘上谁开的都有),多用户时要按人过滤:
|
|
223
251
|
// 记下每个 agent 会话 id 是哪个用户的对话里开出来的。没记过的(飞书、终端里开的、升级前的)归管理员。
|
|
224
252
|
|
|
225
|
-
const sessionOwnersPath = path.join(dataDir, "session-owners.json");
|
|
226
|
-
let sessionOwners: Record<string, string> | null = null;
|
|
227
|
-
|
|
228
|
-
function owners(): Record<string, string> {
|
|
229
|
-
sessionOwners ??= readJson<Record<string, string>>(sessionOwnersPath, {});
|
|
230
|
-
return sessionOwners;
|
|
231
|
-
}
|
|
232
|
-
|
|
233
253
|
/** 第一次见到这个会话 id 时记下主人;已经有主了不改 */
|
|
234
254
|
export function recordSessionOwner(sessionId: string | undefined, userId: string | undefined): void {
|
|
235
255
|
if (!sessionId || !userId) return;
|
|
236
|
-
|
|
237
|
-
if (all[sessionId]) return;
|
|
238
|
-
all[sessionId] = userId;
|
|
239
|
-
writeJsonAtomic(sessionOwnersPath, all);
|
|
256
|
+
sql("INSERT OR IGNORE INTO session_owners (session_id, user_id) VALUES (?, ?)").run(sessionId, userId);
|
|
240
257
|
}
|
|
241
258
|
|
|
242
259
|
export function sessionOwner(sessionId: string): string | undefined {
|
|
243
|
-
|
|
260
|
+
const row = sql("SELECT user_id FROM session_owners WHERE session_id = ?").get(sessionId) as { user_id: string } | undefined;
|
|
261
|
+
return row?.user_id;
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
/** 别的进程(记忆工具)要对话的标题和是否隐藏:直接查库,不走缓存,Web 上刚删的立刻生效 */
|
|
265
|
+
export function conversationLabels(): Map<string, { title?: string; hidden?: boolean }> {
|
|
266
|
+
const rows = sql("SELECT key, json_extract(data, '$.title') AS title, json_extract(data, '$.hidden') AS hidden FROM conversations").all() as {
|
|
267
|
+
key: string;
|
|
268
|
+
title: string | null;
|
|
269
|
+
hidden: number | null;
|
|
270
|
+
}[];
|
|
271
|
+
return new Map(rows.map((r) => [r.key, { ...(r.title ? { title: r.title } : {}), ...(r.hidden ? { hidden: true } : {}) }]));
|
|
244
272
|
}
|