agent-loft 0.2.59 → 0.2.61
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +12 -9
- package/package.json +2 -2
- package/src/channels/web/api.ts +31 -5
- package/src/channels/web/library-api.ts +59 -0
- package/src/core/chat-history.ts +92 -71
- package/src/core/claude-model-prices.json +23 -0
- package/src/core/db.ts +367 -0
- package/src/core/experts.ts +60 -2
- package/src/core/footers.ts +21 -43
- package/src/core/hub.ts +3 -16
- package/src/core/library.ts +401 -0
- package/src/core/local-usage.ts +275 -0
- package/src/core/memory.ts +34 -60
- package/src/core/store.ts +47 -19
- package/src/core/uploads.ts +3 -46
- package/src/core/usage.ts +63 -73
- package/src/main.ts +7 -6
- package/src/mcp/memory-server.ts +4 -7
- package/web/css/library.css +266 -0
- package/web/index.html +1 -0
- package/web/lib/dom.js +2 -0
- package/web/settings/usage.js +9 -3
- package/web/state.js +3 -3
- package/web/views/composer.js +36 -1
- package/web/views/layout.js +11 -0
- package/web/views/library.js +309 -0
- package/web/views/messages.js +3 -3
- package/web/views/remote.js +2 -1
- package/web/views/sidebar.js +1 -0
- package/web/views/topbar.js +3 -2
|
@@ -0,0 +1,401 @@
|
|
|
1
|
+
// 资料库:用户发过的附件、agent 生成的文件,按人汇总在一处,可以浏览、搜索、删掉,或者拿到别的对话里再用。
|
|
2
|
+
//
|
|
3
|
+
// 文件本身不搬家,还在上传目录里(data/uploads/<对话>/…,见 uploads.ts):历史消息里写着的路径照样能预览,
|
|
4
|
+
// agent 读文件的范围也还是「这条对话的上传目录」。这里只在数据库的 files 表里记一份索引:谁的、哪条对话、
|
|
5
|
+
// 什么时候、从哪来(upload:用户发的;agent:agent 回复里给出的新文件,复制了一份进来)。
|
|
6
|
+
//
|
|
7
|
+
// 收录跟着 hub 的事件流走(见 startLibraryCapture):
|
|
8
|
+
// - 用户消息带的附件:发出去时收进来。上传了没发的不收,一天后清掉(见 tidyLibrary);
|
|
9
|
+
// - agent 的回复里提到的、这一轮新写出来的图片 / 文档 / 音视频:复制一份进这条对话的上传目录再收进来,
|
|
10
|
+
// 之后 agent 改了、删了原文件,资料库里的不受影响。
|
|
11
|
+
// 从资料库拿到别的对话里用:硬链接一份进那条对话的上传目录(见 attachToConversation),不占双份空间。
|
|
12
|
+
// 这份也记一行(source = copy,origin 指向原来那条),列表里不显示;删原件时连它一起删,空间才真的腾出来。
|
|
13
|
+
//
|
|
14
|
+
// 归属跟着对话的主人走:没有主人的对话(飞书群聊、没绑定账号的私聊)的文件归管理员,规则同 users.ts 的 ownedBy。
|
|
15
|
+
import crypto from "node:crypto";
|
|
16
|
+
import fs from "node:fs";
|
|
17
|
+
import path from "node:path";
|
|
18
|
+
import { sql, transaction } from "./db.js";
|
|
19
|
+
import { hub } from "./hub.js";
|
|
20
|
+
import { replyFileType, resolveReplyFile } from "./reply-files.js";
|
|
21
|
+
import { getConversation } from "./store.js";
|
|
22
|
+
import { ensureUploadsDir, kindOf, safeName, sniff, uploadsRoot } from "./uploads.js";
|
|
23
|
+
import { listUsers, ownedBy, type User } from "./users.js";
|
|
24
|
+
|
|
25
|
+
type Viewer = Pick<User, "id" | "role">;
|
|
26
|
+
import type { AttachmentKind, SavedAttachment } from "./types.js";
|
|
27
|
+
|
|
28
|
+
/** upload:用户发的;agent:agent 生成的;copy:从资料库放进别的对话的那份(列表里不显示) */
|
|
29
|
+
export type LibrarySource = "upload" | "agent" | "copy";
|
|
30
|
+
|
|
31
|
+
export type LibraryFile = {
|
|
32
|
+
id: string;
|
|
33
|
+
/** 用户 id;空串 = 没有主人的对话里的,归管理员 */
|
|
34
|
+
owner: string;
|
|
35
|
+
convKey: string;
|
|
36
|
+
path: string;
|
|
37
|
+
name: string;
|
|
38
|
+
kind: AttachmentKind;
|
|
39
|
+
size: number;
|
|
40
|
+
source: LibrarySource;
|
|
41
|
+
createdAt: number;
|
|
42
|
+
};
|
|
43
|
+
|
|
44
|
+
/** 筛选:图片 / 文档(其余的文件)/ 音视频 */
|
|
45
|
+
export type LibraryKind = "image" | "file" | "media";
|
|
46
|
+
|
|
47
|
+
type Row = { id: string; owner: string; conv_key: string; path: string; name: string; kind: AttachmentKind; size: number; source: LibrarySource; origin: string | null; created_at: number };
|
|
48
|
+
|
|
49
|
+
const COLUMNS = "id, owner, conv_key, path, name, kind, size, source, origin, created_at";
|
|
50
|
+
|
|
51
|
+
function toFile(r: Row): LibraryFile {
|
|
52
|
+
return { id: r.id, owner: r.owner, convKey: r.conv_key, path: r.path, name: r.name, kind: r.kind, size: r.size, source: r.source, createdAt: r.created_at };
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/** agent 生成的文件最多收这么大的(复制一份要占地方) */
|
|
56
|
+
const AGENT_FILE_MAX = 50 * 1024 * 1024;
|
|
57
|
+
/** 一轮最多收这么多个 agent 生成的文件 */
|
|
58
|
+
const AGENT_FILES_PER_TURN = 10;
|
|
59
|
+
/** 上传了一直没发出去的,过这么久清掉 */
|
|
60
|
+
const UNSENT_TTL = 24 * 60 * 60 * 1000;
|
|
61
|
+
|
|
62
|
+
export function recordFile(params: {
|
|
63
|
+
owner?: string;
|
|
64
|
+
convKey: string;
|
|
65
|
+
path: string;
|
|
66
|
+
name: string;
|
|
67
|
+
kind: AttachmentKind;
|
|
68
|
+
size: number;
|
|
69
|
+
source: LibrarySource;
|
|
70
|
+
origin?: string;
|
|
71
|
+
createdAt?: number;
|
|
72
|
+
}): boolean {
|
|
73
|
+
const res = sql(`INSERT OR IGNORE INTO files (${COLUMNS}) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(
|
|
74
|
+
`f_${crypto.randomBytes(8).toString("hex")}`,
|
|
75
|
+
params.owner ?? "",
|
|
76
|
+
params.convKey,
|
|
77
|
+
path.resolve(params.path),
|
|
78
|
+
params.name,
|
|
79
|
+
params.kind,
|
|
80
|
+
params.size,
|
|
81
|
+
params.source,
|
|
82
|
+
params.origin ?? null,
|
|
83
|
+
params.createdAt ?? Date.now(),
|
|
84
|
+
);
|
|
85
|
+
return Number(res.changes) > 0;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
export function getLibraryFile(id: string): LibraryFile | undefined {
|
|
89
|
+
const row = sql(`SELECT ${COLUMNS} FROM files WHERE id = ?`).get(id) as Row | undefined;
|
|
90
|
+
return row ? toFile(row) : undefined;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
export function canSeeFile(user: Viewer, file: LibraryFile): boolean {
|
|
94
|
+
return ownedBy(user, file.owner || undefined);
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* 这个用户看得到的那部分,拼成 SQL 条件:自己的;管理员还有没主人的、主人已经被删掉的
|
|
99
|
+
* (和 ownedBy 一样:别的管理员的不算)
|
|
100
|
+
*/
|
|
101
|
+
function visibleWhere(user: Viewer): { where: string; args: string[] } {
|
|
102
|
+
if (user.role !== "admin") return { where: "owner = ?", args: [user.id] };
|
|
103
|
+
const others = listUsers()
|
|
104
|
+
.map((u) => u.id)
|
|
105
|
+
.filter((id) => id !== user.id);
|
|
106
|
+
return { where: `(owner = ? OR owner NOT IN (${others.map(() => "?").join(", ") || "''"}))`, args: [user.id, ...others] };
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
const KIND_WHERE: Record<LibraryKind, string> = {
|
|
110
|
+
image: "kind = 'image'",
|
|
111
|
+
file: "kind = 'file'",
|
|
112
|
+
media: "kind IN ('audio', 'video')",
|
|
113
|
+
};
|
|
114
|
+
|
|
115
|
+
/** 按时间从新到旧一页一页地列。before:上一页最后一个的 { createdAt, id } */
|
|
116
|
+
export function listLibrary(
|
|
117
|
+
user: Viewer,
|
|
118
|
+
opts: { kind?: LibraryKind; q?: string; before?: { at: number; id: string }; limit?: number } = {},
|
|
119
|
+
): { files: LibraryFile[]; more: boolean } {
|
|
120
|
+
const { where, args } = visibleWhere(user);
|
|
121
|
+
const conds = [where, "source != 'copy'"];
|
|
122
|
+
const params: (string | number)[] = [...args];
|
|
123
|
+
if (opts.kind) conds.push(KIND_WHERE[opts.kind]);
|
|
124
|
+
const q = opts.q?.trim();
|
|
125
|
+
if (q) {
|
|
126
|
+
conds.push("instr(lower(name), ?) > 0");
|
|
127
|
+
params.push(q.toLowerCase());
|
|
128
|
+
}
|
|
129
|
+
if (opts.before) {
|
|
130
|
+
conds.push("(created_at < ? OR (created_at = ? AND id < ?))");
|
|
131
|
+
params.push(opts.before.at, opts.before.at, opts.before.id);
|
|
132
|
+
}
|
|
133
|
+
const limit = Math.max(1, Math.min(200, opts.limit ?? 60));
|
|
134
|
+
const rows = sql(`SELECT ${COLUMNS} FROM files WHERE ${conds.join(" AND ")} ORDER BY created_at DESC, id DESC LIMIT ?`).all(
|
|
135
|
+
...params,
|
|
136
|
+
limit + 1,
|
|
137
|
+
) as Row[];
|
|
138
|
+
return { files: rows.slice(0, limit).map(toFile), more: rows.length > limit };
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
/** 一共多少个、多大 */
|
|
142
|
+
export function librarySummary(user: Viewer): { count: number; size: number } {
|
|
143
|
+
const { where, args } = visibleWhere(user);
|
|
144
|
+
const row = sql(`SELECT count(*) AS n, coalesce(sum(size), 0) AS s FROM files WHERE ${where} AND source != 'copy'`).get(...args) as { n: number; s: number };
|
|
145
|
+
return { count: row.n, size: row.s };
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/**
|
|
149
|
+
* 从资料库删掉:文件和索引都删,放进别的对话的那几份也一起删(硬链接不全删掉空间腾不出来)。
|
|
150
|
+
* 放它们的小目录(每次上传一个)空了也一并删掉
|
|
151
|
+
*/
|
|
152
|
+
export function deleteLibraryFile(file: LibraryFile): void {
|
|
153
|
+
const all = sql("SELECT id, path FROM files WHERE id = ? OR origin = ?").all(file.id, file.id) as { id: string; path: string }[];
|
|
154
|
+
for (const f of all) {
|
|
155
|
+
fs.rmSync(f.path, { force: true });
|
|
156
|
+
removeEmptyDirs(path.dirname(f.path));
|
|
157
|
+
}
|
|
158
|
+
sql("DELETE FROM files WHERE id = ? OR origin = ?").run(file.id, file.id);
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
/** 往上删空目录,到上传目录的根为止 */
|
|
162
|
+
function removeEmptyDirs(dir: string): void {
|
|
163
|
+
const root = path.resolve(uploadsRoot);
|
|
164
|
+
for (let d = path.resolve(dir); d.startsWith(root + path.sep); d = path.dirname(d)) {
|
|
165
|
+
try {
|
|
166
|
+
fs.rmdirSync(d);
|
|
167
|
+
} catch {
|
|
168
|
+
return; // 不空、或者已经没了
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
// ---------- 拿到对话里用 ----------
|
|
174
|
+
|
|
175
|
+
/**
|
|
176
|
+
* 资料库里的文件放进这条对话:硬链接进它的上传目录(跨不了文件系统就复制),agent 照常能读。
|
|
177
|
+
* 返回和刚上传的附件一样的东西,交给发消息时认领
|
|
178
|
+
*/
|
|
179
|
+
export function attachToConversation(file: LibraryFile, key: string): SavedAttachment {
|
|
180
|
+
const dir = path.join(ensureUploadsDir(key), `lib-${crypto.randomBytes(6).toString("hex")}`);
|
|
181
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
182
|
+
const target = path.join(dir, safeName(file.name));
|
|
183
|
+
try {
|
|
184
|
+
fs.linkSync(file.path, target);
|
|
185
|
+
} catch {
|
|
186
|
+
fs.copyFileSync(file.path, target);
|
|
187
|
+
}
|
|
188
|
+
// 记下这一份:发出去时不会再当成新上传的收一遍,也不会被当成「上传了没发」清掉
|
|
189
|
+
const conv = getConversation(key);
|
|
190
|
+
recordFile({ owner: conv?.owner, convKey: key, path: target, name: path.basename(target), kind: file.kind, size: file.size, source: "copy", origin: file.id });
|
|
191
|
+
const mediaType = sniffFile(target);
|
|
192
|
+
return { kind: file.kind, path: target, name: path.basename(target), size: fs.statSync(target).size, ...(mediaType ? { mediaType } : {}) };
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
function sniffFile(file: string): string | undefined {
|
|
196
|
+
const head = Buffer.alloc(12);
|
|
197
|
+
let fd: number | undefined;
|
|
198
|
+
try {
|
|
199
|
+
fd = fs.openSync(file, "r");
|
|
200
|
+
fs.readSync(fd, head, 0, head.length, 0);
|
|
201
|
+
return sniff(head)?.mediaType;
|
|
202
|
+
} catch {
|
|
203
|
+
return undefined;
|
|
204
|
+
} finally {
|
|
205
|
+
if (fd !== undefined) fs.closeSync(fd);
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
/** 按文件头,认不出再按后缀 */
|
|
210
|
+
function kindOfFile(file: string): AttachmentKind {
|
|
211
|
+
return kindOf(sniffFile(file) ?? replyFileType(file));
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
// ---------- 收录 ----------
|
|
215
|
+
|
|
216
|
+
/** 回复里提到的这些后缀才当成「交给用户的东西」收进来:代码、配置之类的不算 */
|
|
217
|
+
const DELIVERABLE = "png|jpe?g|gif|webp|svg|pdf|docx?|xlsx?|pptx?|csv|zip|mp3|m4a|wav|mp4|mov|webm";
|
|
218
|
+
const CANDIDATE = new RegExp(`[^\\s<>"'\`()()\\[\\]{},。;:!?、“”‘’「」|*]+\\.(?:${DELIVERABLE})(?![\\p{L}\\p{N}_/-])`, "giu");
|
|
219
|
+
|
|
220
|
+
/** 回复里像是文件路径、后缀是图片 / 文档 / 音视频的 */
|
|
221
|
+
export function deliverablePaths(text: string): string[] {
|
|
222
|
+
return [...new Set(text.match(CANDIDATE) ?? [])];
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
function within(file: string, dir: string): boolean {
|
|
226
|
+
return file.startsWith(path.resolve(dir) + path.sep);
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
/**
|
|
230
|
+
* 这一轮 agent 新写出来的文件(修改时间不早于这一轮开始)复制进对话的上传目录,收进资料库。
|
|
231
|
+
* 用户自己传的(本来就在上传目录里)、太大的、敏感的(resolveReplyFile 挡掉)不收
|
|
232
|
+
*/
|
|
233
|
+
export async function captureAgentFiles(key: string, text: string, since: number): Promise<number> {
|
|
234
|
+
const conv = getConversation(key);
|
|
235
|
+
if (!conv) return 0;
|
|
236
|
+
const seen = new Set<string>();
|
|
237
|
+
let n = 0;
|
|
238
|
+
for (const p of deliverablePaths(text)) {
|
|
239
|
+
if (n >= AGENT_FILES_PER_TURN) break;
|
|
240
|
+
const real = resolveReplyFile(p, conv.cwd);
|
|
241
|
+
if (!real || seen.has(real) || within(real, uploadsRoot)) continue;
|
|
242
|
+
seen.add(real);
|
|
243
|
+
let stat: fs.Stats;
|
|
244
|
+
try {
|
|
245
|
+
stat = fs.statSync(real);
|
|
246
|
+
} catch {
|
|
247
|
+
continue;
|
|
248
|
+
}
|
|
249
|
+
// 修改时间留两秒余量:有的文件系统只精确到秒
|
|
250
|
+
if (stat.mtimeMs < since - 2000 || stat.size === 0 || stat.size > AGENT_FILE_MAX) continue;
|
|
251
|
+
const dir = path.join(ensureUploadsDir(key), `agent-${crypto.randomBytes(6).toString("hex")}`);
|
|
252
|
+
const target = path.join(dir, safeName(path.basename(real)));
|
|
253
|
+
try {
|
|
254
|
+
await fs.promises.mkdir(dir, { recursive: true });
|
|
255
|
+
await fs.promises.copyFile(real, target);
|
|
256
|
+
} catch (err) {
|
|
257
|
+
console.warn(`[资料库] 收录 ${real} 失败:`, err instanceof Error ? err.message : err);
|
|
258
|
+
continue;
|
|
259
|
+
}
|
|
260
|
+
recordFile({ owner: conv.owner, convKey: key, path: target, name: path.basename(target), kind: kindOfFile(target), size: stat.size, source: "agent" });
|
|
261
|
+
n++;
|
|
262
|
+
}
|
|
263
|
+
return n;
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
let capturing = false;
|
|
267
|
+
|
|
268
|
+
export function startLibraryCapture(): void {
|
|
269
|
+
if (capturing) return;
|
|
270
|
+
capturing = true;
|
|
271
|
+
/** 每条对话这一轮从什么时候开始(agent 生成的文件按修改时间认) */
|
|
272
|
+
const turnStart = new Map<string, number>();
|
|
273
|
+
hub.on("event", (env) => {
|
|
274
|
+
const ev = env.ev;
|
|
275
|
+
try {
|
|
276
|
+
if (ev.type === "turn_start") {
|
|
277
|
+
turnStart.set(env.key, env.ts);
|
|
278
|
+
return;
|
|
279
|
+
}
|
|
280
|
+
if (ev.type === "user") {
|
|
281
|
+
if (!ev.files?.length) return;
|
|
282
|
+
const conv = getConversation(env.key);
|
|
283
|
+
for (const f of ev.files) {
|
|
284
|
+
const file = path.resolve(f.path);
|
|
285
|
+
if (!within(file, uploadsRoot) || !fs.existsSync(file)) continue;
|
|
286
|
+
recordFile({ owner: conv?.owner, convKey: env.key, path: file, name: f.name, kind: f.kind, size: f.size ?? fs.statSync(file).size, source: "upload", createdAt: env.ts });
|
|
287
|
+
}
|
|
288
|
+
return;
|
|
289
|
+
}
|
|
290
|
+
if (ev.type === "result" && ev.text) {
|
|
291
|
+
const since = turnStart.get(env.key) ?? env.ts - 10 * 60_000;
|
|
292
|
+
turnStart.delete(env.key);
|
|
293
|
+
void captureAgentFiles(env.key, ev.text, since).catch((err) => console.warn("[资料库] 收录 agent 文件失败:", err));
|
|
294
|
+
}
|
|
295
|
+
} catch (err) {
|
|
296
|
+
console.warn("[资料库] 收录失败:", err instanceof Error ? err.message : err);
|
|
297
|
+
}
|
|
298
|
+
});
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
// ---------- 启动时 ----------
|
|
302
|
+
|
|
303
|
+
function walk(dir: string, out: string[] = []): string[] {
|
|
304
|
+
let entries: fs.Dirent[];
|
|
305
|
+
try {
|
|
306
|
+
entries = fs.readdirSync(dir, { withFileTypes: true });
|
|
307
|
+
} catch {
|
|
308
|
+
return out;
|
|
309
|
+
}
|
|
310
|
+
for (const e of entries) {
|
|
311
|
+
const full = path.join(dir, e.name);
|
|
312
|
+
if (e.isDirectory()) walk(full, out);
|
|
313
|
+
else if (e.isFile()) out.push(full);
|
|
314
|
+
}
|
|
315
|
+
return out;
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
/** 上传目录里的文件属于哪条对话(第一级目录名) */
|
|
319
|
+
function convKeyOf(file: string): string | undefined {
|
|
320
|
+
const first = path.relative(uploadsRoot, file).split(path.sep)[0];
|
|
321
|
+
if (!first) return undefined;
|
|
322
|
+
try {
|
|
323
|
+
return decodeURIComponent(first);
|
|
324
|
+
} catch {
|
|
325
|
+
return undefined;
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
/**
|
|
330
|
+
* 升级时把上传目录里已有的文件收进资料库,只做一次。以前切换会话时会清掉旧附件,留下来的都是还在用的那些
|
|
331
|
+
*/
|
|
332
|
+
export function backfillLibrary(): number {
|
|
333
|
+
return transaction(() => {
|
|
334
|
+
if (sql("SELECT 1 FROM migrations WHERE name = 'library-backfill'").get()) return 0;
|
|
335
|
+
let n = 0;
|
|
336
|
+
for (const file of walk(uploadsRoot)) {
|
|
337
|
+
const key = convKeyOf(file);
|
|
338
|
+
const conv = key ? getConversation(key) : undefined;
|
|
339
|
+
if (!key || !conv) continue;
|
|
340
|
+
const stat = fs.statSync(file);
|
|
341
|
+
const agentMade = path.relative(uploadsRoot, file).split(path.sep).some((s) => s.startsWith("agent-"));
|
|
342
|
+
const added = recordFile({
|
|
343
|
+
owner: conv.owner,
|
|
344
|
+
convKey: key,
|
|
345
|
+
path: file,
|
|
346
|
+
name: path.basename(file),
|
|
347
|
+
kind: kindOfFile(file),
|
|
348
|
+
size: stat.size,
|
|
349
|
+
source: agentMade ? "agent" : "upload",
|
|
350
|
+
createdAt: Math.round(stat.mtimeMs),
|
|
351
|
+
});
|
|
352
|
+
if (added) n++;
|
|
353
|
+
}
|
|
354
|
+
sql("INSERT INTO migrations (name, at) VALUES ('library-backfill', ?)").run(Date.now());
|
|
355
|
+
return n;
|
|
356
|
+
});
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
/**
|
|
360
|
+
* 启动时收拾:对话已经不在了的,上传目录和索引一起删;文件已经没了的索引删掉;
|
|
361
|
+
* 上传了一直没发出去(不在索引里)、超过一天的文件删掉
|
|
362
|
+
*/
|
|
363
|
+
export function tidyLibrary(now = Date.now()): { removed: number } {
|
|
364
|
+
let removed = 0;
|
|
365
|
+
let entries: string[] = [];
|
|
366
|
+
try {
|
|
367
|
+
entries = fs.readdirSync(uploadsRoot);
|
|
368
|
+
} catch {
|
|
369
|
+
// 还没有上传目录
|
|
370
|
+
}
|
|
371
|
+
for (const entry of entries) {
|
|
372
|
+
let key: string;
|
|
373
|
+
try {
|
|
374
|
+
key = decodeURIComponent(entry);
|
|
375
|
+
} catch {
|
|
376
|
+
continue;
|
|
377
|
+
}
|
|
378
|
+
if (getConversation(key)) continue;
|
|
379
|
+
fs.rmSync(path.join(uploadsRoot, entry), { recursive: true, force: true });
|
|
380
|
+
removed += Number(sql("DELETE FROM files WHERE conv_key = ?").run(key).changes);
|
|
381
|
+
}
|
|
382
|
+
const indexed = new Set<string>();
|
|
383
|
+
for (const row of sql("SELECT id, path FROM files").all() as { id: string; path: string }[]) {
|
|
384
|
+
if (fs.existsSync(row.path)) indexed.add(row.path);
|
|
385
|
+
else {
|
|
386
|
+
sql("DELETE FROM files WHERE id = ?").run(row.id);
|
|
387
|
+
removed++;
|
|
388
|
+
}
|
|
389
|
+
}
|
|
390
|
+
for (const file of walk(uploadsRoot)) {
|
|
391
|
+
if (indexed.has(file)) continue;
|
|
392
|
+
try {
|
|
393
|
+
if (now - fs.statSync(file).mtimeMs < UNSENT_TTL) continue;
|
|
394
|
+
fs.rmSync(file, { force: true });
|
|
395
|
+
removeEmptyDirs(path.dirname(file));
|
|
396
|
+
} catch {
|
|
397
|
+
// 刚好被删了
|
|
398
|
+
}
|
|
399
|
+
}
|
|
400
|
+
return { removed };
|
|
401
|
+
}
|
|
@@ -0,0 +1,275 @@
|
|
|
1
|
+
// 服务端所在机器上直接用的 Claude Code、Codex(各自的桌面端、命令行)的用量,通过 /api/usage 给网页和桌面端的用量汇总用。
|
|
2
|
+
//
|
|
3
|
+
// 不经过 agent-loft,只能读它们自己落盘的会话记录:
|
|
4
|
+
// - Claude Code(命令行和 Claude 桌面端里的 Code 都写这里):~/.claude/projects/**/*.jsonl,每条 assistant
|
|
5
|
+
// 消息带 usage。同一条消息按内容块拆成好几行、恢复 / 分叉的会话还会把旧消息抄进新文件,按 message.id + requestId 去重。
|
|
6
|
+
// entrypoint 区分来源:claude-desktop 是桌面端,cli 是命令行,sdk-ts 是 Agent SDK(agent-loft 就是用它跑的)。
|
|
7
|
+
// - Codex:~/.codex/sessions/**/*.jsonl(和 archived_sessions),每次模型请求一条 token_count,带这次的
|
|
8
|
+
// last_token_usage 和整条会话的累计 total。第一行 session_meta 的 originator 区分来源,agent-loft 跑的是 "agent-loft"。
|
|
9
|
+
// agent-loft 自己跑的会话已经记在服务器的账里,这里跳过,免得算两遍(别的用 Agent SDK 的程序也一起跳过了)。
|
|
10
|
+
//
|
|
11
|
+
// 新版 Claude Code 不再记费用,金额都按模型单价估(Claude 用内置快照,OpenAI 共用 pricing.ts 每天更新的价格表)。解析过的文件按 mtime + 大小缓存在内存里,没变的不再读。
|
|
12
|
+
import fs from "node:fs";
|
|
13
|
+
import os from "node:os";
|
|
14
|
+
import path from "node:path";
|
|
15
|
+
import { priceOf as openaiPriceOf } from "./pricing.js";
|
|
16
|
+
import { dayOf, type DailyRow } from "./usage.js";
|
|
17
|
+
|
|
18
|
+
type Source = "desktop" | "cli" | "other";
|
|
19
|
+
type Agent = "claude" | "codex";
|
|
20
|
+
type UsageEvent = [key: string, at: number, source: Source, model: string, input: number, cacheRead: number,
|
|
21
|
+
write5m: number, write1h: number, output: number, reasoning: number, fast: number];
|
|
22
|
+
type Price = [number, number | null, number, ...number[]];
|
|
23
|
+
export type LocalUsageRow = DailyRow & { source: Source };
|
|
24
|
+
export type UsageRoots = { claude: string[]; codex: string[] };
|
|
25
|
+
|
|
26
|
+
const CLAUDE_PRICES: Record<string, Price> = JSON.parse(fs.readFileSync(new URL("./claude-model-prices.json", import.meta.url), "utf8"));
|
|
27
|
+
/** 缓存写入相对输入单价的倍数:5 分钟 1.25 倍,1 小时 2 倍 */
|
|
28
|
+
const WRITE_5M = 1.25;
|
|
29
|
+
const WRITE_1H = 2;
|
|
30
|
+
/** 快速模式按两倍价 */
|
|
31
|
+
const FAST = 2;
|
|
32
|
+
|
|
33
|
+
const num = (v: unknown): number => (typeof v === "number" && Number.isFinite(v) && v > 0 ? v : 0);
|
|
34
|
+
|
|
35
|
+
/** Claude 的 entrypoint → 来源;null 是 agent-loft(Agent SDK)跑的,跳过 */
|
|
36
|
+
export function claudeSource(entrypoint: string | undefined): Source | null {
|
|
37
|
+
if (entrypoint === "sdk-ts") return null;
|
|
38
|
+
if (entrypoint === "claude-desktop") return "desktop";
|
|
39
|
+
// 老版本没有 entrypoint,那时只有命令行
|
|
40
|
+
if (!entrypoint || entrypoint === "cli") return "cli";
|
|
41
|
+
return "other";
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/** Codex 的 originator → 来源;null 是 agent-loft 跑的 */
|
|
45
|
+
export function codexSource(originator: string | undefined): Source | null {
|
|
46
|
+
if (originator === "agent-loft") return null;
|
|
47
|
+
if (/desktop/i.test(originator ?? "")) return "desktop";
|
|
48
|
+
if (!originator || /^codex[_-](exec|cli|tui)/i.test(originator)) return "cli";
|
|
49
|
+
return "other";
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* 一个 Claude 会话文件里的用量。每条:
|
|
54
|
+
* [去重键, 时间 ms, 来源, 模型, 输入, 缓存读取, 缓存写入(5 分钟), 缓存写入(1 小时), 输出, 推理, 快速模式]
|
|
55
|
+
*/
|
|
56
|
+
export function parseClaude(text: string): UsageEvent[] {
|
|
57
|
+
const out: UsageEvent[] = [];
|
|
58
|
+
for (const line of text.split("\n")) {
|
|
59
|
+
if (!line.includes('"usage"')) continue;
|
|
60
|
+
let d;
|
|
61
|
+
try {
|
|
62
|
+
d = JSON.parse(line);
|
|
63
|
+
} catch {
|
|
64
|
+
continue;
|
|
65
|
+
}
|
|
66
|
+
const m = d?.message;
|
|
67
|
+
const u = m?.usage;
|
|
68
|
+
if (d.type !== "assistant" || !u || !m.id) continue;
|
|
69
|
+
// 本地报错、被打断时的占位消息
|
|
70
|
+
if (m.model === "<synthetic>") continue;
|
|
71
|
+
const source = claudeSource(d.entrypoint);
|
|
72
|
+
const at = Date.parse(d.timestamp);
|
|
73
|
+
if (!source || !Number.isFinite(at)) continue;
|
|
74
|
+
const write = num(u.cache_creation_input_tokens);
|
|
75
|
+
const w1h = Math.min(write, num(u.cache_creation?.ephemeral_1h_input_tokens));
|
|
76
|
+
out.push([
|
|
77
|
+
`${m.id}\t${d.requestId ?? ""}`,
|
|
78
|
+
at,
|
|
79
|
+
source,
|
|
80
|
+
m.model ?? "",
|
|
81
|
+
num(u.input_tokens),
|
|
82
|
+
num(u.cache_read_input_tokens),
|
|
83
|
+
write - w1h,
|
|
84
|
+
w1h,
|
|
85
|
+
num(u.output_tokens),
|
|
86
|
+
num(u.output_tokens_details?.thinking_tokens),
|
|
87
|
+
u.speed === "fast" ? 1 : 0,
|
|
88
|
+
]);
|
|
89
|
+
}
|
|
90
|
+
return out;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/** 一个 Codex 会话文件里的用量,格式同 parseClaude(Codex 不分缓存时长,都记在 5 分钟那格) */
|
|
94
|
+
export function parseCodex(text: string): UsageEvent[] {
|
|
95
|
+
const out: UsageEvent[] = [];
|
|
96
|
+
let source: Source | null | undefined;
|
|
97
|
+
let sessionId = "";
|
|
98
|
+
let model = "";
|
|
99
|
+
let seenTotal: number | undefined;
|
|
100
|
+
for (const line of text.split("\n")) {
|
|
101
|
+
const meta = source === undefined && line.includes('"session_meta"');
|
|
102
|
+
if (!meta && !line.includes('"turn_context"') && !line.includes('"token_count"')) continue;
|
|
103
|
+
let d;
|
|
104
|
+
try {
|
|
105
|
+
d = JSON.parse(line);
|
|
106
|
+
} catch {
|
|
107
|
+
continue;
|
|
108
|
+
}
|
|
109
|
+
const p = d?.payload;
|
|
110
|
+
if (d.type === "session_meta") {
|
|
111
|
+
source = codexSource(p?.originator);
|
|
112
|
+
if (source === null) return [];
|
|
113
|
+
if (typeof p?.id === "string") sessionId = p.id;
|
|
114
|
+
continue;
|
|
115
|
+
}
|
|
116
|
+
if (d.type === "turn_context") {
|
|
117
|
+
if (typeof p?.model === "string") model = p.model;
|
|
118
|
+
continue;
|
|
119
|
+
}
|
|
120
|
+
if (p?.type !== "token_count") continue;
|
|
121
|
+
const total = p.info?.total_token_usage?.total_tokens;
|
|
122
|
+
const l = p.info?.last_token_usage;
|
|
123
|
+
// 同一次请求会报好几遍(累计值没变),只认第一遍
|
|
124
|
+
if (!l || typeof total !== "number" || total === seenTotal) continue;
|
|
125
|
+
seenTotal = total;
|
|
126
|
+
const at = Date.parse(d.timestamp);
|
|
127
|
+
if (!Number.isFinite(at)) continue;
|
|
128
|
+
// OpenAI 的 input_tokens 含命中缓存的部分,拆出来(和 src/agents/codex/session.ts 一样)
|
|
129
|
+
const cached = num(l.cached_input_tokens);
|
|
130
|
+
const written = num(l.cache_write_input_tokens);
|
|
131
|
+
out.push([
|
|
132
|
+
`${sessionId}\t${d.timestamp}\t${total}`,
|
|
133
|
+
at,
|
|
134
|
+
source ?? "cli",
|
|
135
|
+
model,
|
|
136
|
+
Math.max(0, num(l.input_tokens) - cached - written),
|
|
137
|
+
cached,
|
|
138
|
+
written,
|
|
139
|
+
0,
|
|
140
|
+
num(l.output_tokens),
|
|
141
|
+
num(l.reasoning_output_tokens),
|
|
142
|
+
0,
|
|
143
|
+
]);
|
|
144
|
+
}
|
|
145
|
+
return out;
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/** 模型的单价(美元 / 百万 token):[输入, 缓存读取(null 按输入算), 输出, 长提示词门槛?, 输入, 缓存读取, 输出]。认不出返回 undefined */
|
|
149
|
+
export function priceOf(agent: Agent, model: string): Price | undefined {
|
|
150
|
+
if (!model) return undefined;
|
|
151
|
+
if (agent === "claude") {
|
|
152
|
+
// claude-opus-5-5[1m]、claude-haiku-4-5-20251001、us.anthropic.claude-…-v1:0 都认
|
|
153
|
+
const base = model
|
|
154
|
+
.toLowerCase()
|
|
155
|
+
.replace(/\[.*\]$/, "")
|
|
156
|
+
.replace(/^(\w+\.)?anthropic\./, "")
|
|
157
|
+
.replace(/-v\d+:\d+$/, "")
|
|
158
|
+
.replace(/[-@]\d{8}$/, "");
|
|
159
|
+
return CLAUDE_PRICES[base];
|
|
160
|
+
}
|
|
161
|
+
return openaiPriceOf(model);
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
/** 一条用量的金额(美元),认不出模型是 0 */
|
|
165
|
+
export function costOf(agent: Agent, e: UsageEvent): number {
|
|
166
|
+
const p = priceOf(agent, e[3]);
|
|
167
|
+
if (!p) return 0;
|
|
168
|
+
const [, , , , input, cacheRead, w5, w1h, output, , fast] = e;
|
|
169
|
+
// Haiku 5.5 这类:一次请求的提示词超过门槛就整次按另一档算
|
|
170
|
+
const long = p[3] !== undefined && input + cacheRead + w5 + w1h > p[3];
|
|
171
|
+
const [pi, pr, po] = long ? [p[4]!, p[5]!, p[6]!] : p;
|
|
172
|
+
const cost = input * pi + cacheRead * (pr ?? pi) + w5 * pi * (agent === "claude" ? WRITE_5M : 1) + w1h * pi * WRITE_1H + output * po;
|
|
173
|
+
return (cost / 1e6) * (fast ? FAST : 1);
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
/** 会话记录在哪:尊重 CLAUDE_CONFIG_DIR / CODEX_HOME,默认在家目录下 */
|
|
177
|
+
export function usageRoots(): UsageRoots {
|
|
178
|
+
const home = os.homedir();
|
|
179
|
+
const claude = [...new Set([process.env.CLAUDE_CONFIG_DIR, path.join(home, ".claude")].filter((d): d is string => !!d))];
|
|
180
|
+
const codexHome = process.env.CODEX_HOME || path.join(home, ".codex");
|
|
181
|
+
return {
|
|
182
|
+
claude: claude.map((d) => path.join(d, "projects")),
|
|
183
|
+
codex: [path.join(codexHome, "sessions"), path.join(codexHome, "archived_sessions")],
|
|
184
|
+
};
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
async function listJsonl(dir: string): Promise<string[]> {
|
|
188
|
+
let entries;
|
|
189
|
+
try {
|
|
190
|
+
entries = await fs.promises.readdir(dir, { recursive: true, withFileTypes: true });
|
|
191
|
+
} catch {
|
|
192
|
+
return [];
|
|
193
|
+
}
|
|
194
|
+
return entries.filter((e) => e.isFile() && e.name.endsWith(".jsonl")).map((e) => path.join(e.parentPath, e.name));
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
/** 路径 → { mtimeMs, size, events } */
|
|
198
|
+
const cache = new Map<string, { mtimeMs: number; size: number; events: UsageEvent[] }>();
|
|
199
|
+
/** 同时只扫一遍,后来的排队等 */
|
|
200
|
+
let queue: Promise<unknown> = Promise.resolve();
|
|
201
|
+
|
|
202
|
+
async function eventsOf(file: string, parse: (text: string) => UsageEvent[], sinceMs: number): Promise<UsageEvent[]> {
|
|
203
|
+
let st;
|
|
204
|
+
try {
|
|
205
|
+
st = await fs.promises.stat(file);
|
|
206
|
+
} catch {
|
|
207
|
+
cache.delete(file);
|
|
208
|
+
return [];
|
|
209
|
+
}
|
|
210
|
+
// 最后改动在这段时间之前的文件不会有这段时间的用量
|
|
211
|
+
if (st.mtimeMs < sinceMs) return [];
|
|
212
|
+
const hit = cache.get(file);
|
|
213
|
+
if (hit && hit.mtimeMs === st.mtimeMs && hit.size === st.size) return hit.events;
|
|
214
|
+
let events: UsageEvent[] = [];
|
|
215
|
+
try {
|
|
216
|
+
events = parse(await fs.promises.readFile(file, "utf8"));
|
|
217
|
+
} catch {
|
|
218
|
+
// 读的时候正好被删了之类,跳过
|
|
219
|
+
}
|
|
220
|
+
cache.set(file, { mtimeMs: st.mtimeMs, size: st.size, events });
|
|
221
|
+
// 一个文件一个文件来,别把主进程卡住
|
|
222
|
+
await new Promise((r) => setImmediate(r));
|
|
223
|
+
return events;
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
/**
|
|
227
|
+
* from ~ to(含两头,YYYY-MM-DD,本地时间)的用量,按「日期 + agent + 来源 + 模型」合计,
|
|
228
|
+
* 格式和服务端 /api/usage 的 rows 一样,多一个 source(desktop / cli / other)。金额都是估算的
|
|
229
|
+
*/
|
|
230
|
+
export function localUsage(from: string, to: string, dirs = usageRoots()): Promise<LocalUsageRow[]> {
|
|
231
|
+
const run = queue.then(() => scan(from, to, dirs));
|
|
232
|
+
queue = run.catch(() => {});
|
|
233
|
+
return run;
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
async function scan(from: string, to: string, dirs: UsageRoots): Promise<LocalUsageRow[]> {
|
|
237
|
+
// 往前多留一天,时区、跨零点的会话不至于漏
|
|
238
|
+
const sinceMs = new Date(`${from}T00:00:00`).getTime() - 86400_000;
|
|
239
|
+
const groups = new Map<string, LocalUsageRow>();
|
|
240
|
+
for (const [agent, parse, list] of [
|
|
241
|
+
["claude", parseClaude, dirs.claude],
|
|
242
|
+
["codex", parseCodex, dirs.codex],
|
|
243
|
+
] as const) {
|
|
244
|
+
/** 去重键 → 这条用量(重复的取输出多的那条,流式中途写下的那行输出可能还没写全) */
|
|
245
|
+
const seen = new Map<string, UsageEvent>();
|
|
246
|
+
for (const dir of list) {
|
|
247
|
+
for (const file of await listJsonl(dir)) {
|
|
248
|
+
for (const e of await eventsOf(file, parse, sinceMs)) {
|
|
249
|
+
const day = dayOf(e[1]);
|
|
250
|
+
if (day < from || day > to) continue;
|
|
251
|
+
const prev = seen.get(e[0]);
|
|
252
|
+
if (!prev || e[8] > prev[8]) seen.set(e[0], e);
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
for (const e of seen.values()) {
|
|
257
|
+
const day = dayOf(e[1]);
|
|
258
|
+
const key = `${day}\t${e[2]}\t${e[3]}`;
|
|
259
|
+
let g = groups.get(`${agent}\t${key}`);
|
|
260
|
+
if (!g) {
|
|
261
|
+
g = { day, agent, source: e[2], ...(e[3] ? { model: e[3] } : {}), input: 0, cacheRead: 0, cacheWrite: 0, output: 0, reasoning: 0, costUsd: 0 };
|
|
262
|
+
groups.set(`${agent}\t${key}`, g);
|
|
263
|
+
}
|
|
264
|
+
g.input += e[4];
|
|
265
|
+
g.cacheRead += e[5];
|
|
266
|
+
g.cacheWrite += e[6] + e[7];
|
|
267
|
+
g.output += e[8];
|
|
268
|
+
g.reasoning = (g.reasoning ?? 0) + e[9];
|
|
269
|
+
g.costUsd = (g.costUsd ?? 0) + costOf(agent, e);
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
return [...groups.values()]
|
|
273
|
+
.map(({ costUsd, ...g }) => (costUsd ? { ...g, costUsd, estimated: true as const } : g))
|
|
274
|
+
.sort((a, b) => a.day.localeCompare(b.day));
|
|
275
|
+
}
|