dsh-team-bots 0.3.80 → 0.3.105
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cordis.patch.yml +100 -0
- package/index.js +5 -2
- package/package.json +1 -1
- package/src/api.js +5 -1
- package/src/db-migrations.js +5 -1
- package/src/db.js +52 -0
- package/src/feishu/channel.js +15 -11
- package/src/feishu/format.js +37 -1
- package/src/runtime.js +13 -1
- package/src/service.js +5 -0
- package/src/services/approvals.js +7 -2
- package/src/services/bots.js +8 -1
- package/src/services/chat-stores.js +564 -0
- package/src/services/chat.js +60 -28
- package/src/services/core.js +33 -12
- package/src/services/feishu.js +1 -1
- package/src/services/memory.js +14 -12
- package/src/services/model-fallback.js +3 -1
- package/src/services/notify.js +18 -8
- package/src/services/projects.js +25 -19
- package/src/services/questions.js +37 -12
- package/src/services/review-watchdog.js +31 -12
- package/src/services/run-finish.js +21 -14
- package/src/services/runs.js +29 -41
- package/src/services/runtime/context-window.js +6 -4
- package/src/services/runtime/lifecycle.js +1 -0
- package/src/services/runtime/prompt.js +21 -9
- package/src/services/runtime/sandbox-policy.js +1 -1
- package/src/services/runtime/session.js +27 -15
- package/src/services/runtime/trajectory.js +3 -1
- package/src/services/scheduler.js +48 -49
- package/src/services/system.js +9 -7
- package/src/services/task-review.js +27 -19
- package/src/services/task-runs.js +40 -21
- package/src/services/tasks.js +18 -10
- package/src/services/terminal.js +25 -14
- package/src/services/vision-proxy.js +67 -9
- package/src/services/wechat.js +24 -8
- package/src/subagent-ledger.js +23 -6
- package/src/usage-reconcile.js +93 -32
- package/src/usage.js +125 -50
- package/src/util.js +17 -0
- package/src/wechat/channel.js +3 -1
- package/src/wechat/commands.js +13 -1
- package/ui/modules/events-click.js +14 -1
- package/ui/modules/file-preview.js +22 -10
- package/ui/modules/markdown.js +10 -8
- package/ui/modules/tasks.js +1 -1
- package/ui/styles/cards.css +13 -0
- package/ui/styles/modal.css +4 -0
|
@@ -0,0 +1,564 @@
|
|
|
1
|
+
// src/services/chat-stores.js —— 聊天分库域(chat-db-split Phase 1)
|
|
2
|
+
// 每个群聊(=项目)与每个单聊(=bot)各一个库文件,正文类数据(messages/
|
|
3
|
+
// solo_messages/message_images/run_trajectory)住在聊天库里;顶库只留控制面与全局数据。
|
|
4
|
+
// 库文件名是引用 id(项目 id / bot id,均不可变)的纯函数,不需要注册表。
|
|
5
|
+
// 一次性迁移 migrateChatSplitP1 按聊天库逐库「幂等拷贝 → 记账」,全部完成后顶库
|
|
6
|
+
// 清空四张表并 VACUUM;任意步崩溃重启后按账本续跑(拷贝 INSERT OR IGNORE 幂等)。
|
|
7
|
+
import { mkdirSync, readdirSync, rmSync } from 'node:fs'
|
|
8
|
+
import { dirname, join } from 'node:path'
|
|
9
|
+
import { DatabaseSync } from 'node:sqlite'
|
|
10
|
+
import { Store } from '../db.js'
|
|
11
|
+
import { jsonParse, nowIso } from '../util.js'
|
|
12
|
+
|
|
13
|
+
// 句柄上限:聊天库懒打开、LRU 关闭。当前规模(项目数 + bot 数)远低于上限,
|
|
14
|
+
// 上限只防长期运行进程的句柄漂移。
|
|
15
|
+
const CHAT_STORE_LIMIT = 32
|
|
16
|
+
|
|
17
|
+
// 迁移拷贝分块:每片一个聊天库事务,避免整表载入内存(生产轨迹 18.7 万行 / 186MB)。
|
|
18
|
+
const COPY_CHUNK_ROWS = 500
|
|
19
|
+
|
|
20
|
+
// 聊天库结构:与顶库同名表同列集,但去掉指向顶库表(projects/bots/runs)的外键——
|
|
21
|
+
// 跨库引用由服务层守门(与 messages.reply_to_id 悬空容忍同一口径)。库内引用
|
|
22
|
+
// (message_images → messages/solo_messages)保留外键与级联。
|
|
23
|
+
const CHAT_SCHEMA = `
|
|
24
|
+
CREATE TABLE IF NOT EXISTS messages (
|
|
25
|
+
id TEXT PRIMARY KEY,
|
|
26
|
+
project_id TEXT NOT NULL,
|
|
27
|
+
group_chat_id TEXT,
|
|
28
|
+
sender_type TEXT NOT NULL,
|
|
29
|
+
sender_id TEXT NOT NULL,
|
|
30
|
+
body TEXT NOT NULL,
|
|
31
|
+
mentions_json TEXT NOT NULL DEFAULT '[]',
|
|
32
|
+
task_id TEXT,
|
|
33
|
+
run_id TEXT,
|
|
34
|
+
reply_to_id TEXT,
|
|
35
|
+
kind TEXT NOT NULL DEFAULT 'text',
|
|
36
|
+
question_id TEXT,
|
|
37
|
+
approval_id TEXT,
|
|
38
|
+
created_at TEXT NOT NULL
|
|
39
|
+
);
|
|
40
|
+
CREATE INDEX IF NOT EXISTS messages_project_idx ON messages(project_id, created_at DESC);
|
|
41
|
+
CREATE INDEX IF NOT EXISTS messages_task_idx ON messages(task_id, created_at);
|
|
42
|
+
|
|
43
|
+
CREATE TABLE IF NOT EXISTS solo_messages (
|
|
44
|
+
id TEXT PRIMARY KEY,
|
|
45
|
+
bot_id TEXT NOT NULL,
|
|
46
|
+
sender_type TEXT NOT NULL,
|
|
47
|
+
sender_id TEXT NOT NULL,
|
|
48
|
+
body TEXT NOT NULL,
|
|
49
|
+
run_id TEXT,
|
|
50
|
+
kind TEXT NOT NULL DEFAULT 'text',
|
|
51
|
+
question_id TEXT,
|
|
52
|
+
approval_id TEXT,
|
|
53
|
+
created_at TEXT NOT NULL
|
|
54
|
+
);
|
|
55
|
+
CREATE INDEX IF NOT EXISTS solo_messages_bot_idx ON solo_messages(bot_id, created_at DESC);
|
|
56
|
+
|
|
57
|
+
CREATE TABLE IF NOT EXISTS message_images (
|
|
58
|
+
id TEXT PRIMARY KEY,
|
|
59
|
+
message_id TEXT REFERENCES messages(id) ON DELETE CASCADE,
|
|
60
|
+
solo_message_id TEXT REFERENCES solo_messages(id) ON DELETE CASCADE,
|
|
61
|
+
project_id TEXT,
|
|
62
|
+
bot_id TEXT,
|
|
63
|
+
media_type TEXT NOT NULL,
|
|
64
|
+
data TEXT NOT NULL,
|
|
65
|
+
bytes INTEGER NOT NULL,
|
|
66
|
+
name TEXT,
|
|
67
|
+
created_at TEXT NOT NULL,
|
|
68
|
+
CHECK ((message_id IS NULL) <> (solo_message_id IS NULL))
|
|
69
|
+
);
|
|
70
|
+
CREATE INDEX IF NOT EXISTS message_images_message_idx ON message_images(message_id);
|
|
71
|
+
CREATE INDEX IF NOT EXISTS message_images_solo_idx ON message_images(solo_message_id);
|
|
72
|
+
|
|
73
|
+
CREATE TABLE IF NOT EXISTS run_trajectory (
|
|
74
|
+
id TEXT PRIMARY KEY,
|
|
75
|
+
run_id TEXT NOT NULL,
|
|
76
|
+
seq INTEGER NOT NULL,
|
|
77
|
+
kind TEXT NOT NULL,
|
|
78
|
+
title TEXT NOT NULL DEFAULT '',
|
|
79
|
+
detail TEXT NOT NULL DEFAULT '',
|
|
80
|
+
turn INTEGER,
|
|
81
|
+
step INTEGER,
|
|
82
|
+
created_at TEXT NOT NULL
|
|
83
|
+
);
|
|
84
|
+
CREATE INDEX IF NOT EXISTS run_trajectory_run_idx ON run_trajectory(run_id, seq);
|
|
85
|
+
|
|
86
|
+
CREATE TABLE IF NOT EXISTS usage_events (
|
|
87
|
+
id TEXT PRIMARY KEY,
|
|
88
|
+
source_key TEXT NOT NULL UNIQUE,
|
|
89
|
+
source_event_id TEXT,
|
|
90
|
+
run_id TEXT,
|
|
91
|
+
project_id TEXT,
|
|
92
|
+
bot_id TEXT,
|
|
93
|
+
task_id TEXT,
|
|
94
|
+
provider TEXT NOT NULL DEFAULT '',
|
|
95
|
+
model TEXT NOT NULL DEFAULT '',
|
|
96
|
+
purpose TEXT NOT NULL DEFAULT 'run',
|
|
97
|
+
currency TEXT NOT NULL DEFAULT 'CNY',
|
|
98
|
+
input_tokens INTEGER NOT NULL DEFAULT 0,
|
|
99
|
+
cache_read_tokens INTEGER NOT NULL DEFAULT 0,
|
|
100
|
+
cache_write_tokens INTEGER NOT NULL DEFAULT 0,
|
|
101
|
+
output_tokens INTEGER NOT NULL DEFAULT 0,
|
|
102
|
+
reasoning_tokens INTEGER NOT NULL DEFAULT 0,
|
|
103
|
+
input_cost REAL,
|
|
104
|
+
output_cost REAL,
|
|
105
|
+
total_cost REAL,
|
|
106
|
+
occurred_at TEXT NOT NULL,
|
|
107
|
+
created_at TEXT NOT NULL
|
|
108
|
+
);
|
|
109
|
+
CREATE INDEX IF NOT EXISTS usage_project_idx ON usage_events(project_id, occurred_at);
|
|
110
|
+
CREATE INDEX IF NOT EXISTS usage_bot_idx ON usage_events(bot_id, occurred_at);
|
|
111
|
+
CREATE INDEX IF NOT EXISTS usage_events_run_idx ON usage_events(run_id);
|
|
112
|
+
CREATE INDEX IF NOT EXISTS usage_events_task_idx ON usage_events(task_id);
|
|
113
|
+
CREATE INDEX IF NOT EXISTS usage_events_occurred_idx ON usage_events(occurred_at);
|
|
114
|
+
|
|
115
|
+
CREATE TABLE IF NOT EXISTS run_events (
|
|
116
|
+
id TEXT PRIMARY KEY,
|
|
117
|
+
run_id TEXT NOT NULL,
|
|
118
|
+
type TEXT NOT NULL,
|
|
119
|
+
message TEXT NOT NULL DEFAULT '',
|
|
120
|
+
created_at TEXT NOT NULL
|
|
121
|
+
);
|
|
122
|
+
CREATE INDEX IF NOT EXISTS run_events_run_idx ON run_events(run_id, created_at);
|
|
123
|
+
CREATE INDEX IF NOT EXISTS run_events_type_idx ON run_events(type);
|
|
124
|
+
|
|
125
|
+
-- run 胖载荷(Phase 3):runs 瘦身留顶库当调度索引,output/artifacts 住这里。
|
|
126
|
+
CREATE TABLE IF NOT EXISTS run_payload (
|
|
127
|
+
run_id TEXT PRIMARY KEY,
|
|
128
|
+
output TEXT,
|
|
129
|
+
artifacts TEXT,
|
|
130
|
+
updated_at TEXT NOT NULL
|
|
131
|
+
);
|
|
132
|
+
|
|
133
|
+
CREATE TABLE IF NOT EXISTS task_comments (
|
|
134
|
+
id TEXT PRIMARY KEY,
|
|
135
|
+
task_id TEXT NOT NULL,
|
|
136
|
+
author_type TEXT NOT NULL,
|
|
137
|
+
author_id TEXT NOT NULL,
|
|
138
|
+
body TEXT NOT NULL,
|
|
139
|
+
created_at TEXT NOT NULL
|
|
140
|
+
);
|
|
141
|
+
CREATE INDEX IF NOT EXISTS task_comments_task_idx ON task_comments(task_id, created_at);
|
|
142
|
+
|
|
143
|
+
CREATE TABLE IF NOT EXISTS task_events (
|
|
144
|
+
id TEXT PRIMARY KEY,
|
|
145
|
+
task_id TEXT NOT NULL,
|
|
146
|
+
type TEXT NOT NULL,
|
|
147
|
+
actor_type TEXT,
|
|
148
|
+
actor_id TEXT,
|
|
149
|
+
data_json TEXT,
|
|
150
|
+
created_at TEXT NOT NULL
|
|
151
|
+
);
|
|
152
|
+
CREATE INDEX IF NOT EXISTS task_events_task_idx ON task_events(task_id, created_at);
|
|
153
|
+
|
|
154
|
+
CREATE TABLE IF NOT EXISTS terminal_commands (
|
|
155
|
+
id TEXT PRIMARY KEY,
|
|
156
|
+
project_id TEXT NOT NULL,
|
|
157
|
+
command TEXT NOT NULL,
|
|
158
|
+
status TEXT NOT NULL DEFAULT 'running',
|
|
159
|
+
exit_code INTEGER,
|
|
160
|
+
output TEXT,
|
|
161
|
+
error_message TEXT,
|
|
162
|
+
created_at TEXT NOT NULL,
|
|
163
|
+
ended_at TEXT
|
|
164
|
+
);
|
|
165
|
+
CREATE INDEX IF NOT EXISTS terminal_commands_project_idx ON terminal_commands(project_id, created_at);
|
|
166
|
+
|
|
167
|
+
CREATE TABLE IF NOT EXISTS usage_reconciliation (
|
|
168
|
+
id TEXT PRIMARY KEY,
|
|
169
|
+
run_id TEXT NOT NULL,
|
|
170
|
+
event_calls INTEGER NOT NULL DEFAULT 0,
|
|
171
|
+
event_tokens INTEGER NOT NULL DEFAULT 0,
|
|
172
|
+
run_tokens INTEGER NOT NULL DEFAULT 0,
|
|
173
|
+
delta_tokens INTEGER NOT NULL DEFAULT 0,
|
|
174
|
+
status TEXT NOT NULL DEFAULT 'ok',
|
|
175
|
+
note TEXT NOT NULL DEFAULT '',
|
|
176
|
+
checked_at TEXT NOT NULL,
|
|
177
|
+
UNIQUE(run_id)
|
|
178
|
+
);
|
|
179
|
+
`
|
|
180
|
+
|
|
181
|
+
export function openChatDatabase(path) {
|
|
182
|
+
mkdirSync(dirname(path), { recursive: true })
|
|
183
|
+
const db = new DatabaseSync(path)
|
|
184
|
+
db.exec('PRAGMA journal_mode = WAL;')
|
|
185
|
+
db.exec('PRAGMA foreign_keys = ON;')
|
|
186
|
+
db.exec('PRAGMA busy_timeout = 5000;')
|
|
187
|
+
db.exec('PRAGMA synchronous = NORMAL;')
|
|
188
|
+
db.exec(CHAT_SCHEMA)
|
|
189
|
+
return new Store(db, {})
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
// 分块幂等拷贝:按 rowid 键序翻页读源表,逐片一个目标库事务 INSERT OR IGNORE。
|
|
193
|
+
function copyRowsBetween(from, to, table, columns, where, params) {
|
|
194
|
+
const cols = columns.split(',').map(item => item.trim())
|
|
195
|
+
const insert = `INSERT OR IGNORE INTO ${table}(${columns}) VALUES (${cols.map(() => '?').join(', ')})`
|
|
196
|
+
let last = 0
|
|
197
|
+
for (;;) {
|
|
198
|
+
const rows = from.all(
|
|
199
|
+
`SELECT rowid AS __rid, ${columns} FROM ${table} WHERE (${where}) AND rowid > ? ORDER BY rowid LIMIT ${COPY_CHUNK_ROWS}`,
|
|
200
|
+
...params, last,
|
|
201
|
+
)
|
|
202
|
+
if (!rows.length) return
|
|
203
|
+
to.transaction(() => {
|
|
204
|
+
for (const row of rows) to.run(insert, ...cols.map(col => row[col] ?? null))
|
|
205
|
+
})
|
|
206
|
+
last = rows[rows.length - 1].__rid
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
const CHAT_TABLE_COLUMNS = {
|
|
211
|
+
messages: 'id, project_id, group_chat_id, sender_type, sender_id, body, mentions_json, task_id, run_id, reply_to_id, kind, question_id, approval_id, created_at',
|
|
212
|
+
solo_messages: 'id, bot_id, sender_type, sender_id, body, run_id, kind, question_id, approval_id, created_at',
|
|
213
|
+
message_images: 'id, message_id, solo_message_id, project_id, bot_id, media_type, data, bytes, name, created_at',
|
|
214
|
+
run_trajectory: 'id, run_id, seq, kind, title, detail, turn, step, created_at',
|
|
215
|
+
usage_events: 'id, source_key, source_event_id, run_id, project_id, bot_id, task_id, provider, model, purpose, currency, input_tokens, cache_read_tokens, cache_write_tokens, output_tokens, reasoning_tokens, input_cost, output_cost, total_cost, occurred_at, created_at',
|
|
216
|
+
usage_reconciliation: 'id, run_id, event_calls, event_tokens, run_tokens, delta_tokens, status, note, checked_at',
|
|
217
|
+
run_events: 'id, run_id, type, message, created_at',
|
|
218
|
+
task_comments: 'id, task_id, author_type, author_id, body, created_at',
|
|
219
|
+
task_events: 'id, task_id, type, actor_type, actor_id, data_json, created_at',
|
|
220
|
+
terminal_commands: 'id, project_id, command, status, exit_code, output, error_message, created_at, ended_at',
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
export const chatStoreMethods = {
|
|
224
|
+
chatStoresDir() {
|
|
225
|
+
return join(this.config.dataDir, 'chats')
|
|
226
|
+
},
|
|
227
|
+
groupChatFileName(projectId) {
|
|
228
|
+
return `p-${projectId}.sqlite`
|
|
229
|
+
},
|
|
230
|
+
soloChatFileName(botId) {
|
|
231
|
+
return `solo-${botId}.sqlite`
|
|
232
|
+
},
|
|
233
|
+
openChatStoreFile(file, { evict = true } = {}) {
|
|
234
|
+
if (!this.chatStoreCache) this.chatStoreCache = new Map()
|
|
235
|
+
const hit = this.chatStoreCache.get(file)
|
|
236
|
+
if (hit) {
|
|
237
|
+
hit.at = Date.now()
|
|
238
|
+
return hit.store
|
|
239
|
+
}
|
|
240
|
+
const store = openChatDatabase(join(this.chatStoresDir(), file))
|
|
241
|
+
this.chatStoreCache.set(file, { store, at: Date.now() })
|
|
242
|
+
if (evict && this.chatStoreCache.size > CHAT_STORE_LIMIT) {
|
|
243
|
+
let oldestKey = null
|
|
244
|
+
let oldestAt = Infinity
|
|
245
|
+
for (const [key, entry] of this.chatStoreCache) {
|
|
246
|
+
if (entry.at < oldestAt) {
|
|
247
|
+
oldestAt = entry.at
|
|
248
|
+
oldestKey = key
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
if (oldestKey && oldestKey !== file) {
|
|
252
|
+
const evicted = this.chatStoreCache.get(oldestKey)
|
|
253
|
+
this.chatStoreCache.delete(oldestKey)
|
|
254
|
+
try { evicted.store.close() } catch { /* 已关闭的库再关一次不算错误 */ }
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
return store
|
|
258
|
+
},
|
|
259
|
+
groupChatStore(projectId) {
|
|
260
|
+
return this.openChatStoreFile(this.groupChatFileName(projectId))
|
|
261
|
+
},
|
|
262
|
+
soloChatStore(botId) {
|
|
263
|
+
return this.openChatStoreFile(this.soloChatFileName(botId))
|
|
264
|
+
},
|
|
265
|
+
// 运行归属的聊天库:project_id 非空 → 该项目群聊库;为空 → 运行 bot 的单聊库。
|
|
266
|
+
runChatStore(run) {
|
|
267
|
+
if (!run) return this.store
|
|
268
|
+
return run.project_id ? this.groupChatStore(run.project_id) : this.soloChatStore(run.bot_id)
|
|
269
|
+
},
|
|
270
|
+
listChatStoreFiles() {
|
|
271
|
+
try {
|
|
272
|
+
return readdirSync(this.chatStoresDir()).filter(file => file.endsWith('.sqlite'))
|
|
273
|
+
} catch {
|
|
274
|
+
return []
|
|
275
|
+
}
|
|
276
|
+
},
|
|
277
|
+
allChatStores() {
|
|
278
|
+
// 批量快照期间不做 LRU 淘汰:淘汰会当场关闭本轮已收集进数组的句柄,
|
|
279
|
+
// 随后的扇出清理(applyRetention/未读数/图片回填)就会打到已关闭的库,
|
|
280
|
+
// 报 "database is not open"(聊天库数超过 LRU 上限时必现)。
|
|
281
|
+
// 批量后缓存可能暂时超上限,由后续单开的 LRU 逐步收敛。
|
|
282
|
+
return this.listChatStoreFiles().map(file => this.openChatStoreFile(file, { evict: false }))
|
|
283
|
+
},
|
|
284
|
+
// state 版本签名段:每个聊天库一条「引用 id:本连接累计变更数」。
|
|
285
|
+
// 取聊天实体清单(顶库的 projects/bots)而不是文件系统列表:文件清单会随「首次打开某聊天库」
|
|
286
|
+
// 而增长,导致版本串在两次轮询间漂移、短路失效。实体清单只随项目/bot 增删变化(那是真实变更)。
|
|
287
|
+
// 未缓存的聊天库记 0——它没有被打开过,自然也没有写入;写必先打开(缓存),故不漏检。
|
|
288
|
+
chatVersionParts() {
|
|
289
|
+
const parts = []
|
|
290
|
+
for (const row of this.store.all('SELECT id FROM projects')) {
|
|
291
|
+
parts.push(`p-${row.id}:${this.cachedChatChanges(this.groupChatFileName(row.id))}`)
|
|
292
|
+
}
|
|
293
|
+
for (const row of this.store.all('SELECT id FROM bots')) {
|
|
294
|
+
parts.push(`s-${row.id}:${this.cachedChatChanges(this.soloChatFileName(row.id))}`)
|
|
295
|
+
}
|
|
296
|
+
return parts
|
|
297
|
+
},
|
|
298
|
+
cachedChatChanges(file) {
|
|
299
|
+
const entry = this.chatStoreCache?.get(file)
|
|
300
|
+
if (!entry) return 0
|
|
301
|
+
try {
|
|
302
|
+
return entry.store.changes()
|
|
303
|
+
} catch {
|
|
304
|
+
return 0
|
|
305
|
+
}
|
|
306
|
+
},
|
|
307
|
+
closeChatStores() {
|
|
308
|
+
if (!this.chatStoreCache) return
|
|
309
|
+
for (const entry of this.chatStoreCache.values()) {
|
|
310
|
+
try { entry.store.close() } catch { /* 关闭失败不阻塞停机 */ }
|
|
311
|
+
}
|
|
312
|
+
this.chatStoreCache.clear()
|
|
313
|
+
},
|
|
314
|
+
// 移除追踪:整个聊天库文件随之删除(正文数据全在其中),连 -wal/-shm 一起清。
|
|
315
|
+
dropGroupChatDb(projectId) {
|
|
316
|
+
const file = this.groupChatFileName(projectId)
|
|
317
|
+
const entry = this.chatStoreCache?.get(file)
|
|
318
|
+
if (entry) {
|
|
319
|
+
try { entry.store.close() } catch { /* 已关闭的库再关一次不算错误 */ }
|
|
320
|
+
this.chatStoreCache.delete(file)
|
|
321
|
+
}
|
|
322
|
+
const base = join(this.chatStoresDir(), file)
|
|
323
|
+
for (const suffix of ['', '-wal', '-shm']) {
|
|
324
|
+
try { rmSync(`${base}${suffix}`, { force: true }) } catch { /* 删除失败留人工处理 */ }
|
|
325
|
+
}
|
|
326
|
+
},
|
|
327
|
+
// Phase 3 路由助手:run 胖载荷住运行归属的聊天库。
|
|
328
|
+
// 顶库 runs 已瘦身(output/artifacts 住 run_payload):读取经助手并到载荷。
|
|
329
|
+
runOutputOf(run) {
|
|
330
|
+
if (!run) return ''
|
|
331
|
+
const payload = this.runPayloadOf(run)
|
|
332
|
+
return payload?.output ?? run.output ?? ''
|
|
333
|
+
},
|
|
334
|
+
runArtifactsOf(run) {
|
|
335
|
+
if (!run) return []
|
|
336
|
+
const payload = this.runPayloadOf(run)
|
|
337
|
+
const list = jsonParse(payload?.artifacts ?? run.artifacts, [])
|
|
338
|
+
return Array.isArray(list) ? list : []
|
|
339
|
+
},
|
|
340
|
+
runPayloadOf(run) {
|
|
341
|
+
if (!run) return null
|
|
342
|
+
return this.runChatStore(run).get('SELECT * FROM run_payload WHERE run_id = ?', run.id)
|
|
343
|
+
},
|
|
344
|
+
saveRunPayload(run, { output = null, artifacts = null }) {
|
|
345
|
+
this.runChatStore(run).run(
|
|
346
|
+
`INSERT INTO run_payload(run_id, output, artifacts, updated_at) VALUES (?, ?, ?, ?)
|
|
347
|
+
ON CONFLICT(run_id) DO UPDATE SET output = COALESCE(excluded.output, output),
|
|
348
|
+
artifacts = COALESCE(excluded.artifacts, artifacts), updated_at = excluded.updated_at`,
|
|
349
|
+
run.id, output, artifacts, nowIso(),
|
|
350
|
+
)
|
|
351
|
+
},
|
|
352
|
+
|
|
353
|
+
// Phase 1 一次性迁移:见文件头注释。任意步崩溃重启续跑,最终记账后不再进入。
|
|
354
|
+
migrateChatSplitP1() {
|
|
355
|
+
const ledgerName = 'chat_split_p1'
|
|
356
|
+
if (this.store.get('SELECT 1 FROM schema_migrations WHERE name = ?', ledgerName)) return
|
|
357
|
+
mkdirSync(this.chatStoresDir(), { recursive: true })
|
|
358
|
+
const targets = []
|
|
359
|
+
for (const project of this.store.all('SELECT id FROM projects')) {
|
|
360
|
+
const pid = project.id
|
|
361
|
+
targets.push({
|
|
362
|
+
file: this.groupChatFileName(pid),
|
|
363
|
+
ledger: `chat_split_p1:p:${pid}`,
|
|
364
|
+
copies: [
|
|
365
|
+
{ table: 'messages', where: 'project_id = ?', params: [pid] },
|
|
366
|
+
{ table: 'message_images', where: 'project_id = ? OR message_id IN (SELECT id FROM messages WHERE project_id = ?)', params: [pid, pid] },
|
|
367
|
+
{ table: 'run_trajectory', where: 'run_id IN (SELECT id FROM runs WHERE project_id = ?)', params: [pid] },
|
|
368
|
+
],
|
|
369
|
+
})
|
|
370
|
+
}
|
|
371
|
+
for (const bot of this.store.all('SELECT id FROM bots')) {
|
|
372
|
+
const bid = bot.id
|
|
373
|
+
targets.push({
|
|
374
|
+
file: this.soloChatFileName(bid),
|
|
375
|
+
ledger: `chat_split_p1:s:${bid}`,
|
|
376
|
+
copies: [
|
|
377
|
+
{ table: 'solo_messages', where: 'bot_id = ?', params: [bid] },
|
|
378
|
+
{ table: 'message_images', where: 'bot_id = ? OR solo_message_id IN (SELECT id FROM solo_messages WHERE bot_id = ?)', params: [bid, bid] },
|
|
379
|
+
{ table: 'run_trajectory', where: 'run_id IN (SELECT id FROM runs WHERE project_id IS NULL AND bot_id = ?)', params: [bid] },
|
|
380
|
+
],
|
|
381
|
+
})
|
|
382
|
+
}
|
|
383
|
+
for (const target of targets) {
|
|
384
|
+
if (this.store.get('SELECT 1 FROM schema_migrations WHERE name = ?', target.ledger)) continue
|
|
385
|
+
const chat = this.openChatStoreFile(target.file)
|
|
386
|
+
for (const copy of target.copies) {
|
|
387
|
+
copyRowsBetween(this.store, chat, copy.table, CHAT_TABLE_COLUMNS[copy.table], copy.where, copy.params)
|
|
388
|
+
}
|
|
389
|
+
this.store.run(
|
|
390
|
+
'INSERT INTO schema_migrations(name, applied_at) VALUES (?, ?) ON CONFLICT(name) DO NOTHING',
|
|
391
|
+
target.ledger, nowIso(),
|
|
392
|
+
)
|
|
393
|
+
}
|
|
394
|
+
// 四张表的每一行都属且仅属一个聊天库(图片按归属列兜底关联拷贝),全库拷贝完成后
|
|
395
|
+
// 顶库一次性清空;拷贝幂等,本事务与逐库记账之间的崩溃只会导致重拷,不会丢行。
|
|
396
|
+
this.store.transaction(() => {
|
|
397
|
+
this.store.run('DELETE FROM message_images')
|
|
398
|
+
this.store.run('DELETE FROM messages')
|
|
399
|
+
this.store.run('DELETE FROM solo_messages')
|
|
400
|
+
this.store.run('DELETE FROM run_trajectory')
|
|
401
|
+
this.store.run(
|
|
402
|
+
'INSERT INTO schema_migrations(name, applied_at) VALUES (?, ?) ON CONFLICT(name) DO NOTHING',
|
|
403
|
+
ledgerName, nowIso(),
|
|
404
|
+
)
|
|
405
|
+
})
|
|
406
|
+
// 删除只进 freelist,主文件不缩:一次性 VACUUM 回收(生产库量级数十秒,仅本迁移触发)。
|
|
407
|
+
this.store.exec('VACUUM')
|
|
408
|
+
this.logger?.info?.('dsh-team: 聊天分库 Phase 1 迁移完成:messages/solo_messages/message_images/run_trajectory 已迁入各聊天库')
|
|
409
|
+
},
|
|
410
|
+
// Phase 2 一次性迁移:用量明细与对账迁入聊天库,顶库一次性构建 usage_rollup 汇总
|
|
411
|
+
// (视图/预算自此只读汇总)。同样账本闸控、崩溃可续跑。
|
|
412
|
+
migrateChatSplitP2() {
|
|
413
|
+
const ledgerName = 'chat_split_p2'
|
|
414
|
+
if (this.store.get('SELECT 1 FROM schema_migrations WHERE name = ?', ledgerName)) return
|
|
415
|
+
mkdirSync(this.chatStoresDir(), { recursive: true })
|
|
416
|
+
// 历史行 task_id 预回填(DSHBOTS-212 之前的存量行):聊天库里没有 runs 表可 JOIN
|
|
417
|
+
// 追认,迁移时在顶库一次性补齐,之后明细拷贝就是自足的。
|
|
418
|
+
this.store.run(
|
|
419
|
+
`UPDATE usage_events SET task_id = (SELECT task_id FROM runs WHERE id = usage_events.run_id)
|
|
420
|
+
WHERE task_id IS NULL AND run_id IS NOT NULL`,
|
|
421
|
+
)
|
|
422
|
+
const targets = []
|
|
423
|
+
for (const project of this.store.all('SELECT id FROM projects')) {
|
|
424
|
+
const pid = project.id
|
|
425
|
+
targets.push({
|
|
426
|
+
file: this.groupChatFileName(pid),
|
|
427
|
+
ledger: `chat_split_p2:p:${pid}`,
|
|
428
|
+
copies: [
|
|
429
|
+
{ table: 'usage_events', where: 'project_id = ? OR run_id IN (SELECT id FROM runs WHERE project_id = ?)', params: [pid, pid] },
|
|
430
|
+
{ table: 'usage_reconciliation', where: 'run_id IN (SELECT id FROM runs WHERE project_id = ?)', params: [pid] },
|
|
431
|
+
],
|
|
432
|
+
})
|
|
433
|
+
}
|
|
434
|
+
for (const bot of this.store.all('SELECT id FROM bots')) {
|
|
435
|
+
const bid = bot.id
|
|
436
|
+
targets.push({
|
|
437
|
+
file: this.soloChatFileName(bid),
|
|
438
|
+
ledger: `chat_split_p2:s:${bid}`,
|
|
439
|
+
copies: [
|
|
440
|
+
{ table: 'usage_events', where: '(project_id IS NULL AND bot_id = ?) OR run_id IN (SELECT id FROM runs WHERE project_id IS NULL AND bot_id = ?)', params: [bid, bid] },
|
|
441
|
+
{ table: 'usage_reconciliation', where: 'run_id IN (SELECT id FROM runs WHERE project_id IS NULL AND bot_id = ?)', params: [bid] },
|
|
442
|
+
],
|
|
443
|
+
})
|
|
444
|
+
}
|
|
445
|
+
for (const target of targets) {
|
|
446
|
+
if (this.store.get('SELECT 1 FROM schema_migrations WHERE name = ?', target.ledger)) continue
|
|
447
|
+
const chat = this.openChatStoreFile(target.file)
|
|
448
|
+
for (const copy of target.copies) {
|
|
449
|
+
copyRowsBetween(this.store, chat, copy.table, CHAT_TABLE_COLUMNS[copy.table], copy.where, copy.params)
|
|
450
|
+
}
|
|
451
|
+
this.store.run(
|
|
452
|
+
'INSERT INTO schema_migrations(name, applied_at) VALUES (?, ?) ON CONFLICT(name) DO NOTHING',
|
|
453
|
+
target.ledger, nowIso(),
|
|
454
|
+
)
|
|
455
|
+
}
|
|
456
|
+
// 顶库汇总一次性构建(单条 INSERT..SELECT 原子完成,但重跑会重复累加——
|
|
457
|
+
// 单独账本闸兜崩溃续跑:建桶即记账,续跑只在闸缺失时重建)。
|
|
458
|
+
// project_key 冗余存档:项目移除后历史汇总仍可读。未落在任何聊天库的无主行
|
|
459
|
+
// (无 project/bot/run 归属)也进汇总——用量是全局历史,不因无法路由而丢账。
|
|
460
|
+
if (!this.store.get('SELECT 1 FROM schema_migrations WHERE name = ?', `${ledgerName}:rollup`)) {
|
|
461
|
+
this.store.exec(`
|
|
462
|
+
INSERT INTO usage_rollup(hour_utc, project_id, project_key, bot_id, provider, model, currency, priced,
|
|
463
|
+
calls, input_tokens, cache_read_tokens, cache_write_tokens, output_tokens, reasoning_tokens, stored_cost)
|
|
464
|
+
SELECT substr(occurred_at, 1, 13),
|
|
465
|
+
COALESCE(project_id, ''),
|
|
466
|
+
COALESCE((SELECT key FROM projects WHERE id = usage_events.project_id), ''),
|
|
467
|
+
COALESCE(bot_id, ''), provider, model, currency,
|
|
468
|
+
CASE WHEN total_cost IS NOT NULL THEN 1 ELSE 0 END,
|
|
469
|
+
COUNT(*), SUM(input_tokens), SUM(cache_read_tokens), SUM(cache_write_tokens),
|
|
470
|
+
SUM(output_tokens), SUM(reasoning_tokens),
|
|
471
|
+
COALESCE(SUM(CASE WHEN total_cost IS NOT NULL THEN total_cost ELSE 0 END), 0)
|
|
472
|
+
FROM usage_events
|
|
473
|
+
GROUP BY 1, 2, 3, 4, 5, 6, 7, 8
|
|
474
|
+
ON CONFLICT(hour_utc, project_id, bot_id, provider, model, currency, priced) DO UPDATE SET
|
|
475
|
+
calls = calls + excluded.calls,
|
|
476
|
+
input_tokens = input_tokens + excluded.input_tokens,
|
|
477
|
+
cache_read_tokens = cache_read_tokens + excluded.cache_read_tokens,
|
|
478
|
+
cache_write_tokens = cache_write_tokens + excluded.cache_write_tokens,
|
|
479
|
+
output_tokens = output_tokens + excluded.output_tokens,
|
|
480
|
+
reasoning_tokens = reasoning_tokens + excluded.reasoning_tokens,
|
|
481
|
+
stored_cost = stored_cost + excluded.stored_cost
|
|
482
|
+
`)
|
|
483
|
+
this.store.run(
|
|
484
|
+
'INSERT INTO schema_migrations(name, applied_at) VALUES (?, ?) ON CONFLICT(name) DO NOTHING',
|
|
485
|
+
`${ledgerName}:rollup`, nowIso(),
|
|
486
|
+
)
|
|
487
|
+
}
|
|
488
|
+
// 拷贝幂等 + 汇总闸幂等,清空与记账一个事务;无主行留在顶库由保留期清理兜底。
|
|
489
|
+
this.store.transaction(() => {
|
|
490
|
+
this.store.run('DELETE FROM usage_events WHERE project_id IS NOT NULL OR bot_id IS NOT NULL OR run_id IS NOT NULL')
|
|
491
|
+
this.store.run('DELETE FROM usage_reconciliation WHERE run_id IN (SELECT id FROM runs)')
|
|
492
|
+
this.store.run(
|
|
493
|
+
'INSERT INTO schema_migrations(name, applied_at) VALUES (?, ?) ON CONFLICT(name) DO NOTHING',
|
|
494
|
+
ledgerName, nowIso(),
|
|
495
|
+
)
|
|
496
|
+
})
|
|
497
|
+
this.store.exec('VACUUM')
|
|
498
|
+
this.logger?.info?.('dsh-team: 聊天分库 Phase 2 迁移完成:usage_events/usage_reconciliation 已迁入各聊天库,顶库改读 usage_rollup 汇总')
|
|
499
|
+
},
|
|
500
|
+
// Phase 3 一次性迁移:run_events/胖载荷/任务评论事件/终端命令迁入聊天库;
|
|
501
|
+
// runs 瘦身留顶库当调度索引(output/artifacts 清空转 run_payload)。
|
|
502
|
+
migrateChatSplitP3() {
|
|
503
|
+
const ledgerName = 'chat_split_p3'
|
|
504
|
+
if (this.store.get('SELECT 1 FROM schema_migrations WHERE name = ?', ledgerName)) return
|
|
505
|
+
mkdirSync(this.chatStoresDir(), { recursive: true })
|
|
506
|
+
const targets = []
|
|
507
|
+
for (const project of this.store.all('SELECT id FROM projects')) {
|
|
508
|
+
targets.push({ file: this.groupChatFileName(project.id), ledger: `chat_split_p3:p:${project.id}`, pid: project.id })
|
|
509
|
+
}
|
|
510
|
+
for (const bot of this.store.all('SELECT id FROM bots')) {
|
|
511
|
+
targets.push({ file: this.soloChatFileName(bot.id), ledger: `chat_split_p3:s:${bot.id}`, bid: bot.id })
|
|
512
|
+
}
|
|
513
|
+
for (const target of targets) {
|
|
514
|
+
if (this.store.get('SELECT 1 FROM schema_migrations WHERE name = ?', target.ledger)) continue
|
|
515
|
+
const chat = this.openChatStoreFile(target.file)
|
|
516
|
+
const copies = target.pid
|
|
517
|
+
? [
|
|
518
|
+
{ table: 'run_events', where: 'run_id IN (SELECT id FROM runs WHERE project_id = ?)', params: [target.pid] },
|
|
519
|
+
{ table: 'task_comments', where: 'task_id IN (SELECT id FROM tasks WHERE project_id = ?)', params: [target.pid] },
|
|
520
|
+
{ table: 'task_events', where: 'task_id IN (SELECT id FROM tasks WHERE project_id = ?)', params: [target.pid] },
|
|
521
|
+
{ table: 'terminal_commands', where: 'project_id = ?', params: [target.pid] },
|
|
522
|
+
]
|
|
523
|
+
: [
|
|
524
|
+
{ table: 'run_events', where: 'run_id IN (SELECT id FROM runs WHERE project_id IS NULL AND bot_id = ?)', params: [target.bid] },
|
|
525
|
+
]
|
|
526
|
+
for (const copy of copies) {
|
|
527
|
+
copyRowsBetween(this.store, chat, copy.table, CHAT_TABLE_COLUMNS[copy.table], copy.where, copy.params)
|
|
528
|
+
}
|
|
529
|
+
// run 胖载荷:output/artifacts 非空的运行搬进 run_payload(幂等)。
|
|
530
|
+
chat.transaction(() => {
|
|
531
|
+
const rows = this.store.all(
|
|
532
|
+
`SELECT id, output, artifacts FROM runs WHERE ${target.pid ? 'project_id = ?' : 'project_id IS NULL AND bot_id = ?'}
|
|
533
|
+
AND (output IS NOT NULL OR artifacts IS NOT NULL)`,
|
|
534
|
+
target.pid ?? target.bid,
|
|
535
|
+
)
|
|
536
|
+
for (const row of rows) {
|
|
537
|
+
chat.run(
|
|
538
|
+
`INSERT INTO run_payload(run_id, output, artifacts, updated_at) VALUES (?, ?, ?, ?)
|
|
539
|
+
ON CONFLICT(run_id) DO NOTHING`,
|
|
540
|
+
row.id, row.output, row.artifacts, nowIso(),
|
|
541
|
+
)
|
|
542
|
+
}
|
|
543
|
+
})
|
|
544
|
+
this.store.run(
|
|
545
|
+
'INSERT INTO schema_migrations(name, applied_at) VALUES (?, ?) ON CONFLICT(name) DO NOTHING',
|
|
546
|
+
target.ledger, nowIso(),
|
|
547
|
+
)
|
|
548
|
+
}
|
|
549
|
+
// 清空与 runs 瘦身一个事务;无主行(join 不上 runs/tasks 的孤儿)留顶库由保留期兜底。
|
|
550
|
+
this.store.transaction(() => {
|
|
551
|
+
this.store.run('DELETE FROM run_events WHERE run_id IN (SELECT id FROM runs)')
|
|
552
|
+
this.store.run('DELETE FROM task_comments WHERE task_id IN (SELECT id FROM tasks)')
|
|
553
|
+
this.store.run('DELETE FROM task_events WHERE task_id IN (SELECT id FROM tasks)')
|
|
554
|
+
this.store.run('DELETE FROM terminal_commands WHERE project_id IS NOT NULL')
|
|
555
|
+
this.store.run('UPDATE runs SET output = NULL, artifacts = NULL WHERE output IS NOT NULL OR artifacts IS NOT NULL')
|
|
556
|
+
this.store.run(
|
|
557
|
+
'INSERT INTO schema_migrations(name, applied_at) VALUES (?, ?) ON CONFLICT(name) DO NOTHING',
|
|
558
|
+
ledgerName, nowIso(),
|
|
559
|
+
)
|
|
560
|
+
})
|
|
561
|
+
this.store.exec('VACUUM')
|
|
562
|
+
this.logger?.info?.('dsh-team: 聊天分库 Phase 3 迁移完成:run_events/胖载荷/任务评论事件/终端命令已迁入各聊天库,runs 已瘦身')
|
|
563
|
+
},
|
|
564
|
+
}
|