@mzzsfy/dsh-usage-dash 0.3.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +55 -14
- package/package.json +2 -2
- package/src/archive-reader.js +138 -0
- package/src/client.js +539 -128
- package/src/collector.js +175 -29
- package/src/direct-log-reader.js +126 -0
- package/src/pricing.js +5 -6
- package/src/query.js +38 -14
- package/src/routes.js +14 -1
- package/src/store.js +86 -4
- package/test/archive-reader.test.mjs +365 -0
- package/test/client.test.mjs +210 -12
- package/test/collector.test.mjs +458 -19
- package/test/direct-log-reader.test.mjs +104 -0
- package/test/pricing-parity.test.mjs +38 -4
- package/test/pricing.test.mjs +50 -18
- package/test/query.test.mjs +50 -1
- package/test/routes.test.mjs +9 -2
- package/test/stats-line.test.mjs +6 -6
- package/test/store.test.mjs +116 -1
- package/test/stream-parity.test.mjs +75 -0
- package/test/turn-tail.test.mjs +161 -41
package/src/collector.js
CHANGED
|
@@ -5,19 +5,87 @@
|
|
|
5
5
|
// 游标(liveFirstSeq 分区 + backfilledSessions)保证。采集是观测性的:一切
|
|
6
6
|
// store 失败计入 recordFailures,绝不成为逃逸拒绝。
|
|
7
7
|
|
|
8
|
+
import { createArchiveReader } from './archive-reader.js'
|
|
9
|
+
|
|
8
10
|
const UNKNOWN_SESSION_ID = '(unknown-session)'
|
|
9
11
|
const SEQ_UNKNOWN = -1
|
|
10
12
|
const DEFAULT_BACKFILL_CONCURRENCY = 4
|
|
11
13
|
const MARK_BATCH = 32
|
|
12
14
|
const SCAN_LOG_MAX_ENTRIES = 200
|
|
13
15
|
|
|
16
|
+
// 不可读会话分类:宿主可见性/可读性边界的三代实测形态(见 README 存档兼容)
|
|
17
|
+
const SKIP_KINDS = { descriptor: 'descriptor', corrupt: 'corrupt', legacy: 'legacy', other: 'other' }
|
|
18
|
+
const SKIP_PATTERNS = [
|
|
19
|
+
// 宿主新校验拒读历史档(0.1.5 descriptor v2),宿主修复后自动补扫
|
|
20
|
+
[SKIP_KINDS.descriptor, /unsupported descriptor version/i],
|
|
21
|
+
// 存档损坏(seq gap 等 fail-closed 拒绝),宿主 repair 后自动补扫
|
|
22
|
+
[SKIP_KINDS.corrupt, /corrupt|seq gap|SessionFormatError/i],
|
|
23
|
+
// 超旧格式词汇(v0 时代字段),宿主迁移链覆盖后自动补扫
|
|
24
|
+
[SKIP_KINDS.legacy, /format v0|unexpected member|does not provide|not a function/i],
|
|
25
|
+
]
|
|
26
|
+
|
|
27
|
+
function classifySkip(detail) {
|
|
28
|
+
for (const [kind, pattern] of SKIP_PATTERNS) {
|
|
29
|
+
if (pattern.test(detail)) return kind
|
|
30
|
+
}
|
|
31
|
+
return SKIP_KINDS.other
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
function emptySkipCounts() {
|
|
35
|
+
return { [SKIP_KINDS.descriptor]: 0, [SKIP_KINDS.corrupt]: 0, [SKIP_KINDS.legacy]: 0, [SKIP_KINDS.other]: 0 }
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
// 官方 dsh-llm 助手流读取器同构镜像(isTokenDelta/runFirstTokenTime/
|
|
39
|
+
// assistantStreamFirstTokenTime):从紧凑记录还原首个产出 token 的时刻。
|
|
40
|
+
// 语义由 test/stream-parity.test.mjs 锁定,改一侧必须同步 parity
|
|
41
|
+
function isTokenDelta(chunk) {
|
|
42
|
+
if (chunk?.type === 'text-delta' || chunk?.type === 'reasoning-delta') return chunk.text !== ''
|
|
43
|
+
return chunk?.type === 'tool-call-delta' && (chunk.argumentsDelta !== '' || chunk.name !== undefined)
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
function runFirstTokenTime(run) {
|
|
47
|
+
if (run.type === 'tool-call-chunks' && run.name !== undefined) return run.time0
|
|
48
|
+
const fragments = run.type === 'tool-call-chunks' ? run.args : run.texts
|
|
49
|
+
let time = run.time0
|
|
50
|
+
for (let index = 0; index < fragments.length; index += 1) {
|
|
51
|
+
if (index > 0) {
|
|
52
|
+
const gap = run.dt[index - 1]
|
|
53
|
+
// 差分短缺属存储形态损坏,按无首 token 处理,保留后续报告锁定机会
|
|
54
|
+
if (typeof gap !== 'number') return undefined
|
|
55
|
+
time += gap
|
|
56
|
+
}
|
|
57
|
+
if (fragments[index] !== '') return time
|
|
58
|
+
}
|
|
59
|
+
return undefined
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
// 已知 packed run 形态白名单:未知记录形态(宿主未来新字段)安全跳过,
|
|
63
|
+
// 采集是观测性的,绝不因流记录形态漂移而崩溃
|
|
64
|
+
const RUN_RECORD_TYPES = new Set(['text-chunks', 'reasoning-chunks', 'tool-call-chunks'])
|
|
65
|
+
|
|
66
|
+
export function assistantStreamFirstTokenTime(stream) {
|
|
67
|
+
if (!Array.isArray(stream)) return undefined
|
|
68
|
+
for (const record of stream) {
|
|
69
|
+
const time = record?.type === 'chunk'
|
|
70
|
+
? (isTokenDelta(record.chunk) ? record.time : undefined)
|
|
71
|
+
: RUN_RECORD_TYPES.has(record?.type) ? runFirstTokenTime(record) : undefined
|
|
72
|
+
if (time !== undefined) return time
|
|
73
|
+
}
|
|
74
|
+
return undefined
|
|
75
|
+
}
|
|
76
|
+
|
|
14
77
|
// 会话内单 pass 折叠:跟踪每个 (turn,step) 槽的最新报告,只把首次发射交 store
|
|
15
78
|
export class UsageFold {
|
|
16
79
|
constructor() {
|
|
17
80
|
this.seen = new Map()
|
|
18
|
-
// (turn,step)
|
|
19
|
-
//
|
|
81
|
+
// (turn,step) 模型启动时刻:仅 step/start 设定(官方口径 TTFT 含失败尝试,
|
|
82
|
+
// 不随 retry-started 重置);(turn,step) 首 token 时刻:由首个产出 token
|
|
83
|
+
// 的 attempt 锁定,存活于步内重试。时长均为官方 decode 口径:durationMs
|
|
84
|
+
// = 汇报 - 首 token(吞吐分母),ttftMs = 首 token - 启动
|
|
20
85
|
this.starts = new Map()
|
|
86
|
+
this.firstTokens = new Map()
|
|
87
|
+
// 已补发 timing 的键:token 先发后只补一次,重复报告不重复配对
|
|
88
|
+
this.timingDone = new Set()
|
|
21
89
|
}
|
|
22
90
|
|
|
23
91
|
keyOf(event) {
|
|
@@ -40,9 +108,9 @@ export class UsageFold {
|
|
|
40
108
|
}
|
|
41
109
|
if (event.type === 'step/start' || event.type === 'llm/retry-started') {
|
|
42
110
|
// step/start 恰开一次模型调用,retry-started 标记每次实际启动的重试;
|
|
43
|
-
// 请求只由标记计数,与 token
|
|
111
|
+
// 请求只由标记计数,与 token 样本双计;时长起点不随重试重置
|
|
44
112
|
const key = this.keyOf(event)
|
|
45
|
-
if (key !== null) this.starts.set(key, event.time)
|
|
113
|
+
if (event.type === 'step/start' && key !== null) this.starts.set(key, event.time)
|
|
46
114
|
return {
|
|
47
115
|
time: event.time,
|
|
48
116
|
inputTokens: 0,
|
|
@@ -52,6 +120,15 @@ export class UsageFold {
|
|
|
52
120
|
request: true,
|
|
53
121
|
}
|
|
54
122
|
}
|
|
123
|
+
if (event.type === 'assistant/attempt') {
|
|
124
|
+
// 首 token 锁定:仅首个产出 token 的 attempt 生效,后续 attempt 不覆盖
|
|
125
|
+
const key = this.keyOf(event)
|
|
126
|
+
if (key !== null && !this.firstTokens.has(key)) {
|
|
127
|
+
const first = assistantStreamFirstTokenTime(event.data?.stream)
|
|
128
|
+
if (typeof first === 'number') this.firstTokens.set(key, first)
|
|
129
|
+
}
|
|
130
|
+
return null
|
|
131
|
+
}
|
|
55
132
|
if (event.type === 'assistant/chunk') {
|
|
56
133
|
const usage = event.data?.chunk?.type === 'usage' ? event.data.chunk.usage : undefined
|
|
57
134
|
return usage ? this.replaceSample(event, usage) : null
|
|
@@ -63,11 +140,39 @@ export class UsageFold {
|
|
|
63
140
|
}
|
|
64
141
|
|
|
65
142
|
replaceSample(event, usage) {
|
|
66
|
-
|
|
143
|
+
const key = this.keyOf(event)
|
|
144
|
+
const first = key !== null
|
|
145
|
+
? this.firstTokens.get(key) ?? assistantStreamFirstTokenTime(event.data?.stream)
|
|
146
|
+
: assistantStreamFirstTokenTime(event.data?.stream)
|
|
147
|
+
// decode 配对有效性对齐官方 usageOutputTokens 守卫:输出 token 非有效数值
|
|
148
|
+
// 不建配对(官方同款),首字延迟不受 usage 影响照常采集;
|
|
149
|
+
// 负差值按官方 Math.max(0) 钳 0(时钟回拨计 0 延迟样本),0 时长不附配对
|
|
150
|
+
// (存储侧 0 时长配对天然惰性,防 0 分母放大)
|
|
151
|
+
const decodeable = typeof usage.outputTokens === 'number'
|
|
152
|
+
&& Number.isFinite(usage.outputTokens) && usage.outputTokens >= 0
|
|
153
|
+
const timing = typeof first === 'number'
|
|
154
|
+
? {
|
|
155
|
+
durationMs: Math.max(0, event.time - first),
|
|
156
|
+
ttftMs: key !== null && this.starts.has(key) ? Math.max(0, first - this.starts.get(key)) : undefined,
|
|
157
|
+
decodeable,
|
|
158
|
+
}
|
|
159
|
+
: undefined
|
|
160
|
+
const prev = key !== null ? this.seen.get(key) : undefined
|
|
161
|
+
// 首样本生效:内部无条件跟踪最新报告,但交 store 的只有首次发射;
|
|
162
|
+
// 首 token 时刻不在 chunk 事件上,token 先发后由后续报告补纯 timing 增量,
|
|
163
|
+
// 已补发的键不重复补(timingDone 独立集合持久跟踪,防重复报告双计);
|
|
164
|
+
// 补发判定先于四桶和门:usage 全零的报告仍补 timing(token 已由首发承载)
|
|
165
|
+
if (prev) {
|
|
166
|
+
if (timing === undefined || this.timingDone.has(key)) return null
|
|
167
|
+
this.timingDone.add(key)
|
|
168
|
+
return this.emitTimingOnly(usage, timing, event.time)
|
|
169
|
+
}
|
|
170
|
+
// 四桶全零为噪声(不占去重键),纯缓存调用(仅缓存桶非零)仍有效。
|
|
171
|
+
// retry 场景 attempt 先于 chunk 落流时,chunk 首发即带 timing,分母以
|
|
172
|
+
// chunk 时刻近似官方汇报时刻(毫秒级组装间隔),主路径仍由 message 补发
|
|
67
173
|
const sum = (usage.inputTokens ?? 0) + (usage.outputTokens ?? 0)
|
|
68
174
|
+ (usage.cacheReadTokens ?? 0) + (usage.cacheWriteTokens ?? 0)
|
|
69
175
|
if (sum <= 0) return null
|
|
70
|
-
const key = this.keyOf(event)
|
|
71
176
|
const sample = {
|
|
72
177
|
time: event.time,
|
|
73
178
|
inputTokens: usage.inputTokens ?? 0,
|
|
@@ -75,18 +180,37 @@ export class UsageFold {
|
|
|
75
180
|
cacheReadTokens: usage.cacheReadTokens ?? 0,
|
|
76
181
|
cacheWriteTokens: usage.cacheWriteTokens ?? 0,
|
|
77
182
|
}
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
183
|
+
if (key !== null) this.seen.set(key, sample)
|
|
184
|
+
return this.emitSample(sample, timing, key)
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
emitSample(sample, timing, key) {
|
|
188
|
+
if (timing === undefined) return { ...sample }
|
|
189
|
+
// decode 口径聚合对:分子 decodeTokens 与分母 durationMs 同源配对
|
|
190
|
+
if (timing.decodeable && timing.durationMs > 0) {
|
|
191
|
+
sample.decodeTokens = sample.outputTokens
|
|
192
|
+
sample.durationMs = timing.durationMs
|
|
193
|
+
}
|
|
194
|
+
if (timing.ttftMs !== undefined) sample.ttftMs = timing.ttftMs
|
|
195
|
+
if (key !== undefined && key !== null) this.timingDone.add(key)
|
|
196
|
+
return { ...sample }
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
emitTimingOnly(usage, timing, time) {
|
|
200
|
+
// 纯 timing 增量:token 桶全零不重复计数,decodeTokens 单独承载速度分子
|
|
201
|
+
const delta = {
|
|
202
|
+
time,
|
|
203
|
+
inputTokens: 0,
|
|
204
|
+
outputTokens: 0,
|
|
205
|
+
cacheReadTokens: 0,
|
|
206
|
+
cacheWriteTokens: 0,
|
|
207
|
+
}
|
|
208
|
+
if (timing.decodeable && timing.durationMs > 0) {
|
|
209
|
+
delta.decodeTokens = usage.outputTokens
|
|
210
|
+
delta.durationMs = timing.durationMs
|
|
84
211
|
}
|
|
85
|
-
if (
|
|
86
|
-
|
|
87
|
-
this.seen.set(key, sample)
|
|
88
|
-
// 首样本生效:内部无条件跟踪最新报告,但交 store 的只有首次发射的独立拷贝
|
|
89
|
-
return prev ? null : { ...sample }
|
|
212
|
+
if (timing.ttftMs !== undefined) delta.ttftMs = timing.ttftMs
|
|
213
|
+
return delta
|
|
90
214
|
}
|
|
91
215
|
}
|
|
92
216
|
|
|
@@ -95,8 +219,9 @@ function sessionIdOf(target) {
|
|
|
95
219
|
return typeof id === 'string' && id !== '' ? id : UNKNOWN_SESSION_ID
|
|
96
220
|
}
|
|
97
221
|
|
|
98
|
-
// 规范 provider/model 引用:双全拼引用,仅 model
|
|
99
|
-
|
|
222
|
+
// 规范 provider/model 引用:双全拼引用,仅 model 用裸名;
|
|
223
|
+
// 与 client 侧注入点B turnModelOf 双实现同源(routes[].model 是裸模型名,展示侧才拼 provider),改一侧必须同步 parity
|
|
224
|
+
export function refOf(route) {
|
|
100
225
|
if (!route) return undefined
|
|
101
226
|
if (route.provider && route.model) return `${route.provider}/${route.model}`
|
|
102
227
|
return route.model || undefined
|
|
@@ -126,6 +251,10 @@ export class UsageCollector {
|
|
|
126
251
|
lastSessionId: undefined,
|
|
127
252
|
error: undefined,
|
|
128
253
|
log: [],
|
|
254
|
+
// 计数独立累加,不受日志截断影响;skipped 细分供面板归因
|
|
255
|
+
skippedTotal: 0,
|
|
256
|
+
recordTotal: 0,
|
|
257
|
+
skipCounts: emptySkipCounts(),
|
|
129
258
|
}
|
|
130
259
|
// error 保留给采集器自身故障;单会话读取失败走日志 skipped 条目,计数由日志派生
|
|
131
260
|
}
|
|
@@ -141,17 +270,24 @@ export class UsageCollector {
|
|
|
141
270
|
scannedSessions: this.#state.scannedSessions,
|
|
142
271
|
lastSessionId: this.#state.lastSessionId,
|
|
143
272
|
error: this.#state.error,
|
|
144
|
-
skippedSessions:
|
|
145
|
-
recordFailures:
|
|
273
|
+
skippedSessions: this.#state.skippedTotal,
|
|
274
|
+
recordFailures: this.#state.recordTotal,
|
|
275
|
+
skippedBreakdown: { ...this.#state.skipCounts },
|
|
146
276
|
log,
|
|
147
277
|
}
|
|
148
278
|
}
|
|
149
279
|
|
|
150
|
-
//
|
|
280
|
+
// 扫描异常日志:供面板明细展示,超上限丢最旧;计数独立累加不随截断漂移
|
|
151
281
|
pushLog(kind, detail) {
|
|
152
282
|
const log = this.#state.log
|
|
153
283
|
log.push({ time: Date.now(), kind, detail })
|
|
154
284
|
if (log.length > SCAN_LOG_MAX_ENTRIES) log.splice(0, log.length - SCAN_LOG_MAX_ENTRIES)
|
|
285
|
+
if (kind === 'skipped') {
|
|
286
|
+
this.#state.skippedTotal += 1
|
|
287
|
+
this.#state.skipCounts[classifySkip(detail)] += 1
|
|
288
|
+
} else if (kind === 'record') {
|
|
289
|
+
this.#state.recordTotal += 1
|
|
290
|
+
}
|
|
155
291
|
}
|
|
156
292
|
|
|
157
293
|
get running() {
|
|
@@ -239,7 +375,9 @@ export class UsageCollector {
|
|
|
239
375
|
const controller = new AbortController()
|
|
240
376
|
this.scanController = controller
|
|
241
377
|
await this.store.readyPromise()
|
|
242
|
-
|
|
378
|
+
// 适配发生在宿主服务边界:回扫主体只面向 ArchiveReader 内部契约
|
|
379
|
+
const reader = createArchiveReader(this.ctx.sessionPersistence)
|
|
380
|
+
await this.backfill(reader, this.ctx.sessions, controller.signal)
|
|
243
381
|
}
|
|
244
382
|
|
|
245
383
|
abort() {
|
|
@@ -271,22 +409,30 @@ export class UsageCollector {
|
|
|
271
409
|
return this.resetInFlight
|
|
272
410
|
}
|
|
273
411
|
|
|
274
|
-
// 回扫:
|
|
275
|
-
// liveFirstSeq
|
|
276
|
-
// inheritedCut 跳过 fork
|
|
277
|
-
|
|
412
|
+
// 回扫:ArchiveReader 契约驱动(宿主 persistence 多版本适配见 archive-reader),
|
|
413
|
+
// 逐会话全新 fold 重放;seen 独自决定是否重扫,liveFirstSeq 边界划走实时
|
|
414
|
+
// 已拥区间,liveness 复查防陈旧快照放大重放范围,inheritedCut 跳过 fork
|
|
415
|
+
// 继承前缀。只有干净重放完的会话进游标,失败下轮重试
|
|
416
|
+
async backfill(reader, sessions, signal) {
|
|
278
417
|
if (this.#state.running) return
|
|
279
418
|
if (signal?.aborted) return
|
|
280
419
|
this.#state.running = true
|
|
281
420
|
this.#state.error = undefined
|
|
282
421
|
try {
|
|
283
|
-
const headers = await
|
|
422
|
+
const headers = await reader.list(signal).catch((error) => {
|
|
423
|
+
// 枚举失败(含宿主 API 未识别)按采集器自身故障呈现,面板可见
|
|
424
|
+
this.#state.error = error instanceof Error ? error.message : String(error)
|
|
425
|
+
throw error
|
|
426
|
+
})
|
|
284
427
|
const seen = await this.store.seenSessions()
|
|
285
428
|
const liveSeq = await this.store.liveSequences()
|
|
286
429
|
const targets = headers.filter((header) => !seen.has(header.id))
|
|
287
430
|
this.#state.total = targets.length
|
|
288
431
|
this.#state.done = 0
|
|
289
432
|
this.#state.log = []
|
|
433
|
+
this.#state.skippedTotal = 0
|
|
434
|
+
this.#state.recordTotal = 0
|
|
435
|
+
this.#state.skipCounts = emptySkipCounts()
|
|
290
436
|
const workerCount = Math.min(DEFAULT_BACKFILL_CONCURRENCY, Math.max(1, targets.length))
|
|
291
437
|
let next = 0
|
|
292
438
|
const completed = []
|
|
@@ -321,7 +467,7 @@ export class UsageCollector {
|
|
|
321
467
|
const fold = new UsageFold()
|
|
322
468
|
let route = ''
|
|
323
469
|
try {
|
|
324
|
-
const inspection = await
|
|
470
|
+
const inspection = await reader.readLog(header.id, signal)
|
|
325
471
|
const inheritedCut = inspection.inheritedEventCount
|
|
326
472
|
for (const event of inspection.events) {
|
|
327
473
|
if (signal?.aborted) return
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
// 会话档案文件直读器:宿主 persistence 服务 fail-closed 拒读(descriptor 校验、
|
|
2
|
+
// 格式校验、seq gap 等)时,对磁盘档案做降级直读。统计只消费 usage 词汇
|
|
3
|
+
// (assistant/message 的 usage 与 request/context 路由),无需完整会话语义,
|
|
4
|
+
// 宽松解析即可恢复;corrupt(seq gap)与 legacy(未知成员)对统计无影响。
|
|
5
|
+
//
|
|
6
|
+
// 物理格式(与宿主 jsonl 后端同构):首行 header JSON + 事件行 JSONL 文本,
|
|
7
|
+
// 按 zstd 帧独立压缩顺序拼接;帧以 magic 28 b5 2f fd 起始。压缩由 Node 内置
|
|
8
|
+
// node:zlib 提供(Node >= 24),不依赖宿主模块,不受宿主升级影响。
|
|
9
|
+
|
|
10
|
+
import { existsSync, readdirSync, readFileSync } from 'node:fs'
|
|
11
|
+
import { join } from 'node:path'
|
|
12
|
+
import { homedir } from 'node:os'
|
|
13
|
+
import { zstdDecompressSync } from 'node:zlib'
|
|
14
|
+
|
|
15
|
+
const ZSTD_MAGIC = Buffer.from([0x28, 0xb5, 0x2f, 0xfd])
|
|
16
|
+
|
|
17
|
+
// 档案根目录:DSH_HOME 可重定向(多版本共存测试的隔离约定),默认 ~/.dsh
|
|
18
|
+
function sessionsRoot() {
|
|
19
|
+
const home = process.env.DSH_HOME || join(homedir(), '.dsh')
|
|
20
|
+
return join(home, 'sessions')
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
// 按帧边界切分 zstd 流:帧首 magic 定位,各帧解压范围到下一 magic 或流尾
|
|
24
|
+
function frameRanges(bytes) {
|
|
25
|
+
const ranges = []
|
|
26
|
+
let at = bytes.indexOf(ZSTD_MAGIC)
|
|
27
|
+
while (at >= 0) {
|
|
28
|
+
if (ranges.length > 0) ranges[ranges.length - 1].end = at
|
|
29
|
+
ranges.push({ start: at, end: bytes.length })
|
|
30
|
+
at = bytes.indexOf(ZSTD_MAGIC, at + ZSTD_MAGIC.length)
|
|
31
|
+
}
|
|
32
|
+
return ranges
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
// 多帧解压;任何一帧失败即整体失败(宁缺勿错,调用方按原拒读路径上报)
|
|
36
|
+
export function decodeZstdFile(bytes) {
|
|
37
|
+
const ranges = frameRanges(bytes)
|
|
38
|
+
if (ranges.length === 0) throw new Error('no zstd frame found')
|
|
39
|
+
let text = ''
|
|
40
|
+
for (const range of ranges) {
|
|
41
|
+
text += zstdDecompressSync(bytes.subarray(range.start, range.end)).toString('utf8')
|
|
42
|
+
}
|
|
43
|
+
return text
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
// JSONL 文本 → 宽松事件流:首行为档案 header,其余行逐行解析,坏行跳过;
|
|
47
|
+
// 只要求行能解出 JSON 对象,词汇/代际差异由上层折叠消化
|
|
48
|
+
export function parseJsonlEvents(text) {
|
|
49
|
+
const events = []
|
|
50
|
+
for (const line of text.split('\n')) {
|
|
51
|
+
const trimmed = line.trim()
|
|
52
|
+
if (trimmed === '') continue
|
|
53
|
+
try {
|
|
54
|
+
const row = JSON.parse(trimmed)
|
|
55
|
+
if (row && typeof row === 'object' && row.type !== undefined && row.type !== 'session/header') {
|
|
56
|
+
events.push(row)
|
|
57
|
+
}
|
|
58
|
+
} catch {}
|
|
59
|
+
}
|
|
60
|
+
return events
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
// 会话目录定位:sessions/<cwd 编码桶>/<id>/;桶名是宿主对 cwd 的编码,不解析,
|
|
64
|
+
// 逐桶探测 id 子目录
|
|
65
|
+
export function findSessionDir(id, root = sessionsRoot()) {
|
|
66
|
+
const sessions = root
|
|
67
|
+
if (!existsSync(sessions)) return undefined
|
|
68
|
+
for (const bucket of readdirSync(sessions)) {
|
|
69
|
+
const candidate = join(sessions, bucket, id)
|
|
70
|
+
if (existsSync(candidate) && statDirSync(candidate)) return candidate
|
|
71
|
+
}
|
|
72
|
+
return undefined
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
function statDirSync(path) {
|
|
76
|
+
try {
|
|
77
|
+
return readdirSync(path).length >= 0
|
|
78
|
+
} catch {
|
|
79
|
+
return false
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
// 会话目录内选最新代档案:V3 专用名优先,退化为任意 jsonl(压缩/明文)
|
|
84
|
+
function pickLogFile(dir) {
|
|
85
|
+
const names = readdirSync(dir)
|
|
86
|
+
const v3 = names.find((name) => name.startsWith('session.v3.') && name.endsWith('.zstd'))
|
|
87
|
+
if (v3) return join(dir, v3)
|
|
88
|
+
const compressed = names.find((name) => name.endsWith('.jsonl.zstd'))
|
|
89
|
+
if (compressed) return join(dir, compressed)
|
|
90
|
+
const plain = names.find((name) => name.endsWith('.jsonl'))
|
|
91
|
+
if (plain) return join(dir, plain)
|
|
92
|
+
return undefined
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
// 直读单会话事件流;文件缺失/无法解压抛错,由调用方决定降级链终止
|
|
96
|
+
export function readSessionLogDirect(id, root) {
|
|
97
|
+
const dir = findSessionDir(id, root)
|
|
98
|
+
if (dir === undefined) throw new Error(`session artifact directory not found: ${id}`)
|
|
99
|
+
const file = pickLogFile(dir)
|
|
100
|
+
if (file === undefined) throw new Error(`session artifact file not found: ${id}`)
|
|
101
|
+
const bytes = readFileSync(file)
|
|
102
|
+
const text = file.endsWith('.zstd') ? decodeZstdFile(bytes) : bytes.toString('utf8')
|
|
103
|
+
return parseJsonlEvents(text)
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
// 磁盘全量会话 id:sessions/<cwd 编码桶>/<id>/;旧代宿主的 list 不枚举新代
|
|
107
|
+
// 文件名的档案,直读侧补齐可见集
|
|
108
|
+
export function listSessionIdsDirect(root = sessionsRoot()) {
|
|
109
|
+
if (!existsSync(root)) return []
|
|
110
|
+
const ids = []
|
|
111
|
+
for (const bucket of readdirSync(root)) {
|
|
112
|
+
const bucketPath = join(root, bucket)
|
|
113
|
+
let entries
|
|
114
|
+
try {
|
|
115
|
+
entries = readdirSync(bucketPath)
|
|
116
|
+
} catch {
|
|
117
|
+
continue
|
|
118
|
+
}
|
|
119
|
+
for (const id of entries) {
|
|
120
|
+
if (existsSync(join(bucketPath, id, 'session.jsonl.zstd')) || existsSync(join(bucketPath, id, 'session.v3.jsonl.zstd'))) {
|
|
121
|
+
ids.push(id)
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
return ids
|
|
126
|
+
}
|
package/src/pricing.js
CHANGED
|
@@ -30,15 +30,14 @@ const toMinutesOfDay = (hhmm) => {
|
|
|
30
30
|
return Number.isFinite(h) && Number.isFinite(m) ? h * MINUTES_PER_HOUR + m : Number.NaN
|
|
31
31
|
}
|
|
32
32
|
|
|
33
|
-
// from
|
|
33
|
+
// 所有范围条件统一双侧包含;from>to 跨午夜/跨月环绕;from===to 单点/单日
|
|
34
34
|
const dailyWindowMatches = (condition, date) => {
|
|
35
35
|
const from = toMinutesOfDay(condition.from)
|
|
36
36
|
const to = toMinutesOfDay(condition.to)
|
|
37
37
|
if (Number.isNaN(from) || Number.isNaN(to)) return false
|
|
38
38
|
const m = minutesOfDay(date)
|
|
39
|
-
if (from
|
|
40
|
-
|
|
41
|
-
return true
|
|
39
|
+
if (from <= to) return m >= from && m <= to
|
|
40
|
+
return m >= from || m <= to
|
|
42
41
|
}
|
|
43
42
|
|
|
44
43
|
// days 空数组不成立;0=周日,取 getDay()
|
|
@@ -47,7 +46,7 @@ const weekdaysMatches = (condition, date) => {
|
|
|
47
46
|
return Array.isArray(days) && days.length > 0 && days.includes(date.getDay())
|
|
48
47
|
}
|
|
49
48
|
|
|
50
|
-
//
|
|
49
|
+
// 号段双侧包含;from>to 跨月环绕(账单周期);from===to 单日,2 月无 31 号自然不触发
|
|
51
50
|
const monthDaysMatches = (condition, date) => {
|
|
52
51
|
const { from, to } = condition
|
|
53
52
|
if (!Number.isInteger(from) || !Number.isInteger(to)) return false
|
|
@@ -55,7 +54,7 @@ const monthDaysMatches = (condition, date) => {
|
|
|
55
54
|
return from <= to ? d >= from && d <= to : d >= from || d <= to
|
|
56
55
|
}
|
|
57
56
|
|
|
58
|
-
//
|
|
57
|
+
// 零填充 YYYY-MM-DD 字典序双侧包含;from>to 配置错误不成立(编辑器校验拦截)
|
|
59
58
|
const dateRangeMatches = (condition, date) => {
|
|
60
59
|
const { from, to } = condition
|
|
61
60
|
if (typeof from !== 'string' || typeof to !== 'string') return false
|
package/src/query.js
CHANGED
|
@@ -98,6 +98,10 @@ const percentOf = (part, total) => (total === 0 ? 0 : (part / total) * PERCENT_S
|
|
|
98
98
|
|
|
99
99
|
const rowTokens = (row) => row.inputTokens + row.outputTokens + row.cacheReadTokens + row.cacheWriteTokens
|
|
100
100
|
|
|
101
|
+
// 速度配对分子:decode 口径取 decodeTokens;存量旧格式行(带时长无 decodeTokens)
|
|
102
|
+
// 回落 outputTokens,聚合随新数据自然收敛
|
|
103
|
+
const speedTokensOf = (row) => (row.durationMs ? row.decodeTokens ?? row.outputTokens : 0)
|
|
104
|
+
|
|
101
105
|
export function aggregateRange(rows, g, from, to) {
|
|
102
106
|
const form = BUCKET_FORMS[g]
|
|
103
107
|
const slots = enumerateBucketKeys(form)(from, to).map((key) => emptySlot(key))
|
|
@@ -105,43 +109,59 @@ export function aggregateRange(rows, g, from, to) {
|
|
|
105
109
|
const modelTotals = new Map()
|
|
106
110
|
const providerTotals = new Map()
|
|
107
111
|
const activeBuckets = new Set()
|
|
108
|
-
//
|
|
112
|
+
// 槽级配对:桶串 → 速度对 {decodeTokens, durationMs} 与首字对 {ttftMs, ttftSteps},
|
|
113
|
+
// 与模型级同口径(仅带配对数据的行计入)
|
|
109
114
|
const slotSpeeds = new Map()
|
|
115
|
+
const slotTtfts = new Map()
|
|
110
116
|
for (const row of rows) {
|
|
111
117
|
const slot = slotByKey.get(row.bucket)
|
|
112
118
|
// 桶串未落在枚举序列(如改粒度前的历史残行)不可归属,跳过防崩
|
|
113
119
|
if (!slot) continue
|
|
114
120
|
const tokens = rowTokens(row)
|
|
115
121
|
addRowToSlot(slot, row, tokens)
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
122
|
+
// 纯 timing 行(零 token 桶 + decode 配对)不参与归因,但仍进配对聚合
|
|
123
|
+
if (tokens > 0) {
|
|
124
|
+
activeBuckets.add(row.bucket)
|
|
125
|
+
slot.byModel[row.model] = (slot.byModel[row.model] ?? 0) + tokens
|
|
126
|
+
slot.byProvider[row.provider] = (slot.byProvider[row.provider] ?? 0) + tokens
|
|
127
|
+
}
|
|
120
128
|
if (row.durationMs) {
|
|
121
|
-
const pair = slotSpeeds.get(row.bucket) ?? {
|
|
122
|
-
pair.
|
|
129
|
+
const pair = slotSpeeds.get(row.bucket) ?? { decodeTokens: 0, durationMs: 0 }
|
|
130
|
+
pair.decodeTokens += speedTokensOf(row)
|
|
123
131
|
pair.durationMs += row.durationMs
|
|
124
132
|
slotSpeeds.set(row.bucket, pair)
|
|
125
133
|
}
|
|
134
|
+
if (row.ttftSteps > 0) {
|
|
135
|
+
const pair = slotTtfts.get(row.bucket) ?? { ttftMs: 0, ttftSteps: 0 }
|
|
136
|
+
pair.ttftMs += row.ttftMs ?? 0
|
|
137
|
+
pair.ttftSteps += row.ttftSteps
|
|
138
|
+
slotTtfts.set(row.bucket, pair)
|
|
139
|
+
}
|
|
126
140
|
const modelTotal = modelTotals.get(row.model)
|
|
127
141
|
if (modelTotal) {
|
|
128
142
|
modelTotal.tokens += tokens
|
|
129
143
|
modelTotal.speedDurationMs += row.durationMs ?? 0
|
|
130
|
-
modelTotal.speedOutputTokens += row
|
|
144
|
+
modelTotal.speedOutputTokens += speedTokensOf(row)
|
|
145
|
+
modelTotal.ttftMs += row.ttftMs ?? 0
|
|
146
|
+
modelTotal.ttftSteps += row.ttftSteps ?? 0
|
|
131
147
|
} else {
|
|
132
148
|
modelTotals.set(row.model, {
|
|
133
149
|
provider: row.provider,
|
|
134
150
|
tokens,
|
|
135
151
|
speedDurationMs: row.durationMs ?? 0,
|
|
136
|
-
speedOutputTokens: row
|
|
152
|
+
speedOutputTokens: speedTokensOf(row),
|
|
153
|
+
ttftMs: row.ttftMs ?? 0,
|
|
154
|
+
ttftSteps: row.ttftSteps ?? 0,
|
|
137
155
|
})
|
|
138
156
|
}
|
|
139
157
|
providerTotals.set(row.provider, (providerTotals.get(row.provider) ?? 0) + tokens)
|
|
140
158
|
}
|
|
141
|
-
// 槽级 speed
|
|
159
|
+
// 槽级 speed/ttft 条件挂:无配对数据的槽不挂字段(存量槽形契约不变)
|
|
142
160
|
for (const slot of slots) {
|
|
143
|
-
const
|
|
144
|
-
if (
|
|
161
|
+
const speedPair = slotSpeeds.get(slot.day)
|
|
162
|
+
if (speedPair && speedPair.durationMs > 0) slot.speed = speedPair.decodeTokens / (speedPair.durationMs / MS_PER_SECOND)
|
|
163
|
+
const ttftPair = slotTtfts.get(slot.day)
|
|
164
|
+
if (ttftPair && ttftPair.ttftSteps > 0) slot.ttft = ttftPair.ttftMs / ttftPair.ttftSteps
|
|
145
165
|
}
|
|
146
166
|
const totals = { tokens: 0, requests: 0, turns: 0, cacheHit: 0, cacheMiss: 0 }
|
|
147
167
|
for (const slot of slots) {
|
|
@@ -151,18 +171,22 @@ export function aggregateRange(rows, g, from, to) {
|
|
|
151
171
|
totals.cacheHit += slot.cacheHit
|
|
152
172
|
totals.cacheMiss += slot.cacheMiss
|
|
153
173
|
}
|
|
154
|
-
// speed =
|
|
155
|
-
//
|
|
174
|
+
// speed = decode 配对口径(decodeTokens ÷ 时长秒);ttft = 首 token 延迟
|
|
175
|
+
// 加权平均(毫秒);仅配对数据存在的条目挂字段,无数据条目不挂;
|
|
176
|
+
// 纯 timing 行可能产生 0-token 条目,列表保持只含 token 行(存量契约)
|
|
156
177
|
const models = [...modelTotals.entries()]
|
|
178
|
+
.filter(([, agg]) => agg.tokens > 0)
|
|
157
179
|
.map(([model, agg]) => ({
|
|
158
180
|
model,
|
|
159
181
|
provider: agg.provider,
|
|
160
182
|
tokens: agg.tokens,
|
|
161
183
|
percent: percentOf(agg.tokens, totals.tokens),
|
|
162
184
|
...(agg.speedDurationMs > 0 ? { speed: agg.speedOutputTokens / (agg.speedDurationMs / MS_PER_SECOND) } : {}),
|
|
185
|
+
...(agg.ttftSteps > 0 ? { ttft: agg.ttftMs / agg.ttftSteps } : {}),
|
|
163
186
|
}))
|
|
164
187
|
.sort((a, b) => b.tokens - a.tokens)
|
|
165
188
|
const providers = [...providerTotals.entries()]
|
|
189
|
+
.filter(([, tokens]) => tokens > 0)
|
|
166
190
|
.map(([provider, tokens]) => ({ provider, tokens, percent: percentOf(tokens, totals.tokens) }))
|
|
167
191
|
.sort((a, b) => b.tokens - a.tokens)
|
|
168
192
|
const truncated = slots.length > MAX_SLOTS
|
package/src/routes.js
CHANGED
|
@@ -274,7 +274,10 @@ const pricingHandler = (deps) => async (req, res) => {
|
|
|
274
274
|
}
|
|
275
275
|
|
|
276
276
|
const statusHandler = (deps) => async (req, res) => {
|
|
277
|
-
writeJson(res, {
|
|
277
|
+
writeJson(res, {
|
|
278
|
+
ok: true,
|
|
279
|
+
value: { ...deps.collector.status(), backup: deps.store.backupInfo() },
|
|
280
|
+
})
|
|
278
281
|
}
|
|
279
282
|
|
|
280
283
|
// 409 只拒 boot 期扫描;reset 自身引发的重扫同样 running 为真,经 rebuilding 放行
|
|
@@ -286,6 +289,15 @@ const resetHandler = (deps) => async (req, res) => {
|
|
|
286
289
|
writeJson(res, { ok: true, value: deps.collector.status() })
|
|
287
290
|
}
|
|
288
291
|
|
|
292
|
+
// 回退到最近一次重建(或恢复)前的快照;扫描进行中同样拒绝,避免写交错
|
|
293
|
+
const restoreHandler = (deps) => async (req, res) => {
|
|
294
|
+
if (deps.collector.running) {
|
|
295
|
+
throw usageError(HTTP_STATUS_CONFLICT, MESSAGE_BACKFILL_RUNNING)
|
|
296
|
+
}
|
|
297
|
+
const result = await deps.store.restoreFromBackup()
|
|
298
|
+
writeJson(res, { ok: true, value: result })
|
|
299
|
+
}
|
|
300
|
+
|
|
289
301
|
const STANDARD_GUARDS = [rejectWrongMethod, rejectCrossOrigin, rejectNonJson]
|
|
290
302
|
const PRICING_GUARDS = [rejectPricingMethod, rejectCrossOrigin]
|
|
291
303
|
|
|
@@ -295,6 +307,7 @@ const ENDPOINTS = [
|
|
|
295
307
|
{ path: `${ROUTE_PREFIX}/minutes`, mount: minuteHandler },
|
|
296
308
|
{ path: `${ROUTE_PREFIX}/status`, mount: statusHandler },
|
|
297
309
|
{ path: `${ROUTE_PREFIX}/reset`, mount: resetHandler },
|
|
310
|
+
{ path: `${ROUTE_PREFIX}/restore`, mount: restoreHandler },
|
|
298
311
|
{ path: `${ROUTE_PREFIX}/pricing`, mount: pricingHandler, guards: PRICING_GUARDS },
|
|
299
312
|
]
|
|
300
313
|
|