@mzzsfy/dsh-usage-dash 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/package.json +1 -1
- package/src/client.js +163 -40
- package/src/collector.js +117 -17
- package/src/query.js +38 -14
- package/src/store.js +16 -2
- package/test/client.test.mjs +57 -0
- package/test/collector.test.mjs +306 -11
- package/test/query.test.mjs +48 -0
- package/test/store.test.mjs +58 -0
- package/test/stream-parity.test.mjs +75 -0
- package/test/turn-tail.test.mjs +66 -0
package/src/query.js
CHANGED
|
@@ -98,6 +98,10 @@ const percentOf = (part, total) => (total === 0 ? 0 : (part / total) * PERCENT_S
|
|
|
98
98
|
|
|
99
99
|
const rowTokens = (row) => row.inputTokens + row.outputTokens + row.cacheReadTokens + row.cacheWriteTokens
|
|
100
100
|
|
|
101
|
+
// 速度配对分子:decode 口径取 decodeTokens;存量旧格式行(带时长无 decodeTokens)
|
|
102
|
+
// 回落 outputTokens,聚合随新数据自然收敛
|
|
103
|
+
const speedTokensOf = (row) => (row.durationMs ? row.decodeTokens ?? row.outputTokens : 0)
|
|
104
|
+
|
|
101
105
|
export function aggregateRange(rows, g, from, to) {
|
|
102
106
|
const form = BUCKET_FORMS[g]
|
|
103
107
|
const slots = enumerateBucketKeys(form)(from, to).map((key) => emptySlot(key))
|
|
@@ -105,43 +109,59 @@ export function aggregateRange(rows, g, from, to) {
|
|
|
105
109
|
const modelTotals = new Map()
|
|
106
110
|
const providerTotals = new Map()
|
|
107
111
|
const activeBuckets = new Set()
|
|
108
|
-
//
|
|
112
|
+
// 槽级配对:桶串 → 速度对 {decodeTokens, durationMs} 与首字对 {ttftMs, ttftSteps},
|
|
113
|
+
// 与模型级同口径(仅带配对数据的行计入)
|
|
109
114
|
const slotSpeeds = new Map()
|
|
115
|
+
const slotTtfts = new Map()
|
|
110
116
|
for (const row of rows) {
|
|
111
117
|
const slot = slotByKey.get(row.bucket)
|
|
112
118
|
// 桶串未落在枚举序列(如改粒度前的历史残行)不可归属,跳过防崩
|
|
113
119
|
if (!slot) continue
|
|
114
120
|
const tokens = rowTokens(row)
|
|
115
121
|
addRowToSlot(slot, row, tokens)
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
122
|
+
// 纯 timing 行(零 token 桶 + decode 配对)不参与归因,但仍进配对聚合
|
|
123
|
+
if (tokens > 0) {
|
|
124
|
+
activeBuckets.add(row.bucket)
|
|
125
|
+
slot.byModel[row.model] = (slot.byModel[row.model] ?? 0) + tokens
|
|
126
|
+
slot.byProvider[row.provider] = (slot.byProvider[row.provider] ?? 0) + tokens
|
|
127
|
+
}
|
|
120
128
|
if (row.durationMs) {
|
|
121
|
-
const pair = slotSpeeds.get(row.bucket) ?? {
|
|
122
|
-
pair.
|
|
129
|
+
const pair = slotSpeeds.get(row.bucket) ?? { decodeTokens: 0, durationMs: 0 }
|
|
130
|
+
pair.decodeTokens += speedTokensOf(row)
|
|
123
131
|
pair.durationMs += row.durationMs
|
|
124
132
|
slotSpeeds.set(row.bucket, pair)
|
|
125
133
|
}
|
|
134
|
+
if (row.ttftSteps > 0) {
|
|
135
|
+
const pair = slotTtfts.get(row.bucket) ?? { ttftMs: 0, ttftSteps: 0 }
|
|
136
|
+
pair.ttftMs += row.ttftMs ?? 0
|
|
137
|
+
pair.ttftSteps += row.ttftSteps
|
|
138
|
+
slotTtfts.set(row.bucket, pair)
|
|
139
|
+
}
|
|
126
140
|
const modelTotal = modelTotals.get(row.model)
|
|
127
141
|
if (modelTotal) {
|
|
128
142
|
modelTotal.tokens += tokens
|
|
129
143
|
modelTotal.speedDurationMs += row.durationMs ?? 0
|
|
130
|
-
modelTotal.speedOutputTokens += row
|
|
144
|
+
modelTotal.speedOutputTokens += speedTokensOf(row)
|
|
145
|
+
modelTotal.ttftMs += row.ttftMs ?? 0
|
|
146
|
+
modelTotal.ttftSteps += row.ttftSteps ?? 0
|
|
131
147
|
} else {
|
|
132
148
|
modelTotals.set(row.model, {
|
|
133
149
|
provider: row.provider,
|
|
134
150
|
tokens,
|
|
135
151
|
speedDurationMs: row.durationMs ?? 0,
|
|
136
|
-
speedOutputTokens: row
|
|
152
|
+
speedOutputTokens: speedTokensOf(row),
|
|
153
|
+
ttftMs: row.ttftMs ?? 0,
|
|
154
|
+
ttftSteps: row.ttftSteps ?? 0,
|
|
137
155
|
})
|
|
138
156
|
}
|
|
139
157
|
providerTotals.set(row.provider, (providerTotals.get(row.provider) ?? 0) + tokens)
|
|
140
158
|
}
|
|
141
|
-
// 槽级 speed
|
|
159
|
+
// 槽级 speed/ttft 条件挂:无配对数据的槽不挂字段(存量槽形契约不变)
|
|
142
160
|
for (const slot of slots) {
|
|
143
|
-
const
|
|
144
|
-
if (
|
|
161
|
+
const speedPair = slotSpeeds.get(slot.day)
|
|
162
|
+
if (speedPair && speedPair.durationMs > 0) slot.speed = speedPair.decodeTokens / (speedPair.durationMs / MS_PER_SECOND)
|
|
163
|
+
const ttftPair = slotTtfts.get(slot.day)
|
|
164
|
+
if (ttftPair && ttftPair.ttftSteps > 0) slot.ttft = ttftPair.ttftMs / ttftPair.ttftSteps
|
|
145
165
|
}
|
|
146
166
|
const totals = { tokens: 0, requests: 0, turns: 0, cacheHit: 0, cacheMiss: 0 }
|
|
147
167
|
for (const slot of slots) {
|
|
@@ -151,18 +171,22 @@ export function aggregateRange(rows, g, from, to) {
|
|
|
151
171
|
totals.cacheHit += slot.cacheHit
|
|
152
172
|
totals.cacheMiss += slot.cacheMiss
|
|
153
173
|
}
|
|
154
|
-
// speed =
|
|
155
|
-
//
|
|
174
|
+
// speed = decode 配对口径(decodeTokens ÷ 时长秒);ttft = 首 token 延迟
|
|
175
|
+
// 加权平均(毫秒);仅配对数据存在的条目挂字段,无数据条目不挂;
|
|
176
|
+
// 纯 timing 行可能产生 0-token 条目,列表保持只含 token 行(存量契约)
|
|
156
177
|
const models = [...modelTotals.entries()]
|
|
178
|
+
.filter(([, agg]) => agg.tokens > 0)
|
|
157
179
|
.map(([model, agg]) => ({
|
|
158
180
|
model,
|
|
159
181
|
provider: agg.provider,
|
|
160
182
|
tokens: agg.tokens,
|
|
161
183
|
percent: percentOf(agg.tokens, totals.tokens),
|
|
162
184
|
...(agg.speedDurationMs > 0 ? { speed: agg.speedOutputTokens / (agg.speedDurationMs / MS_PER_SECOND) } : {}),
|
|
185
|
+
...(agg.ttftSteps > 0 ? { ttft: agg.ttftMs / agg.ttftSteps } : {}),
|
|
163
186
|
}))
|
|
164
187
|
.sort((a, b) => b.tokens - a.tokens)
|
|
165
188
|
const providers = [...providerTotals.entries()]
|
|
189
|
+
.filter(([, tokens]) => tokens > 0)
|
|
166
190
|
.map(([provider, tokens]) => ({ provider, tokens, percent: percentOf(tokens, totals.tokens) }))
|
|
167
191
|
.sort((a, b) => b.tokens - a.tokens)
|
|
168
192
|
const truncated = slots.length > MAX_SLOTS
|
package/src/store.js
CHANGED
|
@@ -82,8 +82,12 @@ export const usageRowSchema = z.object({
|
|
|
82
82
|
outputTokens: z.number(),
|
|
83
83
|
cacheReadTokens: z.number(),
|
|
84
84
|
cacheWriteTokens: z.number(),
|
|
85
|
-
// 模型时长累计(毫秒)
|
|
85
|
+
// 模型时长累计(毫秒)与 decode 口径速度分子:optional 兼容存量记录(域 open 逐记录 parse)
|
|
86
86
|
durationMs: z.number().optional(),
|
|
87
|
+
decodeTokens: z.number().optional(),
|
|
88
|
+
// 首 token 延迟累计(毫秒)与样本步数:optional 兼容存量记录
|
|
89
|
+
ttftMs: z.number().optional(),
|
|
90
|
+
ttftSteps: z.number().optional(),
|
|
87
91
|
requests: z.number(),
|
|
88
92
|
turns: z.number(),
|
|
89
93
|
lastSeen: z.number(),
|
|
@@ -136,13 +140,17 @@ function emptyRow(bucket, provider, model, nowMs) {
|
|
|
136
140
|
cacheReadTokens: 0,
|
|
137
141
|
cacheWriteTokens: 0,
|
|
138
142
|
durationMs: 0,
|
|
143
|
+
decodeTokens: 0,
|
|
144
|
+
ttftMs: 0,
|
|
145
|
+
ttftSteps: 0,
|
|
139
146
|
requests: 0,
|
|
140
147
|
turns: 0,
|
|
141
148
|
lastSeen: nowMs,
|
|
142
149
|
}
|
|
143
150
|
}
|
|
144
151
|
|
|
145
|
-
// 样本折叠为计数增量:turn/request 只计次,token
|
|
152
|
+
// 样本折叠为计数增量:turn/request 只计次,token 样本累加四类桶;
|
|
153
|
+
// decodeTokens 与 durationMs 同源配对构成速度,timing 缺失按零累计
|
|
146
154
|
function deltaOf(sample, nowMs) {
|
|
147
155
|
const delta = emptyRow('', '', '', nowMs)
|
|
148
156
|
delete delta.bucket
|
|
@@ -156,6 +164,9 @@ function deltaOf(sample, nowMs) {
|
|
|
156
164
|
delta.cacheReadTokens = sample.cacheReadTokens
|
|
157
165
|
delta.cacheWriteTokens = sample.cacheWriteTokens
|
|
158
166
|
delta.durationMs = sample.durationMs ?? 0
|
|
167
|
+
delta.decodeTokens = sample.decodeTokens ?? 0
|
|
168
|
+
delta.ttftMs = sample.ttftMs ?? 0
|
|
169
|
+
delta.ttftSteps = sample.ttftMs !== undefined ? 1 : 0
|
|
159
170
|
}
|
|
160
171
|
return delta
|
|
161
172
|
}
|
|
@@ -170,6 +181,9 @@ function addDelta(base, delta) {
|
|
|
170
181
|
cacheWriteTokens: base.cacheWriteTokens + delta.cacheWriteTokens,
|
|
171
182
|
// base 侧 ?? 0 容存量旧格式行(缺字段);delta 侧经 deltaOf 恒为数值
|
|
172
183
|
durationMs: (base.durationMs ?? 0) + delta.durationMs,
|
|
184
|
+
decodeTokens: (base.decodeTokens ?? 0) + delta.decodeTokens,
|
|
185
|
+
ttftMs: (base.ttftMs ?? 0) + delta.ttftMs,
|
|
186
|
+
ttftSteps: (base.ttftSteps ?? 0) + delta.ttftSteps,
|
|
173
187
|
requests: base.requests + delta.requests,
|
|
174
188
|
turns: base.turns + delta.turns,
|
|
175
189
|
lastSeen: Math.max(base.lastSeen, delta.lastSeen),
|
package/test/client.test.mjs
CHANGED
|
@@ -90,6 +90,10 @@ const {
|
|
|
90
90
|
trendLayout,
|
|
91
91
|
trendRatePoints,
|
|
92
92
|
trendSpeedPoints,
|
|
93
|
+
trendTtftPoints,
|
|
94
|
+
ttftScaleMax,
|
|
95
|
+
ttftTipText,
|
|
96
|
+
modelTtftText,
|
|
93
97
|
speedTipText,
|
|
94
98
|
speedScaleMax,
|
|
95
99
|
legendToggle,
|
|
@@ -597,6 +601,44 @@ test('speedScaleMax 全零或空槽钳底防除零,混合取最大速度', () =>
|
|
|
597
601
|
assert.equal(speedScaleMax([{ day: 'd0', speed: 30 }, { day: 'd1', speed: 12.5 }]), 30)
|
|
598
602
|
})
|
|
599
603
|
|
|
604
|
+
test('trendTtftPoints 点映射列中心与刻度上限比例高度且无 ttft 槽跳过', () => {
|
|
605
|
+
const slots = [
|
|
606
|
+
{ day: 'd0', ttft: 2000 },
|
|
607
|
+
{ day: 'd1', total: 10 },
|
|
608
|
+
{ day: 'd2', ttft: 1000 },
|
|
609
|
+
]
|
|
610
|
+
const bars = [{ x: 10 }, { x: 20 }, { x: 30 }]
|
|
611
|
+
const plotHeight = CHART_HEIGHT - CHART_PAD.top - CHART_PAD.bottom
|
|
612
|
+
const points = trendTtftPoints(slots, bars, plotHeight, 2200)
|
|
613
|
+
assert.equal(points.length, 2)
|
|
614
|
+
assert.equal(points[0].day, 'd0')
|
|
615
|
+
assert.equal(points[0].x, 10)
|
|
616
|
+
approx(points[0].y, CHART_PAD.top + plotHeight - (2000 / 2200) * plotHeight)
|
|
617
|
+
assert.equal(points[1].day, 'd2')
|
|
618
|
+
assert.equal(points[1].x, 30)
|
|
619
|
+
approx(points[1].y, CHART_PAD.top + plotHeight - (1000 / 2200) * plotHeight)
|
|
620
|
+
})
|
|
621
|
+
|
|
622
|
+
test('ttftScaleMax 全零或空槽钳底防除零,混合取最大延迟', () => {
|
|
623
|
+
assert.equal(ttftScaleMax([]), 1)
|
|
624
|
+
assert.equal(ttftScaleMax([{ day: 'd0' }, { day: 'd1', ttft: 0 }]), 1)
|
|
625
|
+
assert.equal(ttftScaleMax([{ day: 'd0', ttft: 3000 }, { day: 'd1', ttft: 1250 }]), 3000)
|
|
626
|
+
})
|
|
627
|
+
|
|
628
|
+
test('ttftTipText 无 ttft 为占位符有 ttft 为官方时长口径', () => {
|
|
629
|
+
const zhT = createTranslator(MESSAGES_ZH)
|
|
630
|
+
const enT = createTranslator(MESSAGES_EN)
|
|
631
|
+
assert.equal(ttftTipText(undefined, zhT), '—')
|
|
632
|
+
assert.equal(ttftTipText(200, zhT), '0.2秒')
|
|
633
|
+
assert.equal(ttftTipText(200, enT), '0.2s')
|
|
634
|
+
assert.equal(ttftTipText(9500, zhT), '9.5秒')
|
|
635
|
+
})
|
|
636
|
+
|
|
637
|
+
test('ttftLegend 中英文案注册', () => {
|
|
638
|
+
assert.equal(MESSAGES_ZH.ttftLegend, '首 token 延迟')
|
|
639
|
+
assert.equal(MESSAGES_EN.ttftLegend, 'First-token latency')
|
|
640
|
+
})
|
|
641
|
+
|
|
600
642
|
test('smoothPath 空点集为空串单点为移动命令', () => {
|
|
601
643
|
assert.equal(smoothPath([]), '')
|
|
602
644
|
assert.equal(smoothPath([{ x: 1, y: 2 }]), 'M 1 2')
|
|
@@ -612,6 +654,14 @@ test('smoothPath 三点后段取真实邻点', () => {
|
|
|
612
654
|
assert.equal(d, 'M 0 0 C 10 5, 40 30, 60 30 C 80 30, 110 5, 120 0')
|
|
613
655
|
})
|
|
614
656
|
|
|
657
|
+
test('smoothPath 相邻点间距超 maxGap 折线断开成新段', () => {
|
|
658
|
+
const points = [{ x: 0, y: 0 }, { x: 60, y: 0 }, { x: 300, y: 0 }, { x: 360, y: 0 }]
|
|
659
|
+
const d = smoothPath(points, 90)
|
|
660
|
+
assert.equal(d, 'M 0 0 C 10 0, 10 0, 60 0 M 300 0 C 350 0, 350 0, 360 0')
|
|
661
|
+
// 无 maxGap 时不分段,行为与旧签名一致
|
|
662
|
+
assert.equal(smoothPath(points).includes('M 300'), false)
|
|
663
|
+
})
|
|
664
|
+
|
|
615
665
|
// —— S8 模型 donut 与列表 ——
|
|
616
666
|
|
|
617
667
|
test('donut 常量锁定视口与环几何', () => {
|
|
@@ -687,6 +737,13 @@ test('模型速度文本:官方吞吐口径格式化,无速度为空串', () =>
|
|
|
687
737
|
assert.equal(modelSpeedText(undefined), '')
|
|
688
738
|
})
|
|
689
739
|
|
|
740
|
+
test('模型首字文本:语言中立短时长,无 ttft 为空串', () => {
|
|
741
|
+
assert.equal(modelTtftText(200), 'TTFT 0.2s')
|
|
742
|
+
assert.equal(modelTtftText(162000), 'TTFT 2m42s')
|
|
743
|
+
assert.equal(modelTtftText(0), 'TTFT 0s')
|
|
744
|
+
assert.equal(modelTtftText(undefined), '')
|
|
745
|
+
})
|
|
746
|
+
|
|
690
747
|
// —— S14 费用格式化与展示辅助(镜像函数核心语义见 pricing-parity.test.mjs) ——
|
|
691
748
|
|
|
692
749
|
test('formatCost 千分位与两位小数', () => {
|
package/test/collector.test.mjs
CHANGED
|
@@ -17,6 +17,12 @@ const usage = (extra = {}) => ({
|
|
|
17
17
|
cacheWriteTokens: 4,
|
|
18
18
|
...extra,
|
|
19
19
|
})
|
|
20
|
+
// 官方助手流紧凑记录构造(与 dsh-llm AssistantStreamAccumulator.snapshot 同形):
|
|
21
|
+
// packed run 以 time0 + dt 差分还原成员时刻,裸 chunk 携带原始 time
|
|
22
|
+
const textRun = (time0, texts, dt = []) => ({ type: 'text-chunks', time0, index: 0, dt, texts })
|
|
23
|
+
const toolRun = (time0, args, extra = {}) => ({ type: 'tool-call-chunks', time0, index: 0, dt: [], id: 't1', args, ...extra })
|
|
24
|
+
const rawChunk = (time, chunk) => ({ type: 'chunk', time, chunk })
|
|
25
|
+
const attempt = (seq, time, turn, step, stream) => ev('assistant/attempt', seq, time, { turn, step, stream })
|
|
20
26
|
const session = (id) => ({ id })
|
|
21
27
|
const tick = () => new Promise((resolve) => setImmediate(resolve))
|
|
22
28
|
|
|
@@ -168,12 +174,12 @@ test('step/start 与 llm/retry-started 产生 request 标记样本', async () =>
|
|
|
168
174
|
}
|
|
169
175
|
})
|
|
170
176
|
|
|
171
|
-
test('
|
|
177
|
+
test('decode 口径:durationMs 为首 token 到汇报,ttftMs 为起点到首 token', async () => {
|
|
172
178
|
const { ctx, store } = liveCollector()
|
|
173
179
|
const start = local(2026, 8, 2, 10, 0)
|
|
174
|
-
const sampleAt = start + 6 * 1000 + 500
|
|
175
180
|
ctx.emit('session/event', session('s1'), ev('step/start', 1, start, { turn: 0, step: 0 }))
|
|
176
|
-
ctx.emit('session/event', session('s1'),
|
|
181
|
+
ctx.emit('session/event', session('s1'), attempt(2, start + 35 * 1000, 0, 0, [textRun(start + 10 * 1000, ['你好'])]))
|
|
182
|
+
ctx.emit('session/event', session('s1'), ev('assistant/message', 3, start + 40 * 1000, {
|
|
177
183
|
turn: 0,
|
|
178
184
|
step: 0,
|
|
179
185
|
usage: usage(),
|
|
@@ -181,7 +187,98 @@ test('step/start 观测起点:首个 usage 样本附 durationMs(start 到样本
|
|
|
181
187
|
}))
|
|
182
188
|
await tick()
|
|
183
189
|
const usageSample = store.samples.find((sample) => !sample.request)
|
|
184
|
-
assert.equal(usageSample.durationMs,
|
|
190
|
+
assert.equal(usageSample.durationMs, 30 * 1000)
|
|
191
|
+
assert.equal(usageSample.ttftMs, 10 * 1000)
|
|
192
|
+
assert.equal(usageSample.decodeTokens, 20)
|
|
193
|
+
assert.equal(usageSample.outputTokens, 20)
|
|
194
|
+
})
|
|
195
|
+
|
|
196
|
+
test('无 attempt 时回落 message 自带 stream 取首 token', async () => {
|
|
197
|
+
const { ctx, store } = liveCollector()
|
|
198
|
+
const start = local(2026, 8, 2, 10, 0)
|
|
199
|
+
ctx.emit('session/event', session('s1'), ev('step/start', 1, start, { turn: 0, step: 0 }))
|
|
200
|
+
ctx.emit('session/event', session('s1'), ev('assistant/message', 2, start + 40 * 1000, {
|
|
201
|
+
turn: 0,
|
|
202
|
+
step: 0,
|
|
203
|
+
usage: usage(),
|
|
204
|
+
stream: [textRun(start + 8 * 1000, ['好'])],
|
|
205
|
+
message: {},
|
|
206
|
+
}))
|
|
207
|
+
await tick()
|
|
208
|
+
const usageSample = store.samples.find((sample) => !sample.request)
|
|
209
|
+
assert.equal(usageSample.durationMs, 32 * 1000)
|
|
210
|
+
assert.equal(usageSample.ttftMs, 8 * 1000)
|
|
211
|
+
})
|
|
212
|
+
|
|
213
|
+
test('chunk 先发 token 契约保持,message 补发纯 timing 增量样本', async () => {
|
|
214
|
+
const { ctx, store } = liveCollector()
|
|
215
|
+
const start = local(2026, 8, 2, 10, 0)
|
|
216
|
+
ctx.emit('session/event', session('s1'), ev('step/start', 1, start, { turn: 0, step: 0 }))
|
|
217
|
+
ctx.emit('session/event', session('s1'), ev('assistant/chunk', 2, start + 30 * 1000, {
|
|
218
|
+
turn: 0,
|
|
219
|
+
step: 0,
|
|
220
|
+
chunk: { type: 'usage', usage: usage() },
|
|
221
|
+
}))
|
|
222
|
+
ctx.emit('session/event', session('s1'), attempt(3, start + 35 * 1000, 0, 0, [textRun(start + 10 * 1000, ['你好'])]))
|
|
223
|
+
ctx.emit('session/event', session('s1'), ev('assistant/message', 4, start + 40 * 1000, {
|
|
224
|
+
turn: 0,
|
|
225
|
+
step: 0,
|
|
226
|
+
usage: usage({ inputTokens: 999 }),
|
|
227
|
+
message: {},
|
|
228
|
+
}))
|
|
229
|
+
await tick()
|
|
230
|
+
assert.equal(store.samples.length, 3)
|
|
231
|
+
const chunkSample = store.samples[1]
|
|
232
|
+
assert.equal(chunkSample.inputTokens, 10)
|
|
233
|
+
assert.equal(chunkSample.durationMs, undefined)
|
|
234
|
+
assert.equal(chunkSample.ttftMs, undefined)
|
|
235
|
+
assert.equal(chunkSample.decodeTokens, undefined)
|
|
236
|
+
assert.equal(store.samples[0].request, true)
|
|
237
|
+
const timingSample = store.samples[2]
|
|
238
|
+
assert.equal(timingSample.inputTokens, 0)
|
|
239
|
+
assert.equal(timingSample.outputTokens, 0)
|
|
240
|
+
assert.equal(timingSample.decodeTokens, 20)
|
|
241
|
+
assert.equal(timingSample.durationMs, 30 * 1000)
|
|
242
|
+
assert.equal(timingSample.ttftMs, 10 * 1000)
|
|
243
|
+
})
|
|
244
|
+
|
|
245
|
+
test('attempt 首 token 锁定:后续 attempt 不覆盖首个产出 token 的时刻', async () => {
|
|
246
|
+
const { ctx, store } = liveCollector()
|
|
247
|
+
const start = local(2026, 8, 2, 10, 0)
|
|
248
|
+
ctx.emit('session/event', session('s1'), ev('step/start', 1, start, { turn: 0, step: 0 }))
|
|
249
|
+
ctx.emit('session/event', session('s1'), attempt(2, start + 35 * 1000, 0, 0, [textRun(start + 10 * 1000, ['a'])]))
|
|
250
|
+
ctx.emit('session/event', session('s1'), attempt(3, start + 38 * 1000, 0, 0, [textRun(start + 20 * 1000, ['b'])]))
|
|
251
|
+
ctx.emit('session/event', session('s1'), ev('assistant/message', 4, start + 40 * 1000, {
|
|
252
|
+
turn: 0,
|
|
253
|
+
step: 0,
|
|
254
|
+
usage: usage(),
|
|
255
|
+
message: {},
|
|
256
|
+
}))
|
|
257
|
+
await tick()
|
|
258
|
+
const usageSample = store.samples.find((sample) => !sample.request)
|
|
259
|
+
assert.equal(usageSample.ttftMs, 10 * 1000)
|
|
260
|
+
assert.equal(usageSample.durationMs, 30 * 1000)
|
|
261
|
+
})
|
|
262
|
+
|
|
263
|
+
test('首个 attempt 无 token,由后续产出 token 的 attempt 锁定', async () => {
|
|
264
|
+
const { ctx, store } = liveCollector()
|
|
265
|
+
const start = local(2026, 8, 2, 10, 0)
|
|
266
|
+
ctx.emit('session/event', session('s1'), ev('step/start', 1, start, { turn: 0, step: 0 }))
|
|
267
|
+
ctx.emit('session/event', session('s1'), attempt(2, start + 5 * 1000, 0, 0, [
|
|
268
|
+
textRun(start + 5 * 1000, ['']),
|
|
269
|
+
rawChunk(start + 6 * 1000, { type: 'usage' }),
|
|
270
|
+
]))
|
|
271
|
+
ctx.emit('session/event', session('s1'), attempt(3, start + 38 * 1000, 0, 0, [toolRun(start + 15 * 1000, ['{'], { name: 'fn' })]))
|
|
272
|
+
ctx.emit('session/event', session('s1'), ev('assistant/message', 4, start + 40 * 1000, {
|
|
273
|
+
turn: 0,
|
|
274
|
+
step: 0,
|
|
275
|
+
usage: usage(),
|
|
276
|
+
message: {},
|
|
277
|
+
}))
|
|
278
|
+
await tick()
|
|
279
|
+
const usageSample = store.samples.find((sample) => !sample.request)
|
|
280
|
+
assert.equal(usageSample.ttftMs, 15 * 1000)
|
|
281
|
+
assert.equal(usageSample.durationMs, 25 * 1000)
|
|
185
282
|
})
|
|
186
283
|
|
|
187
284
|
test('(session,turn,step) 去重:同键先到样本生效,重复报告吞掉', async () => {
|
|
@@ -217,14 +314,194 @@ test('无 step/start 起点的 usage 样本不带 durationMs', async () => {
|
|
|
217
314
|
assert.equal(store.samples[0].durationMs, undefined)
|
|
218
315
|
})
|
|
219
316
|
|
|
220
|
-
test('retry
|
|
317
|
+
test('retry 不重置起点:TTFT 含失败尝试,decode 段自成功首 token 起', async () => {
|
|
221
318
|
const { ctx, store } = liveCollector()
|
|
222
319
|
const start = local(2026, 8, 2, 10, 0)
|
|
223
320
|
const retryAt = start + 60 * 1000
|
|
224
|
-
const sampleAt = retryAt + 5 * 1000
|
|
225
321
|
ctx.emit('session/event', session('s1'), ev('step/start', 1, start, { turn: 0, step: 0 }))
|
|
226
322
|
ctx.emit('session/event', session('s1'), ev('llm/retry-started', 2, retryAt, { turn: 0, step: 0 }))
|
|
227
|
-
ctx.emit('session/event', session('s1'),
|
|
323
|
+
ctx.emit('session/event', session('s1'), attempt(3, retryAt + 3 * 1000, 0, 0, [textRun(retryAt + 5 * 1000, ['好'])]))
|
|
324
|
+
ctx.emit('session/event', session('s1'), ev('assistant/message', 4, retryAt + 35 * 1000, {
|
|
325
|
+
turn: 0,
|
|
326
|
+
step: 0,
|
|
327
|
+
usage: usage(),
|
|
328
|
+
message: {},
|
|
329
|
+
}))
|
|
330
|
+
await tick()
|
|
331
|
+
const usageSample = store.samples.find((sample) => !sample.request)
|
|
332
|
+
assert.equal(usageSample.ttftMs, 65 * 1000)
|
|
333
|
+
assert.equal(usageSample.durationMs, 30 * 1000)
|
|
334
|
+
})
|
|
335
|
+
|
|
336
|
+
test('零 TTFT(起点即出 token)附 ttftMs 为 0', async () => {
|
|
337
|
+
const { ctx, store } = liveCollector()
|
|
338
|
+
const start = local(2026, 8, 2, 10, 0)
|
|
339
|
+
ctx.emit('session/event', session('s1'), ev('step/start', 1, start, { turn: 0, step: 0 }))
|
|
340
|
+
ctx.emit('session/event', session('s1'), attempt(2, start + 30 * 1000, 0, 0, [textRun(start, ['快'])]))
|
|
341
|
+
ctx.emit('session/event', session('s1'), ev('assistant/message', 3, start + 30 * 1000, {
|
|
342
|
+
turn: 0,
|
|
343
|
+
step: 0,
|
|
344
|
+
usage: usage(),
|
|
345
|
+
message: {},
|
|
346
|
+
}))
|
|
347
|
+
await tick()
|
|
348
|
+
const usageSample = store.samples.find((sample) => !sample.request)
|
|
349
|
+
assert.equal(usageSample.ttftMs, 0)
|
|
350
|
+
assert.equal(usageSample.durationMs, 30 * 1000)
|
|
351
|
+
})
|
|
352
|
+
|
|
353
|
+
test('时刻倒挂防回拨:首 token 晚于汇报,durationMs 钳 0 不附,ttftMs 照常', async () => {
|
|
354
|
+
const { ctx, store } = liveCollector()
|
|
355
|
+
const start = local(2026, 8, 2, 10, 0)
|
|
356
|
+
ctx.emit('session/event', session('s1'), ev('step/start', 1, start, { turn: 0, step: 0 }))
|
|
357
|
+
ctx.emit('session/event', session('s1'), attempt(2, start + 50 * 1000, 0, 0, [textRun(start + 50 * 1000, ['晚'])]))
|
|
358
|
+
ctx.emit('session/event', session('s1'), ev('assistant/message', 3, start + 3 * 1000, {
|
|
359
|
+
turn: 0,
|
|
360
|
+
step: 0,
|
|
361
|
+
usage: usage(),
|
|
362
|
+
message: {},
|
|
363
|
+
}))
|
|
364
|
+
await tick()
|
|
365
|
+
const usageSample = store.samples.find((sample) => !sample.request)
|
|
366
|
+
assert.equal(usageSample.durationMs, undefined)
|
|
367
|
+
assert.equal(usageSample.decodeTokens, undefined)
|
|
368
|
+
assert.equal(usageSample.ttftMs, 50 * 1000)
|
|
369
|
+
})
|
|
370
|
+
|
|
371
|
+
test('时刻倒挂防回拨:首 token 早于起点,ttftMs 钳 0 照常计步,durationMs 照常', async () => {
|
|
372
|
+
const { ctx, store } = liveCollector()
|
|
373
|
+
const start = local(2026, 8, 2, 10, 0)
|
|
374
|
+
ctx.emit('session/event', session('s1'), ev('step/start', 1, start, { turn: 0, step: 0 }))
|
|
375
|
+
ctx.emit('session/event', session('s1'), attempt(2, start + 1 * 1000, 0, 0, [textRun(start - 5 * 1000, ['回拨'])]))
|
|
376
|
+
ctx.emit('session/event', session('s1'), ev('assistant/message', 3, start + 3 * 1000, {
|
|
377
|
+
turn: 0,
|
|
378
|
+
step: 0,
|
|
379
|
+
usage: usage(),
|
|
380
|
+
message: {},
|
|
381
|
+
}))
|
|
382
|
+
await tick()
|
|
383
|
+
const usageSample = store.samples.find((sample) => !sample.request)
|
|
384
|
+
assert.equal(usageSample.ttftMs, 0)
|
|
385
|
+
assert.equal(usageSample.durationMs, 8 * 1000)
|
|
386
|
+
})
|
|
387
|
+
|
|
388
|
+
test('usage 缺 outputTokens:不建 decode 配对(官方守卫),ttft 照常', async () => {
|
|
389
|
+
const { ctx, store } = liveCollector()
|
|
390
|
+
const start = local(2026, 8, 2, 10, 0)
|
|
391
|
+
ctx.emit('session/event', session('s1'), ev('step/start', 1, start, { turn: 0, step: 0 }))
|
|
392
|
+
ctx.emit('session/event', session('s1'), attempt(2, start + 30 * 1000, 0, 0, [textRun(start + 10 * 1000, ['早'])]))
|
|
393
|
+
ctx.emit('session/event', session('s1'), ev('assistant/message', 3, start + 40 * 1000, {
|
|
394
|
+
turn: 0,
|
|
395
|
+
step: 0,
|
|
396
|
+
usage: usage({ outputTokens: undefined }),
|
|
397
|
+
message: {},
|
|
398
|
+
}))
|
|
399
|
+
await tick()
|
|
400
|
+
const usageSample = store.samples.find((sample) => !sample.request)
|
|
401
|
+
assert.equal(usageSample.outputTokens, 0)
|
|
402
|
+
assert.equal(usageSample.durationMs, undefined)
|
|
403
|
+
assert.equal(usageSample.decodeTokens, undefined)
|
|
404
|
+
assert.equal(usageSample.ttftMs, 10 * 1000)
|
|
405
|
+
})
|
|
406
|
+
|
|
407
|
+
test('重复报告不重复补发 timing:chunk → message → 重复 message 恰一条增量', async () => {
|
|
408
|
+
const { ctx, store } = liveCollector()
|
|
409
|
+
const start = local(2026, 8, 2, 10, 0)
|
|
410
|
+
ctx.emit('session/event', session('s1'), ev('step/start', 1, start, { turn: 0, step: 0 }))
|
|
411
|
+
ctx.emit('session/event', session('s1'), ev('assistant/chunk', 2, start + 30 * 1000, {
|
|
412
|
+
turn: 0,
|
|
413
|
+
step: 0,
|
|
414
|
+
chunk: { type: 'usage', usage: usage() },
|
|
415
|
+
}))
|
|
416
|
+
ctx.emit('session/event', session('s1'), attempt(3, start + 35 * 1000, 0, 0, [textRun(start + 10 * 1000, ['好'])]))
|
|
417
|
+
ctx.emit('session/event', session('s1'), ev('assistant/message', 4, start + 40 * 1000, {
|
|
418
|
+
turn: 0,
|
|
419
|
+
step: 0,
|
|
420
|
+
usage: usage({ inputTokens: 999 }),
|
|
421
|
+
message: {},
|
|
422
|
+
}))
|
|
423
|
+
ctx.emit('session/event', session('s1'), ev('assistant/message', 5, start + 41 * 1000, {
|
|
424
|
+
turn: 0,
|
|
425
|
+
step: 0,
|
|
426
|
+
usage: usage(),
|
|
427
|
+
message: {},
|
|
428
|
+
}))
|
|
429
|
+
await tick()
|
|
430
|
+
const timingSamples = store.samples.filter((sample) => !sample.request && sample.decodeTokens !== undefined)
|
|
431
|
+
assert.equal(timingSamples.length, 1)
|
|
432
|
+
assert.equal(timingSamples[0].decodeTokens, 20)
|
|
433
|
+
assert.equal(timingSamples[0].durationMs, 30 * 1000)
|
|
434
|
+
assert.equal(timingSamples[0].ttftMs, 10 * 1000)
|
|
435
|
+
})
|
|
436
|
+
|
|
437
|
+
test('message 首发后到达的 chunk usage 不补 timing', async () => {
|
|
438
|
+
const { ctx, store } = liveCollector()
|
|
439
|
+
const start = local(2026, 8, 2, 10, 0)
|
|
440
|
+
ctx.emit('session/event', session('s1'), ev('step/start', 1, start, { turn: 0, step: 0 }))
|
|
441
|
+
ctx.emit('session/event', session('s1'), attempt(2, start + 35 * 1000, 0, 0, [textRun(start + 10 * 1000, ['好'])]))
|
|
442
|
+
ctx.emit('session/event', session('s1'), ev('assistant/message', 3, start + 40 * 1000, {
|
|
443
|
+
turn: 0,
|
|
444
|
+
step: 0,
|
|
445
|
+
usage: usage(),
|
|
446
|
+
message: {},
|
|
447
|
+
}))
|
|
448
|
+
ctx.emit('session/event', session('s1'), ev('assistant/chunk', 4, start + 41 * 1000, {
|
|
449
|
+
turn: 0,
|
|
450
|
+
step: 0,
|
|
451
|
+
chunk: { type: 'usage', usage: usage() },
|
|
452
|
+
}))
|
|
453
|
+
await tick()
|
|
454
|
+
const usageSamples = store.samples.filter((sample) => !sample.request)
|
|
455
|
+
assert.equal(usageSamples.length, 1)
|
|
456
|
+
assert.equal(usageSamples[0].decodeTokens, 20)
|
|
457
|
+
assert.equal(usageSamples[0].durationMs, 30 * 1000)
|
|
458
|
+
})
|
|
459
|
+
|
|
460
|
+
test('chunk 非零后 message usage 全零仍补发 timing', async () => {
|
|
461
|
+
const { ctx, store } = liveCollector()
|
|
462
|
+
const start = local(2026, 8, 2, 10, 0)
|
|
463
|
+
const zeroUsage = { inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0 }
|
|
464
|
+
ctx.emit('session/event', session('s1'), ev('step/start', 1, start, { turn: 0, step: 0 }))
|
|
465
|
+
ctx.emit('session/event', session('s1'), ev('assistant/chunk', 2, start + 30 * 1000, {
|
|
466
|
+
turn: 0,
|
|
467
|
+
step: 0,
|
|
468
|
+
chunk: { type: 'usage', usage: usage() },
|
|
469
|
+
}))
|
|
470
|
+
ctx.emit('session/event', session('s1'), attempt(3, start + 35 * 1000, 0, 0, [textRun(start + 10 * 1000, ['好'])]))
|
|
471
|
+
ctx.emit('session/event', session('s1'), ev('assistant/message', 4, start + 40 * 1000, {
|
|
472
|
+
turn: 0,
|
|
473
|
+
step: 0,
|
|
474
|
+
usage: zeroUsage,
|
|
475
|
+
message: {},
|
|
476
|
+
}))
|
|
477
|
+
await tick()
|
|
478
|
+
const timingSamples = store.samples.filter((sample) => !sample.request && sample.ttftMs !== undefined)
|
|
479
|
+
assert.equal(timingSamples.length, 1)
|
|
480
|
+
assert.equal(timingSamples[0].inputTokens, 0)
|
|
481
|
+
assert.equal(timingSamples[0].ttftMs, 10 * 1000)
|
|
482
|
+
assert.equal(timingSamples[0].decodeTokens, 0)
|
|
483
|
+
assert.equal(timingSamples[0].durationMs, 30 * 1000)
|
|
484
|
+
})
|
|
485
|
+
|
|
486
|
+
test('缺 turn/step 的 message 带 stream:首发全量样本含 timing 且不污染去重集合', async () => {
|
|
487
|
+
const { ctx, store } = liveCollector()
|
|
488
|
+
const t = local(2026, 8, 2, 10, 0)
|
|
489
|
+
ctx.emit('session/event', session('s1'), ev('assistant/message', 1, t + 30 * 1000, {
|
|
490
|
+
usage: usage(),
|
|
491
|
+
stream: [textRun(t, ['早'])],
|
|
492
|
+
message: {},
|
|
493
|
+
}))
|
|
494
|
+
await tick()
|
|
495
|
+
assert.equal(store.samples.length, 1)
|
|
496
|
+
assert.equal(store.samples[0].durationMs, 30 * 1000)
|
|
497
|
+
assert.equal(store.samples[0].ttftMs, undefined)
|
|
498
|
+
})
|
|
499
|
+
|
|
500
|
+
test('有首 token 无起点:durationMs 附而 ttftMs 不附', async () => {
|
|
501
|
+
const { ctx, store } = liveCollector()
|
|
502
|
+
const t = local(2026, 8, 2, 10, 0)
|
|
503
|
+
ctx.emit('session/event', session('s1'), attempt(1, t + 10 * 1000, 0, 0, [textRun(t, ['早'])]))
|
|
504
|
+
ctx.emit('session/event', session('s1'), ev('assistant/message', 2, t + 30 * 1000, {
|
|
228
505
|
turn: 0,
|
|
229
506
|
step: 0,
|
|
230
507
|
usage: usage(),
|
|
@@ -232,7 +509,23 @@ test('retry-started 重置起点:时长只含最终尝试', async () => {
|
|
|
232
509
|
}))
|
|
233
510
|
await tick()
|
|
234
511
|
const usageSample = store.samples.find((sample) => !sample.request)
|
|
235
|
-
assert.equal(usageSample.durationMs,
|
|
512
|
+
assert.equal(usageSample.durationMs, 30 * 1000)
|
|
513
|
+
assert.equal(usageSample.ttftMs, undefined)
|
|
514
|
+
})
|
|
515
|
+
|
|
516
|
+
test('无任何 stream 信息不附时长与首字(存量超旧日志形态)', async () => {
|
|
517
|
+
const { ctx, store } = liveCollector()
|
|
518
|
+
const t = local(2026, 8, 2, 10, 0)
|
|
519
|
+
ctx.emit('session/event', session('s1'), ev('assistant/message', 1, t, {
|
|
520
|
+
turn: 0,
|
|
521
|
+
step: 0,
|
|
522
|
+
usage: usage(),
|
|
523
|
+
message: {},
|
|
524
|
+
}))
|
|
525
|
+
await tick()
|
|
526
|
+
assert.equal(store.samples.length, 1)
|
|
527
|
+
assert.equal(store.samples[0].durationMs, undefined)
|
|
528
|
+
assert.equal(store.samples[0].ttftMs, undefined)
|
|
236
529
|
})
|
|
237
530
|
|
|
238
531
|
test('同时刻零时长不附 durationMs,request 标记样本不带时长', async () => {
|
|
@@ -519,14 +812,15 @@ test('回扫:无边界会话全量重放,归因与游标写回', async () => {
|
|
|
519
812
|
assert.equal(collector.status().scannedSessions, 1)
|
|
520
813
|
})
|
|
521
814
|
|
|
522
|
-
test('
|
|
815
|
+
test('回扫:重放路径同样产出 decode 口径时长与首字', async () => {
|
|
523
816
|
const start = local(2026, 8, 2, 14, 0)
|
|
524
817
|
const persistence = fakePersistence([{
|
|
525
818
|
id: 's1',
|
|
526
819
|
events: [
|
|
527
820
|
ev('request/context', 0, start, { provider: 'deepseek', model: 'chat' }),
|
|
528
821
|
ev('step/start', 1, start, { turn: 0, step: 0 }),
|
|
529
|
-
|
|
822
|
+
attempt(2, start + 5 * 1000, 0, 0, [textRun(start + 2 * 1000, ['回'])]),
|
|
823
|
+
ev('assistant/message', 3, start + 7 * 1000, {
|
|
530
824
|
turn: 0,
|
|
531
825
|
step: 0,
|
|
532
826
|
usage: usage(),
|
|
@@ -538,7 +832,8 @@ test('回扫:重放路径同样附加 durationMs', async () => {
|
|
|
538
832
|
const collector = new UsageCollector(fakeCtx({ persistence }), store)
|
|
539
833
|
await collector.backfill(persistence, fakeSessions())
|
|
540
834
|
const usageSample = store.samples.find((sample) => !sample.request)
|
|
541
|
-
assert.equal(usageSample.durationMs,
|
|
835
|
+
assert.equal(usageSample.durationMs, 5 * 1000)
|
|
836
|
+
assert.equal(usageSample.ttftMs, 2 * 1000)
|
|
542
837
|
})
|
|
543
838
|
|
|
544
839
|
test('回扫:已入游标会话不再重扫', async () => {
|