dsh-session-matrix 0.16.5 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -5
- package/lib/client.js +19 -8
- package/lib/index.js +125 -13
- package/lib/usage-split.js +132 -17
- package/package.json +4 -3
- package/scripts/make-reconcile-fixture.mjs +89 -0
- package/test/fixtures/reconcile.json +3063 -0
- package/test/risk-matrix.test.mjs +97 -0
- package/test/usage-reconcile.test.mjs +71 -0
- package/test/usage-split.test.mjs +46 -0
package/README.md
CHANGED
|
@@ -10,6 +10,7 @@
|
|
|
10
10
|
<a href="https://www.npmjs.com/package/dsh-session-matrix"><img src="https://img.shields.io/npm/v/dsh-session-matrix?label=npm&color=blue" alt="npm"></a>
|
|
11
11
|
<a href="https://github.com/Wanqiu12345/dsh-session-matrix/actions/workflows/ci.yml"><img src="https://github.com/Wanqiu12345/dsh-session-matrix/actions/workflows/ci.yml/badge.svg" alt="CI"></a>
|
|
12
12
|
<a href="./LICENSE"><img src="https://img.shields.io/badge/license-MIT-green" alt="MIT"></a>
|
|
13
|
+
<a href="https://dsh.market/"><img src="https://raw.githubusercontent.com/2BingLing/dsh-market/master/assets/readme/badge-listed-zh.svg" alt="DSH Market"></a>
|
|
13
14
|
<img src="https://img.shields.io/badge/DSH-%E2%89%A50.2.2--rc.1-8a2be2" alt="DSH >= 0.2.2-rc.1">
|
|
14
15
|
<img src="https://img.shields.io/badge/runtime%20deps-0-brightgreen" alt="no runtime deps">
|
|
15
16
|
</p>
|
|
@@ -27,7 +28,7 @@
|
|
|
27
28
|
<a href="#vs-context"><b>与 dsh-context</b></a>
|
|
28
29
|
</p>
|
|
29
30
|
|
|
30
|
-

|
|
31
32
|
|
|
32
33
|
---
|
|
33
34
|
|
|
@@ -64,7 +65,9 @@
|
|
|
64
65
|
- 每格挂载的是 DSH 原生 `conversation.content` 工厂实例:消息流、工具调用行、审批、
|
|
65
66
|
Chat/Trajectory 切换、composer 全是原生的,**可以直接打字**。
|
|
66
67
|
|
|
67
|
-

|
|
69
|
+
|
|
70
|
+

|
|
68
71
|
|
|
69
72
|
<a id="graph"></a>
|
|
70
73
|
|
|
@@ -81,7 +84,7 @@
|
|
|
81
84
|
- **节点尺寸 = 累计用量**,谁最重一眼能看出来。
|
|
82
85
|
- 选中任一节点,下方三个模块按该节点的子树重新计算。
|
|
83
86
|
|
|
84
|
-

|
|
85
88
|
|
|
86
89
|
### 三个模块
|
|
87
90
|
|
|
@@ -93,7 +96,7 @@
|
|
|
93
96
|
|
|
94
97
|
「当前占用」和「合计」不是同一个量:前者是此刻占着多少,后者是历史上一共读过多少。
|
|
95
98
|
|
|
96
|
-

|
|
97
100
|
|
|
98
101
|
## 上下文监控
|
|
99
102
|
|
|
@@ -115,6 +118,20 @@ pressureBudget = messageBudget − policy.headroomTokens # headroom 默认
|
|
|
115
118
|
占用率取**服务端返回的 prompt 大小**(未命中 + 缓存读 + 缓存写),叠加最后一次采样后的增量,
|
|
116
119
|
不使用字符数估算。
|
|
117
120
|
|
|
121
|
+
### 占用率之外:压缩次数才是「要不要交接」的依据
|
|
122
|
+
|
|
123
|
+
占用率只回答「离压缩点还有多远」;是否值得交接,看的是这个会话**已经压缩过几次**。
|
|
124
|
+
DSH 每次压缩都会在会话里留下一个检查点(`compact-checkpoint`),插件从会话日志里把它们数出来:
|
|
125
|
+
|
|
126
|
+
| 压缩次数 | 到达临界线时 |
|
|
127
|
+
|:--|:--|
|
|
128
|
+
| **0 次** | 标「首次接近」——历史完整,会话仍可用,不推交接 |
|
|
129
|
+
| **1–2 次** | 标「临界」——建议交接:细节损失已经开始叠加 |
|
|
130
|
+
| **≥3 次** | 一过注意线就升级「临界」——社区共识:两次压缩即任务范围失控 |
|
|
131
|
+
|
|
132
|
+
刚压缩完 10 分钟内摘要最新鲜,提示自动降半级;日志尚未解析到的会话如实显示
|
|
133
|
+
「压缩次数未知」,不猜。
|
|
134
|
+
|
|
118
135
|
<a id="handoff"></a>
|
|
119
136
|
|
|
120
137
|
## 交接协议 v1
|
|
@@ -133,7 +150,7 @@ pressureBudget = messageBudget − policy.headroomTokens # headroom 默认
|
|
|
133
150
|
|
|
134
151
|
插件负责复制与定位:目标窗格会闪烁一次,避免粘错会话。
|
|
135
152
|
|
|
136
|
-

|
|
137
154
|
|
|
138
155
|
## 费用口径
|
|
139
156
|
|
package/lib/client.js
CHANGED
|
@@ -1977,9 +1977,13 @@ window.__ModuleLoader__.load({
|
|
|
1977
1977
|
risk ? h('span', {
|
|
1978
1978
|
key: 'rk',
|
|
1979
1979
|
className: `aw-risk ${risk} tip-r`,
|
|
1980
|
-
'data-tip':
|
|
1981
|
-
?
|
|
1982
|
-
:
|
|
1980
|
+
'data-tip': session.firstApproach
|
|
1981
|
+
? `首次接近临界线(压缩点 × ${Math.round((th?.criticalFactor ?? DEFAULT_FACTORS.criticalFactor) * 100)}%):还没有压缩过,历史完整可用;建议收尾当前这一步后观察`
|
|
1982
|
+
: risk === 'crit'
|
|
1983
|
+
? `已到临界线,建议交接${typeof session.compactions === 'number' && session.compactions > 0
|
|
1984
|
+
? `(已压缩 ${session.compactions} 次,细节损失叠加)` : ''}`
|
|
1985
|
+
: `接近压缩点(压缩点 × ${Math.round((th?.warnFactor ?? DEFAULT_FACTORS.warnFactor) * 100)}%)${typeof session.compactions === 'number' && session.compactions > 0
|
|
1986
|
+
? `,已压缩 ${session.compactions} 次` : ''}:再增长一段就会触发自动压缩`,
|
|
1983
1987
|
}, risk === 'crit' ? T.levelCrit : T.levelWarn) : null,
|
|
1984
1988
|
h('button', {
|
|
1985
1989
|
key: 'open',
|
|
@@ -2599,6 +2603,10 @@ window.__ModuleLoader__.load({
|
|
|
2599
2603
|
h('span', { key: 'h', className: 'aw-hcard-h' }, [
|
|
2600
2604
|
h('span', { key: 't', className: 't' }, s.title || T.noTitle),
|
|
2601
2605
|
paneIds.includes(s.id) ? h('span', { key: 'g', className: 'tag' }, '已载入') : null,
|
|
2606
|
+
s.firstApproach ? h('span', { key: 'fa', className: 'tag' }, '首次接近 · 未压缩') : null,
|
|
2607
|
+
typeof s.compactions === 'number' && s.compactions > 0
|
|
2608
|
+
? h('span', { key: 'k', className: 'tag' }, `已压缩 ${s.compactions} 次`)
|
|
2609
|
+
: null,
|
|
2602
2610
|
].filter(Boolean)),
|
|
2603
2611
|
// One bar, two facts: how full it is, and where DSH would compact.
|
|
2604
2612
|
h('span', { key: 'bar', className: 'aw-hcard-bar' }, [
|
|
@@ -2633,13 +2641,16 @@ window.__ModuleLoader__.load({
|
|
|
2633
2641
|
h('p', { className: 'aw-hsec', key: 'w1' }, T.whyTitle),
|
|
2634
2642
|
h('div', { className: 'aw-hp', key: 'w2' }, [
|
|
2635
2643
|
h('div', { key: 'a' },
|
|
2636
|
-
'
|
|
2637
|
-
+ '
|
|
2644
|
+
'占用率只回答「离压缩点还有多远」;是否值得交接,看的是压缩次数。'
|
|
2645
|
+
+ '每压缩一次,早期对话就被换成一层摘要,路径、命令、报错最先丢——损失是叠加的。'),
|
|
2638
2646
|
h('div', { key: 'b' },
|
|
2639
|
-
'
|
|
2640
|
-
+ '
|
|
2647
|
+
'占用率达到临界线,说的是「该收尾当前这一步了」。但一个从未压缩过的新会话第一次到达临界,'
|
|
2648
|
+
+ '历史是完整的,会话仍然好用——面板只会标「首次接近」,不会推交接。'),
|
|
2641
2649
|
h('div', { key: 'c' },
|
|
2642
|
-
'
|
|
2650
|
+
'已压缩 2 次再达临界,说明任务范围已经超过一个会话该装的量,建议交接;'
|
|
2651
|
+
+ '压缩 ≥3 次的会话,一过注意线就升级提醒。刚压缩完 10 分钟内摘要最新鲜,提示会自动降半级。'),
|
|
2652
|
+
h('div', { key: 'd' },
|
|
2653
|
+
'压缩点本身照搬官方规则:窗口 − 最大输出 − 4K 余量(与 dsh-compaction-basic 一致)。'
|
|
2643
2654
|
+ '1,048,576 的窗口配 256,000 的输出预留,压缩点是 788,480(75.2%)。'),
|
|
2644
2655
|
]),
|
|
2645
2656
|
// The three lines the panel paints, computed for the session that is
|
package/lib/index.js
CHANGED
|
@@ -158,6 +158,12 @@ const DEFAULTS = {
|
|
|
158
158
|
logBudget: 6,
|
|
159
159
|
/** Bytes of log parsed per collect; keeps one poll from stalling on a big file. */
|
|
160
160
|
logBudgetBytes: 8 * 1024 * 1024,
|
|
161
|
+
/**
|
|
162
|
+
* Hard per-file ceiling. The soft budget above is a target, not a promise: a
|
|
163
|
+
* single file larger than it used to be read whole. Anything past this limit
|
|
164
|
+
* is skipped and reported as incomplete instead of blowing up one poll.
|
|
165
|
+
*/
|
|
166
|
+
logMaxFileBytes: 32 * 1024 * 1024,
|
|
161
167
|
priceTtlMs: 24 * 60 * 60 * 1000,
|
|
162
168
|
}
|
|
163
169
|
|
|
@@ -196,16 +202,77 @@ function compactPointOf(window, output) {
|
|
|
196
202
|
* @param cfg - thresholds as fractions of `point`.
|
|
197
203
|
* @returns risk level.
|
|
198
204
|
*/
|
|
199
|
-
|
|
205
|
+
/**
|
|
206
|
+
* Count compaction checkpoints in one session's event stream.
|
|
207
|
+
*
|
|
208
|
+
* Every compaction run persists a `user/message` whose source carries
|
|
209
|
+
* `{ kind: 'compact-checkpoint', compactionId }` — manual /compact included
|
|
210
|
+
* (its source also has `sourceCommandId`). Counting them is the only way to
|
|
211
|
+
* know how many times a session's history has been rebuilt: the host exposes
|
|
212
|
+
* no compaction projection, and `contextTimeline.requests` is a two-slot ring
|
|
213
|
+
* buffer that only ever shows the most recent one.
|
|
214
|
+
* @param events - persisted session events (or parsed log lines).
|
|
215
|
+
* @returns `{ compactions, lastCompactionAt }` — zero when the stream has none.
|
|
216
|
+
*/
|
|
217
|
+
/** A summary younger than this is considered fresh; freshness demotes one level. */
|
|
218
|
+
const COMPACTION_FRESH_MS = 10 * 60_000
|
|
219
|
+
|
|
220
|
+
export function countCheckpoints(events) {
|
|
221
|
+
let compactions = 0
|
|
222
|
+
let lastCompactionAt = null
|
|
223
|
+
for (const event of events) {
|
|
224
|
+
if (event?.type !== 'user/message') continue
|
|
225
|
+
if (event.data?.source?.kind !== 'compact-checkpoint') continue
|
|
226
|
+
compactions += 1
|
|
227
|
+
const time = Number(event.time)
|
|
228
|
+
if (Number.isFinite(time)) lastCompactionAt = Math.max(lastCompactionAt ?? time, time)
|
|
229
|
+
}
|
|
230
|
+
return { compactions, lastCompactionAt }
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
/**
|
|
234
|
+
* Occupancy decides WHEN compaction looms; the compaction count decides
|
|
235
|
+
* WHETHER the session is worth handing off. Community consensus (Claude Code
|
|
236
|
+
* ecosystem) is that two compactions in one session already mean the task was
|
|
237
|
+
* scoped wrong, and every further pass stacks information loss on top.
|
|
238
|
+
*
|
|
239
|
+
* Matrix (docs/32 §3):
|
|
240
|
+
* 0 compactions — a fresh session first crossing the critical line is still
|
|
241
|
+
* fully usable: the history has never been rebuilt, so the alarm is
|
|
242
|
+
* demoted to warn and flagged as firstApproach instead of pushing a
|
|
243
|
+
* handoff the user does not need yet.
|
|
244
|
+
* 1–2 compactions — critical occupancy escalates to a handoff suggestion.
|
|
245
|
+
* ≥3 compactions — warn occupancy is already a handoff suggestion: history
|
|
246
|
+
* has been rebuilt repeatedly and detail loss is stacked.
|
|
247
|
+
* A compaction in the last ten minutes leaves the freshest possible summary,
|
|
248
|
+
* so the level is demoted one step regardless of the matrix.
|
|
249
|
+
*
|
|
250
|
+
* @param stats - `{ compactions, lastCompactionAt }` from the log pipeline;
|
|
251
|
+
* `compactions === null` means "not parsed yet" and keeps the old
|
|
252
|
+
* occupancy-only verdict rather than guessing.
|
|
253
|
+
*/
|
|
254
|
+
export function riskLevel(ratio, point, cfg, stats) {
|
|
200
255
|
if (ratio == null) return 'unknown'
|
|
256
|
+
let level
|
|
201
257
|
if (!point) {
|
|
202
|
-
|
|
203
|
-
|
|
258
|
+
level = ratio >= 0.85 ? 'crit' : ratio >= 0.7 ? 'warn' : 'ok'
|
|
259
|
+
} else if (ratio >= point.ratio * cfg.criticalFactor) {
|
|
260
|
+
level = 'crit'
|
|
261
|
+
} else if (ratio >= point.ratio * cfg.warnFactor) {
|
|
262
|
+
level = 'warn'
|
|
263
|
+
} else {
|
|
204
264
|
return 'ok'
|
|
205
265
|
}
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
266
|
+
|
|
267
|
+
const count = stats?.compactions
|
|
268
|
+
const firstApproach = count === 0 && level === 'crit'
|
|
269
|
+
if (firstApproach) level = 'warn'
|
|
270
|
+
else if (typeof count === 'number' && count >= 3 && level === 'warn') level = 'crit'
|
|
271
|
+
|
|
272
|
+
const fresh = stats?.lastCompactionAt != null
|
|
273
|
+
&& Date.now() - stats.lastCompactionAt < COMPACTION_FRESH_MS
|
|
274
|
+
if (fresh) level = level === 'crit' ? 'warn' : level === 'warn' ? 'ok' : level
|
|
275
|
+
return level
|
|
209
276
|
}
|
|
210
277
|
|
|
211
278
|
/**
|
|
@@ -532,6 +599,10 @@ export function apply(ctx, config = {}) {
|
|
|
532
599
|
/** `${id}|${revision}` → in-flight read, so two collects never open two handles. */
|
|
533
600
|
const inflightSplits = new Map()
|
|
534
601
|
|
|
602
|
+
/** Sessions skipped because their log exceeds `logMaxFileBytes`. */
|
|
603
|
+
const oversize = new Set()
|
|
604
|
+
const maxFileBytes = Math.max(1, cfg.logMaxFileBytes ?? 32 * 1024 * 1024)
|
|
605
|
+
|
|
535
606
|
/** Session-log index and its timestamp; see indexSessionLogs(). */
|
|
536
607
|
let logIndex = null
|
|
537
608
|
let logIndexAt = 0
|
|
@@ -706,7 +777,10 @@ export function apply(ctx, config = {}) {
|
|
|
706
777
|
// dsh-context), use the split reconstructed from the session log; the
|
|
707
778
|
// lifetime totals times `peakFactor` is the last resort.
|
|
708
779
|
const projectionBuckets = costOfBuckets(timeline, priceBook.lookup, deepseekCny)
|
|
709
|
-
|
|
780
|
+
// Compaction counts come from the log pipeline regardless of where the
|
|
781
|
+
// usage buckets came from — a projection-priced session still has a log.
|
|
782
|
+
const logStats = LOG_SPLITS.get(id)
|
|
783
|
+
const split = projectionBuckets ? null : logStats
|
|
710
784
|
const logBuckets = !projectionBuckets && split && (split.peak || split.off)
|
|
711
785
|
? costOfBuckets(
|
|
712
786
|
{ cost: { [providerName ?? 'deepseek']: { [modelName ?? 'unknown']: { peak: split.peak, off: split.off } } } },
|
|
@@ -764,6 +838,7 @@ export function apply(ctx, config = {}) {
|
|
|
764
838
|
? `子代理${stamp ? ' · ' + stamp : ''}`
|
|
765
839
|
: `未命名会话${stamp ? ' · ' + stamp : ''}`
|
|
766
840
|
|
|
841
|
+
const rowRisk = riskLevel(ratio, point, cfg, logStats)
|
|
767
842
|
return {
|
|
768
843
|
id,
|
|
769
844
|
title,
|
|
@@ -800,7 +875,10 @@ export function apply(ctx, config = {}) {
|
|
|
800
875
|
costPeakBuckets: buckets ? buckets.peakBuckets : 0,
|
|
801
876
|
updatedAt: values?.sessionListMetadata?.lastPromptAt ?? header.createdAt ?? null,
|
|
802
877
|
activityDays,
|
|
803
|
-
|
|
878
|
+
compactions: logStats?.compactions ?? null,
|
|
879
|
+
lastCompactionAt: logStats?.lastCompactionAt ?? null,
|
|
880
|
+
firstApproach: rowRisk === 'warn' && logStats?.compactions === 0 && ratio >= (point?.ratio ?? Infinity) * cfg.criticalFactor,
|
|
881
|
+
risk: rowRisk,
|
|
804
882
|
contextAvailable: total != null,
|
|
805
883
|
}
|
|
806
884
|
}
|
|
@@ -1020,6 +1098,13 @@ export function apply(ctx, config = {}) {
|
|
|
1020
1098
|
if (!snapshot) return null
|
|
1021
1099
|
const revision = String(snapshot.revision ?? '')
|
|
1022
1100
|
const size = Number(snapshot.sizeBytes) || 0
|
|
1101
|
+
if (size > maxFileBytes) {
|
|
1102
|
+
// Over the ceiling: leave the previous split in place and record that this
|
|
1103
|
+
// session is not covered, rather than reading it whole.
|
|
1104
|
+
oversize.add(id)
|
|
1105
|
+
return { skipped: true, size }
|
|
1106
|
+
}
|
|
1107
|
+
oversize.delete(id)
|
|
1023
1108
|
const hit = LOG_SPLITS.get(id)
|
|
1024
1109
|
if (revision && hit?.revision === revision) return { entry: hit, read: false, size }
|
|
1025
1110
|
|
|
@@ -1034,13 +1119,15 @@ export function apply(ctx, config = {}) {
|
|
|
1034
1119
|
const handle = await persistence.open(id, 'read')
|
|
1035
1120
|
try {
|
|
1036
1121
|
const result = await handle.read()
|
|
1037
|
-
const
|
|
1122
|
+
const events = result?.events ?? []
|
|
1123
|
+
const split = splitUsageEvents(events)
|
|
1038
1124
|
const entry = {
|
|
1039
1125
|
revision,
|
|
1040
1126
|
size,
|
|
1041
1127
|
at: Date.now(),
|
|
1042
1128
|
peak: split?.peak ?? null,
|
|
1043
1129
|
off: split?.off ?? null,
|
|
1130
|
+
...countCheckpoints(events),
|
|
1044
1131
|
}
|
|
1045
1132
|
LOG_SPLITS.set(id, entry)
|
|
1046
1133
|
cache.delete(id)
|
|
@@ -1075,9 +1162,14 @@ export function apply(ctx, config = {}) {
|
|
|
1075
1162
|
for (const rec of records) {
|
|
1076
1163
|
const id = rec.header?.id
|
|
1077
1164
|
if (!id) continue
|
|
1078
|
-
if (parsed >= maxFiles
|
|
1165
|
+
if (parsed >= maxFiles) { remaining.push(rec); continue }
|
|
1166
|
+
// The soft budget is checked before the read, not after: adding a file and
|
|
1167
|
+
// then discovering it blew the budget is how one 50 MB log used to stall a
|
|
1168
|
+
// whole poll.
|
|
1169
|
+
if (bytes > 0 && bytes >= maxBytes) { remaining.push(rec); continue }
|
|
1079
1170
|
const official = await splitFromPersistence(id)
|
|
1080
1171
|
if (!official) { remaining.push(rec); continue }
|
|
1172
|
+
if (official.skipped) { parsed += 0; continue }
|
|
1081
1173
|
if (official.read) {
|
|
1082
1174
|
parsed += 1
|
|
1083
1175
|
bytes += official.size ?? 0
|
|
@@ -1098,6 +1190,10 @@ export function apply(ctx, config = {}) {
|
|
|
1098
1190
|
if (!file) continue
|
|
1099
1191
|
try {
|
|
1100
1192
|
const stat = await fs.stat(file)
|
|
1193
|
+
if (stat.size > maxFileBytes) {
|
|
1194
|
+
oversize.add(id)
|
|
1195
|
+
continue
|
|
1196
|
+
}
|
|
1101
1197
|
const hit = LOG_SPLITS.get(id)
|
|
1102
1198
|
if (hit && hit.mtime === stat.mtimeMs && hit.size === stat.size) continue
|
|
1103
1199
|
// The fallback has no revision token, so it keeps the growth throttle:
|
|
@@ -1111,14 +1207,26 @@ export function apply(ctx, config = {}) {
|
|
|
1111
1207
|
}
|
|
1112
1208
|
todo.sort((a, b) => a.stat.size - b.stat.size)
|
|
1113
1209
|
for (const item of todo) {
|
|
1114
|
-
if (parsed >= maxFiles
|
|
1210
|
+
if (parsed >= maxFiles) break
|
|
1211
|
+
// Do not start a file that cannot fit in what is left of the budget; a
|
|
1212
|
+
// smaller one later in the queue can still be read this round.
|
|
1213
|
+
if (bytes > 0 && bytes + item.stat.size > maxBytes) continue
|
|
1115
1214
|
parsed += 1
|
|
1116
1215
|
bytes += item.stat.size
|
|
1117
1216
|
const stamp = { mtime: item.stat.mtimeMs, size: item.stat.size }
|
|
1118
1217
|
try {
|
|
1119
1218
|
const buf = await fs.readFile(item.file)
|
|
1120
|
-
const
|
|
1121
|
-
|
|
1219
|
+
const text = decompressFrames(buf, zstd)
|
|
1220
|
+
const split = splitUsageText(text)
|
|
1221
|
+
LOG_SPLITS.set(item.id, {
|
|
1222
|
+
...stamp,
|
|
1223
|
+
at: Date.now(),
|
|
1224
|
+
peak: split?.peak ?? null,
|
|
1225
|
+
off: split?.off ?? null,
|
|
1226
|
+
...countCheckpoints(text.split('\n').map((line) => {
|
|
1227
|
+
try { return JSON.parse(line) } catch { return null }
|
|
1228
|
+
})),
|
|
1229
|
+
})
|
|
1122
1230
|
// The DTO cache may hold a totals-priced copy: drop it so the next
|
|
1123
1231
|
// collect re-prices with the buckets we just found.
|
|
1124
1232
|
cache.delete(item.id)
|
|
@@ -1397,6 +1505,10 @@ export function apply(ctx, config = {}) {
|
|
|
1397
1505
|
persistenceReads: (() => { try { return Boolean(ctx.get?.('sessionPersistence')?.stat) } catch { return false } })(),
|
|
1398
1506
|
logSplits: LOG_SPLITS.size,
|
|
1399
1507
|
logSplitsWithUsage: [...LOG_SPLITS.values()].filter(v => v.peak || v.off).length,
|
|
1508
|
+
// A session over the per-file ceiling is never parsed, so the aggregate
|
|
1509
|
+
// is incomplete while this is non-zero. Saying so beats a silent cap.
|
|
1510
|
+
oversizeSessions: oversize.size,
|
|
1511
|
+
logBudget: { maxFiles: cfg.logBudget, maxBytes: cfg.logBudgetBytes, maxFileBytes },
|
|
1400
1512
|
})
|
|
1401
1513
|
}))
|
|
1402
1514
|
|
package/lib/usage-split.js
CHANGED
|
@@ -9,12 +9,20 @@
|
|
|
9
9
|
* `assistant/message` event carries `time` and `data.usage`.
|
|
10
10
|
*
|
|
11
11
|
* The log is a sequence of concatenated zstd frames, one per append. A single
|
|
12
|
-
* `zstdDecompressSync` on the whole file stops after the first frame, so
|
|
13
|
-
*
|
|
14
|
-
*
|
|
12
|
+
* `zstdDecompressSync` on the whole file stops after the first frame, so the
|
|
13
|
+
* frames have to be walked one by one. The walk follows the frame structure
|
|
14
|
+
* (RFC 8878: descriptor, block headers, block bodies, checksum) rather than
|
|
15
|
+
* searching for the magic bytes, because a compressed block can contain those
|
|
16
|
+
* four bytes and searching would split a healthy frame in two.
|
|
17
|
+
*
|
|
18
|
+
* Only structurally complete frames are decompressed. A tail that is still
|
|
19
|
+
* being appended is truncated mid-block; Node's zstd will happily return a
|
|
20
|
+
* partial result for it, so completeness is decided here, not by the decoder.
|
|
21
|
+
* Results are cached by (system revision | mtime, size) by the caller.
|
|
15
22
|
*/
|
|
16
23
|
|
|
17
24
|
const MAGIC = Buffer.from([0x28, 0xb5, 0x2f, 0xfd])
|
|
25
|
+
const ZSTD_MAGIC = 0xfd2fb528 // little-endian 28 b5 2f fd
|
|
18
26
|
const BEIJING_OFFSET_MS = 8 * 60 * 60 * 1000
|
|
19
27
|
|
|
20
28
|
/**
|
|
@@ -33,26 +41,133 @@ export function isPeakTime(epochMs) {
|
|
|
33
41
|
}
|
|
34
42
|
|
|
35
43
|
/** Decompress every concatenated zstd frame in a buffer. */
|
|
44
|
+
/**
|
|
45
|
+
* Walk one Zstandard frame starting at `start`.
|
|
46
|
+
*
|
|
47
|
+
* Follows the frame layout from RFC 8878 §3.1 instead of trusting the magic
|
|
48
|
+
* bytes: frame header descriptor → optional window descriptor → dictionary id →
|
|
49
|
+
* frame content size → blocks → optional checksum. A block body may contain any
|
|
50
|
+
* bytes, including the magic, so only the length fields can tell us where the
|
|
51
|
+
* next frame begins.
|
|
52
|
+
*
|
|
53
|
+
* @param buffer - the whole log.
|
|
54
|
+
* @param start - offset of the frame magic.
|
|
55
|
+
* @returns `{ end }` when the frame is complete, `{ torn }` when it runs past the
|
|
56
|
+
* end of the buffer (a live append), `{ corrupt }` when the header or a block
|
|
57
|
+
* header is invalid.
|
|
58
|
+
*/
|
|
59
|
+
export function scanFrame(buffer, start) {
|
|
60
|
+
let p = start
|
|
61
|
+
if (buffer.length - p < 5) return { torn: true }
|
|
62
|
+
p += 4 // magic, checked by the caller
|
|
63
|
+
|
|
64
|
+
const descriptor = buffer[p]
|
|
65
|
+
p += 1
|
|
66
|
+
const fcsFlag = descriptor >>> 6
|
|
67
|
+
const singleSegment = (descriptor & 0x20) !== 0
|
|
68
|
+
const unused = (descriptor & 0x10) !== 0
|
|
69
|
+
const reserved = (descriptor & 0x08) !== 0
|
|
70
|
+
const hasChecksum = (descriptor & 0x04) !== 0
|
|
71
|
+
const dictFlag = descriptor & 0x03
|
|
72
|
+
if (unused || reserved) return { corrupt: true }
|
|
73
|
+
|
|
74
|
+
if (!singleSegment) {
|
|
75
|
+
if (p >= buffer.length) return { torn: true }
|
|
76
|
+
p += 1 // window descriptor
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
const dictBytes = dictFlag === 3 ? 4 : dictFlag
|
|
80
|
+
if (p + dictBytes > buffer.length) return { torn: true }
|
|
81
|
+
p += dictBytes
|
|
82
|
+
|
|
83
|
+
const fcsBytes = fcsFlag === 0
|
|
84
|
+
? (singleSegment ? 1 : 0)
|
|
85
|
+
: (fcsFlag === 1 ? 2 : (fcsFlag === 2 ? 4 : 8))
|
|
86
|
+
if (p + fcsBytes > buffer.length) return { torn: true }
|
|
87
|
+
let declaredSize = null
|
|
88
|
+
if (fcsBytes === 1) declaredSize = buffer[p]
|
|
89
|
+
else if (fcsBytes === 2) declaredSize = buffer.readUInt16LE(p) + 256
|
|
90
|
+
else if (fcsBytes === 4) declaredSize = buffer.readUInt32LE(p)
|
|
91
|
+
else if (fcsBytes === 8) declaredSize = Number(buffer.readBigUInt64LE(p))
|
|
92
|
+
p += fcsBytes
|
|
93
|
+
|
|
94
|
+
for (;;) {
|
|
95
|
+
if (p + 3 > buffer.length) return { torn: true }
|
|
96
|
+
const header = buffer[p] | (buffer[p + 1] << 8) | (buffer[p + 2] << 16)
|
|
97
|
+
p += 3
|
|
98
|
+
const last = (header & 1) === 1
|
|
99
|
+
const type = (header >>> 1) & 3
|
|
100
|
+
const size = header >>> 3
|
|
101
|
+
if (type === 3) return { corrupt: true } // reserved block type
|
|
102
|
+
// An RLE block stores a single byte no matter how large its output is.
|
|
103
|
+
const bodyBytes = type === 1 ? 1 : size
|
|
104
|
+
if (p + bodyBytes > buffer.length) return { torn: true }
|
|
105
|
+
p += bodyBytes
|
|
106
|
+
if (last) break
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
if (hasChecksum) {
|
|
110
|
+
if (p + 4 > buffer.length) return { torn: true }
|
|
111
|
+
p += 4
|
|
112
|
+
}
|
|
113
|
+
return { end: p, declaredSize }
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/**
|
|
117
|
+
* Decompress a concatenated zstd log, counting only complete frames.
|
|
118
|
+
*
|
|
119
|
+
* A frame that is structurally incomplete (the tail of a file being appended)
|
|
120
|
+
* is never handed to the decoder: `zstdDecompressSync` returns partial output
|
|
121
|
+
* for some truncated inputs instead of throwing, and that partial output would
|
|
122
|
+
* silently corrupt the usage totals. A frame whose declared content size does
|
|
123
|
+
* not match what came out is treated the same way.
|
|
124
|
+
*
|
|
125
|
+
* @param buffer - raw log bytes.
|
|
126
|
+
* @param zstd - `zlib.zstdDecompressSync`.
|
|
127
|
+
* @returns the concatenated JSONL text of every complete frame.
|
|
128
|
+
*/
|
|
36
129
|
export function decompressFrames(buffer, zstd) {
|
|
37
130
|
let text = ''
|
|
38
131
|
let pos = 0
|
|
39
|
-
while (pos
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
132
|
+
while (pos + 4 <= buffer.length) {
|
|
133
|
+
if (buffer.readUInt32LE(pos) !== ZSTD_MAGIC) {
|
|
134
|
+
// Not at a frame boundary: either the file is corrupt or the previous
|
|
135
|
+
// frame ended early. Resynchronise on the next magic rather than giving
|
|
136
|
+
// up on the rest of the log.
|
|
137
|
+
const next = buffer.indexOf(MAGIC, pos + 1)
|
|
138
|
+
if (next < 0) break
|
|
139
|
+
pos = next
|
|
140
|
+
continue
|
|
141
|
+
}
|
|
142
|
+
const frame = scanFrame(buffer, pos)
|
|
143
|
+
// A frame is only trustworthy when it ends exactly where the log ends or
|
|
144
|
+
// where the next frame's magic begins. A frame truncated in the middle of a
|
|
145
|
+
// file still "parses": its block header overstates the body by the missing
|
|
146
|
+
// bytes, and those bytes are supplied by the next frame, so the computed end
|
|
147
|
+
// lands inside it. The boundary check is what catches that.
|
|
148
|
+
const atBoundary = frame.end != null
|
|
149
|
+
&& (frame.end === buffer.length || buffer.readUInt32LE(frame.end) === ZSTD_MAGIC)
|
|
150
|
+
if (!atBoundary) {
|
|
151
|
+
// Neither a torn nor a corrupt frame is ever counted. Look for the next
|
|
152
|
+
// frame instead of giving up on the file: each append writes its own
|
|
153
|
+
// frame, so one damaged frame does not invalidate the ones after it.
|
|
154
|
+
// A torn tail (a live append) simply has no next magic and ends the walk.
|
|
155
|
+
const next = buffer.indexOf(MAGIC, pos + 4)
|
|
50
156
|
if (next < 0) break
|
|
51
|
-
pos =
|
|
157
|
+
pos = next
|
|
52
158
|
continue
|
|
53
159
|
}
|
|
54
|
-
|
|
55
|
-
|
|
160
|
+
try {
|
|
161
|
+
const out = zstd(buffer.subarray(pos, frame.end))
|
|
162
|
+
// A declared content size that does not match the output means the frame
|
|
163
|
+
// decoded to something other than what the writer intended; drop it.
|
|
164
|
+
if (frame.declaredSize == null || out.length === frame.declaredSize) {
|
|
165
|
+
text += out.toString('utf8')
|
|
166
|
+
}
|
|
167
|
+
} catch {
|
|
168
|
+
// Corrupt payload: skip this frame, keep the ones already decoded.
|
|
169
|
+
}
|
|
170
|
+
pos = frame.end
|
|
56
171
|
}
|
|
57
172
|
return text
|
|
58
173
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "dsh-session-matrix",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.17.0",
|
|
4
4
|
"description": "A multi-agent workspace for DeepSeek Harness: run parallel sessions, visualize agent relationships, monitor context usage, and manage task handoffs.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"deepseek-harness",
|
|
@@ -46,7 +46,8 @@
|
|
|
46
46
|
"README.md",
|
|
47
47
|
"LICENSE",
|
|
48
48
|
"cordis.patch.yml",
|
|
49
|
-
"test/"
|
|
49
|
+
"test/",
|
|
50
|
+
"scripts/"
|
|
50
51
|
],
|
|
51
52
|
"engines": {
|
|
52
53
|
"node": "^22.19.0 || >=24.0.0",
|
|
@@ -69,7 +70,7 @@
|
|
|
69
70
|
}
|
|
70
71
|
},
|
|
71
72
|
"scripts": {
|
|
72
|
-
"test": "node test/host.test.mjs && node test/usage-split.test.mjs && node test/usage-vs-projection.test.mjs",
|
|
73
|
+
"test": "node test/host.test.mjs && node test/usage-split.test.mjs && node test/usage-reconcile.test.mjs && node test/risk-matrix.test.mjs && node test/usage-vs-projection.test.mjs",
|
|
73
74
|
"prepublishOnly": "npm test"
|
|
74
75
|
}
|
|
75
76
|
}
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Build a committed, anonymised reconciliation fixture from real DSH logs.
|
|
3
|
+
*
|
|
4
|
+
* The CI job has no DSH data, so the local 37/37 check skips there. This strips
|
|
5
|
+
* a few real sessions down to the only fields the parser reads — event type,
|
|
6
|
+
* timestamp, and the usage payload — and records the totals DSH itself computed,
|
|
7
|
+
* so the comparison can run anywhere without shipping anyone's conversations.
|
|
8
|
+
*/
|
|
9
|
+
import { readFileSync, writeFileSync, readdirSync, statSync, existsSync, mkdirSync } from 'node:fs'
|
|
10
|
+
import { zstdDecompressSync } from 'node:zlib'
|
|
11
|
+
import { join } from 'node:path'
|
|
12
|
+
import { homedir } from 'node:os'
|
|
13
|
+
import { decompressFrames, splitUsageEvents } from '../plugin/agent-workspa../lib/usage-split.js'
|
|
14
|
+
|
|
15
|
+
const home = process.env.DSH_HOME || join(homedir(), '.dsh')
|
|
16
|
+
const sessionsRoot = join(home, 'sessions')
|
|
17
|
+
const cacheRoot = join(home, 'storages', 'session_projcache', 'sessions')
|
|
18
|
+
const outFile = 'test/fixtures/reconcile.json'
|
|
19
|
+
|
|
20
|
+
const totals = new Map()
|
|
21
|
+
for (const file of readdirSync(cacheRoot)) {
|
|
22
|
+
if (!file.endsWith('.json')) continue
|
|
23
|
+
try {
|
|
24
|
+
const record = JSON.parse(readFileSync(join(cacheRoot, file), 'utf8'))
|
|
25
|
+
const value = record?.record?.rows?.tokenUsage?.val ?? record?.record?.rows?.tokenUsage
|
|
26
|
+
if (value?.totals) totals.set(file.replace(/\.json$/, '').replace(/^session-/, ''), value.totals)
|
|
27
|
+
} catch { /* ignore a half-written cache entry */ }
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
const samples = []
|
|
31
|
+
for (const dir of readdirSync(sessionsRoot)) {
|
|
32
|
+
if (!statSync(join(sessionsRoot, dir)).isDirectory()) continue
|
|
33
|
+
for (const entry of readdirSync(join(sessionsRoot, dir), { withFileTypes: true })) {
|
|
34
|
+
if (!entry.isDirectory()) continue
|
|
35
|
+
const file = join(sessionsRoot, dir, entry.name, 'session.v4.jsonl.zstd')
|
|
36
|
+
if (!existsSync(file)) continue
|
|
37
|
+
const want = totals.get(entry.name)
|
|
38
|
+
if (!want) continue
|
|
39
|
+
let text = ''
|
|
40
|
+
try { text = decompressFrames(readFileSync(file), zstdDecompressSync) } catch { continue }
|
|
41
|
+
|
|
42
|
+
const events = []
|
|
43
|
+
for (const line of text.split('\n')) {
|
|
44
|
+
if (!line.includes('assistant/')) continue
|
|
45
|
+
let event
|
|
46
|
+
try { event = JSON.parse(line) } catch { continue }
|
|
47
|
+
if (event.type !== 'assistant/message' && event.type !== 'assistant/attempt') continue
|
|
48
|
+
// Keep the shape the parser walks and nothing else: no message text, no
|
|
49
|
+
// prompt, no tool arguments, no titles.
|
|
50
|
+
const kept = { type: event.type, time: event.time, data: {} }
|
|
51
|
+
if (event.data?.usage) kept.data.usage = event.data.usage
|
|
52
|
+
const chunks = (event.data?.stream ?? [])
|
|
53
|
+
.filter(r => r?.chunk?.type === 'usage')
|
|
54
|
+
.map(r => ({ time: r.time, chunk: { type: 'usage', usage: r.chunk.usage } }))
|
|
55
|
+
if (chunks.length) kept.data.stream = chunks
|
|
56
|
+
events.push(kept)
|
|
57
|
+
}
|
|
58
|
+
if (!events.length) continue
|
|
59
|
+
|
|
60
|
+
const split = splitUsageEvents(events)
|
|
61
|
+
if (!split) continue
|
|
62
|
+
const sum = bucket => ({
|
|
63
|
+
input: bucket.uncached + bucket.cacheRead + bucket.cacheWrite,
|
|
64
|
+
output: bucket.output,
|
|
65
|
+
})
|
|
66
|
+
samples.push({
|
|
67
|
+
// Stable anonymous label: a hash-free counter, not the real session id.
|
|
68
|
+
id: `sample-${samples.length + 1}`,
|
|
69
|
+
events,
|
|
70
|
+
expected: {
|
|
71
|
+
// DSH's own totals, which the parser must reproduce exactly.
|
|
72
|
+
uncachedInputTokens: want.uncachedInputTokens ?? 0,
|
|
73
|
+
outputTokens: want.outputTokens ?? 0,
|
|
74
|
+
cacheReadTokens: want.cacheReadTokens ?? 0,
|
|
75
|
+
cacheWriteTokens: want.cacheWriteTokens ?? 0,
|
|
76
|
+
requests: split.requests,
|
|
77
|
+
},
|
|
78
|
+
checks: [...(sum(split.peak).input ? ['peak'] : []), ...(sum(split.off).input ? ['off'] : [])],
|
|
79
|
+
})
|
|
80
|
+
if (samples.length >= 4) break
|
|
81
|
+
}
|
|
82
|
+
if (samples.length >= 4) break
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
if (!samples.length) throw new Error('no usable samples; run where DSH data exists')
|
|
86
|
+
mkdirSync('test/fixtures', { recursive: true })
|
|
87
|
+
writeFileSync(outFile, JSON.stringify({ generatedAt: 'fixed', samples }, null, 1) + '\n')
|
|
88
|
+
console.log('wrote', outFile, '| samples:', samples.length,
|
|
89
|
+
'| events:', samples.reduce((n, s) => n + s.events.length, 0))
|