dsh-session-matrix 0.16.5 → 0.17.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -10,6 +10,7 @@
10
10
  <a href="https://www.npmjs.com/package/dsh-session-matrix"><img src="https://img.shields.io/npm/v/dsh-session-matrix?label=npm&color=blue" alt="npm"></a>
11
11
  <a href="https://github.com/Wanqiu12345/dsh-session-matrix/actions/workflows/ci.yml"><img src="https://github.com/Wanqiu12345/dsh-session-matrix/actions/workflows/ci.yml/badge.svg" alt="CI"></a>
12
12
  <a href="./LICENSE"><img src="https://img.shields.io/badge/license-MIT-green" alt="MIT"></a>
13
+ <a href="https://dsh.market/"><img src="https://raw.githubusercontent.com/2BingLing/dsh-market/master/assets/readme/badge-listed-zh.svg" alt="DSH Market"></a>
13
14
  <img src="https://img.shields.io/badge/DSH-%E2%89%A50.2.2--rc.1-8a2be2" alt="DSH >= 0.2.2-rc.1">
14
15
  <img src="https://img.shields.io/badge/runtime%20deps-0-brightgreen" alt="no runtime deps">
15
16
  </p>
@@ -27,7 +28,7 @@
27
28
  <a href="#vs-context"><b>与 dsh-context</b></a>
28
29
  </p>
29
30
 
30
- ![左侧入口与四格工作区](./docs/01-entry.png)
31
+ ![左侧面板入口与智能体脉络](./docs/01-entry.png)
31
32
 
32
33
  ---
33
34
 
@@ -64,7 +65,9 @@
64
65
  - 每格挂载的是 DSH 原生 `conversation.content` 工厂实例:消息流、工具调用行、审批、
65
66
  Chat/Trajectory 切换、composer 全是原生的,**可以直接打字**。
66
67
 
67
- ![四格并排](./docs/02-workspace-4.png)
68
+ ![工作区空态:进入时不自动占用窗格,由你从左侧挑选](./docs/02-workspace-empty.png)
69
+
70
+ ![三栏原生会话并排:消息流、工具调用与 composer 都是 DSH 原生的](./docs/02-workspace.png)
68
71
 
69
72
  <a id="graph"></a>
70
73
 
@@ -81,7 +84,7 @@
81
84
  - **节点尺寸 = 累计用量**,谁最重一眼能看出来。
82
85
  - 选中任一节点,下方三个模块按该节点的子树重新计算。
83
86
 
84
- ![智能体脉络](./docs/03-graph.png)
87
+ ![智能体脉络:节点颜色 = 距压缩点的远近,尺寸 = 累计用量(图中为模拟预警状态)](./docs/03-graph.png)
85
88
 
86
89
  ### 三个模块
87
90
 
@@ -93,7 +96,7 @@
93
96
 
94
97
  「当前占用」和「合计」不是同一个量:前者是此刻占着多少,后者是历史上一共读过多少。
95
98
 
96
- ![选中子代理后的联动](./docs/04-graph-selected.png)
99
+ ![选中节点后的联动:下方三个模块按该节点子树重新计算](./docs/04-graph-selected.png)
97
100
 
98
101
  ## 上下文监控
99
102
 
@@ -115,6 +118,20 @@ pressureBudget = messageBudget − policy.headroomTokens # headroom 默认
115
118
  占用率取**服务端返回的 prompt 大小**(未命中 + 缓存读 + 缓存写),叠加最后一次采样后的增量,
116
119
  不使用字符数估算。
117
120
 
121
+ ### 占用率之外:压缩次数才是「要不要交接」的依据
122
+
123
+ 占用率只回答「离压缩点还有多远」;是否值得交接,看的是这个会话**已经压缩过几次**。
124
+ DSH 每次压缩都会在会话里留下一个检查点(`compact-checkpoint`),插件从会话日志里把它们数出来:
125
+
126
+ | 压缩次数 | 到达临界线时 |
127
+ |:--|:--|
128
+ | **0 次** | 标「首次接近」——历史完整,会话仍可用,不推交接 |
129
+ | **1–2 次** | 标「临界」——建议交接:细节损失已经开始叠加 |
130
+ | **≥3 次** | 一过注意线就升级「临界」——社区共识:两次压缩即任务范围失控 |
131
+
132
+ 刚压缩完 10 分钟内摘要最新鲜,提示自动降半级;日志尚未解析到的会话如实显示
133
+ 「压缩次数未知」,不猜。
134
+
118
135
  <a id="handoff"></a>
119
136
 
120
137
  ## 交接协议 v1
@@ -133,7 +150,7 @@ pressureBudget = messageBudget − policy.headroomTokens # headroom 默认
133
150
 
134
151
  插件负责复制与定位:目标窗格会闪烁一次,避免粘错会话。
135
152
 
136
- ![通用交接提示词](./docs/05-handoff.png)
153
+ ![通用交接提示词:左侧是为什么建议交接(压缩点阈值表),右侧是可编辑的协议全文,右下角「复制」](./docs/05-handoff.png)
137
154
 
138
155
  ## 费用口径
139
156
 
package/lib/client.js CHANGED
@@ -1977,9 +1977,13 @@ window.__ModuleLoader__.load({
1977
1977
  risk ? h('span', {
1978
1978
  key: 'rk',
1979
1979
  className: `aw-risk ${risk} tip-r`,
1980
- 'data-tip': risk === 'crit'
1981
- ? `已到预警线(压缩点 × ${Math.round((th?.criticalFactor ?? DEFAULT_FACTORS.criticalFactor) * 100)}%):继续对话很可能触发自动压缩`
1982
- : `接近压缩点(压缩点 × ${Math.round((th?.warnFactor ?? DEFAULT_FACTORS.warnFactor) * 100)}%):再增长一段就会触发自动压缩`,
1980
+ 'data-tip': session.firstApproach
1981
+ ? `首次接近临界线(压缩点 × ${Math.round((th?.criticalFactor ?? DEFAULT_FACTORS.criticalFactor) * 100)}%):还没有压缩过,历史完整可用;建议收尾当前这一步后观察`
1982
+ : risk === 'crit'
1983
+ ? `已到临界线,建议交接${typeof session.compactions === 'number' && session.compactions > 0
1984
+ ? `(已压缩 ${session.compactions} 次,细节损失叠加)` : ''}`
1985
+ : `接近压缩点(压缩点 × ${Math.round((th?.warnFactor ?? DEFAULT_FACTORS.warnFactor) * 100)}%)${typeof session.compactions === 'number' && session.compactions > 0
1986
+ ? `,已压缩 ${session.compactions} 次` : ''}:再增长一段就会触发自动压缩`,
1983
1987
  }, risk === 'crit' ? T.levelCrit : T.levelWarn) : null,
1984
1988
  h('button', {
1985
1989
  key: 'open',
@@ -2599,6 +2603,10 @@ window.__ModuleLoader__.load({
2599
2603
  h('span', { key: 'h', className: 'aw-hcard-h' }, [
2600
2604
  h('span', { key: 't', className: 't' }, s.title || T.noTitle),
2601
2605
  paneIds.includes(s.id) ? h('span', { key: 'g', className: 'tag' }, '已载入') : null,
2606
+ s.firstApproach ? h('span', { key: 'fa', className: 'tag' }, '首次接近 · 未压缩') : null,
2607
+ typeof s.compactions === 'number' && s.compactions > 0
2608
+ ? h('span', { key: 'k', className: 'tag' }, `已压缩 ${s.compactions} 次`)
2609
+ : null,
2602
2610
  ].filter(Boolean)),
2603
2611
  // One bar, two facts: how full it is, and where DSH would compact.
2604
2612
  h('span', { key: 'bar', className: 'aw-hcard-bar' }, [
@@ -2633,13 +2641,16 @@ window.__ModuleLoader__.load({
2633
2641
  h('p', { className: 'aw-hsec', key: 'w1' }, T.whyTitle),
2634
2642
  h('div', { className: 'aw-hp', key: 'w2' }, [
2635
2643
  h('div', { key: 'a' },
2636
- '上下文是这个会话里模型能看到的全部内容——你说的话、它的回复、读过的文件、跑过的命令。'
2637
- + '当前模型上限约 100 万 token。'),
2644
+ '占用率只回答「离压缩点还有多远」;是否值得交接,看的是压缩次数。'
2645
+ + '每压缩一次,早期对话就被换成一层摘要,路径、命令、报错最先丢——损失是叠加的。'),
2638
2646
  h('div', { key: 'b' },
2639
- '每个会话有一个压缩点,DSH 到这里会把早期对话换成摘要。压缩点 = 窗口 − 最大输出 − 4K 余量,'
2640
- + '和 dsh-compaction-basic 的 messageBudget − headroomTokens 一致。'),
2647
+ '占用率达到临界线,说的是「该收尾当前这一步了」。但一个从未压缩过的新会话第一次到达临界,'
2648
+ + '历史是完整的,会话仍然好用——面板只会标「首次接近」,不会推交接。'),
2641
2649
  h('div', { key: 'c' },
2642
- '减去输出预留,是因为压缩之后仍要生成内容;4K 余量是压缩器自己留的安全垫。'
2650
+ '已压缩 2 次再达临界,说明任务范围已经超过一个会话该装的量,建议交接;'
2651
+ + '压缩 ≥3 次的会话,一过注意线就升级提醒。刚压缩完 10 分钟内摘要最新鲜,提示会自动降半级。'),
2652
+ h('div', { key: 'd' },
2653
+ '压缩点本身照搬官方规则:窗口 − 最大输出 − 4K 余量(与 dsh-compaction-basic 一致)。'
2643
2654
  + '1,048,576 的窗口配 256,000 的输出预留,压缩点是 788,480(75.2%)。'),
2644
2655
  ]),
2645
2656
  // The three lines the panel paints, computed for the session that is
package/lib/index.js CHANGED
@@ -158,6 +158,12 @@ const DEFAULTS = {
158
158
  logBudget: 6,
159
159
  /** Bytes of log parsed per collect; keeps one poll from stalling on a big file. */
160
160
  logBudgetBytes: 8 * 1024 * 1024,
161
+ /**
162
+ * Hard per-file ceiling. The soft budget above is a target, not a promise: a
163
+ * single file larger than it used to be read whole. Anything past this limit
164
+ * is skipped and reported as incomplete instead of blowing up one poll.
165
+ */
166
+ logMaxFileBytes: 32 * 1024 * 1024,
161
167
  priceTtlMs: 24 * 60 * 60 * 1000,
162
168
  }
163
169
 
@@ -196,16 +202,77 @@ function compactPointOf(window, output) {
196
202
  * @param cfg - thresholds as fractions of `point`.
197
203
  * @returns risk level.
198
204
  */
199
- function riskLevel(ratio, point, cfg) {
205
+ /**
206
+ * Count compaction checkpoints in one session's event stream.
207
+ *
208
+ * Every compaction run persists a `user/message` whose source carries
209
+ * `{ kind: 'compact-checkpoint', compactionId }` — manual /compact included
210
+ * (its source also has `sourceCommandId`). Counting them is the only way to
211
+ * know how many times a session's history has been rebuilt: the host exposes
212
+ * no compaction projection, and `contextTimeline.requests` is a two-slot ring
213
+ * buffer that only ever shows the most recent one.
214
+ * @param events - persisted session events (or parsed log lines).
215
+ * @returns `{ compactions, lastCompactionAt }` — zero when the stream has none.
216
+ */
217
+ /** A summary younger than this is considered fresh; freshness demotes one level. */
218
+ const COMPACTION_FRESH_MS = 10 * 60_000
219
+
220
+ export function countCheckpoints(events) {
221
+ let compactions = 0
222
+ let lastCompactionAt = null
223
+ for (const event of events) {
224
+ if (event?.type !== 'user/message') continue
225
+ if (event.data?.source?.kind !== 'compact-checkpoint') continue
226
+ compactions += 1
227
+ const time = Number(event.time)
228
+ if (Number.isFinite(time)) lastCompactionAt = Math.max(lastCompactionAt ?? time, time)
229
+ }
230
+ return { compactions, lastCompactionAt }
231
+ }
232
+
233
+ /**
234
+ * Occupancy decides WHEN compaction looms; the compaction count decides
235
+ * WHETHER the session is worth handing off. Community consensus (Claude Code
236
+ * ecosystem) is that two compactions in one session already mean the task was
237
+ * scoped wrong, and every further pass stacks information loss on top.
238
+ *
239
+ * Matrix (docs/32 §3):
240
+ * 0 compactions — a fresh session first crossing the critical line is still
241
+ * fully usable: the history has never been rebuilt, so the alarm is
242
+ * demoted to warn and flagged as firstApproach instead of pushing a
243
+ * handoff the user does not need yet.
244
+ * 1–2 compactions — critical occupancy escalates to a handoff suggestion.
245
+ * ≥3 compactions — warn occupancy is already a handoff suggestion: history
246
+ * has been rebuilt repeatedly and detail loss is stacked.
247
+ * A compaction in the last ten minutes leaves the freshest possible summary,
248
+ * so the level is demoted one step regardless of the matrix.
249
+ *
250
+ * @param stats - `{ compactions, lastCompactionAt }` from the log pipeline;
251
+ * `compactions === null` means "not parsed yet" and keeps the old
252
+ * occupancy-only verdict rather than guessing.
253
+ */
254
+ export function riskLevel(ratio, point, cfg, stats) {
200
255
  if (ratio == null) return 'unknown'
256
+ let level
201
257
  if (!point) {
202
- if (ratio >= 0.85) return 'crit'
203
- if (ratio >= 0.7) return 'warn'
258
+ level = ratio >= 0.85 ? 'crit' : ratio >= 0.7 ? 'warn' : 'ok'
259
+ } else if (ratio >= point.ratio * cfg.criticalFactor) {
260
+ level = 'crit'
261
+ } else if (ratio >= point.ratio * cfg.warnFactor) {
262
+ level = 'warn'
263
+ } else {
204
264
  return 'ok'
205
265
  }
206
- if (ratio >= point.ratio * cfg.criticalFactor) return 'crit'
207
- if (ratio >= point.ratio * cfg.warnFactor) return 'warn'
208
- return 'ok'
266
+
267
+ const count = stats?.compactions
268
+ const firstApproach = count === 0 && level === 'crit'
269
+ if (firstApproach) level = 'warn'
270
+ else if (typeof count === 'number' && count >= 3 && level === 'warn') level = 'crit'
271
+
272
+ const fresh = stats?.lastCompactionAt != null
273
+ && Date.now() - stats.lastCompactionAt < COMPACTION_FRESH_MS
274
+ if (fresh) level = level === 'crit' ? 'warn' : level === 'warn' ? 'ok' : level
275
+ return level
209
276
  }
210
277
 
211
278
  /**
@@ -532,6 +599,10 @@ export function apply(ctx, config = {}) {
532
599
  /** `${id}|${revision}` → in-flight read, so two collects never open two handles. */
533
600
  const inflightSplits = new Map()
534
601
 
602
+ /** Sessions skipped because their log exceeds `logMaxFileBytes`. */
603
+ const oversize = new Set()
604
+ const maxFileBytes = Math.max(1, cfg.logMaxFileBytes ?? 32 * 1024 * 1024)
605
+
535
606
  /** Session-log index and its timestamp; see indexSessionLogs(). */
536
607
  let logIndex = null
537
608
  let logIndexAt = 0
@@ -706,7 +777,10 @@ export function apply(ctx, config = {}) {
706
777
  // dsh-context), use the split reconstructed from the session log; the
707
778
  // lifetime totals times `peakFactor` is the last resort.
708
779
  const projectionBuckets = costOfBuckets(timeline, priceBook.lookup, deepseekCny)
709
- const split = projectionBuckets ? null : LOG_SPLITS.get(id)
780
+ // Compaction counts come from the log pipeline regardless of where the
781
+ // usage buckets came from — a projection-priced session still has a log.
782
+ const logStats = LOG_SPLITS.get(id)
783
+ const split = projectionBuckets ? null : logStats
710
784
  const logBuckets = !projectionBuckets && split && (split.peak || split.off)
711
785
  ? costOfBuckets(
712
786
  { cost: { [providerName ?? 'deepseek']: { [modelName ?? 'unknown']: { peak: split.peak, off: split.off } } } },
@@ -764,6 +838,7 @@ export function apply(ctx, config = {}) {
764
838
  ? `子代理${stamp ? ' · ' + stamp : ''}`
765
839
  : `未命名会话${stamp ? ' · ' + stamp : ''}`
766
840
 
841
+ const rowRisk = riskLevel(ratio, point, cfg, logStats)
767
842
  return {
768
843
  id,
769
844
  title,
@@ -800,7 +875,10 @@ export function apply(ctx, config = {}) {
800
875
  costPeakBuckets: buckets ? buckets.peakBuckets : 0,
801
876
  updatedAt: values?.sessionListMetadata?.lastPromptAt ?? header.createdAt ?? null,
802
877
  activityDays,
803
- risk: riskLevel(ratio, point, cfg),
878
+ compactions: logStats?.compactions ?? null,
879
+ lastCompactionAt: logStats?.lastCompactionAt ?? null,
880
+ firstApproach: rowRisk === 'warn' && logStats?.compactions === 0 && ratio >= (point?.ratio ?? Infinity) * cfg.criticalFactor,
881
+ risk: rowRisk,
804
882
  contextAvailable: total != null,
805
883
  }
806
884
  }
@@ -1020,6 +1098,13 @@ export function apply(ctx, config = {}) {
1020
1098
  if (!snapshot) return null
1021
1099
  const revision = String(snapshot.revision ?? '')
1022
1100
  const size = Number(snapshot.sizeBytes) || 0
1101
+ if (size > maxFileBytes) {
1102
+ // Over the ceiling: leave the previous split in place and record that this
1103
+ // session is not covered, rather than reading it whole.
1104
+ oversize.add(id)
1105
+ return { skipped: true, size }
1106
+ }
1107
+ oversize.delete(id)
1023
1108
  const hit = LOG_SPLITS.get(id)
1024
1109
  if (revision && hit?.revision === revision) return { entry: hit, read: false, size }
1025
1110
 
@@ -1034,13 +1119,15 @@ export function apply(ctx, config = {}) {
1034
1119
  const handle = await persistence.open(id, 'read')
1035
1120
  try {
1036
1121
  const result = await handle.read()
1037
- const split = splitUsageEvents(result?.events ?? [])
1122
+ const events = result?.events ?? []
1123
+ const split = splitUsageEvents(events)
1038
1124
  const entry = {
1039
1125
  revision,
1040
1126
  size,
1041
1127
  at: Date.now(),
1042
1128
  peak: split?.peak ?? null,
1043
1129
  off: split?.off ?? null,
1130
+ ...countCheckpoints(events),
1044
1131
  }
1045
1132
  LOG_SPLITS.set(id, entry)
1046
1133
  cache.delete(id)
@@ -1075,9 +1162,14 @@ export function apply(ctx, config = {}) {
1075
1162
  for (const rec of records) {
1076
1163
  const id = rec.header?.id
1077
1164
  if (!id) continue
1078
- if (parsed >= maxFiles || bytes >= maxBytes) { remaining.push(rec); continue }
1165
+ if (parsed >= maxFiles) { remaining.push(rec); continue }
1166
+ // The soft budget is checked before the read, not after: adding a file and
1167
+ // then discovering it blew the budget is how one 50 MB log used to stall a
1168
+ // whole poll.
1169
+ if (bytes > 0 && bytes >= maxBytes) { remaining.push(rec); continue }
1079
1170
  const official = await splitFromPersistence(id)
1080
1171
  if (!official) { remaining.push(rec); continue }
1172
+ if (official.skipped) { parsed += 0; continue }
1081
1173
  if (official.read) {
1082
1174
  parsed += 1
1083
1175
  bytes += official.size ?? 0
@@ -1098,6 +1190,10 @@ export function apply(ctx, config = {}) {
1098
1190
  if (!file) continue
1099
1191
  try {
1100
1192
  const stat = await fs.stat(file)
1193
+ if (stat.size > maxFileBytes) {
1194
+ oversize.add(id)
1195
+ continue
1196
+ }
1101
1197
  const hit = LOG_SPLITS.get(id)
1102
1198
  if (hit && hit.mtime === stat.mtimeMs && hit.size === stat.size) continue
1103
1199
  // The fallback has no revision token, so it keeps the growth throttle:
@@ -1111,14 +1207,26 @@ export function apply(ctx, config = {}) {
1111
1207
  }
1112
1208
  todo.sort((a, b) => a.stat.size - b.stat.size)
1113
1209
  for (const item of todo) {
1114
- if (parsed >= maxFiles || bytes >= maxBytes) break
1210
+ if (parsed >= maxFiles) break
1211
+ // Do not start a file that cannot fit in what is left of the budget; a
1212
+ // smaller one later in the queue can still be read this round.
1213
+ if (bytes > 0 && bytes + item.stat.size > maxBytes) continue
1115
1214
  parsed += 1
1116
1215
  bytes += item.stat.size
1117
1216
  const stamp = { mtime: item.stat.mtimeMs, size: item.stat.size }
1118
1217
  try {
1119
1218
  const buf = await fs.readFile(item.file)
1120
- const split = splitUsageText(decompressFrames(buf, zstd))
1121
- LOG_SPLITS.set(item.id, { ...stamp, at: Date.now(), peak: split?.peak ?? null, off: split?.off ?? null })
1219
+ const text = decompressFrames(buf, zstd)
1220
+ const split = splitUsageText(text)
1221
+ LOG_SPLITS.set(item.id, {
1222
+ ...stamp,
1223
+ at: Date.now(),
1224
+ peak: split?.peak ?? null,
1225
+ off: split?.off ?? null,
1226
+ ...countCheckpoints(text.split('\n').map((line) => {
1227
+ try { return JSON.parse(line) } catch { return null }
1228
+ })),
1229
+ })
1122
1230
  // The DTO cache may hold a totals-priced copy: drop it so the next
1123
1231
  // collect re-prices with the buckets we just found.
1124
1232
  cache.delete(item.id)
@@ -1397,6 +1505,10 @@ export function apply(ctx, config = {}) {
1397
1505
  persistenceReads: (() => { try { return Boolean(ctx.get?.('sessionPersistence')?.stat) } catch { return false } })(),
1398
1506
  logSplits: LOG_SPLITS.size,
1399
1507
  logSplitsWithUsage: [...LOG_SPLITS.values()].filter(v => v.peak || v.off).length,
1508
+ // A session over the per-file ceiling is never parsed, so the aggregate
1509
+ // is incomplete while this is non-zero. Saying so beats a silent cap.
1510
+ oversizeSessions: oversize.size,
1511
+ logBudget: { maxFiles: cfg.logBudget, maxBytes: cfg.logBudgetBytes, maxFileBytes },
1400
1512
  })
1401
1513
  }))
1402
1514
 
@@ -9,12 +9,20 @@
9
9
  * `assistant/message` event carries `time` and `data.usage`.
10
10
  *
11
11
  * The log is a sequence of concatenated zstd frames, one per append. A single
12
- * `zstdDecompressSync` on the whole file stops after the first frame, so this
13
- * walks the frame magic instead. Parsing a 20 MB session takes under a second;
14
- * results are cached by (mtime, size) by the caller.
12
+ * `zstdDecompressSync` on the whole file stops after the first frame, so the
13
+ * frames have to be walked one by one. The walk follows the frame structure
14
+ * (RFC 8878: descriptor, block headers, block bodies, checksum) rather than
15
+ * searching for the magic bytes, because a compressed block can contain those
16
+ * four bytes and searching would split a healthy frame in two.
17
+ *
18
+ * Only structurally complete frames are decompressed. A tail that is still
19
+ * being appended is truncated mid-block; Node's zstd will happily return a
20
+ * partial result for it, so completeness is decided here, not by the decoder.
21
+ * Results are cached by (system revision | mtime, size) by the caller.
15
22
  */
16
23
 
17
24
  const MAGIC = Buffer.from([0x28, 0xb5, 0x2f, 0xfd])
25
+ const ZSTD_MAGIC = 0xfd2fb528 // little-endian 28 b5 2f fd
18
26
  const BEIJING_OFFSET_MS = 8 * 60 * 60 * 1000
19
27
 
20
28
  /**
@@ -33,26 +41,133 @@ export function isPeakTime(epochMs) {
33
41
  }
34
42
 
35
43
  /** Decompress every concatenated zstd frame in a buffer. */
44
+ /**
45
+ * Walk one Zstandard frame starting at `start`.
46
+ *
47
+ * Follows the frame layout from RFC 8878 §3.1 instead of trusting the magic
48
+ * bytes: frame header descriptor → optional window descriptor → dictionary id →
49
+ * frame content size → blocks → optional checksum. A block body may contain any
50
+ * bytes, including the magic, so only the length fields can tell us where the
51
+ * next frame begins.
52
+ *
53
+ * @param buffer - the whole log.
54
+ * @param start - offset of the frame magic.
55
+ * @returns `{ end }` when the frame is complete, `{ torn }` when it runs past the
56
+ * end of the buffer (a live append), `{ corrupt }` when the header or a block
57
+ * header is invalid.
58
+ */
59
+ export function scanFrame(buffer, start) {
60
+ let p = start
61
+ if (buffer.length - p < 5) return { torn: true }
62
+ p += 4 // magic, checked by the caller
63
+
64
+ const descriptor = buffer[p]
65
+ p += 1
66
+ const fcsFlag = descriptor >>> 6
67
+ const singleSegment = (descriptor & 0x20) !== 0
68
+ const unused = (descriptor & 0x10) !== 0
69
+ const reserved = (descriptor & 0x08) !== 0
70
+ const hasChecksum = (descriptor & 0x04) !== 0
71
+ const dictFlag = descriptor & 0x03
72
+ if (unused || reserved) return { corrupt: true }
73
+
74
+ if (!singleSegment) {
75
+ if (p >= buffer.length) return { torn: true }
76
+ p += 1 // window descriptor
77
+ }
78
+
79
+ const dictBytes = dictFlag === 3 ? 4 : dictFlag
80
+ if (p + dictBytes > buffer.length) return { torn: true }
81
+ p += dictBytes
82
+
83
+ const fcsBytes = fcsFlag === 0
84
+ ? (singleSegment ? 1 : 0)
85
+ : (fcsFlag === 1 ? 2 : (fcsFlag === 2 ? 4 : 8))
86
+ if (p + fcsBytes > buffer.length) return { torn: true }
87
+ let declaredSize = null
88
+ if (fcsBytes === 1) declaredSize = buffer[p]
89
+ else if (fcsBytes === 2) declaredSize = buffer.readUInt16LE(p) + 256
90
+ else if (fcsBytes === 4) declaredSize = buffer.readUInt32LE(p)
91
+ else if (fcsBytes === 8) declaredSize = Number(buffer.readBigUInt64LE(p))
92
+ p += fcsBytes
93
+
94
+ for (;;) {
95
+ if (p + 3 > buffer.length) return { torn: true }
96
+ const header = buffer[p] | (buffer[p + 1] << 8) | (buffer[p + 2] << 16)
97
+ p += 3
98
+ const last = (header & 1) === 1
99
+ const type = (header >>> 1) & 3
100
+ const size = header >>> 3
101
+ if (type === 3) return { corrupt: true } // reserved block type
102
+ // An RLE block stores a single byte no matter how large its output is.
103
+ const bodyBytes = type === 1 ? 1 : size
104
+ if (p + bodyBytes > buffer.length) return { torn: true }
105
+ p += bodyBytes
106
+ if (last) break
107
+ }
108
+
109
+ if (hasChecksum) {
110
+ if (p + 4 > buffer.length) return { torn: true }
111
+ p += 4
112
+ }
113
+ return { end: p, declaredSize }
114
+ }
115
+
116
+ /**
117
+ * Decompress a concatenated zstd log, counting only complete frames.
118
+ *
119
+ * A frame that is structurally incomplete (the tail of a file being appended)
120
+ * is never handed to the decoder: `zstdDecompressSync` returns partial output
121
+ * for some truncated inputs instead of throwing, and that partial output would
122
+ * silently corrupt the usage totals. A frame whose declared content size does
123
+ * not match what came out is treated the same way.
124
+ *
125
+ * @param buffer - raw log bytes.
126
+ * @param zstd - `zlib.zstdDecompressSync`.
127
+ * @returns the concatenated JSONL text of every complete frame.
128
+ */
36
129
  export function decompressFrames(buffer, zstd) {
37
130
  let text = ''
38
131
  let pos = 0
39
- while (pos < buffer.length) {
40
- const start = buffer.indexOf(MAGIC, pos)
41
- if (start < 0) break
42
- const next = buffer.indexOf(MAGIC, start + 4)
43
- const frame = buffer.subarray(start, next < 0 ? buffer.length : next)
44
- try {
45
- text += zstd(frame).toString('utf8')
46
- } catch {
47
- // Two cases land here: a genuinely broken frame, and a false magic that
48
- // split a real frame in two. Skipping one candidate and retrying keeps
49
- // the rest of the log readable instead of truncating the whole file.
132
+ while (pos + 4 <= buffer.length) {
133
+ if (buffer.readUInt32LE(pos) !== ZSTD_MAGIC) {
134
+ // Not at a frame boundary: either the file is corrupt or the previous
135
+ // frame ended early. Resynchronise on the next magic rather than giving
136
+ // up on the rest of the log.
137
+ const next = buffer.indexOf(MAGIC, pos + 1)
138
+ if (next < 0) break
139
+ pos = next
140
+ continue
141
+ }
142
+ const frame = scanFrame(buffer, pos)
143
+ // A frame is only trustworthy when it ends exactly where the log ends or
144
+ // where the next frame's magic begins. A frame truncated in the middle of a
145
+ // file still "parses": its block header overstates the body by the missing
146
+ // bytes, and those bytes are supplied by the next frame, so the computed end
147
+ // lands inside it. The boundary check is what catches that.
148
+ const atBoundary = frame.end != null
149
+ && (frame.end === buffer.length || buffer.readUInt32LE(frame.end) === ZSTD_MAGIC)
150
+ if (!atBoundary) {
151
+ // Neither a torn nor a corrupt frame is ever counted. Look for the next
152
+ // frame instead of giving up on the file: each append writes its own
153
+ // frame, so one damaged frame does not invalidate the ones after it.
154
+ // A torn tail (a live append) simply has no next magic and ends the walk.
155
+ const next = buffer.indexOf(MAGIC, pos + 4)
50
156
  if (next < 0) break
51
- pos = start + 4
157
+ pos = next
52
158
  continue
53
159
  }
54
- if (next < 0) break
55
- pos = next
160
+ try {
161
+ const out = zstd(buffer.subarray(pos, frame.end))
162
+ // A declared content size that does not match the output means the frame
163
+ // decoded to something other than what the writer intended; drop it.
164
+ if (frame.declaredSize == null || out.length === frame.declaredSize) {
165
+ text += out.toString('utf8')
166
+ }
167
+ } catch {
168
+ // Corrupt payload: skip this frame, keep the ones already decoded.
169
+ }
170
+ pos = frame.end
56
171
  }
57
172
  return text
58
173
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "dsh-session-matrix",
3
- "version": "0.16.5",
3
+ "version": "0.17.0",
4
4
  "description": "A multi-agent workspace for DeepSeek Harness: run parallel sessions, visualize agent relationships, monitor context usage, and manage task handoffs.",
5
5
  "keywords": [
6
6
  "deepseek-harness",
@@ -46,7 +46,8 @@
46
46
  "README.md",
47
47
  "LICENSE",
48
48
  "cordis.patch.yml",
49
- "test/"
49
+ "test/",
50
+ "scripts/"
50
51
  ],
51
52
  "engines": {
52
53
  "node": "^22.19.0 || >=24.0.0",
@@ -69,7 +70,7 @@
69
70
  }
70
71
  },
71
72
  "scripts": {
72
- "test": "node test/host.test.mjs && node test/usage-split.test.mjs && node test/usage-vs-projection.test.mjs",
73
+ "test": "node test/host.test.mjs && node test/usage-split.test.mjs && node test/usage-reconcile.test.mjs && node test/risk-matrix.test.mjs && node test/usage-vs-projection.test.mjs",
73
74
  "prepublishOnly": "npm test"
74
75
  }
75
76
  }
@@ -0,0 +1,89 @@
1
+ /**
2
+ * Build a committed, anonymised reconciliation fixture from real DSH logs.
3
+ *
4
+ * The CI job has no DSH data, so the local 37/37 check skips there. This strips
5
+ * a few real sessions down to the only fields the parser reads — event type,
6
+ * timestamp, and the usage payload — and records the totals DSH itself computed,
7
+ * so the comparison can run anywhere without shipping anyone's conversations.
8
+ */
9
+ import { readFileSync, writeFileSync, readdirSync, statSync, existsSync, mkdirSync } from 'node:fs'
10
+ import { zstdDecompressSync } from 'node:zlib'
11
+ import { join } from 'node:path'
12
+ import { homedir } from 'node:os'
13
+ import { decompressFrames, splitUsageEvents } from '../plugin/agent-workspa../lib/usage-split.js'
14
+
15
+ const home = process.env.DSH_HOME || join(homedir(), '.dsh')
16
+ const sessionsRoot = join(home, 'sessions')
17
+ const cacheRoot = join(home, 'storages', 'session_projcache', 'sessions')
18
+ const outFile = 'test/fixtures/reconcile.json'
19
+
20
+ const totals = new Map()
21
+ for (const file of readdirSync(cacheRoot)) {
22
+ if (!file.endsWith('.json')) continue
23
+ try {
24
+ const record = JSON.parse(readFileSync(join(cacheRoot, file), 'utf8'))
25
+ const value = record?.record?.rows?.tokenUsage?.val ?? record?.record?.rows?.tokenUsage
26
+ if (value?.totals) totals.set(file.replace(/\.json$/, '').replace(/^session-/, ''), value.totals)
27
+ } catch { /* ignore a half-written cache entry */ }
28
+ }
29
+
30
+ const samples = []
31
+ for (const dir of readdirSync(sessionsRoot)) {
32
+ if (!statSync(join(sessionsRoot, dir)).isDirectory()) continue
33
+ for (const entry of readdirSync(join(sessionsRoot, dir), { withFileTypes: true })) {
34
+ if (!entry.isDirectory()) continue
35
+ const file = join(sessionsRoot, dir, entry.name, 'session.v4.jsonl.zstd')
36
+ if (!existsSync(file)) continue
37
+ const want = totals.get(entry.name)
38
+ if (!want) continue
39
+ let text = ''
40
+ try { text = decompressFrames(readFileSync(file), zstdDecompressSync) } catch { continue }
41
+
42
+ const events = []
43
+ for (const line of text.split('\n')) {
44
+ if (!line.includes('assistant/')) continue
45
+ let event
46
+ try { event = JSON.parse(line) } catch { continue }
47
+ if (event.type !== 'assistant/message' && event.type !== 'assistant/attempt') continue
48
+ // Keep the shape the parser walks and nothing else: no message text, no
49
+ // prompt, no tool arguments, no titles.
50
+ const kept = { type: event.type, time: event.time, data: {} }
51
+ if (event.data?.usage) kept.data.usage = event.data.usage
52
+ const chunks = (event.data?.stream ?? [])
53
+ .filter(r => r?.chunk?.type === 'usage')
54
+ .map(r => ({ time: r.time, chunk: { type: 'usage', usage: r.chunk.usage } }))
55
+ if (chunks.length) kept.data.stream = chunks
56
+ events.push(kept)
57
+ }
58
+ if (!events.length) continue
59
+
60
+ const split = splitUsageEvents(events)
61
+ if (!split) continue
62
+ const sum = bucket => ({
63
+ input: bucket.uncached + bucket.cacheRead + bucket.cacheWrite,
64
+ output: bucket.output,
65
+ })
66
+ samples.push({
67
+ // Stable anonymous label: a hash-free counter, not the real session id.
68
+ id: `sample-${samples.length + 1}`,
69
+ events,
70
+ expected: {
71
+ // DSH's own totals, which the parser must reproduce exactly.
72
+ uncachedInputTokens: want.uncachedInputTokens ?? 0,
73
+ outputTokens: want.outputTokens ?? 0,
74
+ cacheReadTokens: want.cacheReadTokens ?? 0,
75
+ cacheWriteTokens: want.cacheWriteTokens ?? 0,
76
+ requests: split.requests,
77
+ },
78
+ checks: [...(sum(split.peak).input ? ['peak'] : []), ...(sum(split.off).input ? ['off'] : [])],
79
+ })
80
+ if (samples.length >= 4) break
81
+ }
82
+ if (samples.length >= 4) break
83
+ }
84
+
85
+ if (!samples.length) throw new Error('no usable samples; run where DSH data exists')
86
+ mkdirSync('test/fixtures', { recursive: true })
87
+ writeFileSync(outFile, JSON.stringify({ generatedAt: 'fixed', samples }, null, 1) + '\n')
88
+ console.log('wrote', outFile, '| samples:', samples.length,
89
+ '| events:', samples.reduce((n, s) => n + s.events.length, 0))