dsh-session-matrix 0.16.4 → 0.16.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/lib/index.js CHANGED
@@ -158,6 +158,12 @@ const DEFAULTS = {
158
158
  logBudget: 6,
159
159
  /** Bytes of log parsed per collect; keeps one poll from stalling on a big file. */
160
160
  logBudgetBytes: 8 * 1024 * 1024,
161
+ /**
162
+ * Hard per-file ceiling. The soft budget above is a target, not a promise: a
163
+ * single file larger than it used to be read whole. Anything past this limit
164
+ * is skipped and reported as incomplete instead of blowing up one poll.
165
+ */
166
+ logMaxFileBytes: 32 * 1024 * 1024,
161
167
  priceTtlMs: 24 * 60 * 60 * 1000,
162
168
  }
163
169
 
@@ -532,6 +538,10 @@ export function apply(ctx, config = {}) {
532
538
  /** `${id}|${revision}` → in-flight read, so two collects never open two handles. */
533
539
  const inflightSplits = new Map()
534
540
 
541
+ /** Sessions skipped because their log exceeds `logMaxFileBytes`. */
542
+ const oversize = new Set()
543
+ const maxFileBytes = Math.max(1, cfg.logMaxFileBytes ?? 32 * 1024 * 1024)
544
+
535
545
  /** Session-log index and its timestamp; see indexSessionLogs(). */
536
546
  let logIndex = null
537
547
  let logIndexAt = 0
@@ -1020,6 +1030,13 @@ export function apply(ctx, config = {}) {
1020
1030
  if (!snapshot) return null
1021
1031
  const revision = String(snapshot.revision ?? '')
1022
1032
  const size = Number(snapshot.sizeBytes) || 0
1033
+ if (size > maxFileBytes) {
1034
+ // Over the ceiling: leave the previous split in place and record that this
1035
+ // session is not covered, rather than reading it whole.
1036
+ oversize.add(id)
1037
+ return { skipped: true, size }
1038
+ }
1039
+ oversize.delete(id)
1023
1040
  const hit = LOG_SPLITS.get(id)
1024
1041
  if (revision && hit?.revision === revision) return { entry: hit, read: false, size }
1025
1042
 
@@ -1075,9 +1092,14 @@ export function apply(ctx, config = {}) {
1075
1092
  for (const rec of records) {
1076
1093
  const id = rec.header?.id
1077
1094
  if (!id) continue
1078
- if (parsed >= maxFiles || bytes >= maxBytes) { remaining.push(rec); continue }
1095
+ if (parsed >= maxFiles) { remaining.push(rec); continue }
1096
+ // The soft budget is checked before the read, not after: adding a file and
1097
+ // then discovering it blew the budget is how one 50 MB log used to stall a
1098
+ // whole poll.
1099
+ if (bytes > 0 && bytes >= maxBytes) { remaining.push(rec); continue }
1079
1100
  const official = await splitFromPersistence(id)
1080
1101
  if (!official) { remaining.push(rec); continue }
1102
+ if (official.skipped) { parsed += 0; continue }
1081
1103
  if (official.read) {
1082
1104
  parsed += 1
1083
1105
  bytes += official.size ?? 0
@@ -1098,6 +1120,10 @@ export function apply(ctx, config = {}) {
1098
1120
  if (!file) continue
1099
1121
  try {
1100
1122
  const stat = await fs.stat(file)
1123
+ if (stat.size > maxFileBytes) {
1124
+ oversize.add(id)
1125
+ continue
1126
+ }
1101
1127
  const hit = LOG_SPLITS.get(id)
1102
1128
  if (hit && hit.mtime === stat.mtimeMs && hit.size === stat.size) continue
1103
1129
  // The fallback has no revision token, so it keeps the growth throttle:
@@ -1111,7 +1137,10 @@ export function apply(ctx, config = {}) {
1111
1137
  }
1112
1138
  todo.sort((a, b) => a.stat.size - b.stat.size)
1113
1139
  for (const item of todo) {
1114
- if (parsed >= maxFiles || bytes >= maxBytes) break
1140
+ if (parsed >= maxFiles) break
1141
+ // Do not start a file that cannot fit in what is left of the budget; a
1142
+ // smaller one later in the queue can still be read this round.
1143
+ if (bytes > 0 && bytes + item.stat.size > maxBytes) continue
1115
1144
  parsed += 1
1116
1145
  bytes += item.stat.size
1117
1146
  const stamp = { mtime: item.stat.mtimeMs, size: item.stat.size }
@@ -1397,6 +1426,10 @@ export function apply(ctx, config = {}) {
1397
1426
  persistenceReads: (() => { try { return Boolean(ctx.get?.('sessionPersistence')?.stat) } catch { return false } })(),
1398
1427
  logSplits: LOG_SPLITS.size,
1399
1428
  logSplitsWithUsage: [...LOG_SPLITS.values()].filter(v => v.peak || v.off).length,
1429
+ // A session over the per-file ceiling is never parsed, so the aggregate
1430
+ // is incomplete while this is non-zero. Saying so beats a silent cap.
1431
+ oversizeSessions: oversize.size,
1432
+ logBudget: { maxFiles: cfg.logBudget, maxBytes: cfg.logBudgetBytes, maxFileBytes },
1400
1433
  })
1401
1434
  }))
1402
1435
 
@@ -9,12 +9,20 @@
9
9
  * `assistant/message` event carries `time` and `data.usage`.
10
10
  *
11
11
  * The log is a sequence of concatenated zstd frames, one per append. A single
12
- * `zstdDecompressSync` on the whole file stops after the first frame, so this
13
- * walks the frame magic instead. Parsing a 20 MB session takes under a second;
14
- * results are cached by (mtime, size) by the caller.
12
+ * `zstdDecompressSync` on the whole file stops after the first frame, so the
13
+ * frames have to be walked one by one. The walk follows the frame structure
14
+ * (RFC 8878: descriptor, block headers, block bodies, checksum) rather than
15
+ * searching for the magic bytes, because a compressed block can contain those
16
+ * four bytes and searching would split a healthy frame in two.
17
+ *
18
+ * Only structurally complete frames are decompressed. A tail that is still
19
+ * being appended is truncated mid-block; Node's zstd will happily return a
20
+ * partial result for it, so completeness is decided here, not by the decoder.
21
+ * Results are cached by (system revision | mtime, size) by the caller.
15
22
  */
16
23
 
17
24
  const MAGIC = Buffer.from([0x28, 0xb5, 0x2f, 0xfd])
25
+ const ZSTD_MAGIC = 0xfd2fb528 // little-endian 28 b5 2f fd
18
26
  const BEIJING_OFFSET_MS = 8 * 60 * 60 * 1000
19
27
 
20
28
  /**
@@ -33,26 +41,133 @@ export function isPeakTime(epochMs) {
33
41
  }
34
42
 
35
43
  /** Decompress every concatenated zstd frame in a buffer. */
44
+ /**
45
+ * Walk one Zstandard frame starting at `start`.
46
+ *
47
+ * Follows the frame layout from RFC 8878 §3.1 instead of trusting the magic
48
+ * bytes: frame header descriptor → optional window descriptor → dictionary id →
49
+ * frame content size → blocks → optional checksum. A block body may contain any
50
+ * bytes, including the magic, so only the length fields can tell us where the
51
+ * next frame begins.
52
+ *
53
+ * @param buffer - the whole log.
54
+ * @param start - offset of the frame magic.
55
+ * @returns `{ end }` when the frame is complete, `{ torn }` when it runs past the
56
+ * end of the buffer (a live append), `{ corrupt }` when the header or a block
57
+ * header is invalid.
58
+ */
59
+ export function scanFrame(buffer, start) {
60
+ let p = start
61
+ if (buffer.length - p < 5) return { torn: true }
62
+ p += 4 // magic, checked by the caller
63
+
64
+ const descriptor = buffer[p]
65
+ p += 1
66
+ const fcsFlag = descriptor >>> 6
67
+ const singleSegment = (descriptor & 0x20) !== 0
68
+ const unused = (descriptor & 0x10) !== 0
69
+ const reserved = (descriptor & 0x08) !== 0
70
+ const hasChecksum = (descriptor & 0x04) !== 0
71
+ const dictFlag = descriptor & 0x03
72
+ if (unused || reserved) return { corrupt: true }
73
+
74
+ if (!singleSegment) {
75
+ if (p >= buffer.length) return { torn: true }
76
+ p += 1 // window descriptor
77
+ }
78
+
79
+ const dictBytes = dictFlag === 3 ? 4 : dictFlag
80
+ if (p + dictBytes > buffer.length) return { torn: true }
81
+ p += dictBytes
82
+
83
+ const fcsBytes = fcsFlag === 0
84
+ ? (singleSegment ? 1 : 0)
85
+ : (fcsFlag === 1 ? 2 : (fcsFlag === 2 ? 4 : 8))
86
+ if (p + fcsBytes > buffer.length) return { torn: true }
87
+ let declaredSize = null
88
+ if (fcsBytes === 1) declaredSize = buffer[p]
89
+ else if (fcsBytes === 2) declaredSize = buffer.readUInt16LE(p) + 256
90
+ else if (fcsBytes === 4) declaredSize = buffer.readUInt32LE(p)
91
+ else if (fcsBytes === 8) declaredSize = Number(buffer.readBigUInt64LE(p))
92
+ p += fcsBytes
93
+
94
+ for (;;) {
95
+ if (p + 3 > buffer.length) return { torn: true }
96
+ const header = buffer[p] | (buffer[p + 1] << 8) | (buffer[p + 2] << 16)
97
+ p += 3
98
+ const last = (header & 1) === 1
99
+ const type = (header >>> 1) & 3
100
+ const size = header >>> 3
101
+ if (type === 3) return { corrupt: true } // reserved block type
102
+ // An RLE block stores a single byte no matter how large its output is.
103
+ const bodyBytes = type === 1 ? 1 : size
104
+ if (p + bodyBytes > buffer.length) return { torn: true }
105
+ p += bodyBytes
106
+ if (last) break
107
+ }
108
+
109
+ if (hasChecksum) {
110
+ if (p + 4 > buffer.length) return { torn: true }
111
+ p += 4
112
+ }
113
+ return { end: p, declaredSize }
114
+ }
115
+
116
+ /**
117
+ * Decompress a concatenated zstd log, counting only complete frames.
118
+ *
119
+ * A frame that is structurally incomplete (the tail of a file being appended)
120
+ * is never handed to the decoder: `zstdDecompressSync` returns partial output
121
+ * for some truncated inputs instead of throwing, and that partial output would
122
+ * silently corrupt the usage totals. A frame whose declared content size does
123
+ * not match what came out is treated the same way.
124
+ *
125
+ * @param buffer - raw log bytes.
126
+ * @param zstd - `zlib.zstdDecompressSync`.
127
+ * @returns the concatenated JSONL text of every complete frame.
128
+ */
36
129
  export function decompressFrames(buffer, zstd) {
37
130
  let text = ''
38
131
  let pos = 0
39
- while (pos < buffer.length) {
40
- const start = buffer.indexOf(MAGIC, pos)
41
- if (start < 0) break
42
- const next = buffer.indexOf(MAGIC, start + 4)
43
- const frame = buffer.subarray(start, next < 0 ? buffer.length : next)
44
- try {
45
- text += zstd(frame).toString('utf8')
46
- } catch {
47
- // Two cases land here: a genuinely broken frame, and a false magic that
48
- // split a real frame in two. Skipping one candidate and retrying keeps
49
- // the rest of the log readable instead of truncating the whole file.
132
+ while (pos + 4 <= buffer.length) {
133
+ if (buffer.readUInt32LE(pos) !== ZSTD_MAGIC) {
134
+ // Not at a frame boundary: either the file is corrupt or the previous
135
+ // frame ended early. Resynchronise on the next magic rather than giving
136
+ // up on the rest of the log.
137
+ const next = buffer.indexOf(MAGIC, pos + 1)
138
+ if (next < 0) break
139
+ pos = next
140
+ continue
141
+ }
142
+ const frame = scanFrame(buffer, pos)
143
+ // A frame is only trustworthy when it ends exactly where the log ends or
144
+ // where the next frame's magic begins. A frame truncated in the middle of a
145
+ // file still "parses": its block header overstates the body by the missing
146
+ // bytes, and those bytes are supplied by the next frame, so the computed end
147
+ // lands inside it. The boundary check is what catches that.
148
+ const atBoundary = frame.end != null
149
+ && (frame.end === buffer.length || buffer.readUInt32LE(frame.end) === ZSTD_MAGIC)
150
+ if (!atBoundary) {
151
+ // Neither a torn nor a corrupt frame is ever counted. Look for the next
152
+ // frame instead of giving up on the file: each append writes its own
153
+ // frame, so one damaged frame does not invalidate the ones after it.
154
+ // A torn tail (a live append) simply has no next magic and ends the walk.
155
+ const next = buffer.indexOf(MAGIC, pos + 4)
50
156
  if (next < 0) break
51
- pos = start + 4
157
+ pos = next
52
158
  continue
53
159
  }
54
- if (next < 0) break
55
- pos = next
160
+ try {
161
+ const out = zstd(buffer.subarray(pos, frame.end))
162
+ // A declared content size that does not match the output means the frame
163
+ // decoded to something other than what the writer intended; drop it.
164
+ if (frame.declaredSize == null || out.length === frame.declaredSize) {
165
+ text += out.toString('utf8')
166
+ }
167
+ } catch {
168
+ // Corrupt payload: skip this frame, keep the ones already decoded.
169
+ }
170
+ pos = frame.end
56
171
  }
57
172
  return text
58
173
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "dsh-session-matrix",
3
- "version": "0.16.4",
3
+ "version": "0.16.6",
4
4
  "description": "A multi-agent workspace for DeepSeek Harness: run parallel sessions, visualize agent relationships, monitor context usage, and manage task handoffs.",
5
5
  "keywords": [
6
6
  "deepseek-harness",
@@ -46,7 +46,8 @@
46
46
  "README.md",
47
47
  "LICENSE",
48
48
  "cordis.patch.yml",
49
- "test/"
49
+ "test/",
50
+ "scripts/"
50
51
  ],
51
52
  "engines": {
52
53
  "node": "^22.19.0 || >=24.0.0",
@@ -69,6 +70,7 @@
69
70
  }
70
71
  },
71
72
  "scripts": {
72
- "test": "node test/host.test.mjs && node test/usage-split.test.mjs && node test/usage-vs-projection.test.mjs"
73
+ "test": "node test/host.test.mjs && node test/usage-split.test.mjs && node test/usage-reconcile.test.mjs && node test/usage-vs-projection.test.mjs",
74
+ "prepublishOnly": "npm test"
73
75
  }
74
76
  }
@@ -0,0 +1,89 @@
1
+ /**
2
+ * Build a committed, anonymised reconciliation fixture from real DSH logs.
3
+ *
4
+ * The CI job has no DSH data, so the local 37/37 check skips there. This strips
5
+ * a few real sessions down to the only fields the parser reads — event type,
6
+ * timestamp, and the usage payload — and records the totals DSH itself computed,
7
+ * so the comparison can run anywhere without shipping anyone's conversations.
8
+ */
9
+ import { readFileSync, writeFileSync, readdirSync, statSync, existsSync, mkdirSync } from 'node:fs'
10
+ import { zstdDecompressSync } from 'node:zlib'
11
+ import { join } from 'node:path'
12
+ import { homedir } from 'node:os'
13
+ import { decompressFrames, splitUsageEvents } from '../plugin/agent-workspa../lib/usage-split.js'
14
+
15
+ const home = process.env.DSH_HOME || join(homedir(), '.dsh')
16
+ const sessionsRoot = join(home, 'sessions')
17
+ const cacheRoot = join(home, 'storages', 'session_projcache', 'sessions')
18
+ const outFile = 'test/fixtures/reconcile.json'
19
+
20
+ const totals = new Map()
21
+ for (const file of readdirSync(cacheRoot)) {
22
+ if (!file.endsWith('.json')) continue
23
+ try {
24
+ const record = JSON.parse(readFileSync(join(cacheRoot, file), 'utf8'))
25
+ const value = record?.record?.rows?.tokenUsage?.val ?? record?.record?.rows?.tokenUsage
26
+ if (value?.totals) totals.set(file.replace(/\.json$/, '').replace(/^session-/, ''), value.totals)
27
+ } catch { /* ignore a half-written cache entry */ }
28
+ }
29
+
30
+ const samples = []
31
+ for (const dir of readdirSync(sessionsRoot)) {
32
+ if (!statSync(join(sessionsRoot, dir)).isDirectory()) continue
33
+ for (const entry of readdirSync(join(sessionsRoot, dir), { withFileTypes: true })) {
34
+ if (!entry.isDirectory()) continue
35
+ const file = join(sessionsRoot, dir, entry.name, 'session.v4.jsonl.zstd')
36
+ if (!existsSync(file)) continue
37
+ const want = totals.get(entry.name)
38
+ if (!want) continue
39
+ let text = ''
40
+ try { text = decompressFrames(readFileSync(file), zstdDecompressSync) } catch { continue }
41
+
42
+ const events = []
43
+ for (const line of text.split('\n')) {
44
+ if (!line.includes('assistant/')) continue
45
+ let event
46
+ try { event = JSON.parse(line) } catch { continue }
47
+ if (event.type !== 'assistant/message' && event.type !== 'assistant/attempt') continue
48
+ // Keep the shape the parser walks and nothing else: no message text, no
49
+ // prompt, no tool arguments, no titles.
50
+ const kept = { type: event.type, time: event.time, data: {} }
51
+ if (event.data?.usage) kept.data.usage = event.data.usage
52
+ const chunks = (event.data?.stream ?? [])
53
+ .filter(r => r?.chunk?.type === 'usage')
54
+ .map(r => ({ time: r.time, chunk: { type: 'usage', usage: r.chunk.usage } }))
55
+ if (chunks.length) kept.data.stream = chunks
56
+ events.push(kept)
57
+ }
58
+ if (!events.length) continue
59
+
60
+ const split = splitUsageEvents(events)
61
+ if (!split) continue
62
+ const sum = bucket => ({
63
+ input: bucket.uncached + bucket.cacheRead + bucket.cacheWrite,
64
+ output: bucket.output,
65
+ })
66
+ samples.push({
67
+ // Stable anonymous label: a hash-free counter, not the real session id.
68
+ id: `sample-${samples.length + 1}`,
69
+ events,
70
+ expected: {
71
+ // DSH's own totals, which the parser must reproduce exactly.
72
+ uncachedInputTokens: want.uncachedInputTokens ?? 0,
73
+ outputTokens: want.outputTokens ?? 0,
74
+ cacheReadTokens: want.cacheReadTokens ?? 0,
75
+ cacheWriteTokens: want.cacheWriteTokens ?? 0,
76
+ requests: split.requests,
77
+ },
78
+ checks: [...(sum(split.peak).input ? ['peak'] : []), ...(sum(split.off).input ? ['off'] : [])],
79
+ })
80
+ if (samples.length >= 4) break
81
+ }
82
+ if (samples.length >= 4) break
83
+ }
84
+
85
+ if (!samples.length) throw new Error('no usable samples; run where DSH data exists')
86
+ mkdirSync('test/fixtures', { recursive: true })
87
+ writeFileSync(outFile, JSON.stringify({ generatedAt: 'fixed', samples }, null, 1) + '\n')
88
+ console.log('wrote', outFile, '| samples:', samples.length,
89
+ '| events:', samples.reduce((n, s) => n + s.events.length, 0))