dsh-session-matrix 0.16.5 → 0.16.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/index.js +35 -2
- package/lib/usage-split.js +132 -17
- package/package.json +4 -3
- package/scripts/make-reconcile-fixture.mjs +89 -0
- package/test/fixtures/reconcile.json +3063 -0
- package/test/usage-reconcile.test.mjs +71 -0
- package/test/usage-split.test.mjs +46 -0
package/lib/index.js
CHANGED
|
@@ -158,6 +158,12 @@ const DEFAULTS = {
|
|
|
158
158
|
logBudget: 6,
|
|
159
159
|
/** Bytes of log parsed per collect; keeps one poll from stalling on a big file. */
|
|
160
160
|
logBudgetBytes: 8 * 1024 * 1024,
|
|
161
|
+
/**
|
|
162
|
+
* Hard per-file ceiling. The soft budget above is a target, not a promise: a
|
|
163
|
+
* single file larger than it used to be read whole. Anything past this limit
|
|
164
|
+
* is skipped and reported as incomplete instead of blowing up one poll.
|
|
165
|
+
*/
|
|
166
|
+
logMaxFileBytes: 32 * 1024 * 1024,
|
|
161
167
|
priceTtlMs: 24 * 60 * 60 * 1000,
|
|
162
168
|
}
|
|
163
169
|
|
|
@@ -532,6 +538,10 @@ export function apply(ctx, config = {}) {
|
|
|
532
538
|
/** `${id}|${revision}` → in-flight read, so two collects never open two handles. */
|
|
533
539
|
const inflightSplits = new Map()
|
|
534
540
|
|
|
541
|
+
/** Sessions skipped because their log exceeds `logMaxFileBytes`. */
|
|
542
|
+
const oversize = new Set()
|
|
543
|
+
const maxFileBytes = Math.max(1, cfg.logMaxFileBytes ?? 32 * 1024 * 1024)
|
|
544
|
+
|
|
535
545
|
/** Session-log index and its timestamp; see indexSessionLogs(). */
|
|
536
546
|
let logIndex = null
|
|
537
547
|
let logIndexAt = 0
|
|
@@ -1020,6 +1030,13 @@ export function apply(ctx, config = {}) {
|
|
|
1020
1030
|
if (!snapshot) return null
|
|
1021
1031
|
const revision = String(snapshot.revision ?? '')
|
|
1022
1032
|
const size = Number(snapshot.sizeBytes) || 0
|
|
1033
|
+
if (size > maxFileBytes) {
|
|
1034
|
+
// Over the ceiling: leave the previous split in place and record that this
|
|
1035
|
+
// session is not covered, rather than reading it whole.
|
|
1036
|
+
oversize.add(id)
|
|
1037
|
+
return { skipped: true, size }
|
|
1038
|
+
}
|
|
1039
|
+
oversize.delete(id)
|
|
1023
1040
|
const hit = LOG_SPLITS.get(id)
|
|
1024
1041
|
if (revision && hit?.revision === revision) return { entry: hit, read: false, size }
|
|
1025
1042
|
|
|
@@ -1075,9 +1092,14 @@ export function apply(ctx, config = {}) {
|
|
|
1075
1092
|
for (const rec of records) {
|
|
1076
1093
|
const id = rec.header?.id
|
|
1077
1094
|
if (!id) continue
|
|
1078
|
-
if (parsed >= maxFiles
|
|
1095
|
+
if (parsed >= maxFiles) { remaining.push(rec); continue }
|
|
1096
|
+
// The soft budget is checked before the read, not after: adding a file and
|
|
1097
|
+
// then discovering it blew the budget is how one 50 MB log used to stall a
|
|
1098
|
+
// whole poll.
|
|
1099
|
+
if (bytes > 0 && bytes >= maxBytes) { remaining.push(rec); continue }
|
|
1079
1100
|
const official = await splitFromPersistence(id)
|
|
1080
1101
|
if (!official) { remaining.push(rec); continue }
|
|
1102
|
+
if (official.skipped) { parsed += 0; continue }
|
|
1081
1103
|
if (official.read) {
|
|
1082
1104
|
parsed += 1
|
|
1083
1105
|
bytes += official.size ?? 0
|
|
@@ -1098,6 +1120,10 @@ export function apply(ctx, config = {}) {
|
|
|
1098
1120
|
if (!file) continue
|
|
1099
1121
|
try {
|
|
1100
1122
|
const stat = await fs.stat(file)
|
|
1123
|
+
if (stat.size > maxFileBytes) {
|
|
1124
|
+
oversize.add(id)
|
|
1125
|
+
continue
|
|
1126
|
+
}
|
|
1101
1127
|
const hit = LOG_SPLITS.get(id)
|
|
1102
1128
|
if (hit && hit.mtime === stat.mtimeMs && hit.size === stat.size) continue
|
|
1103
1129
|
// The fallback has no revision token, so it keeps the growth throttle:
|
|
@@ -1111,7 +1137,10 @@ export function apply(ctx, config = {}) {
|
|
|
1111
1137
|
}
|
|
1112
1138
|
todo.sort((a, b) => a.stat.size - b.stat.size)
|
|
1113
1139
|
for (const item of todo) {
|
|
1114
|
-
if (parsed >= maxFiles
|
|
1140
|
+
if (parsed >= maxFiles) break
|
|
1141
|
+
// Do not start a file that cannot fit in what is left of the budget; a
|
|
1142
|
+
// smaller one later in the queue can still be read this round.
|
|
1143
|
+
if (bytes > 0 && bytes + item.stat.size > maxBytes) continue
|
|
1115
1144
|
parsed += 1
|
|
1116
1145
|
bytes += item.stat.size
|
|
1117
1146
|
const stamp = { mtime: item.stat.mtimeMs, size: item.stat.size }
|
|
@@ -1397,6 +1426,10 @@ export function apply(ctx, config = {}) {
|
|
|
1397
1426
|
persistenceReads: (() => { try { return Boolean(ctx.get?.('sessionPersistence')?.stat) } catch { return false } })(),
|
|
1398
1427
|
logSplits: LOG_SPLITS.size,
|
|
1399
1428
|
logSplitsWithUsage: [...LOG_SPLITS.values()].filter(v => v.peak || v.off).length,
|
|
1429
|
+
// A session over the per-file ceiling is never parsed, so the aggregate
|
|
1430
|
+
// is incomplete while this is non-zero. Saying so beats a silent cap.
|
|
1431
|
+
oversizeSessions: oversize.size,
|
|
1432
|
+
logBudget: { maxFiles: cfg.logBudget, maxBytes: cfg.logBudgetBytes, maxFileBytes },
|
|
1400
1433
|
})
|
|
1401
1434
|
}))
|
|
1402
1435
|
|
package/lib/usage-split.js
CHANGED
|
@@ -9,12 +9,20 @@
|
|
|
9
9
|
* `assistant/message` event carries `time` and `data.usage`.
|
|
10
10
|
*
|
|
11
11
|
* The log is a sequence of concatenated zstd frames, one per append. A single
|
|
12
|
-
* `zstdDecompressSync` on the whole file stops after the first frame, so
|
|
13
|
-
*
|
|
14
|
-
*
|
|
12
|
+
* `zstdDecompressSync` on the whole file stops after the first frame, so the
|
|
13
|
+
* frames have to be walked one by one. The walk follows the frame structure
|
|
14
|
+
* (RFC 8878: descriptor, block headers, block bodies, checksum) rather than
|
|
15
|
+
* searching for the magic bytes, because a compressed block can contain those
|
|
16
|
+
* four bytes and searching would split a healthy frame in two.
|
|
17
|
+
*
|
|
18
|
+
* Only structurally complete frames are decompressed. A tail that is still
|
|
19
|
+
* being appended is truncated mid-block; Node's zstd will happily return a
|
|
20
|
+
* partial result for it, so completeness is decided here, not by the decoder.
|
|
21
|
+
* Results are cached by (system revision | mtime, size) by the caller.
|
|
15
22
|
*/
|
|
16
23
|
|
|
17
24
|
const MAGIC = Buffer.from([0x28, 0xb5, 0x2f, 0xfd])
|
|
25
|
+
const ZSTD_MAGIC = 0xfd2fb528 // little-endian 28 b5 2f fd
|
|
18
26
|
const BEIJING_OFFSET_MS = 8 * 60 * 60 * 1000
|
|
19
27
|
|
|
20
28
|
/**
|
|
@@ -33,26 +41,133 @@ export function isPeakTime(epochMs) {
|
|
|
33
41
|
}
|
|
34
42
|
|
|
35
43
|
/** Decompress every concatenated zstd frame in a buffer. */
|
|
44
|
+
/**
|
|
45
|
+
* Walk one Zstandard frame starting at `start`.
|
|
46
|
+
*
|
|
47
|
+
* Follows the frame layout from RFC 8878 §3.1 instead of trusting the magic
|
|
48
|
+
* bytes: frame header descriptor → optional window descriptor → dictionary id →
|
|
49
|
+
* frame content size → blocks → optional checksum. A block body may contain any
|
|
50
|
+
* bytes, including the magic, so only the length fields can tell us where the
|
|
51
|
+
* next frame begins.
|
|
52
|
+
*
|
|
53
|
+
* @param buffer - the whole log.
|
|
54
|
+
* @param start - offset of the frame magic.
|
|
55
|
+
* @returns `{ end }` when the frame is complete, `{ torn }` when it runs past the
|
|
56
|
+
* end of the buffer (a live append), `{ corrupt }` when the header or a block
|
|
57
|
+
* header is invalid.
|
|
58
|
+
*/
|
|
59
|
+
export function scanFrame(buffer, start) {
|
|
60
|
+
let p = start
|
|
61
|
+
if (buffer.length - p < 5) return { torn: true }
|
|
62
|
+
p += 4 // magic, checked by the caller
|
|
63
|
+
|
|
64
|
+
const descriptor = buffer[p]
|
|
65
|
+
p += 1
|
|
66
|
+
const fcsFlag = descriptor >>> 6
|
|
67
|
+
const singleSegment = (descriptor & 0x20) !== 0
|
|
68
|
+
const unused = (descriptor & 0x10) !== 0
|
|
69
|
+
const reserved = (descriptor & 0x08) !== 0
|
|
70
|
+
const hasChecksum = (descriptor & 0x04) !== 0
|
|
71
|
+
const dictFlag = descriptor & 0x03
|
|
72
|
+
if (unused || reserved) return { corrupt: true }
|
|
73
|
+
|
|
74
|
+
if (!singleSegment) {
|
|
75
|
+
if (p >= buffer.length) return { torn: true }
|
|
76
|
+
p += 1 // window descriptor
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
const dictBytes = dictFlag === 3 ? 4 : dictFlag
|
|
80
|
+
if (p + dictBytes > buffer.length) return { torn: true }
|
|
81
|
+
p += dictBytes
|
|
82
|
+
|
|
83
|
+
const fcsBytes = fcsFlag === 0
|
|
84
|
+
? (singleSegment ? 1 : 0)
|
|
85
|
+
: (fcsFlag === 1 ? 2 : (fcsFlag === 2 ? 4 : 8))
|
|
86
|
+
if (p + fcsBytes > buffer.length) return { torn: true }
|
|
87
|
+
let declaredSize = null
|
|
88
|
+
if (fcsBytes === 1) declaredSize = buffer[p]
|
|
89
|
+
else if (fcsBytes === 2) declaredSize = buffer.readUInt16LE(p) + 256
|
|
90
|
+
else if (fcsBytes === 4) declaredSize = buffer.readUInt32LE(p)
|
|
91
|
+
else if (fcsBytes === 8) declaredSize = Number(buffer.readBigUInt64LE(p))
|
|
92
|
+
p += fcsBytes
|
|
93
|
+
|
|
94
|
+
for (;;) {
|
|
95
|
+
if (p + 3 > buffer.length) return { torn: true }
|
|
96
|
+
const header = buffer[p] | (buffer[p + 1] << 8) | (buffer[p + 2] << 16)
|
|
97
|
+
p += 3
|
|
98
|
+
const last = (header & 1) === 1
|
|
99
|
+
const type = (header >>> 1) & 3
|
|
100
|
+
const size = header >>> 3
|
|
101
|
+
if (type === 3) return { corrupt: true } // reserved block type
|
|
102
|
+
// An RLE block stores a single byte no matter how large its output is.
|
|
103
|
+
const bodyBytes = type === 1 ? 1 : size
|
|
104
|
+
if (p + bodyBytes > buffer.length) return { torn: true }
|
|
105
|
+
p += bodyBytes
|
|
106
|
+
if (last) break
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
if (hasChecksum) {
|
|
110
|
+
if (p + 4 > buffer.length) return { torn: true }
|
|
111
|
+
p += 4
|
|
112
|
+
}
|
|
113
|
+
return { end: p, declaredSize }
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/**
|
|
117
|
+
* Decompress a concatenated zstd log, counting only complete frames.
|
|
118
|
+
*
|
|
119
|
+
* A frame that is structurally incomplete (the tail of a file being appended)
|
|
120
|
+
* is never handed to the decoder: `zstdDecompressSync` returns partial output
|
|
121
|
+
* for some truncated inputs instead of throwing, and that partial output would
|
|
122
|
+
* silently corrupt the usage totals. A frame whose declared content size does
|
|
123
|
+
* not match what came out is treated the same way.
|
|
124
|
+
*
|
|
125
|
+
* @param buffer - raw log bytes.
|
|
126
|
+
* @param zstd - `zlib.zstdDecompressSync`.
|
|
127
|
+
* @returns the concatenated JSONL text of every complete frame.
|
|
128
|
+
*/
|
|
36
129
|
export function decompressFrames(buffer, zstd) {
|
|
37
130
|
let text = ''
|
|
38
131
|
let pos = 0
|
|
39
|
-
while (pos
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
132
|
+
while (pos + 4 <= buffer.length) {
|
|
133
|
+
if (buffer.readUInt32LE(pos) !== ZSTD_MAGIC) {
|
|
134
|
+
// Not at a frame boundary: either the file is corrupt or the previous
|
|
135
|
+
// frame ended early. Resynchronise on the next magic rather than giving
|
|
136
|
+
// up on the rest of the log.
|
|
137
|
+
const next = buffer.indexOf(MAGIC, pos + 1)
|
|
138
|
+
if (next < 0) break
|
|
139
|
+
pos = next
|
|
140
|
+
continue
|
|
141
|
+
}
|
|
142
|
+
const frame = scanFrame(buffer, pos)
|
|
143
|
+
// A frame is only trustworthy when it ends exactly where the log ends or
|
|
144
|
+
// where the next frame's magic begins. A frame truncated in the middle of a
|
|
145
|
+
// file still "parses": its block header overstates the body by the missing
|
|
146
|
+
// bytes, and those bytes are supplied by the next frame, so the computed end
|
|
147
|
+
// lands inside it. The boundary check is what catches that.
|
|
148
|
+
const atBoundary = frame.end != null
|
|
149
|
+
&& (frame.end === buffer.length || buffer.readUInt32LE(frame.end) === ZSTD_MAGIC)
|
|
150
|
+
if (!atBoundary) {
|
|
151
|
+
// Neither a torn nor a corrupt frame is ever counted. Look for the next
|
|
152
|
+
// frame instead of giving up on the file: each append writes its own
|
|
153
|
+
// frame, so one damaged frame does not invalidate the ones after it.
|
|
154
|
+
// A torn tail (a live append) simply has no next magic and ends the walk.
|
|
155
|
+
const next = buffer.indexOf(MAGIC, pos + 4)
|
|
50
156
|
if (next < 0) break
|
|
51
|
-
pos =
|
|
157
|
+
pos = next
|
|
52
158
|
continue
|
|
53
159
|
}
|
|
54
|
-
|
|
55
|
-
|
|
160
|
+
try {
|
|
161
|
+
const out = zstd(buffer.subarray(pos, frame.end))
|
|
162
|
+
// A declared content size that does not match the output means the frame
|
|
163
|
+
// decoded to something other than what the writer intended; drop it.
|
|
164
|
+
if (frame.declaredSize == null || out.length === frame.declaredSize) {
|
|
165
|
+
text += out.toString('utf8')
|
|
166
|
+
}
|
|
167
|
+
} catch {
|
|
168
|
+
// Corrupt payload: skip this frame, keep the ones already decoded.
|
|
169
|
+
}
|
|
170
|
+
pos = frame.end
|
|
56
171
|
}
|
|
57
172
|
return text
|
|
58
173
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "dsh-session-matrix",
|
|
3
|
-
"version": "0.16.
|
|
3
|
+
"version": "0.16.6",
|
|
4
4
|
"description": "A multi-agent workspace for DeepSeek Harness: run parallel sessions, visualize agent relationships, monitor context usage, and manage task handoffs.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"deepseek-harness",
|
|
@@ -46,7 +46,8 @@
|
|
|
46
46
|
"README.md",
|
|
47
47
|
"LICENSE",
|
|
48
48
|
"cordis.patch.yml",
|
|
49
|
-
"test/"
|
|
49
|
+
"test/",
|
|
50
|
+
"scripts/"
|
|
50
51
|
],
|
|
51
52
|
"engines": {
|
|
52
53
|
"node": "^22.19.0 || >=24.0.0",
|
|
@@ -69,7 +70,7 @@
|
|
|
69
70
|
}
|
|
70
71
|
},
|
|
71
72
|
"scripts": {
|
|
72
|
-
"test": "node test/host.test.mjs && node test/usage-split.test.mjs && node test/usage-vs-projection.test.mjs",
|
|
73
|
+
"test": "node test/host.test.mjs && node test/usage-split.test.mjs && node test/usage-reconcile.test.mjs && node test/usage-vs-projection.test.mjs",
|
|
73
74
|
"prepublishOnly": "npm test"
|
|
74
75
|
}
|
|
75
76
|
}
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Build a committed, anonymised reconciliation fixture from real DSH logs.
|
|
3
|
+
*
|
|
4
|
+
* The CI job has no DSH data, so the local 37/37 check skips there. This strips
|
|
5
|
+
* a few real sessions down to the only fields the parser reads — event type,
|
|
6
|
+
* timestamp, and the usage payload — and records the totals DSH itself computed,
|
|
7
|
+
* so the comparison can run anywhere without shipping anyone's conversations.
|
|
8
|
+
*/
|
|
9
|
+
import { readFileSync, writeFileSync, readdirSync, statSync, existsSync, mkdirSync } from 'node:fs'
|
|
10
|
+
import { zstdDecompressSync } from 'node:zlib'
|
|
11
|
+
import { join } from 'node:path'
|
|
12
|
+
import { homedir } from 'node:os'
|
|
13
|
+
import { decompressFrames, splitUsageEvents } from '../plugin/agent-workspa../lib/usage-split.js'
|
|
14
|
+
|
|
15
|
+
const home = process.env.DSH_HOME || join(homedir(), '.dsh')
|
|
16
|
+
const sessionsRoot = join(home, 'sessions')
|
|
17
|
+
const cacheRoot = join(home, 'storages', 'session_projcache', 'sessions')
|
|
18
|
+
const outFile = 'test/fixtures/reconcile.json'
|
|
19
|
+
|
|
20
|
+
const totals = new Map()
|
|
21
|
+
for (const file of readdirSync(cacheRoot)) {
|
|
22
|
+
if (!file.endsWith('.json')) continue
|
|
23
|
+
try {
|
|
24
|
+
const record = JSON.parse(readFileSync(join(cacheRoot, file), 'utf8'))
|
|
25
|
+
const value = record?.record?.rows?.tokenUsage?.val ?? record?.record?.rows?.tokenUsage
|
|
26
|
+
if (value?.totals) totals.set(file.replace(/\.json$/, '').replace(/^session-/, ''), value.totals)
|
|
27
|
+
} catch { /* ignore a half-written cache entry */ }
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
const samples = []
|
|
31
|
+
for (const dir of readdirSync(sessionsRoot)) {
|
|
32
|
+
if (!statSync(join(sessionsRoot, dir)).isDirectory()) continue
|
|
33
|
+
for (const entry of readdirSync(join(sessionsRoot, dir), { withFileTypes: true })) {
|
|
34
|
+
if (!entry.isDirectory()) continue
|
|
35
|
+
const file = join(sessionsRoot, dir, entry.name, 'session.v4.jsonl.zstd')
|
|
36
|
+
if (!existsSync(file)) continue
|
|
37
|
+
const want = totals.get(entry.name)
|
|
38
|
+
if (!want) continue
|
|
39
|
+
let text = ''
|
|
40
|
+
try { text = decompressFrames(readFileSync(file), zstdDecompressSync) } catch { continue }
|
|
41
|
+
|
|
42
|
+
const events = []
|
|
43
|
+
for (const line of text.split('\n')) {
|
|
44
|
+
if (!line.includes('assistant/')) continue
|
|
45
|
+
let event
|
|
46
|
+
try { event = JSON.parse(line) } catch { continue }
|
|
47
|
+
if (event.type !== 'assistant/message' && event.type !== 'assistant/attempt') continue
|
|
48
|
+
// Keep the shape the parser walks and nothing else: no message text, no
|
|
49
|
+
// prompt, no tool arguments, no titles.
|
|
50
|
+
const kept = { type: event.type, time: event.time, data: {} }
|
|
51
|
+
if (event.data?.usage) kept.data.usage = event.data.usage
|
|
52
|
+
const chunks = (event.data?.stream ?? [])
|
|
53
|
+
.filter(r => r?.chunk?.type === 'usage')
|
|
54
|
+
.map(r => ({ time: r.time, chunk: { type: 'usage', usage: r.chunk.usage } }))
|
|
55
|
+
if (chunks.length) kept.data.stream = chunks
|
|
56
|
+
events.push(kept)
|
|
57
|
+
}
|
|
58
|
+
if (!events.length) continue
|
|
59
|
+
|
|
60
|
+
const split = splitUsageEvents(events)
|
|
61
|
+
if (!split) continue
|
|
62
|
+
const sum = bucket => ({
|
|
63
|
+
input: bucket.uncached + bucket.cacheRead + bucket.cacheWrite,
|
|
64
|
+
output: bucket.output,
|
|
65
|
+
})
|
|
66
|
+
samples.push({
|
|
67
|
+
// Stable anonymous label: a hash-free counter, not the real session id.
|
|
68
|
+
id: `sample-${samples.length + 1}`,
|
|
69
|
+
events,
|
|
70
|
+
expected: {
|
|
71
|
+
// DSH's own totals, which the parser must reproduce exactly.
|
|
72
|
+
uncachedInputTokens: want.uncachedInputTokens ?? 0,
|
|
73
|
+
outputTokens: want.outputTokens ?? 0,
|
|
74
|
+
cacheReadTokens: want.cacheReadTokens ?? 0,
|
|
75
|
+
cacheWriteTokens: want.cacheWriteTokens ?? 0,
|
|
76
|
+
requests: split.requests,
|
|
77
|
+
},
|
|
78
|
+
checks: [...(sum(split.peak).input ? ['peak'] : []), ...(sum(split.off).input ? ['off'] : [])],
|
|
79
|
+
})
|
|
80
|
+
if (samples.length >= 4) break
|
|
81
|
+
}
|
|
82
|
+
if (samples.length >= 4) break
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
if (!samples.length) throw new Error('no usable samples; run where DSH data exists')
|
|
86
|
+
mkdirSync('test/fixtures', { recursive: true })
|
|
87
|
+
writeFileSync(outFile, JSON.stringify({ generatedAt: 'fixed', samples }, null, 1) + '\n')
|
|
88
|
+
console.log('wrote', outFile, '| samples:', samples.length,
|
|
89
|
+
'| events:', samples.reduce((n, s) => n + s.events.length, 0))
|