dsh-token-usage-ledger 0.0.0-stage → 0.0.36

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/CHANGELOG.md +1067 -0
  2. package/LICENSE +21 -0
  3. package/README.md +237 -2
  4. package/cordis.patch.yml +30 -0
  5. package/lib/client.js +4469 -0
  6. package/lib/host/aggregate.d.ts +132 -0
  7. package/lib/host/aggregate.js +840 -0
  8. package/lib/host/api.d.ts +118 -0
  9. package/lib/host/api.js +346 -0
  10. package/lib/host/backfill.d.ts +54 -0
  11. package/lib/host/backfill.js +265 -0
  12. package/lib/host/collector.d.ts +139 -0
  13. package/lib/host/collector.js +441 -0
  14. package/lib/host/ledger-io.d.ts +125 -0
  15. package/lib/host/ledger-io.js +211 -0
  16. package/lib/host/operation-error.d.ts +22 -0
  17. package/lib/host/operation-error.js +28 -0
  18. package/lib/host/paths.d.ts +35 -0
  19. package/lib/host/paths.js +63 -0
  20. package/lib/host/pricing-module.d.ts +112 -0
  21. package/lib/host/pricing-module.js +63 -0
  22. package/lib/host/pricing-modules/deepseek-2026-09-10.d.ts +31 -0
  23. package/lib/host/pricing-modules/deepseek-2026-09-10.js +67 -0
  24. package/lib/host/pricing-modules/index.d.ts +27 -0
  25. package/lib/host/pricing-modules/index.js +30 -0
  26. package/lib/host/pricing.d.ts +192 -0
  27. package/lib/host/pricing.js +287 -0
  28. package/lib/host/records.d.ts +47 -0
  29. package/lib/host/records.js +116 -0
  30. package/lib/host/runtime-marker.d.ts +67 -0
  31. package/lib/host/runtime-marker.js +71 -0
  32. package/lib/host/schema.d.ts +77 -0
  33. package/lib/host/schema.js +158 -0
  34. package/lib/host/store.d.ts +171 -0
  35. package/lib/host/store.js +337 -0
  36. package/lib/host/workspace-titles.d.ts +38 -0
  37. package/lib/host/workspace-titles.js +136 -0
  38. package/lib/host/zstd.d.ts +41 -0
  39. package/lib/host/zstd.js +134 -0
  40. package/lib/index.d.ts +82 -0
  41. package/lib/index.js +512 -0
  42. package/lib/shared/format.d.ts +415 -0
  43. package/lib/shared/format.js +714 -0
  44. package/lib/shared/protocol.d.ts +646 -0
  45. package/lib/shared/protocol.js +129 -0
  46. package/package.json +85 -4
@@ -0,0 +1,337 @@
1
+ /**
2
+ * Append-only ledger storage.
3
+ *
4
+ * One JSON object per line under the harness home, so the ledger is
5
+ * human-inspectable, append-only, and survives a crash mid-write (a torn final
6
+ * line is skipped and counted, never fatal). Writes are batched onto a
7
+ * `setImmediate` drain so the `session/event` dispatch that produced a record
8
+ * never performs file I/O on the conversation's critical path.
9
+ *
10
+ * Every entry point swallows its own failure and reports it through
11
+ * {@link UsageStore.stats}: the ledger is a diagnostic side-channel and must
12
+ * never be able to fail a Turn.
13
+ *
14
+ * @module dsh-token-usage-ledger/host/store
15
+ */
16
+ import { createReadStream } from 'node:fs';
17
+ import { appendFile, mkdir, stat } from 'node:fs/promises';
18
+ import { join } from 'node:path';
19
+ import { createInterface } from 'node:readline';
20
+ import { API_LIMITS } from '../shared/protocol.js';
21
+ import { parseRecordLine } from './schema.js';
22
+ /** Directory (under the harness home) that holds the ledger. */
23
+ export const LEDGER_DIR = 'token-usage';
24
+ /** File name of the append-only ledger. */
25
+ export const LEDGER_FILE = 'records.jsonl';
26
+ /** Bound on queued records; beyond it the oldest are dropped rather than growing without limit. */
27
+ const DEFAULT_MAX_PENDING = 4096;
28
+ /**
29
+ * The ledger: a write queue plus a cached reader.
30
+ *
31
+ * Instances are created per plugin activation and dropped with the fiber; a
32
+ * re-activation re-reads the file, so the ledger — not memory — is the source
33
+ * of truth.
34
+ */
35
+ export class UsageStore {
36
+ dir;
37
+ file;
38
+ path;
39
+ maxPending;
40
+ warn;
41
+ queue = [];
42
+ drainScheduled = false;
43
+ writeErrors = 0;
44
+ lastWriteError = '';
45
+ malformedLines = 0;
46
+ unsupportedRecords = 0;
47
+ truncated = false;
48
+ droppedRecords = 0;
49
+ /** Set while a failed batch is awaiting its single retry. */
50
+ retryArmed = false;
51
+ warnedOnce = false;
52
+ snapshot = undefined;
53
+ snapshotKey = '';
54
+ loadInFlight = undefined;
55
+ /**
56
+ * @param options - directory, file name, queue bound, and warning sink.
57
+ */
58
+ constructor(options) {
59
+ this.dir = options.dir;
60
+ this.file = options.file ?? LEDGER_FILE;
61
+ this.path = join(this.dir, this.file);
62
+ this.maxPending = options.maxPending ?? DEFAULT_MAX_PENDING;
63
+ this.warn = options.warn ?? (() => { });
64
+ }
65
+ /** Absolute path of the ledger file. */
66
+ get filePath() {
67
+ return this.path;
68
+ }
69
+ /**
70
+ * Queue one record for the next append drain.
71
+ *
72
+ * Returns immediately; the caller is on the conversation's dispatch path and
73
+ * must not block. Never throws.
74
+ *
75
+ * @param record - the record to persist.
76
+ */
77
+ enqueue(record) {
78
+ try {
79
+ if (this.queue.length >= this.maxPending) {
80
+ // Shed the oldest record rather than grow without bound: recent usage
81
+ // is the more valuable half of a diagnostic ledger.
82
+ this.queue.shift();
83
+ this.report('ledger queue full; oldest pending record dropped');
84
+ }
85
+ this.queue.push(record);
86
+ this.scheduleDrain();
87
+ }
88
+ catch (error) {
89
+ this.recordWriteError(error);
90
+ }
91
+ }
92
+ /**
93
+ * Flush the queue and wait for it, for shutdown paths and tests.
94
+ * @returns the number of records written, or 0 when the flush failed.
95
+ */
96
+ async flush() {
97
+ return await this.drain();
98
+ }
99
+ /**
100
+ * Append records produced outside the live collector (import/backfill).
101
+ *
102
+ * Waits for the live queue first so the file stays in one append order, then
103
+ * appends the batch in a single write. The caller has already validated the
104
+ * records and checked `(sessionId, turn)` duplicates; this method only reports
105
+ * whether the OS write succeeded. It never throws into the caller's request
106
+ * path — the API maps the returned failure to a structured error.
107
+ *
108
+ * @param records - records to append.
109
+ * @returns the written count, or the write error message on failure.
110
+ */
111
+ async appendRecords(records) {
112
+ if (records.length === 0)
113
+ return { written: 0 };
114
+ try {
115
+ await this.flush();
116
+ await mkdir(this.dir, { recursive: true });
117
+ const payload = records.map((record) => `${JSON.stringify(record)}\n`).join('');
118
+ await appendFile(this.path, payload, { encoding: 'utf8', mode: 0o600 });
119
+ this.snapshotKey = '';
120
+ this.retryArmed = false;
121
+ return { written: records.length };
122
+ }
123
+ catch (error) {
124
+ this.recordWriteError(error);
125
+ return {
126
+ written: 0,
127
+ error: error instanceof Error ? error.message : String(error),
128
+ };
129
+ }
130
+ }
131
+ /**
132
+ * Load and cache the ledger.
133
+ *
134
+ * The cache is keyed on the file's size and mtime, so a second request after
135
+ * a write re-reads while repeated reads stay cheap. A concurrent caller
136
+ * joins the in-flight read instead of starting a second one.
137
+ *
138
+ * @returns the parsed snapshot; an empty snapshot when the file is absent.
139
+ */
140
+ async load() {
141
+ if (this.loadInFlight !== undefined)
142
+ return await this.loadInFlight;
143
+ const run = this.loadOnce();
144
+ this.loadInFlight = run;
145
+ try {
146
+ return await run;
147
+ }
148
+ finally {
149
+ this.loadInFlight = undefined;
150
+ }
151
+ }
152
+ /**
153
+ * Report current diagnostics.
154
+ * @param extra - additional counters owned by the caller.
155
+ * @returns the health report embedded in the summary payload.
156
+ */
157
+ stats(extra) {
158
+ return {
159
+ malformedLines: this.malformedLines,
160
+ unsupportedRecords: this.unsupportedRecords,
161
+ writeErrors: this.writeErrors,
162
+ lastWriteError: this.lastWriteError,
163
+ droppedRecords: this.droppedRecords,
164
+ pendingWrites: this.queue.length,
165
+ skippedTurns: extra.skippedTurns,
166
+ // Recomputed by the aggregator, which is the only place that knows which
167
+ // currency the caller asked for.
168
+ currencyMismatches: 0,
169
+ ledgerTruncated: this.truncated,
170
+ };
171
+ }
172
+ /** Read + parse the ledger, refreshing the cache. */
173
+ async loadOnce() {
174
+ let key = '';
175
+ try {
176
+ const info = await stat(this.path);
177
+ if (!info.isFile()) {
178
+ return { records: [], malformedLines: 0, unsupportedRecords: 0, duplicateLines: 0, truncated: false };
179
+ }
180
+ key = `${String(info.size)}:${String(info.mtimeMs)}`;
181
+ }
182
+ catch {
183
+ // Absent ledger is the normal first-run state, not an error.
184
+ const empty = {
185
+ records: [],
186
+ malformedLines: 0,
187
+ unsupportedRecords: 0,
188
+ duplicateLines: 0,
189
+ truncated: false,
190
+ };
191
+ this.snapshot = empty;
192
+ this.snapshotKey = 'absent';
193
+ this.malformedLines = 0;
194
+ this.unsupportedRecords = 0;
195
+ this.truncated = false;
196
+ return empty;
197
+ }
198
+ if (this.snapshot !== undefined && this.snapshotKey === key)
199
+ return this.snapshot;
200
+ const parsed = await this.parseLedger();
201
+ this.snapshot = parsed;
202
+ this.snapshotKey = key;
203
+ this.malformedLines = parsed.malformedLines;
204
+ this.unsupportedRecords = parsed.unsupportedRecords;
205
+ this.truncated = parsed.truncated;
206
+ return parsed;
207
+ }
208
+ /**
209
+ * Stream the ledger into de-duplicated records.
210
+ *
211
+ * Streamed rather than read whole so memory is bounded by the RECORD cap and
212
+ * not by the file size: a large ledger degrades to the newest
213
+ * {@link API_LIMITS.maxRecords} records with `ledgerTruncated` set, instead of
214
+ * silently reporting an empty dashboard. Every line is still visited, so
215
+ * de-duplication and the malformed count stay exact.
216
+ *
217
+ * @returns the snapshot.
218
+ */
219
+ async parseLedger() {
220
+ const byKey = new Map();
221
+ let malformedLines = 0;
222
+ let unsupportedRecords = 0;
223
+ let duplicateLines = 0;
224
+ let truncated = false;
225
+ const accept = (line) => {
226
+ if (line.length === 0)
227
+ return;
228
+ const outcome = parseRecordLine(line);
229
+ if (!outcome.ok) {
230
+ if (outcome.reason === 'unsupported-version')
231
+ unsupportedRecords += 1;
232
+ else
233
+ malformedLines += 1;
234
+ return;
235
+ }
236
+ const key = `${outcome.record.sessionId}\u0000${String(outcome.record.turn)}`;
237
+ if (byKey.has(key))
238
+ duplicateLines += 1;
239
+ byKey.set(key, outcome.record);
240
+ // Keep the NEWEST records: a Map iterates in insertion order, so the
241
+ // first key is the oldest surviving one.
242
+ if (byKey.size > API_LIMITS.maxRecords) {
243
+ const oldest = byKey.keys().next();
244
+ if (oldest.done !== true)
245
+ byKey.delete(oldest.value);
246
+ truncated = true;
247
+ }
248
+ };
249
+ try {
250
+ const reader = createInterface({
251
+ input: createReadStream(this.path, { encoding: 'utf8' }),
252
+ crlfDelay: Number.POSITIVE_INFINITY,
253
+ });
254
+ for await (const line of reader)
255
+ accept(line);
256
+ }
257
+ catch (error) {
258
+ this.recordWriteError(error);
259
+ return { records: [], malformedLines, unsupportedRecords, duplicateLines, truncated };
260
+ }
261
+ const records = [...byKey.values()].sort((a, b) => a.ts - b.ts);
262
+ return { records, malformedLines, unsupportedRecords, duplicateLines, truncated };
263
+ }
264
+ /** Schedule one drain on the next macrotask boundary. */
265
+ scheduleDrain() {
266
+ if (this.drainScheduled)
267
+ return;
268
+ this.drainScheduled = true;
269
+ setImmediate(() => {
270
+ this.drainScheduled = false;
271
+ void this.drain();
272
+ });
273
+ }
274
+ /**
275
+ * Write every queued record in one append.
276
+ * @returns the number of records written.
277
+ */
278
+ async drain() {
279
+ if (this.queue.length === 0)
280
+ return 0;
281
+ const batch = this.queue;
282
+ this.queue = [];
283
+ try {
284
+ await mkdir(this.dir, { recursive: true });
285
+ const payload = batch.map((record) => `${JSON.stringify(record)}\n`).join('');
286
+ // `mode` applies only when the file is created; the harness home is 0700,
287
+ // and a private ledger keeps that posture for the file itself too.
288
+ await appendFile(this.path, payload, { encoding: 'utf8', mode: 0o600 });
289
+ // The cache key must not match a stale read.
290
+ this.snapshotKey = '';
291
+ this.retryArmed = false;
292
+ return batch.length;
293
+ }
294
+ catch (error) {
295
+ this.recordWriteError(error);
296
+ if (!this.retryArmed) {
297
+ // One retry absorbs a transient failure (a busy fd, a momentary EAGAIN)
298
+ // without risking a retry loop: the second failure drops the batch and
299
+ // records exactly how many records were lost.
300
+ this.retryArmed = true;
301
+ this.queue.unshift(...batch);
302
+ this.scheduleDrain();
303
+ }
304
+ else {
305
+ this.retryArmed = false;
306
+ this.droppedRecords += batch.length;
307
+ }
308
+ return 0;
309
+ }
310
+ }
311
+ /**
312
+ * Account one append failure, reporting it at most once per process so a
313
+ * broken disk cannot flood the log.
314
+ * @param error - the failure to record.
315
+ */
316
+ recordWriteError(error) {
317
+ this.writeErrors += 1;
318
+ this.lastWriteError = error instanceof Error ? error.message : String(error);
319
+ this.report(`ledger write failed: ${this.lastWriteError}`);
320
+ }
321
+ /**
322
+ * Emit one warning line, throttled to a single occurrence per activation.
323
+ * @param message - warning text.
324
+ */
325
+ report(message) {
326
+ if (this.warnedOnce)
327
+ return;
328
+ this.warnedOnce = true;
329
+ try {
330
+ this.warn(message);
331
+ }
332
+ catch {
333
+ // A throwing log sink must not escalate.
334
+ }
335
+ }
336
+ }
337
+ //# sourceMappingURL=store.js.map
@@ -0,0 +1,38 @@
1
+ /**
2
+ * Resolve a ledger's raw workspace paths to the Workspace titles that are
3
+ * current at summary time.
4
+ *
5
+ * The ledger stores the Session working directory and, at write time, its
6
+ * basename. A user can rename a Workspace without changing its directory, so a
7
+ * stored label is not a durable display name. `ctx.workspaceRegistry` is the
8
+ * Host's authoritative Workspace registry: `Workspace.path` stays canonical
9
+ * across renames and `Workspace.title` is the current user-visible title.
10
+ *
11
+ * This module is deliberately failure-tolerant. A missing service, a registry
12
+ * that is still initializing, an unreadable directory, or a foreign-platform
13
+ * path all degrade to "no current title", in which case aggregation keeps the
14
+ * stored `workspaceLabel` (the old behavior).
15
+ *
16
+ * @module dsh-token-usage-ledger/host/workspace-titles
17
+ */
18
+ import type { Context } from '@deepseek-ai/cordis';
19
+ import type { UsageRecord } from '../shared/protocol.js';
20
+ /**
21
+ * Look up a current display title for one raw workspace path.
22
+ * @param workspace - raw `session.header.cwd` value stored on a record.
23
+ * @returns the current title, or undefined when no registry entry matches.
24
+ */
25
+ export type WorkspaceTitleLookup = (workspace: string) => string | undefined;
26
+ /**
27
+ * Build a `raw workspace -> current title` map for the records in one summary.
28
+ *
29
+ * Only unique record paths are looked up. Exact spellings hit the registry
30
+ * without I/O; the rest try `fs.realpath` once so a differently spelled or
31
+ * symlinked path still finds its Workspace. Every failure is contained: this
32
+ * function never rejects.
33
+ *
34
+ * @param ctx - Host context; the Workspace registry is optional.
35
+ * @param records - ledger records in the summary.
36
+ * @returns titles keyed by the exact `record.workspace` spelling.
37
+ */
38
+ export declare function resolveWorkspaceTitles(ctx: Context, records: readonly UsageRecord[]): Promise<Map<string, string>>;
@@ -0,0 +1,136 @@
1
+ /**
2
+ * Resolve a ledger's raw workspace paths to the Workspace titles that are
3
+ * current at summary time.
4
+ *
5
+ * The ledger stores the Session working directory and, at write time, its
6
+ * basename. A user can rename a Workspace without changing its directory, so a
7
+ * stored label is not a durable display name. `ctx.workspaceRegistry` is the
8
+ * Host's authoritative Workspace registry: `Workspace.path` stays canonical
9
+ * across renames and `Workspace.title` is the current user-visible title.
10
+ *
11
+ * This module is deliberately failure-tolerant. A missing service, a registry
12
+ * that is still initializing, an unreadable directory, or a foreign-platform
13
+ * path all degrade to "no current title", in which case aggregation keeps the
14
+ * stored `workspaceLabel` (the old behavior).
15
+ *
16
+ * @module dsh-token-usage-ledger/host/workspace-titles
17
+ */
18
+ import { realpath } from 'node:fs/promises';
19
+ import { isAbsolute } from 'node:path';
20
+ /**
21
+ * Strip the Windows extended-length prefix, which `fs.realpath` can return for
22
+ * long paths. The registry normalizes with the same `fs.realpath`, but a record
23
+ * may have been written through a shorter spelling.
24
+ * @param value - path spelling.
25
+ * @returns the path without a `\\?\` / `\\?\UNC\` prefix.
26
+ */
27
+ function withoutLongPathPrefix(value) {
28
+ if (value.startsWith('\\\\?\\UNC\\'))
29
+ return `\\\\${value.slice('\\\\?\\UNC\\'.length)}`;
30
+ if (value.startsWith('\\\\?\\'))
31
+ return value.slice('\\\\?\\'.length);
32
+ return value;
33
+ }
34
+ /**
35
+ * Comparison key for one path, deliberately not a filesystem canonical form.
36
+ *
37
+ * Trailing separators are ignored, and Windows keys are case-folded because
38
+ * that filesystem is case-insensitive. Separator style, `..`, and symlinks are
39
+ * handled by the `realpath` fallback instead of guessed here.
40
+ *
41
+ * @param value - raw path.
42
+ * @returns the key used for a fast registry lookup.
43
+ */
44
+ function pathKey(value) {
45
+ const stripped = withoutLongPathPrefix(value);
46
+ const trimmed = stripped.length > 1 ? stripped.replace(/[\\/]+$/u, '') : stripped;
47
+ return process.platform === 'win32' ? trimmed.toLowerCase() : trimmed;
48
+ }
49
+ /**
50
+ * Whether a path cannot name a Workspace on the current host.
51
+ *
52
+ * Ledgers are cross-platform: a Windows Host can contain `/mnt/...` paths from
53
+ * WSL, and a POSIX Host can contain `C:\...` paths. Calling `realpath` on those
54
+ * would resolve them relative to the current host and could falsely match an
55
+ * unrelated native Workspace, so they are excluded from the fallback.
56
+ *
57
+ * @param value - raw path.
58
+ * @returns true when the path shape belongs to another platform.
59
+ */
60
+ function isForeignPath(value) {
61
+ if (process.platform === 'win32') {
62
+ // Windows Workspaces are drive- or UNC-qualified; a lone leading slash is
63
+ // a POSIX path from WSL/another machine.
64
+ return value.startsWith('/') && !value.startsWith('//');
65
+ }
66
+ return /^[A-Za-z]:[\\/]/u.test(value) || value.startsWith('\\\\');
67
+ }
68
+ /**
69
+ * Build a `raw workspace -> current title` map for the records in one summary.
70
+ *
71
+ * Only unique record paths are looked up. Exact spellings hit the registry
72
+ * without I/O; the rest try `fs.realpath` once so a differently spelled or
73
+ * symlinked path still finds its Workspace. Every failure is contained: this
74
+ * function never rejects.
75
+ *
76
+ * @param ctx - Host context; the Workspace registry is optional.
77
+ * @param records - ledger records in the summary.
78
+ * @returns titles keyed by the exact `record.workspace` spelling.
79
+ */
80
+ export async function resolveWorkspaceTitles(ctx, records) {
81
+ let registry;
82
+ try {
83
+ registry = ctx.get('workspaceRegistry');
84
+ }
85
+ catch {
86
+ return new Map();
87
+ }
88
+ if (registry === undefined)
89
+ return new Map();
90
+ let workspaces;
91
+ try {
92
+ workspaces = registry.list();
93
+ }
94
+ catch {
95
+ return new Map();
96
+ }
97
+ const byPath = new Map();
98
+ for (const workspace of workspaces) {
99
+ if (typeof workspace?.path !== 'string' ||
100
+ typeof workspace?.title !== 'string' ||
101
+ workspace.path.length === 0 ||
102
+ workspace.title.length === 0) {
103
+ continue;
104
+ }
105
+ byPath.set(pathKey(workspace.path), workspace.title);
106
+ }
107
+ if (byPath.size === 0)
108
+ return new Map();
109
+ const rawPaths = [...new Set(records.map((record) => record.workspace).filter((path) => path.length > 0))];
110
+ const titles = new Map();
111
+ await Promise.all(rawPaths.map(async (raw) => {
112
+ // A hand-written/relative ledger path must not be resolved against this
113
+ // process's cwd into an unrelated Workspace.
114
+ if (!isAbsolute(raw))
115
+ return;
116
+ const direct = byPath.get(pathKey(raw));
117
+ if (direct !== undefined) {
118
+ titles.set(raw, direct);
119
+ return;
120
+ }
121
+ if (isForeignPath(raw))
122
+ return;
123
+ try {
124
+ const canonical = await realpath(raw);
125
+ const title = byPath.get(pathKey(canonical));
126
+ if (title !== undefined)
127
+ titles.set(raw, title);
128
+ }
129
+ catch {
130
+ // Missing/unreadable directory: keep the stored label, never fail the
131
+ // summary for a display-name lookup.
132
+ }
133
+ }));
134
+ return titles;
135
+ }
136
+ //# sourceMappingURL=workspace-titles.js.map
@@ -0,0 +1,41 @@
1
+ /**
2
+ * Multi-frame Zstandard decoding, in pure Node.
3
+ *
4
+ * A session log is a stream of CONCATENATED zstd frames — one per flush — which
5
+ * is why `zlib.zstdDecompressSync` (and its streaming sibling) stop after the
6
+ * first frame: they decode one frame and then fail on the next frame's magic
7
+ * with "Unknown frame descriptor". Measured on a real 4.9 MB log: the CLI
8
+ * decoded 5688 lines, `zstdDecompressSync` decoded 1.
9
+ *
10
+ * Rather than shelling out to the `zstd` binary (a portability dependency), this
11
+ * module walks each frame's own header and block table to find exact boundaries
12
+ * and hands single frames to Node's decoder. The walk is mechanical and
13
+ * self-validating: a mis-computed boundary surfaces immediately as a decode
14
+ * error rather than as silently truncated data.
15
+ *
16
+ * @module dsh-token-usage-ledger/host/zstd
17
+ */
18
+ /** Outcome of one split operation. */
19
+ export interface ZstdSplitResult {
20
+ /** One buffer per frame, in file order. */
21
+ frames: Buffer[];
22
+ /** Trailing bytes that were not a complete frame. */
23
+ trailingBytes: number;
24
+ }
25
+ /**
26
+ * Split a concatenated zstd stream into its individual frames.
27
+ *
28
+ * @param buffer - the complete file contents.
29
+ * @returns the frames plus the count of unparseable trailing bytes.
30
+ * @throws when a frame header or block table is malformed — a truncated or
31
+ * corrupt log must fail loudly instead of yielding partial session history.
32
+ */
33
+ export declare function splitZstdFrames(buffer: Buffer): ZstdSplitResult;
34
+ /**
35
+ * Decompress a concatenated multi-frame zstd file.
36
+ *
37
+ * @param buffer - the complete file contents.
38
+ * @returns the decoded UTF-8 text.
39
+ * @throws when any frame fails to decode.
40
+ */
41
+ export declare function decodeZstdFrames(buffer: Buffer): string;
@@ -0,0 +1,134 @@
1
+ /**
2
+ * Multi-frame Zstandard decoding, in pure Node.
3
+ *
4
+ * A session log is a stream of CONCATENATED zstd frames — one per flush — which
5
+ * is why `zlib.zstdDecompressSync` (and its streaming sibling) stop after the
6
+ * first frame: they decode one frame and then fail on the next frame's magic
7
+ * with "Unknown frame descriptor". Measured on a real 4.9 MB log: the CLI
8
+ * decoded 5688 lines, `zstdDecompressSync` decoded 1.
9
+ *
10
+ * Rather than shelling out to the `zstd` binary (a portability dependency), this
11
+ * module walks each frame's own header and block table to find exact boundaries
12
+ * and hands single frames to Node's decoder. The walk is mechanical and
13
+ * self-validating: a mis-computed boundary surfaces immediately as a decode
14
+ * error rather than as silently truncated data.
15
+ *
16
+ * @module dsh-token-usage-ledger/host/zstd
17
+ */
18
+ import { zstdDecompressSync } from 'node:zlib';
19
+ /** Zstandard frame magic, little-endian on disk. */
20
+ const FRAME_MAGIC = 0xfd2fb528;
21
+ /** Skippable-frame magic range (0x184D2A50..0x184D2A5F). */
22
+ const SKIPPABLE_MASK = 0xfffffff0;
23
+ const SKIPPABLE_BASE = 0x184d2a50;
24
+ /** Bytes in a block header. */
25
+ const BLOCK_HEADER_BYTES = 3;
26
+ /** Dictionary-id field widths, indexed by the 2-bit flag. */
27
+ const DICTIONARY_ID_WIDTHS = [0, 1, 2, 4];
28
+ /** Frame-content-size field widths: flag 0 is special-cased, 1..3 map here. */
29
+ const CONTENT_SIZE_WIDTHS = [2, 4, 8];
30
+ /**
31
+ * Split a concatenated zstd stream into its individual frames.
32
+ *
33
+ * @param buffer - the complete file contents.
34
+ * @returns the frames plus the count of unparseable trailing bytes.
35
+ * @throws when a frame header or block table is malformed — a truncated or
36
+ * corrupt log must fail loudly instead of yielding partial session history.
37
+ */
38
+ export function splitZstdFrames(buffer) {
39
+ const frames = [];
40
+ let offset = 0;
41
+ while (offset + 4 <= buffer.length) {
42
+ const magic = buffer.readUInt32LE(offset);
43
+ if (((magic & SKIPPABLE_MASK) >>> 0) === SKIPPABLE_BASE) {
44
+ // Skippable frame: 4-byte magic + 4-byte little-endian payload length.
45
+ if (offset + 8 > buffer.length)
46
+ break;
47
+ const size = buffer.readUInt32LE(offset + 4);
48
+ offset += 8 + size;
49
+ continue;
50
+ }
51
+ if (magic !== FRAME_MAGIC)
52
+ break;
53
+ const end = frameEnd(buffer, offset);
54
+ frames.push(buffer.subarray(offset, end));
55
+ offset = end;
56
+ }
57
+ return { frames, trailingBytes: buffer.length - offset };
58
+ }
59
+ /**
60
+ * Compute the exclusive end offset of the frame starting at `start`.
61
+ * @param buffer - file contents.
62
+ * @param start - offset of the frame's magic number.
63
+ * @returns the offset just past the frame.
64
+ */
65
+ function frameEnd(buffer, start) {
66
+ let cursor = start + 4;
67
+ const descriptor = buffer.readUInt8(cursor);
68
+ cursor += 1;
69
+ const contentSizeFlag = descriptor >>> 6;
70
+ const singleSegment = (descriptor & 0x20) !== 0;
71
+ const checksumFlag = (descriptor & 0x04) !== 0;
72
+ const dictionaryIdFlag = descriptor & 0x03;
73
+ if (!singleSegment)
74
+ cursor += 1; // Window_Descriptor
75
+ cursor += DICTIONARY_ID_WIDTHS[dictionaryIdFlag] ?? 0;
76
+ if (contentSizeFlag === 0) {
77
+ if (singleSegment)
78
+ cursor += 1;
79
+ }
80
+ else {
81
+ cursor += CONTENT_SIZE_WIDTHS[contentSizeFlag - 1] ?? 0;
82
+ }
83
+ if (cursor > buffer.length)
84
+ throw new Error(`zstd: truncated frame header at ${String(start)}`);
85
+ for (;;) {
86
+ if (cursor + BLOCK_HEADER_BYTES > buffer.length) {
87
+ throw new Error(`zstd: truncated block header at ${String(cursor)}`);
88
+ }
89
+ const header = buffer.readUIntLE(cursor, BLOCK_HEADER_BYTES);
90
+ cursor += BLOCK_HEADER_BYTES;
91
+ const lastBlock = (header & 0x01) !== 0;
92
+ const blockType = (header >>> 1) & 0x03;
93
+ const blockSize = header >>> 3;
94
+ // Raw and Compressed blocks carry `blockSize` bytes; an RLE block carries a
95
+ // single byte expanded to `blockSize`.
96
+ const payload = blockType === 1 ? 1 : blockSize;
97
+ cursor += payload;
98
+ if (cursor > buffer.length)
99
+ throw new Error(`zstd: truncated block at ${String(cursor)}`);
100
+ if (lastBlock)
101
+ break;
102
+ }
103
+ // A frame may append a 4-byte xxhash of the decompressed content; without it
104
+ // the walk lands 4 bytes short and the next frame's magic is never found.
105
+ if (checksumFlag)
106
+ cursor += 4;
107
+ return cursor;
108
+ }
109
+ /**
110
+ * Decompress a concatenated multi-frame zstd file.
111
+ *
112
+ * @param buffer - the complete file contents.
113
+ * @returns the decoded UTF-8 text.
114
+ * @throws when any frame fails to decode.
115
+ */
116
+ export function decodeZstdFrames(buffer) {
117
+ const { frames, trailingBytes } = splitZstdFrames(buffer);
118
+ if (frames.length === 0 && buffer.length > 0) {
119
+ throw new Error('zstd: no frame found in the input');
120
+ }
121
+ const parts = frames.map((frame, index) => {
122
+ try {
123
+ return zstdDecompressSync(frame);
124
+ }
125
+ catch (error) {
126
+ throw new Error(`zstd: frame ${String(index)} failed to decode: ${error instanceof Error ? error.message : String(error)}`);
127
+ }
128
+ });
129
+ if (trailingBytes > 0) {
130
+ throw new Error(`zstd: ${String(trailingBytes)} trailing bytes after the last frame`);
131
+ }
132
+ return Buffer.concat(parts).toString('utf8');
133
+ }
134
+ //# sourceMappingURL=zstd.js.map