dsh-token-usage-ledger 0.0.0-stage → 0.0.37
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +1081 -0
- package/LICENSE +21 -0
- package/README.md +247 -2
- package/cordis.patch.yml +30 -0
- package/docs/token-usage-image01.png +0 -0
- package/docs/token-usage-image02.png +0 -0
- package/lib/client.js +4469 -0
- package/lib/host/aggregate.d.ts +132 -0
- package/lib/host/aggregate.js +840 -0
- package/lib/host/api.d.ts +118 -0
- package/lib/host/api.js +346 -0
- package/lib/host/backfill.d.ts +54 -0
- package/lib/host/backfill.js +265 -0
- package/lib/host/collector.d.ts +139 -0
- package/lib/host/collector.js +441 -0
- package/lib/host/ledger-io.d.ts +125 -0
- package/lib/host/ledger-io.js +211 -0
- package/lib/host/operation-error.d.ts +22 -0
- package/lib/host/operation-error.js +28 -0
- package/lib/host/paths.d.ts +35 -0
- package/lib/host/paths.js +63 -0
- package/lib/host/pricing-module.d.ts +112 -0
- package/lib/host/pricing-module.js +63 -0
- package/lib/host/pricing-modules/deepseek-2026-09-10.d.ts +31 -0
- package/lib/host/pricing-modules/deepseek-2026-09-10.js +67 -0
- package/lib/host/pricing-modules/index.d.ts +27 -0
- package/lib/host/pricing-modules/index.js +30 -0
- package/lib/host/pricing.d.ts +192 -0
- package/lib/host/pricing.js +287 -0
- package/lib/host/records.d.ts +47 -0
- package/lib/host/records.js +116 -0
- package/lib/host/runtime-marker.d.ts +67 -0
- package/lib/host/runtime-marker.js +71 -0
- package/lib/host/schema.d.ts +77 -0
- package/lib/host/schema.js +158 -0
- package/lib/host/store.d.ts +171 -0
- package/lib/host/store.js +337 -0
- package/lib/host/workspace-titles.d.ts +38 -0
- package/lib/host/workspace-titles.js +136 -0
- package/lib/host/zstd.d.ts +41 -0
- package/lib/host/zstd.js +134 -0
- package/lib/index.d.ts +82 -0
- package/lib/index.js +512 -0
- package/lib/shared/format.d.ts +415 -0
- package/lib/shared/format.js +714 -0
- package/lib/shared/protocol.d.ts +646 -0
- package/lib/shared/protocol.js +129 -0
- package/package.json +87 -4
|
@@ -0,0 +1,337 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Append-only ledger storage.
|
|
3
|
+
*
|
|
4
|
+
* One JSON object per line under the harness home, so the ledger is
|
|
5
|
+
* human-inspectable, append-only, and survives a crash mid-write (a torn final
|
|
6
|
+
* line is skipped and counted, never fatal). Writes are batched onto a
|
|
7
|
+
* `setImmediate` drain so the `session/event` dispatch that produced a record
|
|
8
|
+
* never performs file I/O on the conversation's critical path.
|
|
9
|
+
*
|
|
10
|
+
* Every entry point swallows its own failure and reports it through
|
|
11
|
+
* {@link UsageStore.stats}: the ledger is a diagnostic side-channel and must
|
|
12
|
+
* never be able to fail a Turn.
|
|
13
|
+
*
|
|
14
|
+
* @module dsh-token-usage-ledger/host/store
|
|
15
|
+
*/
|
|
16
|
+
import { createReadStream } from 'node:fs';
|
|
17
|
+
import { appendFile, mkdir, stat } from 'node:fs/promises';
|
|
18
|
+
import { join } from 'node:path';
|
|
19
|
+
import { createInterface } from 'node:readline';
|
|
20
|
+
import { API_LIMITS } from '../shared/protocol.js';
|
|
21
|
+
import { parseRecordLine } from './schema.js';
|
|
22
|
+
/** Directory (under the harness home) that holds the ledger. */
|
|
23
|
+
export const LEDGER_DIR = 'token-usage';
|
|
24
|
+
/** File name of the append-only ledger. */
|
|
25
|
+
export const LEDGER_FILE = 'records.jsonl';
|
|
26
|
+
/** Bound on queued records; beyond it the oldest are dropped rather than growing without limit. */
|
|
27
|
+
const DEFAULT_MAX_PENDING = 4096;
|
|
28
|
+
/**
|
|
29
|
+
* The ledger: a write queue plus a cached reader.
|
|
30
|
+
*
|
|
31
|
+
* Instances are created per plugin activation and dropped with the fiber; a
|
|
32
|
+
* re-activation re-reads the file, so the ledger — not memory — is the source
|
|
33
|
+
* of truth.
|
|
34
|
+
*/
|
|
35
|
+
export class UsageStore {
|
|
36
|
+
dir;
|
|
37
|
+
file;
|
|
38
|
+
path;
|
|
39
|
+
maxPending;
|
|
40
|
+
warn;
|
|
41
|
+
queue = [];
|
|
42
|
+
drainScheduled = false;
|
|
43
|
+
writeErrors = 0;
|
|
44
|
+
lastWriteError = '';
|
|
45
|
+
malformedLines = 0;
|
|
46
|
+
unsupportedRecords = 0;
|
|
47
|
+
truncated = false;
|
|
48
|
+
droppedRecords = 0;
|
|
49
|
+
/** Set while a failed batch is awaiting its single retry. */
|
|
50
|
+
retryArmed = false;
|
|
51
|
+
warnedOnce = false;
|
|
52
|
+
snapshot = undefined;
|
|
53
|
+
snapshotKey = '';
|
|
54
|
+
loadInFlight = undefined;
|
|
55
|
+
/**
|
|
56
|
+
* @param options - directory, file name, queue bound, and warning sink.
|
|
57
|
+
*/
|
|
58
|
+
constructor(options) {
|
|
59
|
+
this.dir = options.dir;
|
|
60
|
+
this.file = options.file ?? LEDGER_FILE;
|
|
61
|
+
this.path = join(this.dir, this.file);
|
|
62
|
+
this.maxPending = options.maxPending ?? DEFAULT_MAX_PENDING;
|
|
63
|
+
this.warn = options.warn ?? (() => { });
|
|
64
|
+
}
|
|
65
|
+
/** Absolute path of the ledger file. */
|
|
66
|
+
get filePath() {
|
|
67
|
+
return this.path;
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* Queue one record for the next append drain.
|
|
71
|
+
*
|
|
72
|
+
* Returns immediately; the caller is on the conversation's dispatch path and
|
|
73
|
+
* must not block. Never throws.
|
|
74
|
+
*
|
|
75
|
+
* @param record - the record to persist.
|
|
76
|
+
*/
|
|
77
|
+
enqueue(record) {
|
|
78
|
+
try {
|
|
79
|
+
if (this.queue.length >= this.maxPending) {
|
|
80
|
+
// Shed the oldest record rather than grow without bound: recent usage
|
|
81
|
+
// is the more valuable half of a diagnostic ledger.
|
|
82
|
+
this.queue.shift();
|
|
83
|
+
this.report('ledger queue full; oldest pending record dropped');
|
|
84
|
+
}
|
|
85
|
+
this.queue.push(record);
|
|
86
|
+
this.scheduleDrain();
|
|
87
|
+
}
|
|
88
|
+
catch (error) {
|
|
89
|
+
this.recordWriteError(error);
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
/**
|
|
93
|
+
* Flush the queue and wait for it, for shutdown paths and tests.
|
|
94
|
+
* @returns the number of records written, or 0 when the flush failed.
|
|
95
|
+
*/
|
|
96
|
+
async flush() {
|
|
97
|
+
return await this.drain();
|
|
98
|
+
}
|
|
99
|
+
/**
|
|
100
|
+
* Append records produced outside the live collector (import/backfill).
|
|
101
|
+
*
|
|
102
|
+
* Waits for the live queue first so the file stays in one append order, then
|
|
103
|
+
* appends the batch in a single write. The caller has already validated the
|
|
104
|
+
* records and checked `(sessionId, turn)` duplicates; this method only reports
|
|
105
|
+
* whether the OS write succeeded. It never throws into the caller's request
|
|
106
|
+
* path — the API maps the returned failure to a structured error.
|
|
107
|
+
*
|
|
108
|
+
* @param records - records to append.
|
|
109
|
+
* @returns the written count, or the write error message on failure.
|
|
110
|
+
*/
|
|
111
|
+
async appendRecords(records) {
|
|
112
|
+
if (records.length === 0)
|
|
113
|
+
return { written: 0 };
|
|
114
|
+
try {
|
|
115
|
+
await this.flush();
|
|
116
|
+
await mkdir(this.dir, { recursive: true });
|
|
117
|
+
const payload = records.map((record) => `${JSON.stringify(record)}\n`).join('');
|
|
118
|
+
await appendFile(this.path, payload, { encoding: 'utf8', mode: 0o600 });
|
|
119
|
+
this.snapshotKey = '';
|
|
120
|
+
this.retryArmed = false;
|
|
121
|
+
return { written: records.length };
|
|
122
|
+
}
|
|
123
|
+
catch (error) {
|
|
124
|
+
this.recordWriteError(error);
|
|
125
|
+
return {
|
|
126
|
+
written: 0,
|
|
127
|
+
error: error instanceof Error ? error.message : String(error),
|
|
128
|
+
};
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
/**
|
|
132
|
+
* Load and cache the ledger.
|
|
133
|
+
*
|
|
134
|
+
* The cache is keyed on the file's size and mtime, so a second request after
|
|
135
|
+
* a write re-reads while repeated reads stay cheap. A concurrent caller
|
|
136
|
+
* joins the in-flight read instead of starting a second one.
|
|
137
|
+
*
|
|
138
|
+
* @returns the parsed snapshot; an empty snapshot when the file is absent.
|
|
139
|
+
*/
|
|
140
|
+
async load() {
|
|
141
|
+
if (this.loadInFlight !== undefined)
|
|
142
|
+
return await this.loadInFlight;
|
|
143
|
+
const run = this.loadOnce();
|
|
144
|
+
this.loadInFlight = run;
|
|
145
|
+
try {
|
|
146
|
+
return await run;
|
|
147
|
+
}
|
|
148
|
+
finally {
|
|
149
|
+
this.loadInFlight = undefined;
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
/**
|
|
153
|
+
* Report current diagnostics.
|
|
154
|
+
* @param extra - additional counters owned by the caller.
|
|
155
|
+
* @returns the health report embedded in the summary payload.
|
|
156
|
+
*/
|
|
157
|
+
stats(extra) {
|
|
158
|
+
return {
|
|
159
|
+
malformedLines: this.malformedLines,
|
|
160
|
+
unsupportedRecords: this.unsupportedRecords,
|
|
161
|
+
writeErrors: this.writeErrors,
|
|
162
|
+
lastWriteError: this.lastWriteError,
|
|
163
|
+
droppedRecords: this.droppedRecords,
|
|
164
|
+
pendingWrites: this.queue.length,
|
|
165
|
+
skippedTurns: extra.skippedTurns,
|
|
166
|
+
// Recomputed by the aggregator, which is the only place that knows which
|
|
167
|
+
// currency the caller asked for.
|
|
168
|
+
currencyMismatches: 0,
|
|
169
|
+
ledgerTruncated: this.truncated,
|
|
170
|
+
};
|
|
171
|
+
}
|
|
172
|
+
/** Read + parse the ledger, refreshing the cache. */
|
|
173
|
+
async loadOnce() {
|
|
174
|
+
let key = '';
|
|
175
|
+
try {
|
|
176
|
+
const info = await stat(this.path);
|
|
177
|
+
if (!info.isFile()) {
|
|
178
|
+
return { records: [], malformedLines: 0, unsupportedRecords: 0, duplicateLines: 0, truncated: false };
|
|
179
|
+
}
|
|
180
|
+
key = `${String(info.size)}:${String(info.mtimeMs)}`;
|
|
181
|
+
}
|
|
182
|
+
catch {
|
|
183
|
+
// Absent ledger is the normal first-run state, not an error.
|
|
184
|
+
const empty = {
|
|
185
|
+
records: [],
|
|
186
|
+
malformedLines: 0,
|
|
187
|
+
unsupportedRecords: 0,
|
|
188
|
+
duplicateLines: 0,
|
|
189
|
+
truncated: false,
|
|
190
|
+
};
|
|
191
|
+
this.snapshot = empty;
|
|
192
|
+
this.snapshotKey = 'absent';
|
|
193
|
+
this.malformedLines = 0;
|
|
194
|
+
this.unsupportedRecords = 0;
|
|
195
|
+
this.truncated = false;
|
|
196
|
+
return empty;
|
|
197
|
+
}
|
|
198
|
+
if (this.snapshot !== undefined && this.snapshotKey === key)
|
|
199
|
+
return this.snapshot;
|
|
200
|
+
const parsed = await this.parseLedger();
|
|
201
|
+
this.snapshot = parsed;
|
|
202
|
+
this.snapshotKey = key;
|
|
203
|
+
this.malformedLines = parsed.malformedLines;
|
|
204
|
+
this.unsupportedRecords = parsed.unsupportedRecords;
|
|
205
|
+
this.truncated = parsed.truncated;
|
|
206
|
+
return parsed;
|
|
207
|
+
}
|
|
208
|
+
/**
|
|
209
|
+
* Stream the ledger into de-duplicated records.
|
|
210
|
+
*
|
|
211
|
+
* Streamed rather than read whole so memory is bounded by the RECORD cap and
|
|
212
|
+
* not by the file size: a large ledger degrades to the newest
|
|
213
|
+
* {@link API_LIMITS.maxRecords} records with `ledgerTruncated` set, instead of
|
|
214
|
+
* silently reporting an empty dashboard. Every line is still visited, so
|
|
215
|
+
* de-duplication and the malformed count stay exact.
|
|
216
|
+
*
|
|
217
|
+
* @returns the snapshot.
|
|
218
|
+
*/
|
|
219
|
+
async parseLedger() {
|
|
220
|
+
const byKey = new Map();
|
|
221
|
+
let malformedLines = 0;
|
|
222
|
+
let unsupportedRecords = 0;
|
|
223
|
+
let duplicateLines = 0;
|
|
224
|
+
let truncated = false;
|
|
225
|
+
const accept = (line) => {
|
|
226
|
+
if (line.length === 0)
|
|
227
|
+
return;
|
|
228
|
+
const outcome = parseRecordLine(line);
|
|
229
|
+
if (!outcome.ok) {
|
|
230
|
+
if (outcome.reason === 'unsupported-version')
|
|
231
|
+
unsupportedRecords += 1;
|
|
232
|
+
else
|
|
233
|
+
malformedLines += 1;
|
|
234
|
+
return;
|
|
235
|
+
}
|
|
236
|
+
const key = `${outcome.record.sessionId}\u0000${String(outcome.record.turn)}`;
|
|
237
|
+
if (byKey.has(key))
|
|
238
|
+
duplicateLines += 1;
|
|
239
|
+
byKey.set(key, outcome.record);
|
|
240
|
+
// Keep the NEWEST records: a Map iterates in insertion order, so the
|
|
241
|
+
// first key is the oldest surviving one.
|
|
242
|
+
if (byKey.size > API_LIMITS.maxRecords) {
|
|
243
|
+
const oldest = byKey.keys().next();
|
|
244
|
+
if (oldest.done !== true)
|
|
245
|
+
byKey.delete(oldest.value);
|
|
246
|
+
truncated = true;
|
|
247
|
+
}
|
|
248
|
+
};
|
|
249
|
+
try {
|
|
250
|
+
const reader = createInterface({
|
|
251
|
+
input: createReadStream(this.path, { encoding: 'utf8' }),
|
|
252
|
+
crlfDelay: Number.POSITIVE_INFINITY,
|
|
253
|
+
});
|
|
254
|
+
for await (const line of reader)
|
|
255
|
+
accept(line);
|
|
256
|
+
}
|
|
257
|
+
catch (error) {
|
|
258
|
+
this.recordWriteError(error);
|
|
259
|
+
return { records: [], malformedLines, unsupportedRecords, duplicateLines, truncated };
|
|
260
|
+
}
|
|
261
|
+
const records = [...byKey.values()].sort((a, b) => a.ts - b.ts);
|
|
262
|
+
return { records, malformedLines, unsupportedRecords, duplicateLines, truncated };
|
|
263
|
+
}
|
|
264
|
+
/** Schedule one drain on the next macrotask boundary. */
|
|
265
|
+
scheduleDrain() {
|
|
266
|
+
if (this.drainScheduled)
|
|
267
|
+
return;
|
|
268
|
+
this.drainScheduled = true;
|
|
269
|
+
setImmediate(() => {
|
|
270
|
+
this.drainScheduled = false;
|
|
271
|
+
void this.drain();
|
|
272
|
+
});
|
|
273
|
+
}
|
|
274
|
+
/**
|
|
275
|
+
* Write every queued record in one append.
|
|
276
|
+
* @returns the number of records written.
|
|
277
|
+
*/
|
|
278
|
+
async drain() {
|
|
279
|
+
if (this.queue.length === 0)
|
|
280
|
+
return 0;
|
|
281
|
+
const batch = this.queue;
|
|
282
|
+
this.queue = [];
|
|
283
|
+
try {
|
|
284
|
+
await mkdir(this.dir, { recursive: true });
|
|
285
|
+
const payload = batch.map((record) => `${JSON.stringify(record)}\n`).join('');
|
|
286
|
+
// `mode` applies only when the file is created; the harness home is 0700,
|
|
287
|
+
// and a private ledger keeps that posture for the file itself too.
|
|
288
|
+
await appendFile(this.path, payload, { encoding: 'utf8', mode: 0o600 });
|
|
289
|
+
// The cache key must not match a stale read.
|
|
290
|
+
this.snapshotKey = '';
|
|
291
|
+
this.retryArmed = false;
|
|
292
|
+
return batch.length;
|
|
293
|
+
}
|
|
294
|
+
catch (error) {
|
|
295
|
+
this.recordWriteError(error);
|
|
296
|
+
if (!this.retryArmed) {
|
|
297
|
+
// One retry absorbs a transient failure (a busy fd, a momentary EAGAIN)
|
|
298
|
+
// without risking a retry loop: the second failure drops the batch and
|
|
299
|
+
// records exactly how many records were lost.
|
|
300
|
+
this.retryArmed = true;
|
|
301
|
+
this.queue.unshift(...batch);
|
|
302
|
+
this.scheduleDrain();
|
|
303
|
+
}
|
|
304
|
+
else {
|
|
305
|
+
this.retryArmed = false;
|
|
306
|
+
this.droppedRecords += batch.length;
|
|
307
|
+
}
|
|
308
|
+
return 0;
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
/**
|
|
312
|
+
* Account one append failure, reporting it at most once per process so a
|
|
313
|
+
* broken disk cannot flood the log.
|
|
314
|
+
* @param error - the failure to record.
|
|
315
|
+
*/
|
|
316
|
+
recordWriteError(error) {
|
|
317
|
+
this.writeErrors += 1;
|
|
318
|
+
this.lastWriteError = error instanceof Error ? error.message : String(error);
|
|
319
|
+
this.report(`ledger write failed: ${this.lastWriteError}`);
|
|
320
|
+
}
|
|
321
|
+
/**
|
|
322
|
+
* Emit one warning line, throttled to a single occurrence per activation.
|
|
323
|
+
* @param message - warning text.
|
|
324
|
+
*/
|
|
325
|
+
report(message) {
|
|
326
|
+
if (this.warnedOnce)
|
|
327
|
+
return;
|
|
328
|
+
this.warnedOnce = true;
|
|
329
|
+
try {
|
|
330
|
+
this.warn(message);
|
|
331
|
+
}
|
|
332
|
+
catch {
|
|
333
|
+
// A throwing log sink must not escalate.
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
}
|
|
337
|
+
//# sourceMappingURL=store.js.map
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Resolve a ledger's raw workspace paths to the Workspace titles that are
|
|
3
|
+
* current at summary time.
|
|
4
|
+
*
|
|
5
|
+
* The ledger stores the Session working directory and, at write time, its
|
|
6
|
+
* basename. A user can rename a Workspace without changing its directory, so a
|
|
7
|
+
* stored label is not a durable display name. `ctx.workspaceRegistry` is the
|
|
8
|
+
* Host's authoritative Workspace registry: `Workspace.path` stays canonical
|
|
9
|
+
* across renames and `Workspace.title` is the current user-visible title.
|
|
10
|
+
*
|
|
11
|
+
* This module is deliberately failure-tolerant. A missing service, a registry
|
|
12
|
+
* that is still initializing, an unreadable directory, or a foreign-platform
|
|
13
|
+
* path all degrade to "no current title", in which case aggregation keeps the
|
|
14
|
+
* stored `workspaceLabel` (the old behavior).
|
|
15
|
+
*
|
|
16
|
+
* @module dsh-token-usage-ledger/host/workspace-titles
|
|
17
|
+
*/
|
|
18
|
+
import type { Context } from '@deepseek-ai/cordis';
|
|
19
|
+
import type { UsageRecord } from '../shared/protocol.js';
|
|
20
|
+
/**
|
|
21
|
+
* Look up a current display title for one raw workspace path.
|
|
22
|
+
* @param workspace - raw `session.header.cwd` value stored on a record.
|
|
23
|
+
* @returns the current title, or undefined when no registry entry matches.
|
|
24
|
+
*/
|
|
25
|
+
export type WorkspaceTitleLookup = (workspace: string) => string | undefined;
|
|
26
|
+
/**
|
|
27
|
+
* Build a `raw workspace -> current title` map for the records in one summary.
|
|
28
|
+
*
|
|
29
|
+
* Only unique record paths are looked up. Exact spellings hit the registry
|
|
30
|
+
* without I/O; the rest try `fs.realpath` once so a differently spelled or
|
|
31
|
+
* symlinked path still finds its Workspace. Every failure is contained: this
|
|
32
|
+
* function never rejects.
|
|
33
|
+
*
|
|
34
|
+
* @param ctx - Host context; the Workspace registry is optional.
|
|
35
|
+
* @param records - ledger records in the summary.
|
|
36
|
+
* @returns titles keyed by the exact `record.workspace` spelling.
|
|
37
|
+
*/
|
|
38
|
+
export declare function resolveWorkspaceTitles(ctx: Context, records: readonly UsageRecord[]): Promise<Map<string, string>>;
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Resolve a ledger's raw workspace paths to the Workspace titles that are
|
|
3
|
+
* current at summary time.
|
|
4
|
+
*
|
|
5
|
+
* The ledger stores the Session working directory and, at write time, its
|
|
6
|
+
* basename. A user can rename a Workspace without changing its directory, so a
|
|
7
|
+
* stored label is not a durable display name. `ctx.workspaceRegistry` is the
|
|
8
|
+
* Host's authoritative Workspace registry: `Workspace.path` stays canonical
|
|
9
|
+
* across renames and `Workspace.title` is the current user-visible title.
|
|
10
|
+
*
|
|
11
|
+
* This module is deliberately failure-tolerant. A missing service, a registry
|
|
12
|
+
* that is still initializing, an unreadable directory, or a foreign-platform
|
|
13
|
+
* path all degrade to "no current title", in which case aggregation keeps the
|
|
14
|
+
* stored `workspaceLabel` (the old behavior).
|
|
15
|
+
*
|
|
16
|
+
* @module dsh-token-usage-ledger/host/workspace-titles
|
|
17
|
+
*/
|
|
18
|
+
import { realpath } from 'node:fs/promises';
|
|
19
|
+
import { isAbsolute } from 'node:path';
|
|
20
|
+
/**
|
|
21
|
+
* Strip the Windows extended-length prefix, which `fs.realpath` can return for
|
|
22
|
+
* long paths. The registry normalizes with the same `fs.realpath`, but a record
|
|
23
|
+
* may have been written through a shorter spelling.
|
|
24
|
+
* @param value - path spelling.
|
|
25
|
+
* @returns the path without a `\\?\` / `\\?\UNC\` prefix.
|
|
26
|
+
*/
|
|
27
|
+
function withoutLongPathPrefix(value) {
|
|
28
|
+
if (value.startsWith('\\\\?\\UNC\\'))
|
|
29
|
+
return `\\\\${value.slice('\\\\?\\UNC\\'.length)}`;
|
|
30
|
+
if (value.startsWith('\\\\?\\'))
|
|
31
|
+
return value.slice('\\\\?\\'.length);
|
|
32
|
+
return value;
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Comparison key for one path, deliberately not a filesystem canonical form.
|
|
36
|
+
*
|
|
37
|
+
* Trailing separators are ignored, and Windows keys are case-folded because
|
|
38
|
+
* that filesystem is case-insensitive. Separator style, `..`, and symlinks are
|
|
39
|
+
* handled by the `realpath` fallback instead of guessed here.
|
|
40
|
+
*
|
|
41
|
+
* @param value - raw path.
|
|
42
|
+
* @returns the key used for a fast registry lookup.
|
|
43
|
+
*/
|
|
44
|
+
function pathKey(value) {
|
|
45
|
+
const stripped = withoutLongPathPrefix(value);
|
|
46
|
+
const trimmed = stripped.length > 1 ? stripped.replace(/[\\/]+$/u, '') : stripped;
|
|
47
|
+
return process.platform === 'win32' ? trimmed.toLowerCase() : trimmed;
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Whether a path cannot name a Workspace on the current host.
|
|
51
|
+
*
|
|
52
|
+
* Ledgers are cross-platform: a Windows Host can contain `/mnt/...` paths from
|
|
53
|
+
* WSL, and a POSIX Host can contain `C:\...` paths. Calling `realpath` on those
|
|
54
|
+
* would resolve them relative to the current host and could falsely match an
|
|
55
|
+
* unrelated native Workspace, so they are excluded from the fallback.
|
|
56
|
+
*
|
|
57
|
+
* @param value - raw path.
|
|
58
|
+
* @returns true when the path shape belongs to another platform.
|
|
59
|
+
*/
|
|
60
|
+
function isForeignPath(value) {
|
|
61
|
+
if (process.platform === 'win32') {
|
|
62
|
+
// Windows Workspaces are drive- or UNC-qualified; a lone leading slash is
|
|
63
|
+
// a POSIX path from WSL/another machine.
|
|
64
|
+
return value.startsWith('/') && !value.startsWith('//');
|
|
65
|
+
}
|
|
66
|
+
return /^[A-Za-z]:[\\/]/u.test(value) || value.startsWith('\\\\');
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* Build a `raw workspace -> current title` map for the records in one summary.
|
|
70
|
+
*
|
|
71
|
+
* Only unique record paths are looked up. Exact spellings hit the registry
|
|
72
|
+
* without I/O; the rest try `fs.realpath` once so a differently spelled or
|
|
73
|
+
* symlinked path still finds its Workspace. Every failure is contained: this
|
|
74
|
+
* function never rejects.
|
|
75
|
+
*
|
|
76
|
+
* @param ctx - Host context; the Workspace registry is optional.
|
|
77
|
+
* @param records - ledger records in the summary.
|
|
78
|
+
* @returns titles keyed by the exact `record.workspace` spelling.
|
|
79
|
+
*/
|
|
80
|
+
export async function resolveWorkspaceTitles(ctx, records) {
|
|
81
|
+
let registry;
|
|
82
|
+
try {
|
|
83
|
+
registry = ctx.get('workspaceRegistry');
|
|
84
|
+
}
|
|
85
|
+
catch {
|
|
86
|
+
return new Map();
|
|
87
|
+
}
|
|
88
|
+
if (registry === undefined)
|
|
89
|
+
return new Map();
|
|
90
|
+
let workspaces;
|
|
91
|
+
try {
|
|
92
|
+
workspaces = registry.list();
|
|
93
|
+
}
|
|
94
|
+
catch {
|
|
95
|
+
return new Map();
|
|
96
|
+
}
|
|
97
|
+
const byPath = new Map();
|
|
98
|
+
for (const workspace of workspaces) {
|
|
99
|
+
if (typeof workspace?.path !== 'string' ||
|
|
100
|
+
typeof workspace?.title !== 'string' ||
|
|
101
|
+
workspace.path.length === 0 ||
|
|
102
|
+
workspace.title.length === 0) {
|
|
103
|
+
continue;
|
|
104
|
+
}
|
|
105
|
+
byPath.set(pathKey(workspace.path), workspace.title);
|
|
106
|
+
}
|
|
107
|
+
if (byPath.size === 0)
|
|
108
|
+
return new Map();
|
|
109
|
+
const rawPaths = [...new Set(records.map((record) => record.workspace).filter((path) => path.length > 0))];
|
|
110
|
+
const titles = new Map();
|
|
111
|
+
await Promise.all(rawPaths.map(async (raw) => {
|
|
112
|
+
// A hand-written/relative ledger path must not be resolved against this
|
|
113
|
+
// process's cwd into an unrelated Workspace.
|
|
114
|
+
if (!isAbsolute(raw))
|
|
115
|
+
return;
|
|
116
|
+
const direct = byPath.get(pathKey(raw));
|
|
117
|
+
if (direct !== undefined) {
|
|
118
|
+
titles.set(raw, direct);
|
|
119
|
+
return;
|
|
120
|
+
}
|
|
121
|
+
if (isForeignPath(raw))
|
|
122
|
+
return;
|
|
123
|
+
try {
|
|
124
|
+
const canonical = await realpath(raw);
|
|
125
|
+
const title = byPath.get(pathKey(canonical));
|
|
126
|
+
if (title !== undefined)
|
|
127
|
+
titles.set(raw, title);
|
|
128
|
+
}
|
|
129
|
+
catch {
|
|
130
|
+
// Missing/unreadable directory: keep the stored label, never fail the
|
|
131
|
+
// summary for a display-name lookup.
|
|
132
|
+
}
|
|
133
|
+
}));
|
|
134
|
+
return titles;
|
|
135
|
+
}
|
|
136
|
+
//# sourceMappingURL=workspace-titles.js.map
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Multi-frame Zstandard decoding, in pure Node.
|
|
3
|
+
*
|
|
4
|
+
* A session log is a stream of CONCATENATED zstd frames — one per flush — which
|
|
5
|
+
* is why `zlib.zstdDecompressSync` (and its streaming sibling) stop after the
|
|
6
|
+
* first frame: they decode one frame and then fail on the next frame's magic
|
|
7
|
+
* with "Unknown frame descriptor". Measured on a real 4.9 MB log: the CLI
|
|
8
|
+
* decoded 5688 lines, `zstdDecompressSync` decoded 1.
|
|
9
|
+
*
|
|
10
|
+
* Rather than shelling out to the `zstd` binary (a portability dependency), this
|
|
11
|
+
* module walks each frame's own header and block table to find exact boundaries
|
|
12
|
+
* and hands single frames to Node's decoder. The walk is mechanical and
|
|
13
|
+
* self-validating: a mis-computed boundary surfaces immediately as a decode
|
|
14
|
+
* error rather than as silently truncated data.
|
|
15
|
+
*
|
|
16
|
+
* @module dsh-token-usage-ledger/host/zstd
|
|
17
|
+
*/
|
|
18
|
+
/** Outcome of one split operation. */
|
|
19
|
+
export interface ZstdSplitResult {
|
|
20
|
+
/** One buffer per frame, in file order. */
|
|
21
|
+
frames: Buffer[];
|
|
22
|
+
/** Trailing bytes that were not a complete frame. */
|
|
23
|
+
trailingBytes: number;
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* Split a concatenated zstd stream into its individual frames.
|
|
27
|
+
*
|
|
28
|
+
* @param buffer - the complete file contents.
|
|
29
|
+
* @returns the frames plus the count of unparseable trailing bytes.
|
|
30
|
+
* @throws when a frame header or block table is malformed — a truncated or
|
|
31
|
+
* corrupt log must fail loudly instead of yielding partial session history.
|
|
32
|
+
*/
|
|
33
|
+
export declare function splitZstdFrames(buffer: Buffer): ZstdSplitResult;
|
|
34
|
+
/**
|
|
35
|
+
* Decompress a concatenated multi-frame zstd file.
|
|
36
|
+
*
|
|
37
|
+
* @param buffer - the complete file contents.
|
|
38
|
+
* @returns the decoded UTF-8 text.
|
|
39
|
+
* @throws when any frame fails to decode.
|
|
40
|
+
*/
|
|
41
|
+
export declare function decodeZstdFrames(buffer: Buffer): string;
|
package/lib/host/zstd.js
ADDED
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Multi-frame Zstandard decoding, in pure Node.
|
|
3
|
+
*
|
|
4
|
+
* A session log is a stream of CONCATENATED zstd frames — one per flush — which
|
|
5
|
+
* is why `zlib.zstdDecompressSync` (and its streaming sibling) stop after the
|
|
6
|
+
* first frame: they decode one frame and then fail on the next frame's magic
|
|
7
|
+
* with "Unknown frame descriptor". Measured on a real 4.9 MB log: the CLI
|
|
8
|
+
* decoded 5688 lines, `zstdDecompressSync` decoded 1.
|
|
9
|
+
*
|
|
10
|
+
* Rather than shelling out to the `zstd` binary (a portability dependency), this
|
|
11
|
+
* module walks each frame's own header and block table to find exact boundaries
|
|
12
|
+
* and hands single frames to Node's decoder. The walk is mechanical and
|
|
13
|
+
* self-validating: a mis-computed boundary surfaces immediately as a decode
|
|
14
|
+
* error rather than as silently truncated data.
|
|
15
|
+
*
|
|
16
|
+
* @module dsh-token-usage-ledger/host/zstd
|
|
17
|
+
*/
|
|
18
|
+
import { zstdDecompressSync } from 'node:zlib';
|
|
19
|
+
/** Zstandard frame magic, little-endian on disk. */
|
|
20
|
+
const FRAME_MAGIC = 0xfd2fb528;
|
|
21
|
+
/** Skippable-frame magic range (0x184D2A50..0x184D2A5F). */
|
|
22
|
+
const SKIPPABLE_MASK = 0xfffffff0;
|
|
23
|
+
const SKIPPABLE_BASE = 0x184d2a50;
|
|
24
|
+
/** Bytes in a block header. */
|
|
25
|
+
const BLOCK_HEADER_BYTES = 3;
|
|
26
|
+
/** Dictionary-id field widths, indexed by the 2-bit flag. */
|
|
27
|
+
const DICTIONARY_ID_WIDTHS = [0, 1, 2, 4];
|
|
28
|
+
/** Frame-content-size field widths: flag 0 is special-cased, 1..3 map here. */
|
|
29
|
+
const CONTENT_SIZE_WIDTHS = [2, 4, 8];
|
|
30
|
+
/**
|
|
31
|
+
* Split a concatenated zstd stream into its individual frames.
|
|
32
|
+
*
|
|
33
|
+
* @param buffer - the complete file contents.
|
|
34
|
+
* @returns the frames plus the count of unparseable trailing bytes.
|
|
35
|
+
* @throws when a frame header or block table is malformed — a truncated or
|
|
36
|
+
* corrupt log must fail loudly instead of yielding partial session history.
|
|
37
|
+
*/
|
|
38
|
+
export function splitZstdFrames(buffer) {
|
|
39
|
+
const frames = [];
|
|
40
|
+
let offset = 0;
|
|
41
|
+
while (offset + 4 <= buffer.length) {
|
|
42
|
+
const magic = buffer.readUInt32LE(offset);
|
|
43
|
+
if (((magic & SKIPPABLE_MASK) >>> 0) === SKIPPABLE_BASE) {
|
|
44
|
+
// Skippable frame: 4-byte magic + 4-byte little-endian payload length.
|
|
45
|
+
if (offset + 8 > buffer.length)
|
|
46
|
+
break;
|
|
47
|
+
const size = buffer.readUInt32LE(offset + 4);
|
|
48
|
+
offset += 8 + size;
|
|
49
|
+
continue;
|
|
50
|
+
}
|
|
51
|
+
if (magic !== FRAME_MAGIC)
|
|
52
|
+
break;
|
|
53
|
+
const end = frameEnd(buffer, offset);
|
|
54
|
+
frames.push(buffer.subarray(offset, end));
|
|
55
|
+
offset = end;
|
|
56
|
+
}
|
|
57
|
+
return { frames, trailingBytes: buffer.length - offset };
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* Compute the exclusive end offset of the frame starting at `start`.
|
|
61
|
+
* @param buffer - file contents.
|
|
62
|
+
* @param start - offset of the frame's magic number.
|
|
63
|
+
* @returns the offset just past the frame.
|
|
64
|
+
*/
|
|
65
|
+
function frameEnd(buffer, start) {
|
|
66
|
+
let cursor = start + 4;
|
|
67
|
+
const descriptor = buffer.readUInt8(cursor);
|
|
68
|
+
cursor += 1;
|
|
69
|
+
const contentSizeFlag = descriptor >>> 6;
|
|
70
|
+
const singleSegment = (descriptor & 0x20) !== 0;
|
|
71
|
+
const checksumFlag = (descriptor & 0x04) !== 0;
|
|
72
|
+
const dictionaryIdFlag = descriptor & 0x03;
|
|
73
|
+
if (!singleSegment)
|
|
74
|
+
cursor += 1; // Window_Descriptor
|
|
75
|
+
cursor += DICTIONARY_ID_WIDTHS[dictionaryIdFlag] ?? 0;
|
|
76
|
+
if (contentSizeFlag === 0) {
|
|
77
|
+
if (singleSegment)
|
|
78
|
+
cursor += 1;
|
|
79
|
+
}
|
|
80
|
+
else {
|
|
81
|
+
cursor += CONTENT_SIZE_WIDTHS[contentSizeFlag - 1] ?? 0;
|
|
82
|
+
}
|
|
83
|
+
if (cursor > buffer.length)
|
|
84
|
+
throw new Error(`zstd: truncated frame header at ${String(start)}`);
|
|
85
|
+
for (;;) {
|
|
86
|
+
if (cursor + BLOCK_HEADER_BYTES > buffer.length) {
|
|
87
|
+
throw new Error(`zstd: truncated block header at ${String(cursor)}`);
|
|
88
|
+
}
|
|
89
|
+
const header = buffer.readUIntLE(cursor, BLOCK_HEADER_BYTES);
|
|
90
|
+
cursor += BLOCK_HEADER_BYTES;
|
|
91
|
+
const lastBlock = (header & 0x01) !== 0;
|
|
92
|
+
const blockType = (header >>> 1) & 0x03;
|
|
93
|
+
const blockSize = header >>> 3;
|
|
94
|
+
// Raw and Compressed blocks carry `blockSize` bytes; an RLE block carries a
|
|
95
|
+
// single byte expanded to `blockSize`.
|
|
96
|
+
const payload = blockType === 1 ? 1 : blockSize;
|
|
97
|
+
cursor += payload;
|
|
98
|
+
if (cursor > buffer.length)
|
|
99
|
+
throw new Error(`zstd: truncated block at ${String(cursor)}`);
|
|
100
|
+
if (lastBlock)
|
|
101
|
+
break;
|
|
102
|
+
}
|
|
103
|
+
// A frame may append a 4-byte xxhash of the decompressed content; without it
|
|
104
|
+
// the walk lands 4 bytes short and the next frame's magic is never found.
|
|
105
|
+
if (checksumFlag)
|
|
106
|
+
cursor += 4;
|
|
107
|
+
return cursor;
|
|
108
|
+
}
|
|
109
|
+
/**
|
|
110
|
+
* Decompress a concatenated multi-frame zstd file.
|
|
111
|
+
*
|
|
112
|
+
* @param buffer - the complete file contents.
|
|
113
|
+
* @returns the decoded UTF-8 text.
|
|
114
|
+
* @throws when any frame fails to decode.
|
|
115
|
+
*/
|
|
116
|
+
export function decodeZstdFrames(buffer) {
|
|
117
|
+
const { frames, trailingBytes } = splitZstdFrames(buffer);
|
|
118
|
+
if (frames.length === 0 && buffer.length > 0) {
|
|
119
|
+
throw new Error('zstd: no frame found in the input');
|
|
120
|
+
}
|
|
121
|
+
const parts = frames.map((frame, index) => {
|
|
122
|
+
try {
|
|
123
|
+
return zstdDecompressSync(frame);
|
|
124
|
+
}
|
|
125
|
+
catch (error) {
|
|
126
|
+
throw new Error(`zstd: frame ${String(index)} failed to decode: ${error instanceof Error ? error.message : String(error)}`);
|
|
127
|
+
}
|
|
128
|
+
});
|
|
129
|
+
if (trailingBytes > 0) {
|
|
130
|
+
throw new Error(`zstd: ${String(trailingBytes)} trailing bytes after the last frame`);
|
|
131
|
+
}
|
|
132
|
+
return Buffer.concat(parts).toString('utf8');
|
|
133
|
+
}
|
|
134
|
+
//# sourceMappingURL=zstd.js.map
|