dsh-token-usage-ledger 0.0.0-stage → 0.0.37

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/CHANGELOG.md +1081 -0
  2. package/LICENSE +21 -0
  3. package/README.md +247 -2
  4. package/cordis.patch.yml +30 -0
  5. package/docs/token-usage-image01.png +0 -0
  6. package/docs/token-usage-image02.png +0 -0
  7. package/lib/client.js +4469 -0
  8. package/lib/host/aggregate.d.ts +132 -0
  9. package/lib/host/aggregate.js +840 -0
  10. package/lib/host/api.d.ts +118 -0
  11. package/lib/host/api.js +346 -0
  12. package/lib/host/backfill.d.ts +54 -0
  13. package/lib/host/backfill.js +265 -0
  14. package/lib/host/collector.d.ts +139 -0
  15. package/lib/host/collector.js +441 -0
  16. package/lib/host/ledger-io.d.ts +125 -0
  17. package/lib/host/ledger-io.js +211 -0
  18. package/lib/host/operation-error.d.ts +22 -0
  19. package/lib/host/operation-error.js +28 -0
  20. package/lib/host/paths.d.ts +35 -0
  21. package/lib/host/paths.js +63 -0
  22. package/lib/host/pricing-module.d.ts +112 -0
  23. package/lib/host/pricing-module.js +63 -0
  24. package/lib/host/pricing-modules/deepseek-2026-09-10.d.ts +31 -0
  25. package/lib/host/pricing-modules/deepseek-2026-09-10.js +67 -0
  26. package/lib/host/pricing-modules/index.d.ts +27 -0
  27. package/lib/host/pricing-modules/index.js +30 -0
  28. package/lib/host/pricing.d.ts +192 -0
  29. package/lib/host/pricing.js +287 -0
  30. package/lib/host/records.d.ts +47 -0
  31. package/lib/host/records.js +116 -0
  32. package/lib/host/runtime-marker.d.ts +67 -0
  33. package/lib/host/runtime-marker.js +71 -0
  34. package/lib/host/schema.d.ts +77 -0
  35. package/lib/host/schema.js +158 -0
  36. package/lib/host/store.d.ts +171 -0
  37. package/lib/host/store.js +337 -0
  38. package/lib/host/workspace-titles.d.ts +38 -0
  39. package/lib/host/workspace-titles.js +136 -0
  40. package/lib/host/zstd.d.ts +41 -0
  41. package/lib/host/zstd.js +134 -0
  42. package/lib/index.d.ts +82 -0
  43. package/lib/index.js +512 -0
  44. package/lib/shared/format.d.ts +415 -0
  45. package/lib/shared/format.js +714 -0
  46. package/lib/shared/protocol.d.ts +646 -0
  47. package/lib/shared/protocol.js +129 -0
  48. package/package.json +87 -4
@@ -0,0 +1,265 @@
1
+ /**
2
+ * Backfill the ledger from durable session logs on disk.
3
+ *
4
+ * The live plugin only sees turns after it is installed; every durable session
5
+ * log contains the same `turn/start`/`turn/end` and settlement events, so a
6
+ * replay through the SAME collector/pricing/record builder produces a line
7
+ * indistinguishable from a live one. This module is shared by the CLI script
8
+ * and the dashboard's 「补录」 button so both use one implementation.
9
+ *
10
+ * @module dsh-token-usage-ledger/host/backfill
11
+ */
12
+ import { readFile, readdir } from 'node:fs/promises';
13
+ import { join } from 'node:path';
14
+ import { TurnUsageCollector } from './collector.js';
15
+ import { fingerprintOfParts, readLedgerKeys } from './ledger-io.js';
16
+ import { buildRecord } from './records.js';
17
+ import { decodeZstdFrames } from './zstd.js';
18
+ /** Session-log filenames this build understands, newest first. */
19
+ export const SESSION_LOG_NAMES = ['session.v4.jsonl.zstd', 'session.v3.jsonl.zstd'];
20
+ /**
21
+ * List session logs under the harness home.
22
+ * @param home - harness home.
23
+ * @param lead - optional workspace-directory filter.
24
+ * @param limit - maximum logs (0 = all).
25
+ * @returns readable logs plus unknown-format counts.
26
+ */
27
+ async function findSessionLogs(home, lead, limit) {
28
+ const root = join(home, 'sessions');
29
+ const found = [];
30
+ const unknown = [];
31
+ let workspaces;
32
+ try {
33
+ workspaces = await readdir(root, { withFileTypes: true });
34
+ }
35
+ catch {
36
+ return { logs: found, unknown };
37
+ }
38
+ for (const workspace of workspaces) {
39
+ if (!workspace.isDirectory())
40
+ continue;
41
+ if (lead !== '' && !workspace.name.includes(lead))
42
+ continue;
43
+ const workspaceDir = join(root, workspace.name);
44
+ let sessions;
45
+ try {
46
+ sessions = await readdir(workspaceDir, { withFileTypes: true });
47
+ }
48
+ catch {
49
+ continue;
50
+ }
51
+ for (const session of sessions) {
52
+ if (!session.isDirectory())
53
+ continue;
54
+ const sessionDir = join(workspaceDir, session.name);
55
+ let names;
56
+ try {
57
+ names = (await readdir(sessionDir, { withFileTypes: true }))
58
+ .filter((entry) => entry.isFile())
59
+ .map((entry) => entry.name);
60
+ }
61
+ catch {
62
+ continue;
63
+ }
64
+ for (const name of SESSION_LOG_NAMES) {
65
+ if (!names.includes(name))
66
+ continue;
67
+ found.push(join(sessionDir, name));
68
+ if (limit > 0 && found.length >= limit)
69
+ return { logs: found, unknown };
70
+ }
71
+ for (const name of names) {
72
+ if (/^session\.v\d+\.jsonl\.zstd$/.test(name) &&
73
+ !SESSION_LOG_NAMES.includes(name)) {
74
+ unknown.push(join(sessionDir, name));
75
+ }
76
+ }
77
+ }
78
+ }
79
+ return { logs: found, unknown };
80
+ }
81
+ /**
82
+ * Decode one durable session log into its header and replayable events.
83
+ * @param path - absolute log path.
84
+ * @param applyPrefilter - skip the inherited fork prefix when possible.
85
+ * @returns the parsed log.
86
+ */
87
+ async function readSessionLog(path, applyPrefilter) {
88
+ const text = decodeZstdFrames(await readFile(path));
89
+ const events = [];
90
+ let header = {};
91
+ for (const line of text.split('\n')) {
92
+ if (line.length === 0)
93
+ continue;
94
+ let value;
95
+ try {
96
+ value = JSON.parse(line);
97
+ }
98
+ catch {
99
+ continue;
100
+ }
101
+ if (value === null || typeof value !== 'object')
102
+ continue;
103
+ const record = value;
104
+ if (record['type'] === 'session') {
105
+ header = record;
106
+ continue;
107
+ }
108
+ if (typeof record['type'] === 'string')
109
+ events.push(record);
110
+ }
111
+ // Cheap pre-filter only: a `{inherited:true}` marker delimits a replayed
112
+ // parent prefix. The fingerprint check below is what actually guarantees no
113
+ // double count.
114
+ let inherited = 0;
115
+ for (let index = events.length - 1; applyPrefilter && index >= 0; index -= 1) {
116
+ const event = events[index];
117
+ if (event?.type === 'session/end-seed' && event.data?.inherited === true) {
118
+ inherited = index + 1;
119
+ break;
120
+ }
121
+ }
122
+ return { header, events: events.slice(inherited), inheritedEvents: inherited };
123
+ }
124
+ /**
125
+ * Report whether a turn's end time passes the optional `since` filter.
126
+ * @param endedAt - turn end epoch ms.
127
+ * @param since - inclusive local day key, or `''`.
128
+ * @returns true when the turn should be considered.
129
+ */
130
+ function sinceAllows(endedAt, since) {
131
+ if (since === '')
132
+ return true;
133
+ const date = new Date(endedAt);
134
+ const key = `${String(date.getFullYear())}-${String(date.getMonth() + 1).padStart(2, '0')}-${String(date.getDate()).padStart(2, '0')}`;
135
+ return key >= since;
136
+ }
137
+ /**
138
+ * Build the record context from a session header.
139
+ * @param header - session header.
140
+ * @param sessionId - session id.
141
+ * @returns record context.
142
+ */
143
+ function contextOf(header, sessionId) {
144
+ return {
145
+ sessionId,
146
+ workspace: typeof header['cwd'] === 'string' ? header['cwd'] : '',
147
+ agentPreset: typeof header['agentPreset'] === 'string' ? header['agentPreset'] : '',
148
+ origin: header['origin'] === 'subagent' ? 'subagent' : '',
149
+ delegationDepth: typeof header['delegationDepth'] === 'number' ? header['delegationDepth'] : 0,
150
+ parentSession: typeof header['parentSession'] === 'string' ? header['parentSession'] : '',
151
+ };
152
+ }
153
+ /**
154
+ * Scan durable logs and return the missing records.
155
+ *
156
+ * This function never writes; the caller decides whether to append the
157
+ * returned records. It is safe to call repeatedly: the returned set excludes
158
+ * turns already in the ledger and forked copies of them.
159
+ *
160
+ * @param options - home, ledger, pricing rule, filters and warning sink.
161
+ * @returns the report and timestamp-sorted records.
162
+ */
163
+ export async function backfillLedger(options) {
164
+ const { home, ledgerPath, rule } = options;
165
+ const since = options.since ?? '';
166
+ const workspaceFilter = options.workspace ?? '';
167
+ const lead = options.lead ?? '';
168
+ const limit = options.limit ?? 0;
169
+ const noPrefilter = options.noPrefilter === true;
170
+ const existing = await readLedgerKeys(ledgerPath);
171
+ const { logs, unknown } = await findSessionLogs(home, lead, limit);
172
+ const report = {
173
+ logsScanned: 0,
174
+ logsFailed: 0,
175
+ logsUnknownFormat: unknown.length,
176
+ eventsReplayed: 0,
177
+ inheritedEventsSkipped: 0,
178
+ turnsSeen: 0,
179
+ turnsAlreadyRecorded: 0,
180
+ turnsInheritedCopy: 0,
181
+ turnsUndecidable: 0,
182
+ turnsFilteredOut: 0,
183
+ turnsBackfillable: 0,
184
+ firstTurn: '',
185
+ lastTurn: '',
186
+ };
187
+ // Process sessions oldest-created first so an inherited turn is attributed to
188
+ // the session that actually produced it (the parent), not the fork copy.
189
+ const ordered = [];
190
+ for (const log of logs) {
191
+ try {
192
+ const head = decodeZstdFrames(await readFile(log)).split('\n', 1)[0] ?? '';
193
+ const header = JSON.parse(head);
194
+ ordered.push({
195
+ log,
196
+ createdAt: typeof header['createdAt'] === 'number' ? header['createdAt'] : Number.MAX_SAFE_INTEGER,
197
+ });
198
+ }
199
+ catch {
200
+ ordered.push({ log, createdAt: Number.MAX_SAFE_INTEGER });
201
+ }
202
+ }
203
+ ordered.sort((left, right) => left.createdAt - right.createdAt);
204
+ const records = [];
205
+ for (const { log } of ordered) {
206
+ let parsed;
207
+ try {
208
+ parsed = await readSessionLog(log, !noPrefilter);
209
+ }
210
+ catch (error) {
211
+ report.logsFailed += 1;
212
+ options.onWarning?.(`backfill: could not read a session log (${error instanceof Error ? error.name : 'error'})`);
213
+ continue;
214
+ }
215
+ report.logsScanned += 1;
216
+ report.inheritedEventsSkipped += parsed.inheritedEvents;
217
+ const header = parsed.header;
218
+ const sessionId = typeof header['id'] === 'string' ? header['id'] : '';
219
+ const workspace = typeof header['cwd'] === 'string' ? header['cwd'] : '';
220
+ if (sessionId === '')
221
+ continue;
222
+ if (workspaceFilter !== '' && !workspace.includes(workspaceFilter))
223
+ continue;
224
+ const context = contextOf(header, sessionId);
225
+ const collector = new TurnUsageCollector();
226
+ for (const event of parsed.events) {
227
+ report.eventsReplayed += 1;
228
+ const result = collector.push(event);
229
+ if (result.kind !== 'closed')
230
+ continue;
231
+ report.turnsSeen += 1;
232
+ if (result.outcome === undefined) {
233
+ report.turnsUndecidable += 1;
234
+ continue;
235
+ }
236
+ const outcome = result.outcome;
237
+ const key = `${sessionId}\u0000${String(outcome.turn)}`;
238
+ if (existing.keys.has(key)) {
239
+ report.turnsAlreadyRecorded += 1;
240
+ continue;
241
+ }
242
+ const fingerprint = fingerprintOfParts(outcome.turn, outcome.startedAt, outcome.endedAt, outcome.attempts.map((attempt) => attempt.at), outcome.total);
243
+ if (existing.fingerprints.has(fingerprint)) {
244
+ report.turnsInheritedCopy += 1;
245
+ continue;
246
+ }
247
+ if (!sinceAllows(outcome.endedAt, since)) {
248
+ report.turnsFilteredOut += 1;
249
+ continue;
250
+ }
251
+ existing.keys.add(key);
252
+ existing.fingerprints.add(fingerprint);
253
+ report.turnsBackfillable += 1;
254
+ const stamp = new Date(outcome.endedAt).toISOString();
255
+ if (report.firstTurn === '' || stamp < report.firstTurn)
256
+ report.firstTurn = stamp;
257
+ if (stamp > report.lastTurn)
258
+ report.lastTurn = stamp;
259
+ records.push(buildRecord(outcome, context, rule));
260
+ }
261
+ }
262
+ records.sort((left, right) => left.ts - right.ts);
263
+ return { report, records };
264
+ }
265
+ //# sourceMappingURL=backfill.js.map
@@ -0,0 +1,139 @@
1
+ /**
2
+ * Streaming Turn usage fold — the exact-accounting core of the plugin.
3
+ *
4
+ * This is a faithful, incremental re-implementation of the harness's own
5
+ * `deriveTurnTokenUsage` (`@deepseek-ai/dsh-token-meter/client`): same lifecycle
6
+ * state machine, same validations, same "prove it or drop it" posture. It is
7
+ * re-implemented rather than imported on purpose — see README "版本兼容与升级
8
+ * 要点": an out-of-tree plugin that imports an internal package at runtime
9
+ * inherits that package's resolution and version coupling, while this fold is a
10
+ * few hundred lines of pure arithmetic that cannot break the conversation even
11
+ * if the upstream package moves. `tests/conformance.test.mjs` pins the two
12
+ * against each other on synthetic logs whenever the upstream package is
13
+ * resolvable.
14
+ *
15
+ * Cost of the streaming shape: the collector NEVER retains an event or a
16
+ * stream. It reads the usage sample out of the durable stream at push time and
17
+ * keeps only scalars, so a long Turn costs a constant number of bytes here.
18
+ *
19
+ * @module dsh-token-usage-ledger/host/collector
20
+ */
21
+ import type { SessionEvent } from '@deepseek-ai/dsh-session';
22
+ import type { TokenBuckets } from '../shared/protocol.js';
23
+ /**
24
+ * One billed attempt as the collector observes it — usage and route only.
25
+ *
26
+ * Pricing is applied later, when the record is built, so this fold stays pure
27
+ * token accounting exactly as the upstream derivation is.
28
+ */
29
+ export interface CollectedAttempt {
30
+ /** Provider id of the billed route, `''` when unattributed. */
31
+ readonly provider: string;
32
+ /** Model id of the billed route, `''` when unattributed. */
33
+ readonly model: string;
34
+ /**
35
+ * Local epoch ms of the event that settled this attempt. Kept per attempt so
36
+ * peak/off-peak pricing is decided by when the request actually ran rather
37
+ * than by when the whole Turn ended.
38
+ */
39
+ readonly at: number;
40
+ /** Provider-reported buckets for this attempt. */
41
+ readonly tokens: TokenBuckets;
42
+ /** `totalTokens` reported by the provider for this attempt. */
43
+ readonly total: number;
44
+ }
45
+ /** Outcome of feeding one event to the collector. */
46
+ export type PushResult = {
47
+ readonly kind: 'ignored';
48
+ } | {
49
+ readonly kind: 'collecting';
50
+ } | {
51
+ readonly kind: 'closed';
52
+ readonly outcome: TurnOutcome | undefined;
53
+ };
54
+ /** Exact accounting for one completed Turn. */
55
+ export interface TurnOutcome {
56
+ /** Turn number that closed. */
57
+ readonly turn: number;
58
+ /** Why it ended, as reported by the harness. */
59
+ readonly reason: string;
60
+ /** Local epoch ms of `turn/start`. */
61
+ readonly startedAt: number;
62
+ /** Local epoch ms of `turn/end`. */
63
+ readonly endedAt: number;
64
+ /** Per-attempt detail, in settlement order. */
65
+ readonly attempts: readonly CollectedAttempt[];
66
+ /** Bucket sums; an optional bucket appears only when every attempt reported it. */
67
+ readonly tokens: TokenBuckets;
68
+ /** Prompt + output across attempts. */
69
+ readonly total: number;
70
+ }
71
+ /**
72
+ * Fold one Session's live event feed into exact per-Turn accounting.
73
+ *
74
+ * One instance per Session. The owner feeds every `session/event` it observes
75
+ * and reacts to the `'closed'` result. Collection begins only at a `turn/start`
76
+ * the collector actually saw, so attaching mid-turn drops that Turn instead of
77
+ * recording a half-observed one.
78
+ */
79
+ export declare class TurnUsageCollector {
80
+ private state;
81
+ private attempts;
82
+ private turn;
83
+ private sawEnd;
84
+ private invalid;
85
+ private startedAt;
86
+ private endedAt;
87
+ private reason;
88
+ /** Event time of the event currently being consumed; stamps settled attempts. */
89
+ private eventTime;
90
+ /**
91
+ * Whether a Turn is currently being collected.
92
+ * @returns true while an observed Turn is open.
93
+ */
94
+ get collecting(): boolean;
95
+ /**
96
+ * Consume one durable session event.
97
+ *
98
+ * Never throws for any event shape: a malformed event marks the Turn
99
+ * underivable, which drops it, rather than escaping into the caller's
100
+ * dispatch. That containment is what keeps this plugin off the conversation's
101
+ * failure path.
102
+ *
103
+ * @param event - the appended session event, exactly as published.
104
+ * @returns `ignored`, `collecting`, or `closed` with the outcome.
105
+ */
106
+ push(event: SessionEvent): PushResult;
107
+ /** Discard all state; the next observed `turn/start` begins a fresh Turn. */
108
+ reset(): void;
109
+ /**
110
+ * The state machine itself; kept separate so `push` can armor it.
111
+ * @param event - event to fold.
112
+ * @returns the fold's progress.
113
+ */
114
+ private consume;
115
+ /**
116
+ * Report whether an event's (turn, step) pair matches the open attempt.
117
+ * @param turn - event Turn.
118
+ * @param step - event step.
119
+ * @returns true when the pair matches the current open state.
120
+ */
121
+ private sameAttempt;
122
+ /**
123
+ * Close the open attempt, if it holds a usable sample.
124
+ *
125
+ * The sample is normalized HERE rather than on arrival so a raw sample can be
126
+ * carried across the `assistant/attempt` → `assistant/message` pair exactly
127
+ * as upstream does.
128
+ *
129
+ * @param provider - route provider for the settled attempt.
130
+ * @param model - route model for the settled attempt.
131
+ * @returns true when the attempt was recorded.
132
+ */
133
+ private closeOpen;
134
+ /**
135
+ * Assemble the outcome for the Turn that just closed.
136
+ * @returns exact accounting, or undefined when the fold was invalid.
137
+ */
138
+ private buildOutcome;
139
+ }