dsh-token-use 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.en.md +105 -0
- package/README.md +105 -0
- package/client/client.js +428 -0
- package/cordis.patch.yml +7 -0
- package/lib/index.js +727 -0
- package/lib/scan-worker.js +12 -0
- package/package.json +66 -0
package/lib/index.js
ADDED
|
@@ -0,0 +1,727 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* dsh-token-use host half: an always-live, in-memory fold of every model
|
|
3
|
+
* call's token usage plus a read-only loopback JSON endpoint.
|
|
4
|
+
*
|
|
5
|
+
* Cost model: O(1) arithmetic per `session/event` — no polling, no file
|
|
6
|
+
* watches, no periodic writes. History is rebuilt once at boot by streaming
|
|
7
|
+
* the zstd session logs with cooperative yields; from then on the fold only
|
|
8
|
+
* moves forward through the live event bus. A per-session seq watermark makes
|
|
9
|
+
* the boot scan and the live fold overlap-safe in either order.
|
|
10
|
+
*
|
|
11
|
+
* Queries: `GET /dsh-token-use` returns all-time buckets;
|
|
12
|
+
* `?day=YYYY-MM-DD` and `?month=YYYY-MM` return the matching window,
|
|
13
|
+
* aggregated in memory from per-day buckets (no rescan, no disk).
|
|
14
|
+
*
|
|
15
|
+
* @module dsh-token-use
|
|
16
|
+
*/
|
|
17
|
+
import { homedir } from 'node:os'
|
|
18
|
+
import { readFileSync } from 'node:fs'
|
|
19
|
+
import { readdir, readFile, stat } from 'node:fs/promises'
|
|
20
|
+
import { join } from 'node:path'
|
|
21
|
+
import { zstdDecompressSync } from 'node:zlib'
|
|
22
|
+
import { scheduler } from 'node:timers/promises'
|
|
23
|
+
|
|
24
|
+
export const name = 'dsh-token-use'
|
|
25
|
+
|
|
26
|
+
const VERSION = JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf8')).version
|
|
27
|
+
|
|
28
|
+
/** usage field -> fold key, in display order. */
|
|
29
|
+
const USAGE_FIELDS = [
|
|
30
|
+
['inputTokens', 'input'],
|
|
31
|
+
['outputTokens', 'output'],
|
|
32
|
+
['cacheReadTokens', 'cacheRead'],
|
|
33
|
+
['cacheWriteTokens', 'cacheWrite'],
|
|
34
|
+
['reasoningTokens', 'reasoning'],
|
|
35
|
+
]
|
|
36
|
+
|
|
37
|
+
const LOOPBACK = new Set(['127.0.0.1', '::1', '::ffff:127.0.0.1'])
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* Session artifact names: `session.jsonl[.zstd]` for the original generation
|
|
41
|
+
* and `session.v<N>.jsonl[.zstd]` for a versioned re-encoding of the same log.
|
|
42
|
+
* A migration leaves both on disk (same session, different generations), so a
|
|
43
|
+
* session is read from its highest-version file only.
|
|
44
|
+
*/
|
|
45
|
+
const LOG_NAME_RE = /^session(?:\.v(\d+))?\.jsonl(?:\.zstd)?$/
|
|
46
|
+
|
|
47
|
+
/** Pick the newest generation present in one session directory. */
|
|
48
|
+
function pickSessionLog(entries) {
|
|
49
|
+
let best
|
|
50
|
+
for (const name of entries) {
|
|
51
|
+
const match = LOG_NAME_RE.exec(name)
|
|
52
|
+
if (match === null) continue
|
|
53
|
+
const version = match[1] === undefined ? 0 : Number(match[1])
|
|
54
|
+
const compressed = name.endsWith('.zstd') ? 1 : 0
|
|
55
|
+
if (best === undefined || version > best.version || (version === best.version && compressed > best.compressed)) {
|
|
56
|
+
best = { name, version, compressed }
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
return best === undefined ? undefined : best.name
|
|
60
|
+
}
|
|
61
|
+
const ZSTD_MAGIC = 4247762216
|
|
62
|
+
const DAY_RE = /^\d{4}-\d{2}-\d{2}$/
|
|
63
|
+
const MONTH_RE = /^\d{4}-\d{2}$/
|
|
64
|
+
|
|
65
|
+
function resolveDshHome(env = process.env) {
|
|
66
|
+
const explicit = env.DSH_HOME
|
|
67
|
+
if (explicit !== undefined && explicit.trim().length > 0) return explicit
|
|
68
|
+
return join(homedir(), '.dsh')
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
function shiftDay(day, delta) {
|
|
72
|
+
const [year, month, date] = day.split('-').map(Number)
|
|
73
|
+
const shifted = new Date(year, month - 1, date + delta)
|
|
74
|
+
const p = (n) => String(n).padStart(2, '0')
|
|
75
|
+
return `${shifted.getFullYear()}-${p(shifted.getMonth() + 1)}-${p(shifted.getDate())}`
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
function localDay(timestamp) {
|
|
79
|
+
const d = new Date(timestamp)
|
|
80
|
+
const p = (n) => String(n).padStart(2, '0')
|
|
81
|
+
return `${d.getFullYear()}-${p(d.getMonth() + 1)}-${p(d.getDate())}`
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/**
|
|
85
|
+
* Project of a live session: cwd lives on the detached creation header
|
|
86
|
+
* (`session.header.cwd`), not on the Session object itself.
|
|
87
|
+
*/
|
|
88
|
+
function sessionCwd(session) {
|
|
89
|
+
if (session === null || typeof session !== 'object') return undefined
|
|
90
|
+
if (typeof session.cwd === 'string') return session.cwd
|
|
91
|
+
const header = session.header
|
|
92
|
+
if (header !== null && typeof header === 'object' && typeof header.cwd === 'string') return header.cwd
|
|
93
|
+
return undefined
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/** Fork/subagent lineage of a live session, used to inherit its project. */
|
|
97
|
+
function sessionParent(session) {
|
|
98
|
+
if (session === null || typeof session !== 'object') return undefined
|
|
99
|
+
const header = session.header
|
|
100
|
+
if (header !== null && typeof header === 'object' && typeof header.parentSession === 'string') return header.parentSession
|
|
101
|
+
if (typeof session.parentSession === 'string') return session.parentSession
|
|
102
|
+
return undefined
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
function emptyBucket() {
|
|
106
|
+
return { calls: 0, input: 0, output: 0, cacheRead: 0, cacheWrite: 0, reasoning: 0 }
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
function addUsage(bucket, usage) {
|
|
110
|
+
bucket.calls += 1
|
|
111
|
+
for (const [field, key] of USAGE_FIELDS) {
|
|
112
|
+
const value = usage[field]
|
|
113
|
+
if (typeof value === 'number' && Number.isFinite(value)) bucket[key] += value
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
function mergeInto(target, source) {
|
|
118
|
+
target.calls += source.calls
|
|
119
|
+
for (const key of ['input', 'output', 'cacheRead', 'cacheWrite', 'reasoning']) target[key] += source[key]
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/**
|
|
123
|
+
* Split a concatenated-frame zstd container into complete frame ranges.
|
|
124
|
+
* Session artifacts append one frame per flush; the single-frame Node
|
|
125
|
+
* decompressor stops at the first frame, so the container is walked frame by
|
|
126
|
+
* frame with the same header math as the harness's own persistence backend.
|
|
127
|
+
*/
|
|
128
|
+
function scanZstdFrames(buffer) {
|
|
129
|
+
const frames = []
|
|
130
|
+
let offset = 0
|
|
131
|
+
while (offset < buffer.length) {
|
|
132
|
+
const start = offset
|
|
133
|
+
if (buffer.length - offset < 4) break
|
|
134
|
+
if (buffer.readUInt32LE(offset) !== ZSTD_MAGIC) break
|
|
135
|
+
offset += 4
|
|
136
|
+
const descriptor = buffer.readUInt8(offset)
|
|
137
|
+
offset += 1
|
|
138
|
+
const contentSizeFlag = descriptor >>> 6
|
|
139
|
+
const singleSegment = (descriptor & 32) !== 0
|
|
140
|
+
const checksum = (descriptor & 4) !== 0
|
|
141
|
+
const dictionaryFlag = descriptor & 3
|
|
142
|
+
const dictionaryBytes = dictionaryFlag === 3 ? 4 : dictionaryFlag
|
|
143
|
+
const contentSizeBytes = contentSizeFlag === 0 ? singleSegment ? 1 : 0 : 1 << contentSizeFlag
|
|
144
|
+
offset += (singleSegment ? 0 : 1) + dictionaryBytes + contentSizeBytes
|
|
145
|
+
let complete = false
|
|
146
|
+
for (;;) {
|
|
147
|
+
if (buffer.length - offset < 3) break
|
|
148
|
+
const blockHeader = buffer.readUIntLE(offset, 3)
|
|
149
|
+
offset += 3
|
|
150
|
+
const lastBlock = (blockHeader & 1) !== 0
|
|
151
|
+
const blockType = blockHeader >>> 1 & 3
|
|
152
|
+
const blockSize = blockHeader >>> 3
|
|
153
|
+
if (blockType === 3) break
|
|
154
|
+
const payloadBytes = blockType === 1 ? 1 : blockSize
|
|
155
|
+
if (buffer.length - offset < payloadBytes) break
|
|
156
|
+
offset += payloadBytes
|
|
157
|
+
if (lastBlock) {
|
|
158
|
+
if (checksum) {
|
|
159
|
+
if (buffer.length - offset < 4) break
|
|
160
|
+
offset += 4
|
|
161
|
+
}
|
|
162
|
+
complete = true
|
|
163
|
+
break
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
if (complete) frames.push([start, offset])
|
|
167
|
+
}
|
|
168
|
+
return frames
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
/**
|
|
172
|
+
* Decode one session artifact to utf-8 text. Both persistence encodings are
|
|
173
|
+
* supported: plaintext `session.jsonl` and multi-frame `session.jsonl.zstd`.
|
|
174
|
+
*/
|
|
175
|
+
async function decompressSessionLog(path) {
|
|
176
|
+
const buffer = await readFile(path)
|
|
177
|
+
if (path.endsWith('.jsonl')) return buffer.toString('utf8')
|
|
178
|
+
if (typeof zstdDecompressSync !== 'function') {
|
|
179
|
+
throw new Error('node:zlib has no zstd support on this Node version (needs >= 22.15)')
|
|
180
|
+
}
|
|
181
|
+
let text = ''
|
|
182
|
+
for (const [start, end] of scanZstdFrames(buffer)) {
|
|
183
|
+
text += zstdDecompressSync(buffer.subarray(start, end)).toString('utf8')
|
|
184
|
+
}
|
|
185
|
+
return text
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
async function listSessionLogs(sessionsDir) {
|
|
189
|
+
const logs = []
|
|
190
|
+
let projects
|
|
191
|
+
try {
|
|
192
|
+
projects = await readdir(sessionsDir, { withFileTypes: true })
|
|
193
|
+
} catch {
|
|
194
|
+
return logs
|
|
195
|
+
}
|
|
196
|
+
for (const project of projects) {
|
|
197
|
+
if (!project.isDirectory()) continue
|
|
198
|
+
const projectDir = join(sessionsDir, project.name)
|
|
199
|
+
let sessions
|
|
200
|
+
try {
|
|
201
|
+
sessions = await readdir(projectDir, { withFileTypes: true })
|
|
202
|
+
} catch {
|
|
203
|
+
continue
|
|
204
|
+
}
|
|
205
|
+
for (const session of sessions) {
|
|
206
|
+
if (!session.isDirectory()) continue
|
|
207
|
+
const sessionDir = join(projectDir, session.name)
|
|
208
|
+
let entries
|
|
209
|
+
try {
|
|
210
|
+
entries = await readdir(sessionDir)
|
|
211
|
+
} catch {
|
|
212
|
+
continue
|
|
213
|
+
}
|
|
214
|
+
const picked = pickSessionLog(entries)
|
|
215
|
+
if (picked !== undefined) logs.push(join(sessionDir, picked))
|
|
216
|
+
}
|
|
217
|
+
await scheduler.yield()
|
|
218
|
+
}
|
|
219
|
+
return logs
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
/**
|
|
223
|
+
* In-memory token ledger. Both sources — the live `session/event` bus and the
|
|
224
|
+
* boot-time log scan — funnel through {@link UsageTracker#fold}; the seq
|
|
225
|
+
* watermark in {@link UsageTracker#seen} deduplicates the overlap.
|
|
226
|
+
*/
|
|
227
|
+
export class UsageTracker {
|
|
228
|
+
constructor(dshHome = resolveDshHome(), options = {}) {
|
|
229
|
+
this.dshHome = dshHome
|
|
230
|
+
this.allowRemote = options.allowRemote === true
|
|
231
|
+
this.startedAt = Date.now()
|
|
232
|
+
this.totals = emptyBucket()
|
|
233
|
+
this.byModel = new Map()
|
|
234
|
+
this.byProject = new Map()
|
|
235
|
+
this.byDay = new Map()
|
|
236
|
+
this.dayModel = new Map()
|
|
237
|
+
this.dayProject = new Map()
|
|
238
|
+
this.dayModelProject = new Map()
|
|
239
|
+
this.lastModel = new Map()
|
|
240
|
+
this.seen = new Map()
|
|
241
|
+
this.scanning = false
|
|
242
|
+
this.pending = []
|
|
243
|
+
this.projectOf = new Map()
|
|
244
|
+
this.parentOf = new Map()
|
|
245
|
+
this.orphans = new Map()
|
|
246
|
+
this.scan = { startedAt: 0, done: false, files: 0, sessions: 0, bytes: 0, ms: 0, skipped: 0, error: null }
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
bucket(map, key) {
|
|
250
|
+
let bucket = map.get(key)
|
|
251
|
+
if (bucket === undefined) {
|
|
252
|
+
bucket = emptyBucket()
|
|
253
|
+
map.set(key, bucket)
|
|
254
|
+
}
|
|
255
|
+
return bucket
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
nestedBucket(map, outerKey, innerKey) {
|
|
259
|
+
let inner = map.get(outerKey)
|
|
260
|
+
if (inner === undefined) {
|
|
261
|
+
inner = new Map()
|
|
262
|
+
map.set(outerKey, inner)
|
|
263
|
+
}
|
|
264
|
+
return this.bucket(inner, innerKey)
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
nestedBucket3(map, firstKey, secondKey, thirdKey) {
|
|
268
|
+
let second = map.get(firstKey)
|
|
269
|
+
if (second === undefined) {
|
|
270
|
+
second = new Map()
|
|
271
|
+
map.set(firstKey, second)
|
|
272
|
+
}
|
|
273
|
+
return this.nestedBucket(second, secondKey, thirdKey)
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
/**
|
|
277
|
+
* Fold one decoded session event. `request/header` only updates the
|
|
278
|
+
* per-session model attribution; `assistant/message` with a usage record is
|
|
279
|
+
* counted once. @returns true when the event was counted.
|
|
280
|
+
*/
|
|
281
|
+
fold(session, event) {
|
|
282
|
+
if (event === null || typeof event !== 'object' || session === null || typeof session !== 'object') return false
|
|
283
|
+
if (this.scanning) {
|
|
284
|
+
this.pending.push([session, event])
|
|
285
|
+
return false
|
|
286
|
+
}
|
|
287
|
+
if (event.type === 'request/header') {
|
|
288
|
+
const model = event.data?.header?.config?.model
|
|
289
|
+
if (typeof model === 'string' && model.length > 0) this.lastModel.set(session.id, model)
|
|
290
|
+
return false
|
|
291
|
+
}
|
|
292
|
+
if (event.type !== 'assistant/message') return false
|
|
293
|
+
const usage = event.data?.usage
|
|
294
|
+
if (usage === null || typeof usage !== 'object') return false
|
|
295
|
+
const seq = event.seq
|
|
296
|
+
const seen = this.seen.get(session.id)
|
|
297
|
+
if (typeof seen === 'number' && typeof seq === 'number' && seq <= seen) return false
|
|
298
|
+
if (typeof seq === 'number') this.seen.set(session.id, Math.max(seen ?? 0, seq))
|
|
299
|
+
addUsage(this.totals, usage)
|
|
300
|
+
const model = this.lastModel.get(session.id) ?? 'unknown'
|
|
301
|
+
addUsage(this.bucket(this.byModel, model), usage)
|
|
302
|
+
const project = this.projectFor(session)
|
|
303
|
+
if (project === undefined) {
|
|
304
|
+
// The session's project is not known yet: hold the project-side usage
|
|
305
|
+
// until it resolves (a later event or the parent's resolution).
|
|
306
|
+
this.holdOrphan(session.id, usage, typeof event.time === 'number' ? localDay(event.time) : undefined, model)
|
|
307
|
+
} else {
|
|
308
|
+
addUsage(this.bucket(this.byProject, project), usage)
|
|
309
|
+
}
|
|
310
|
+
if (typeof event.time === 'number') {
|
|
311
|
+
const day = localDay(event.time)
|
|
312
|
+
addUsage(this.bucket(this.byDay, day), usage)
|
|
313
|
+
addUsage(this.nestedBucket(this.dayModel, day, model), usage)
|
|
314
|
+
if (project !== undefined) {
|
|
315
|
+
addUsage(this.nestedBucket(this.dayProject, day, project), usage)
|
|
316
|
+
addUsage(this.nestedBucket3(this.dayModelProject, day, model, project), usage)
|
|
317
|
+
}
|
|
318
|
+
}
|
|
319
|
+
return true
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
/**
|
|
323
|
+
* Rebuild history from `$DSH_HOME/sessions/**` once. Runs on a worker
|
|
324
|
+
* thread by default so a large history never blocks the host event loop;
|
|
325
|
+
* events arriving during the scan are buffered and replayed afterwards, and
|
|
326
|
+
* the seq watermark keeps the two folds from double counting.
|
|
327
|
+
*/
|
|
328
|
+
async scanSessions(options = {}) {
|
|
329
|
+
if (this.scan.startedAt !== 0) return this.scan
|
|
330
|
+
if (options.worker !== false) {
|
|
331
|
+
const done = await this.scanInWorker()
|
|
332
|
+
if (done) return this.scan
|
|
333
|
+
}
|
|
334
|
+
return this.scanInProcess()
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
/** Worker-thread scan; returns false when unavailable so the caller falls back. */
|
|
338
|
+
async scanInWorker() {
|
|
339
|
+
if (typeof Worker !== 'function') return false
|
|
340
|
+
let WorkerCtor
|
|
341
|
+
try {
|
|
342
|
+
;({ Worker: WorkerCtor } = await import('node:worker_threads'))
|
|
343
|
+
} catch {
|
|
344
|
+
return false
|
|
345
|
+
}
|
|
346
|
+
this.scan.startedAt = Date.now()
|
|
347
|
+
this.scanning = true
|
|
348
|
+
try {
|
|
349
|
+
const result = await new Promise((resolve, reject) => {
|
|
350
|
+
const worker = new WorkerCtor(new URL('./scan-worker.js', import.meta.url), { workerData: { dshHome: this.dshHome } })
|
|
351
|
+
worker.once('message', (message) => {
|
|
352
|
+
worker.terminate().catch(() => {})
|
|
353
|
+
resolve(message)
|
|
354
|
+
})
|
|
355
|
+
worker.once('error', (error) => {
|
|
356
|
+
worker.terminate().catch(() => {})
|
|
357
|
+
reject(error)
|
|
358
|
+
})
|
|
359
|
+
})
|
|
360
|
+
this.merge(result)
|
|
361
|
+
this.scan.ms = Date.now() - this.scan.startedAt
|
|
362
|
+
return true
|
|
363
|
+
} catch (error) {
|
|
364
|
+
this.scan.error = String(error?.message ?? error)
|
|
365
|
+
this.scan.startedAt = 0
|
|
366
|
+
return false
|
|
367
|
+
} finally {
|
|
368
|
+
this.scanning = false
|
|
369
|
+
const buffered = this.pending
|
|
370
|
+
this.pending = []
|
|
371
|
+
for (const [session, event] of buffered) this.fold(session, event)
|
|
372
|
+
}
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
async scanInProcess() {
|
|
376
|
+
if (this.scan.startedAt !== 0) return this.scan
|
|
377
|
+
this.scan.startedAt = Date.now()
|
|
378
|
+
const root = join(this.dshHome, 'sessions')
|
|
379
|
+
const logs = await listSessionLogs(root)
|
|
380
|
+
this.scan.files = logs.length
|
|
381
|
+
const sessionDirs = new Set()
|
|
382
|
+
const processLog = async (log) => {
|
|
383
|
+
sessionDirs.add(log.slice(root.length + 1).split('/')[0])
|
|
384
|
+
try {
|
|
385
|
+
this.scan.bytes += (await stat(log)).size
|
|
386
|
+
const decoded = await decompressSessionLog(log)
|
|
387
|
+
let current = null
|
|
388
|
+
for (const line of decoded.split('\n')) {
|
|
389
|
+
if (line.length === 0) continue
|
|
390
|
+
let event
|
|
391
|
+
try {
|
|
392
|
+
event = JSON.parse(line)
|
|
393
|
+
} catch {
|
|
394
|
+
continue
|
|
395
|
+
}
|
|
396
|
+
if (event.type === 'session') {
|
|
397
|
+
current = { id: event.id, cwd: event.cwd }
|
|
398
|
+
if (typeof event.cwd === 'string') this.projectOf.set(event.id, event.cwd)
|
|
399
|
+
if (typeof event.parentSession === 'string') this.parentOf.set(event.id, event.parentSession)
|
|
400
|
+
continue
|
|
401
|
+
}
|
|
402
|
+
if (current !== null) this.fold(current, event)
|
|
403
|
+
}
|
|
404
|
+
} catch (error) {
|
|
405
|
+
this.scan.skipped += 1
|
|
406
|
+
if (this.scan.error === null) this.scan.error = String(error?.message ?? error)
|
|
407
|
+
}
|
|
408
|
+
}
|
|
409
|
+
let cursor = 0
|
|
410
|
+
const worker = async () => {
|
|
411
|
+
while (cursor < logs.length) {
|
|
412
|
+
const log = logs[cursor]
|
|
413
|
+
cursor += 1
|
|
414
|
+
await processLog(log)
|
|
415
|
+
await scheduler.yield()
|
|
416
|
+
}
|
|
417
|
+
}
|
|
418
|
+
await Promise.all(Array.from({ length: Math.min(3, logs.length) }, worker))
|
|
419
|
+
for (const id of [...this.projectOf.keys()]) this.rememberProject(id, this.projectOf.get(id))
|
|
420
|
+
this.scan.sessions = sessionDirs.size
|
|
421
|
+
this.scan.ms = Date.now() - this.scan.startedAt
|
|
422
|
+
this.scan.done = true
|
|
423
|
+
return this.scan
|
|
424
|
+
}
|
|
425
|
+
|
|
426
|
+
sorted(map) {
|
|
427
|
+
return Object.fromEntries([...map.entries()].sort((a, b) => b[1].input - a[1].input))
|
|
428
|
+
}
|
|
429
|
+
|
|
430
|
+
/** Record a session's project and release any usage that was waiting on it. */
|
|
431
|
+
rememberProject(sessionId, project) {
|
|
432
|
+
if (typeof sessionId !== 'string' || typeof project !== 'string') return
|
|
433
|
+
this.projectOf.set(sessionId, project)
|
|
434
|
+
for (const [id, parent] of this.parentOf) {
|
|
435
|
+
if (parent === sessionId && !this.projectOf.has(id)) this.rememberProject(id, project)
|
|
436
|
+
}
|
|
437
|
+
const held = this.orphans.get(sessionId)
|
|
438
|
+
if (held !== undefined) {
|
|
439
|
+
this.orphans.delete(sessionId)
|
|
440
|
+
this.commitProject(project, held)
|
|
441
|
+
}
|
|
442
|
+
}
|
|
443
|
+
|
|
444
|
+
/** Attribute usage that arrived before its session's project was known. */
|
|
445
|
+
holdOrphan(sessionId, usage, day, model) {
|
|
446
|
+
let held = this.orphans.get(sessionId)
|
|
447
|
+
if (held === undefined) {
|
|
448
|
+
held = { totals: emptyBucket(), days: new Map(), dayModels: new Map(), since: Date.now() }
|
|
449
|
+
this.orphans.set(sessionId, held)
|
|
450
|
+
}
|
|
451
|
+
addUsage(held.totals, usage)
|
|
452
|
+
if (day !== undefined) {
|
|
453
|
+
addUsage(this.bucket(held.days, day), usage)
|
|
454
|
+
if (model !== undefined) {
|
|
455
|
+
let byModel = held.dayModels.get(day)
|
|
456
|
+
if (byModel === undefined) {
|
|
457
|
+
byModel = new Map()
|
|
458
|
+
held.dayModels.set(day, byModel)
|
|
459
|
+
}
|
|
460
|
+
addUsage(this.bucket(byModel, model), usage)
|
|
461
|
+
}
|
|
462
|
+
}
|
|
463
|
+
}
|
|
464
|
+
|
|
465
|
+
/** Merge held usage into the by-project, by-day-project and model-project buckets. */
|
|
466
|
+
commitProject(project, held) {
|
|
467
|
+
mergeInto(this.bucket(this.byProject, project), held.totals)
|
|
468
|
+
for (const [day, bucket] of held.days) mergeInto(this.nestedBucket(this.dayProject, day, project), bucket)
|
|
469
|
+
for (const [day, byModel] of held.dayModels) {
|
|
470
|
+
for (const [model, bucket] of byModel) mergeInto(this.nestedBucket3(this.dayModelProject, day, model, project), bucket)
|
|
471
|
+
}
|
|
472
|
+
}
|
|
473
|
+
|
|
474
|
+
/** Anything still unattributed after this long is shown as `(no cwd)`. */
|
|
475
|
+
flushStaleOrphans(maxAgeMs = 30000) {
|
|
476
|
+
const now = Date.now()
|
|
477
|
+
for (const [id, held] of [...this.orphans]) {
|
|
478
|
+
if (now - held.since < maxAgeMs) continue
|
|
479
|
+
this.orphans.delete(id)
|
|
480
|
+
this.commitProject('(no cwd)', held)
|
|
481
|
+
}
|
|
482
|
+
}
|
|
483
|
+
|
|
484
|
+
/**
|
|
485
|
+
* Resolve a session's project: the live header first, then the boot scan's
|
|
486
|
+
* map, then the parent session (subagents/fork children inherit it).
|
|
487
|
+
*/
|
|
488
|
+
projectFor(session) {
|
|
489
|
+
const direct = sessionCwd(session)
|
|
490
|
+
if (direct !== undefined) {
|
|
491
|
+
this.rememberProject(session.id, direct)
|
|
492
|
+
return direct
|
|
493
|
+
}
|
|
494
|
+
const known = this.projectOf.get(session.id)
|
|
495
|
+
if (known !== undefined) return known
|
|
496
|
+
const parent = sessionParent(session)
|
|
497
|
+
if (parent !== undefined) {
|
|
498
|
+
this.parentOf.set(session.id, parent)
|
|
499
|
+
const inherited = this.projectOf.get(parent)
|
|
500
|
+
if (inherited !== undefined) {
|
|
501
|
+
this.rememberProject(session.id, inherited)
|
|
502
|
+
return inherited
|
|
503
|
+
}
|
|
504
|
+
}
|
|
505
|
+
return undefined
|
|
506
|
+
}
|
|
507
|
+
|
|
508
|
+
/** Snapshot the fold for structured-clone transfer out of a worker. */
|
|
509
|
+
serialize() {
|
|
510
|
+
const pairs = (map) => [...map.entries()]
|
|
511
|
+
return {
|
|
512
|
+
totals: { ...this.totals },
|
|
513
|
+
byModel: pairs(this.byModel),
|
|
514
|
+
byProject: pairs(this.byProject),
|
|
515
|
+
byDay: pairs(this.byDay),
|
|
516
|
+
dayModel: pairs(this.dayModel).map(([day, inner]) => [day, pairs(inner)]),
|
|
517
|
+
dayProject: pairs(this.dayProject).map(([day, inner]) => [day, pairs(inner)]),
|
|
518
|
+
dayModelProject: pairs(this.dayModelProject).map(([day, inner]) => [day, pairs(inner).map(([model, byProject]) => [model, pairs(byProject)])]),
|
|
519
|
+
lastModel: pairs(this.lastModel),
|
|
520
|
+
projectOf: pairs(this.projectOf),
|
|
521
|
+
parentOf: pairs(this.parentOf),
|
|
522
|
+
seen: pairs(this.seen),
|
|
523
|
+
scan: { ...this.scan },
|
|
524
|
+
}
|
|
525
|
+
}
|
|
526
|
+
|
|
527
|
+
/** Adopt a serialized fold (worker result); buckets are additive. */
|
|
528
|
+
merge(part) {
|
|
529
|
+
mergeInto(this.totals, part.totals)
|
|
530
|
+
for (const [key, bucket] of part.byModel) mergeInto(this.bucket(this.byModel, key), bucket)
|
|
531
|
+
for (const [key, bucket] of part.byProject) mergeInto(this.bucket(this.byProject, key), bucket)
|
|
532
|
+
for (const [key, bucket] of part.byDay) mergeInto(this.bucket(this.byDay, key), bucket)
|
|
533
|
+
for (const [day, inner] of part.dayModel) for (const [key, bucket] of inner) mergeInto(this.nestedBucket(this.dayModel, day, key), bucket)
|
|
534
|
+
for (const [day, inner] of part.dayProject) for (const [key, bucket] of inner) mergeInto(this.nestedBucket(this.dayProject, day, key), bucket)
|
|
535
|
+
for (const [day, byModel] of part.dayModelProject) {
|
|
536
|
+
for (const [model, byProject] of byModel) for (const [project, bucket] of byProject) {
|
|
537
|
+
mergeInto(this.nestedBucket3(this.dayModelProject, day, model, project), bucket)
|
|
538
|
+
}
|
|
539
|
+
}
|
|
540
|
+
for (const [key, model] of part.lastModel) this.lastModel.set(key, model)
|
|
541
|
+
for (const [key, project] of part.projectOf) this.rememberProject(key, project)
|
|
542
|
+
for (const [key, parent] of part.parentOf) this.parentOf.set(key, parent)
|
|
543
|
+
for (const [key, seq] of part.seen) this.seen.set(key, Math.max(this.seen.get(key) ?? 0, seq))
|
|
544
|
+
this.scan = { ...this.scan, ...part.scan }
|
|
545
|
+
}
|
|
546
|
+
|
|
547
|
+
/** Known model names across every day bucket, sorted for the picker. */
|
|
548
|
+
models() {
|
|
549
|
+
return [...this.byModel.keys()].sort()
|
|
550
|
+
}
|
|
551
|
+
|
|
552
|
+
/**
|
|
553
|
+
* Materialize totals/breakdowns for the optional window + model filter.
|
|
554
|
+
* Filtering only re-aggregates existing per-day buckets — no rescan, no disk.
|
|
555
|
+
*/
|
|
556
|
+
view(filter = null) {
|
|
557
|
+
if (filter === null) {
|
|
558
|
+
const sortedDays = [...this.byDay.keys()].sort()
|
|
559
|
+
const trendEnd = sortedDays.length > 0 ? sortedDays[sortedDays.length - 1] : localDay(Date.now())
|
|
560
|
+
const trendStart = shiftDay(trendEnd, -29)
|
|
561
|
+
const trendDays = []
|
|
562
|
+
for (let day = trendStart; day <= trendEnd; day = shiftDay(day, 1)) {
|
|
563
|
+
const bucket = this.byDay.get(day)
|
|
564
|
+
trendDays.push([day, bucket === undefined ? emptyBucket() : { ...bucket }])
|
|
565
|
+
}
|
|
566
|
+
return {
|
|
567
|
+
totals: { ...this.totals },
|
|
568
|
+
byModel: this.sorted(this.byModel),
|
|
569
|
+
byProject: this.sorted(this.byProject),
|
|
570
|
+
byDay: Object.fromEntries([...this.byDay.entries()].sort()),
|
|
571
|
+
trend: { from: trendStart, to: trendEnd, days: Object.fromEntries(trendDays) },
|
|
572
|
+
}
|
|
573
|
+
}
|
|
574
|
+
const model = filter.model
|
|
575
|
+
const allDays = [...this.byDay.keys()].sort()
|
|
576
|
+
const days = filter.kind === 'all'
|
|
577
|
+
? allDays
|
|
578
|
+
: allDays.filter((day) => (filter.kind === 'day' ? day === filter.value : day.startsWith(filter.value)))
|
|
579
|
+
const totals = emptyBucket()
|
|
580
|
+
const models = new Map()
|
|
581
|
+
const projects = new Map()
|
|
582
|
+
const byDay = {}
|
|
583
|
+
for (const day of days) {
|
|
584
|
+
const dayBucket = model === undefined ? this.byDay.get(day) : this.dayModel.get(day)?.get(model)
|
|
585
|
+
if (dayBucket === undefined) continue
|
|
586
|
+
mergeInto(totals, dayBucket)
|
|
587
|
+
byDay[day] = { ...dayBucket }
|
|
588
|
+
if (model === undefined) {
|
|
589
|
+
for (const [name, bucket] of this.dayModel.get(day) ?? []) mergeInto(this.bucket(models, name), bucket)
|
|
590
|
+
for (const [name, bucket] of this.dayProject.get(day) ?? []) mergeInto(this.bucket(projects, name), bucket)
|
|
591
|
+
} else {
|
|
592
|
+
mergeInto(this.bucket(models, model), dayBucket)
|
|
593
|
+
for (const [name, bucket] of this.dayModelProject.get(day)?.get(model) ?? []) mergeInto(this.bucket(projects, name), bucket)
|
|
594
|
+
}
|
|
595
|
+
}
|
|
596
|
+
const trendDays = []
|
|
597
|
+
const lastDataDay = days.length > 0 ? days[days.length - 1] : undefined
|
|
598
|
+
const trendEnd = filter.kind === 'day'
|
|
599
|
+
? filter.value
|
|
600
|
+
: (lastDataDay ?? localDay(Date.now()))
|
|
601
|
+
const trendStart = shiftDay(trendEnd, -29)
|
|
602
|
+
for (let day = trendStart; day <= trendEnd; day = shiftDay(day, 1)) {
|
|
603
|
+
const bucket = model === undefined ? this.byDay.get(day) : this.dayModel.get(day)?.get(model)
|
|
604
|
+
trendDays.push([day, bucket === undefined ? emptyBucket() : { ...bucket }])
|
|
605
|
+
}
|
|
606
|
+
return {
|
|
607
|
+
totals,
|
|
608
|
+
byModel: this.sorted(models),
|
|
609
|
+
byProject: this.sorted(projects),
|
|
610
|
+
byDay,
|
|
611
|
+
trend: { from: trendStart, to: trendEnd, days: Object.fromEntries(trendDays) },
|
|
612
|
+
}
|
|
613
|
+
}
|
|
614
|
+
|
|
615
|
+
snapshot(filter = null) {
|
|
616
|
+
this.flushStaleOrphans()
|
|
617
|
+
const view = this.view(filter)
|
|
618
|
+
const withTotal = (bucket) => ({ ...bucket, total: bucket.input + bucket.output + bucket.cacheRead + bucket.cacheWrite })
|
|
619
|
+
return {
|
|
620
|
+
ok: true,
|
|
621
|
+
name,
|
|
622
|
+
version: VERSION,
|
|
623
|
+
startedAt: this.startedAt,
|
|
624
|
+
updatedAt: Date.now(),
|
|
625
|
+
dshHome: this.dshHome,
|
|
626
|
+
filter: filter === null ? null : { kind: filter.kind, value: filter.value ?? null, model: filter.model ?? null },
|
|
627
|
+
models: this.models(),
|
|
628
|
+
scan: { ...this.scan },
|
|
629
|
+
totals: withTotal(view.totals),
|
|
630
|
+
byModel: Object.fromEntries(Object.entries(view.byModel).map(([k, v]) => [k, withTotal(v)])),
|
|
631
|
+
byProject: Object.fromEntries(Object.entries(view.byProject).map(([k, v]) => [k, withTotal(v)])),
|
|
632
|
+
byDay: Object.fromEntries(Object.entries(view.byDay).map(([k, v]) => [k, withTotal(v)])),
|
|
633
|
+
trend: {
|
|
634
|
+
from: view.trend.from,
|
|
635
|
+
to: view.trend.to,
|
|
636
|
+
days: Object.fromEntries(Object.entries(view.trend.days).map(([k, v]) => [k, withTotal(v)])),
|
|
637
|
+
},
|
|
638
|
+
}
|
|
639
|
+
}
|
|
640
|
+
|
|
641
|
+
handle(req, res) {
|
|
642
|
+
res.setHeader('cache-control', 'no-store')
|
|
643
|
+
if (req.method !== 'GET') {
|
|
644
|
+
res.writeHead(405, { 'content-type': 'application/json; charset=utf-8' })
|
|
645
|
+
res.end(JSON.stringify({ ok: false, error: 'method not allowed' }))
|
|
646
|
+
return
|
|
647
|
+
}
|
|
648
|
+
if (!LOOPBACK.has(req.socket?.remoteAddress)) {
|
|
649
|
+
const origin = req.headers.origin
|
|
650
|
+
const sameOrigin = req.headers['sec-fetch-site'] === 'same-origin'
|
|
651
|
+
&& typeof origin === 'string'
|
|
652
|
+
&& typeof req.headers.host === 'string'
|
|
653
|
+
let trusted = false
|
|
654
|
+
if (sameOrigin) {
|
|
655
|
+
try {
|
|
656
|
+
trusted = new URL(origin).host === req.headers.host
|
|
657
|
+
} catch {
|
|
658
|
+
trusted = false
|
|
659
|
+
}
|
|
660
|
+
}
|
|
661
|
+
if (!(this.allowRemote && trusted)) {
|
|
662
|
+
res.writeHead(403, { 'content-type': 'application/json; charset=utf-8' })
|
|
663
|
+
res.end(JSON.stringify({ ok: false, error: 'loopback only; set allowRemote for trusted same-origin access' }))
|
|
664
|
+
return
|
|
665
|
+
}
|
|
666
|
+
}
|
|
667
|
+
const params = new URL(req.url, 'http://localhost').searchParams
|
|
668
|
+
const day = params.get('day')
|
|
669
|
+
const month = params.get('month')
|
|
670
|
+
const rawModel = params.get('model')
|
|
671
|
+
const model = rawModel !== null && rawModel.length > 0 && rawModel.length <= 200 ? rawModel : undefined
|
|
672
|
+
let filter = null
|
|
673
|
+
if (day !== null && month !== null) {
|
|
674
|
+
res.writeHead(400, { 'content-type': 'application/json; charset=utf-8' })
|
|
675
|
+
res.end(JSON.stringify({ ok: false, error: 'use exactly one of day or month' }))
|
|
676
|
+
return
|
|
677
|
+
}
|
|
678
|
+
if (day !== null) {
|
|
679
|
+
if (!DAY_RE.test(day)) {
|
|
680
|
+
res.writeHead(400, { 'content-type': 'application/json; charset=utf-8' })
|
|
681
|
+
res.end(JSON.stringify({ ok: false, error: 'day must be YYYY-MM-DD' }))
|
|
682
|
+
return
|
|
683
|
+
}
|
|
684
|
+
filter = { kind: 'day', value: day, model }
|
|
685
|
+
} else if (month !== null) {
|
|
686
|
+
if (!MONTH_RE.test(month)) {
|
|
687
|
+
res.writeHead(400, { 'content-type': 'application/json; charset=utf-8' })
|
|
688
|
+
res.end(JSON.stringify({ ok: false, error: 'month must be YYYY-MM' }))
|
|
689
|
+
return
|
|
690
|
+
}
|
|
691
|
+
filter = { kind: 'month', value: month, model }
|
|
692
|
+
} else if (model !== undefined) {
|
|
693
|
+
filter = { kind: 'all', value: undefined, model }
|
|
694
|
+
}
|
|
695
|
+
res.writeHead(200, { 'content-type': 'application/json; charset=utf-8' })
|
|
696
|
+
res.end(JSON.stringify(this.snapshot(filter)))
|
|
697
|
+
}
|
|
698
|
+
}
|
|
699
|
+
|
|
700
|
+
/**
|
|
701
|
+
* Register the tracker once the profile's webServer service exists.
|
|
702
|
+
* @param ctx - host context from the cordis loader.
|
|
703
|
+
* @param config - loader config: `endpoint` and `scanAtBoot`.
|
|
704
|
+
*/
|
|
705
|
+
export function apply(ctx, config = {}) {
|
|
706
|
+
const endpoint = typeof config.endpoint === 'string' && config.endpoint.length > 0 ? config.endpoint : '/dsh-token-use'
|
|
707
|
+
const scanAtBoot = config.scanAtBoot !== false
|
|
708
|
+
ctx.inject(['webServer'], (host) => {
|
|
709
|
+
const tracker = new UsageTracker(resolveDshHome(), { allowRemote: config.allowRemote === true })
|
|
710
|
+
host.on('session/event', (session, event) => {
|
|
711
|
+
tracker.fold(session, event)
|
|
712
|
+
}, { global: true })
|
|
713
|
+
host.effect(() => {
|
|
714
|
+
const dispose = host.webServer.register({
|
|
715
|
+
kind: 'exact',
|
|
716
|
+
path: endpoint,
|
|
717
|
+
handler: (req, res) => tracker.handle(req, res),
|
|
718
|
+
})
|
|
719
|
+
return () => dispose()
|
|
720
|
+
}, 'dsh-token-use: http route')
|
|
721
|
+
if (scanAtBoot) {
|
|
722
|
+
tracker.scanSessions().catch((error) => {
|
|
723
|
+
tracker.scan.error = String(error?.stack ?? error)
|
|
724
|
+
})
|
|
725
|
+
}
|
|
726
|
+
})
|
|
727
|
+
}
|