modelmeter-collect 0.6.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/collect.mjs +104 -58
- package/lib.mjs +91 -3
- package/package.json +1 -5
- package/lib.test.mjs +0 -173
package/collect.mjs
CHANGED
|
@@ -19,7 +19,8 @@ import {
|
|
|
19
19
|
chmodSync,
|
|
20
20
|
} from 'node:fs'
|
|
21
21
|
import { homedir } from 'node:os'
|
|
22
|
-
import { join } from 'node:path'
|
|
22
|
+
import { join, dirname } from 'node:path'
|
|
23
|
+
import { createHash } from 'node:crypto'
|
|
23
24
|
import {
|
|
24
25
|
findLastTokenCount,
|
|
25
26
|
codexToolFromEvent,
|
|
@@ -29,8 +30,9 @@ import {
|
|
|
29
30
|
aggregateDaily,
|
|
30
31
|
aggregateHourly,
|
|
31
32
|
aggregateTools,
|
|
32
|
-
|
|
33
|
-
|
|
33
|
+
claudeSessionSummary,
|
|
34
|
+
codexSessionSummary,
|
|
35
|
+
sessionSendRow,
|
|
34
36
|
} from './lib.mjs'
|
|
35
37
|
|
|
36
38
|
const HOME = homedir()
|
|
@@ -64,6 +66,11 @@ const TOKEN = process.env.MODELMETER_TOKEN || cfg.token
|
|
|
64
66
|
const INGEST_URL = process.env.MODELMETER_INGEST_URL || cfg.ingestUrl
|
|
65
67
|
if (!TOKEN || !INGEST_URL) process.exit(0) // not configured: do nothing, never block
|
|
66
68
|
|
|
69
|
+
// Repo attribution is opt-in for the human-readable label only. A repo hash is always
|
|
70
|
+
// sent (opaque). The label (the repo folder name, never the full path) is sent only
|
|
71
|
+
// when explicitly enabled, so a repo name never leaves the machine by default.
|
|
72
|
+
const REPO_LABELS = process.env.MODELMETER_REPO_LABELS === '1' || cfg.repoLabels === true
|
|
73
|
+
|
|
67
74
|
// POST JSON with a hard timeout so a stuck network path can never hang a Stop
|
|
68
75
|
// hook or pile up scheduled collectors. Callers handle the thrown abort/error.
|
|
69
76
|
async function postJson(url, body) {
|
|
@@ -81,29 +88,13 @@ async function postJson(url, body) {
|
|
|
81
88
|
}
|
|
82
89
|
}
|
|
83
90
|
|
|
84
|
-
let state = { claude: {}, codex: {}
|
|
91
|
+
let state = { claude: {}, codex: {} }
|
|
85
92
|
try {
|
|
86
|
-
state = {
|
|
87
|
-
claude: {},
|
|
88
|
-
codex: {},
|
|
89
|
-
pendingDetail: { hours: [], tools: [] },
|
|
90
|
-
...JSON.parse(readFileSync(STATE_PATH, 'utf8')),
|
|
91
|
-
}
|
|
93
|
+
state = { claude: {}, codex: {}, ...JSON.parse(readFileSync(STATE_PATH, 'utf8')) }
|
|
92
94
|
} catch {
|
|
93
95
|
// first run
|
|
94
96
|
}
|
|
95
97
|
|
|
96
|
-
function saveState() {
|
|
97
|
-
if (!existsSync(MM_DIR)) mkdirSync(MM_DIR, { recursive: true })
|
|
98
|
-
state.claude = pruneClaudeState(state.claude, CLAUDE_STATE_CAP)
|
|
99
|
-
writeFileSync(STATE_PATH, JSON.stringify(state))
|
|
100
|
-
try {
|
|
101
|
-
chmodSync(STATE_PATH, 0o600) // usage metadata is not secret, but keep it owner-only
|
|
102
|
-
} catch {
|
|
103
|
-
// best effort on platforms without POSIX perms
|
|
104
|
-
}
|
|
105
|
-
}
|
|
106
|
-
|
|
107
98
|
// --- invocation detection -------------------------------------------------
|
|
108
99
|
let hookInput = null
|
|
109
100
|
if (!process.stdin.isTTY) {
|
|
@@ -177,7 +168,7 @@ function scanClaude(files) {
|
|
|
177
168
|
}
|
|
178
169
|
const ev = claudeEventFromLine(o)
|
|
179
170
|
if (!ev || !ev.id || state.claude[ev.id]) continue
|
|
180
|
-
state.claude[ev.id] =
|
|
171
|
+
state.claude[ev.id] = 1
|
|
181
172
|
events.push(ev)
|
|
182
173
|
}
|
|
183
174
|
}
|
|
@@ -234,13 +225,66 @@ function scanCodex(files) {
|
|
|
234
225
|
}
|
|
235
226
|
}
|
|
236
227
|
|
|
228
|
+
let claudeFiles = []
|
|
229
|
+
let codexFiles = []
|
|
237
230
|
if (hookInput?.transcript_path) {
|
|
238
|
-
|
|
231
|
+
claudeFiles = [hookInput.transcript_path] // Claude Code hook: just this session
|
|
239
232
|
} else if (codexNotify) {
|
|
240
|
-
|
|
233
|
+
codexFiles = recentFiles(join(HOME, '.codex', 'sessions'), 2) // Codex notify: newest session(s)
|
|
241
234
|
} else {
|
|
242
|
-
|
|
243
|
-
|
|
235
|
+
claudeFiles = recentFiles(join(HOME, '.claude', 'projects'))
|
|
236
|
+
codexFiles = recentFiles(join(HOME, '.codex', 'sessions'))
|
|
237
|
+
}
|
|
238
|
+
scanClaude(claudeFiles)
|
|
239
|
+
scanCodex(codexFiles)
|
|
240
|
+
|
|
241
|
+
// Per-session summaries (context bloat, comparisons) for the detail endpoint. Hash
|
|
242
|
+
// the session-file basename (the session id), never the path, so no repo/project
|
|
243
|
+
// label leaks here. Recomputed in full each run; the backend overwrites by hash.
|
|
244
|
+
const SESSION_CAP = 500
|
|
245
|
+
function sessionHashFor(file) {
|
|
246
|
+
const base = file.split('/').pop() || file
|
|
247
|
+
return createHash('sha256').update(base).digest('hex').slice(0, 40)
|
|
248
|
+
}
|
|
249
|
+
// Resolve a cwd to its git-root for a stable repo identity (so subdirs of one repo do
|
|
250
|
+
// not fragment). Falls back to the cwd if no .git is found (e.g. the repo is gone).
|
|
251
|
+
function repoRoot(cwd) {
|
|
252
|
+
if (!cwd) return ''
|
|
253
|
+
let dir = cwd
|
|
254
|
+
for (let i = 0; i < 12; i++) {
|
|
255
|
+
try {
|
|
256
|
+
if (existsSync(join(dir, '.git'))) return dir
|
|
257
|
+
} catch {
|
|
258
|
+
// ignore
|
|
259
|
+
}
|
|
260
|
+
const parent = dirname(dir)
|
|
261
|
+
if (parent === dir) break
|
|
262
|
+
dir = parent
|
|
263
|
+
}
|
|
264
|
+
return cwd
|
|
265
|
+
}
|
|
266
|
+
const sessions = []
|
|
267
|
+
for (const [files, summarize] of [
|
|
268
|
+
[claudeFiles, claudeSessionSummary],
|
|
269
|
+
[codexFiles, codexSessionSummary],
|
|
270
|
+
]) {
|
|
271
|
+
for (const file of files.slice(0, SESSION_CAP)) {
|
|
272
|
+
let text = ''
|
|
273
|
+
try {
|
|
274
|
+
text = readFileSync(file, 'utf8')
|
|
275
|
+
} catch {
|
|
276
|
+
continue
|
|
277
|
+
}
|
|
278
|
+
const summary = summarize(text)
|
|
279
|
+
const row = sessionSendRow(summary, sessionHashFor(file))
|
|
280
|
+
if (!row) continue
|
|
281
|
+
const root = repoRoot(summary.cwd)
|
|
282
|
+
if (root) {
|
|
283
|
+
row.repoHash = createHash('sha256').update(root).digest('hex').slice(0, 40)
|
|
284
|
+
if (REPO_LABELS) row.repoLabel = (root.split('/').pop() || root).slice(0, 80)
|
|
285
|
+
}
|
|
286
|
+
sessions.push(row)
|
|
287
|
+
}
|
|
244
288
|
}
|
|
245
289
|
|
|
246
290
|
// Collapse to one row per (provider, model, day) so the request stays small.
|
|
@@ -255,11 +299,10 @@ const HOURLY_URL = deriveHourlyUrl(INGEST_URL, process.env.MODELMETER_HOURLY_ING
|
|
|
255
299
|
// Per-tool / per-MCP attribution. Claude even-splits each turn's tokens across the
|
|
256
300
|
// tools it called; Codex contributes precomputed deltas. Calls are exact.
|
|
257
301
|
const toolsPayload = aggregateTools(events, codexToolDeltas, today)
|
|
258
|
-
const currentDetail = { hours: hourly, tools: toolsPayload }
|
|
259
|
-
const detailBatch = mergeDetailBatches(state.pendingDetail, currentDetail)
|
|
260
|
-
const hasDetail = detailBatch.hours.length > 0 || detailBatch.tools.length > 0
|
|
261
302
|
|
|
262
|
-
|
|
303
|
+
const haveDaily = payload.length > 0
|
|
304
|
+
const haveDetail = hourly.length > 0 || toolsPayload.length > 0 || sessions.length > 0
|
|
305
|
+
if (!haveDaily && !haveDetail) {
|
|
263
306
|
process.exit(0)
|
|
264
307
|
}
|
|
265
308
|
|
|
@@ -267,23 +310,34 @@ if (process.env.MODELMETER_DRYRUN) {
|
|
|
267
310
|
const tally = {}
|
|
268
311
|
for (const e of events) tally[e.provider] = (tally[e.provider] || 0) + 1
|
|
269
312
|
console.log(`DRY RUN: ${events.length} raw events -> ${payload.length} daily rows`, tally)
|
|
270
|
-
console.log(` + ${hourly.length} recent hourly rows, ${toolsPayload.length} tool rows -> ${HOURLY_URL}`)
|
|
271
|
-
if (state.pendingDetail?.hours?.length || state.pendingDetail?.tools?.length) {
|
|
272
|
-
console.log(
|
|
273
|
-
` + pending retry rows: ${state.pendingDetail.hours?.length || 0} hourly, ${state.pendingDetail.tools?.length || 0} tool`,
|
|
274
|
-
)
|
|
275
|
-
}
|
|
313
|
+
console.log(` + ${hourly.length} recent hourly rows, ${toolsPayload.length} tool rows, ${sessions.length} session summaries -> ${HOURLY_URL}`)
|
|
276
314
|
console.log(JSON.stringify(payload, null, 2))
|
|
277
315
|
if (toolsPayload.length) console.log('tools:', JSON.stringify(toolsPayload, null, 2))
|
|
316
|
+
if (sessions.length) console.log('sessions:', JSON.stringify(sessions.slice(0, 3), null, 2))
|
|
278
317
|
process.exit(0)
|
|
279
318
|
}
|
|
280
319
|
|
|
281
|
-
|
|
282
|
-
|
|
320
|
+
// With no new daily events there is nothing to commit, but session summaries (and any
|
|
321
|
+
// timestamped detail) are recomputed in full each run and must still be sent, so treat
|
|
322
|
+
// that case as already committed.
|
|
323
|
+
let committed = !haveDaily
|
|
324
|
+
if (haveDaily) {
|
|
283
325
|
try {
|
|
284
326
|
const res = await postJson(INGEST_URL, { source: 'collector', events: payload })
|
|
285
327
|
if (res.ok) {
|
|
286
|
-
|
|
328
|
+
if (!existsSync(MM_DIR)) mkdirSync(MM_DIR, { recursive: true })
|
|
329
|
+
const claudeIds = Object.keys(state.claude)
|
|
330
|
+
if (claudeIds.length > CLAUDE_STATE_CAP) {
|
|
331
|
+
const next = {}
|
|
332
|
+
for (const id of claudeIds.slice(-CLAUDE_STATE_CAP)) next[id] = 1
|
|
333
|
+
state.claude = next
|
|
334
|
+
}
|
|
335
|
+
writeFileSync(STATE_PATH, JSON.stringify(state))
|
|
336
|
+
try {
|
|
337
|
+
chmodSync(STATE_PATH, 0o600) // usage metadata is not secret, but keep it owner-only
|
|
338
|
+
} catch {
|
|
339
|
+
// best effort on platforms without POSIX perms
|
|
340
|
+
}
|
|
287
341
|
committed = true
|
|
288
342
|
console.error(`modelmeter: reported ${payload.length} usage rows`)
|
|
289
343
|
} else {
|
|
@@ -292,32 +346,24 @@ if (payload.length > 0) {
|
|
|
292
346
|
} catch (err) {
|
|
293
347
|
console.error(`modelmeter: ${err.message}`)
|
|
294
348
|
}
|
|
295
|
-
} else {
|
|
296
|
-
committed = true // retrying previously committed detail rows
|
|
297
349
|
}
|
|
298
350
|
|
|
299
|
-
//
|
|
300
|
-
//
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
state.pendingDetail = detailBatch
|
|
304
|
-
saveState()
|
|
305
|
-
console.error('modelmeter: detail ingest URL could not be derived; set MODELMETER_HOURLY_INGEST_URL')
|
|
306
|
-
process.exit(0)
|
|
307
|
-
}
|
|
308
|
-
let detailSent = false
|
|
351
|
+
// Detail rows go only after the daily batch is committed (or when there was none), so a
|
|
352
|
+
// daily retry cannot race them. The endpoint applies them in one transaction; retry once
|
|
353
|
+
// to cover a transient failure, then give up (the window self-heals as new data flows).
|
|
354
|
+
if (committed && haveDetail) {
|
|
309
355
|
for (let attempt = 0; attempt < 2; attempt++) {
|
|
310
356
|
try {
|
|
311
|
-
const res = await postJson(HOURLY_URL, {
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
357
|
+
const res = await postJson(HOURLY_URL, {
|
|
358
|
+
source: 'collector',
|
|
359
|
+
hours: hourly,
|
|
360
|
+
tools: toolsPayload,
|
|
361
|
+
sessions,
|
|
362
|
+
})
|
|
363
|
+
if (res.ok) break
|
|
316
364
|
} catch {
|
|
317
365
|
// fall through to one retry, then give up
|
|
318
366
|
}
|
|
319
367
|
}
|
|
320
|
-
state.pendingDetail = detailSent ? { hours: [], tools: [] } : detailBatch
|
|
321
|
-
saveState()
|
|
322
368
|
}
|
|
323
369
|
process.exit(0)
|
package/lib.mjs
CHANGED
|
@@ -42,6 +42,25 @@ export function findLastTokenCount(obj) {
|
|
|
42
42
|
return last
|
|
43
43
|
}
|
|
44
44
|
|
|
45
|
+
// Depth-first find of the first string `cwd` in a parsed line (Claude carries it at
|
|
46
|
+
// top level, Codex nests it under payload). Used only to derive a repo identity; the
|
|
47
|
+
// collector hashes it and never sends the raw path.
|
|
48
|
+
export function findCwd(obj) {
|
|
49
|
+
const stack = [obj]
|
|
50
|
+
let guard = 0
|
|
51
|
+
while (stack.length && guard < 5000) {
|
|
52
|
+
guard++
|
|
53
|
+
const d = stack.pop()
|
|
54
|
+
if (Array.isArray(d)) {
|
|
55
|
+
for (const v of d) stack.push(v)
|
|
56
|
+
} else if (d && typeof d === 'object') {
|
|
57
|
+
if (typeof d.cwd === 'string' && d.cwd) return d.cwd
|
|
58
|
+
for (const v of Object.values(d)) stack.push(v)
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
return ''
|
|
62
|
+
}
|
|
63
|
+
|
|
45
64
|
// Detail endpoint from the ingest URL, tolerating a trailing slash, or an override.
|
|
46
65
|
export function deriveHourlyUrl(ingestUrl, override) {
|
|
47
66
|
if (override) return override
|
|
@@ -294,6 +313,9 @@ export function claudeSessionSummary(text) {
|
|
|
294
313
|
let cacheCreate = 0
|
|
295
314
|
let output = 0
|
|
296
315
|
let requests = 0
|
|
316
|
+
let firstTs = ''
|
|
317
|
+
let lastTs = ''
|
|
318
|
+
let cwd = ''
|
|
297
319
|
const contextSeq = []
|
|
298
320
|
const tools = {}
|
|
299
321
|
for (const line of String(text).split('\n')) {
|
|
@@ -304,9 +326,14 @@ export function claudeSessionSummary(text) {
|
|
|
304
326
|
} catch {
|
|
305
327
|
continue
|
|
306
328
|
}
|
|
329
|
+
if (!cwd) cwd = findCwd(o)
|
|
307
330
|
const ev = claudeEventFromLine(o)
|
|
308
331
|
if (!ev) continue
|
|
309
332
|
model = ev.model || model
|
|
333
|
+
if (ev.occurredAt) {
|
|
334
|
+
if (!firstTs || ev.occurredAt < firstTs) firstTs = ev.occurredAt
|
|
335
|
+
if (ev.occurredAt > lastTs) lastTs = ev.occurredAt
|
|
336
|
+
}
|
|
310
337
|
uncached += ev.uncachedInputTokens
|
|
311
338
|
cacheRead += ev.cacheReadInputTokens
|
|
312
339
|
cacheCreate += ev.cacheCreationInputTokens
|
|
@@ -329,7 +356,9 @@ export function claudeSessionSummary(text) {
|
|
|
329
356
|
}
|
|
330
357
|
}
|
|
331
358
|
if (requests === 0) return null
|
|
332
|
-
|
|
359
|
+
// Claude folds extended-thinking into output_tokens with no separate field, so
|
|
360
|
+
// reasoning is 0 here (the breakdown is only available for Codex).
|
|
361
|
+
return { provider: 'anthropic', model, uncached, cacheRead, cacheCreate, output, reasoning: 0, requests, firstTs, lastTs, cwd, contextSeq, tools }
|
|
333
362
|
}
|
|
334
363
|
|
|
335
364
|
// Summarize one Codex session from its final cumulative token_count. Codex totals are
|
|
@@ -338,6 +367,9 @@ export function codexSessionSummary(text) {
|
|
|
338
367
|
let model = 'gpt-5'
|
|
339
368
|
let totals = null
|
|
340
369
|
let requests = 0
|
|
370
|
+
let firstTs = ''
|
|
371
|
+
let lastTs = ''
|
|
372
|
+
let cwd = ''
|
|
341
373
|
const tools = {}
|
|
342
374
|
for (const line of String(text).split('\n')) {
|
|
343
375
|
if (!line.trim()) continue
|
|
@@ -347,10 +379,15 @@ export function codexSessionSummary(text) {
|
|
|
347
379
|
} catch {
|
|
348
380
|
continue
|
|
349
381
|
}
|
|
382
|
+
if (!cwd) cwd = findCwd(o)
|
|
350
383
|
const p = o.payload || o
|
|
351
384
|
const ptype = p.type || o.type
|
|
352
385
|
if (typeof o.model === 'string') model = o.model
|
|
353
386
|
else if (typeof p.model === 'string') model = p.model
|
|
387
|
+
if (typeof o.timestamp === 'string') {
|
|
388
|
+
if (!firstTs || o.timestamp < firstTs) firstTs = o.timestamp
|
|
389
|
+
if (o.timestamp > lastTs) lastTs = o.timestamp
|
|
390
|
+
}
|
|
354
391
|
const tc = findLastTokenCount(o)
|
|
355
392
|
if (tc) {
|
|
356
393
|
totals = tc
|
|
@@ -366,7 +403,11 @@ export function codexSessionSummary(text) {
|
|
|
366
403
|
if (!totals) return null
|
|
367
404
|
const cacheRead = totals.cached_input_tokens || 0
|
|
368
405
|
const uncached = Math.max(0, (totals.input_tokens || 0) - cacheRead)
|
|
369
|
-
|
|
406
|
+
// Reasoning tokens bill as output (the priciest tier) but are reported separately and
|
|
407
|
+
// are otherwise hidden, so keep the breakdown: output is the total billed output and
|
|
408
|
+
// reasoning is the share of it spent thinking.
|
|
409
|
+
const reasoning = totals.reasoning_output_tokens || 0
|
|
410
|
+
const output = (totals.output_tokens || 0) + reasoning
|
|
370
411
|
return {
|
|
371
412
|
provider: 'openai',
|
|
372
413
|
model,
|
|
@@ -374,12 +415,45 @@ export function codexSessionSummary(text) {
|
|
|
374
415
|
cacheRead,
|
|
375
416
|
cacheCreate: 0,
|
|
376
417
|
output,
|
|
418
|
+
reasoning,
|
|
377
419
|
requests: Math.max(1, requests),
|
|
420
|
+
firstTs,
|
|
421
|
+
lastTs,
|
|
422
|
+
cwd,
|
|
378
423
|
contextSeq: [],
|
|
379
424
|
tools,
|
|
380
425
|
}
|
|
381
426
|
}
|
|
382
427
|
|
|
428
|
+
// Shape a session summary into the row the collector sends to ingest-hourly. Derives
|
|
429
|
+
// the bloat metrics from the context sequence and the bucket date from the timestamps.
|
|
430
|
+
// Returns null if there is no date to bucket on. The hash is supplied by the caller
|
|
431
|
+
// (the collector hashes the session-file basename, never the path).
|
|
432
|
+
export function sessionSendRow(summary, sessionHash) {
|
|
433
|
+
if (!summary || !sessionHash) return null
|
|
434
|
+
const bucketDate = (summary.lastTs || summary.firstTs || '').slice(0, 10)
|
|
435
|
+
if (!bucketDate) return null
|
|
436
|
+
const bloat = sessionBloat(summary.contextSeq)
|
|
437
|
+
return {
|
|
438
|
+
sessionHash,
|
|
439
|
+
provider: summary.provider,
|
|
440
|
+
model: summary.model,
|
|
441
|
+
bucketDate,
|
|
442
|
+
firstTs: summary.firstTs || '',
|
|
443
|
+
lastTs: summary.lastTs || '',
|
|
444
|
+
requests: summary.requests,
|
|
445
|
+
uncached: summary.uncached,
|
|
446
|
+
cacheRead: summary.cacheRead,
|
|
447
|
+
cacheCreation: summary.cacheCreate,
|
|
448
|
+
output: summary.output,
|
|
449
|
+
reasoning: summary.reasoning || 0,
|
|
450
|
+
maxInputTurn: bloat ? bloat.max : 0,
|
|
451
|
+
first5Avg: bloat ? bloat.first5 : 0,
|
|
452
|
+
last5Avg: bloat ? bloat.last5 : 0,
|
|
453
|
+
toolCounts: summary.tools,
|
|
454
|
+
}
|
|
455
|
+
}
|
|
456
|
+
|
|
383
457
|
// First-5 vs last-5 average turn size + max, for context-bloat detection.
|
|
384
458
|
export function sessionBloat(seq) {
|
|
385
459
|
if (!Array.isArray(seq) || seq.length < 10) return null
|
|
@@ -398,6 +472,7 @@ export function buildLocalRecommendations(summaries) {
|
|
|
398
472
|
let cacheRead = 0
|
|
399
473
|
let cacheCreate = 0
|
|
400
474
|
let output = 0
|
|
475
|
+
let reasoning = 0
|
|
401
476
|
let requests = 0
|
|
402
477
|
const tools = {}
|
|
403
478
|
for (const s of list) {
|
|
@@ -405,6 +480,7 @@ export function buildLocalRecommendations(summaries) {
|
|
|
405
480
|
cacheRead += s.cacheRead
|
|
406
481
|
cacheCreate += s.cacheCreate
|
|
407
482
|
output += s.output
|
|
483
|
+
reasoning += s.reasoning || 0
|
|
408
484
|
requests += s.requests
|
|
409
485
|
for (const [g, v] of Object.entries(s.tools || {})) {
|
|
410
486
|
const cur = tools[g] || { tokens: 0, calls: 0 }
|
|
@@ -481,7 +557,19 @@ export function buildLocalRecommendations(summaries) {
|
|
|
481
557
|
})
|
|
482
558
|
}
|
|
483
559
|
|
|
484
|
-
// 4.
|
|
560
|
+
// 4. Reasoning share (Codex). Reasoning bills as output, the priciest tier, but is
|
|
561
|
+
// hidden by default, so a large reasoning share is invisible cost.
|
|
562
|
+
if (output > 100_000 && reasoning / output > 0.4) {
|
|
563
|
+
recs.push({
|
|
564
|
+
kind: 'reasoning',
|
|
565
|
+
level: 'warn',
|
|
566
|
+
text: `Reasoning is ${pct(
|
|
567
|
+
reasoning / output,
|
|
568
|
+
)} of your output tokens, the priciest tier, and hidden by default. Lower the reasoning effort for routine tasks and reserve high effort for genuinely hard problems.`,
|
|
569
|
+
})
|
|
570
|
+
}
|
|
571
|
+
|
|
572
|
+
// 5. Context bloat (Claude sessions carry a per-turn sequence).
|
|
485
573
|
let worst = null
|
|
486
574
|
for (const s of list) {
|
|
487
575
|
const b = sessionBloat(s.contextSeq)
|
package/package.json
CHANGED
|
@@ -1,19 +1,15 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "modelmeter-collect",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.9.0",
|
|
4
4
|
"description": "Report LLM token usage from local Claude Code / Codex logs to ModelMeter. Token counts only, never prompts or keys.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
7
7
|
"modelmeter-collect": "cli.mjs"
|
|
8
8
|
},
|
|
9
|
-
"scripts": {
|
|
10
|
-
"test": "node --test lib.test.mjs"
|
|
11
|
-
},
|
|
12
9
|
"files": [
|
|
13
10
|
"cli.mjs",
|
|
14
11
|
"collect.mjs",
|
|
15
12
|
"lib.mjs",
|
|
16
|
-
"lib.test.mjs",
|
|
17
13
|
"README.md"
|
|
18
14
|
],
|
|
19
15
|
"engines": {
|
package/lib.test.mjs
DELETED
|
@@ -1,173 +0,0 @@
|
|
|
1
|
-
import test from 'node:test'
|
|
2
|
-
import assert from 'node:assert/strict'
|
|
3
|
-
|
|
4
|
-
import {
|
|
5
|
-
aggregateDaily,
|
|
6
|
-
aggregateHourly,
|
|
7
|
-
aggregateTools,
|
|
8
|
-
claudeEventFromLine,
|
|
9
|
-
codexDelta,
|
|
10
|
-
deriveHourlyUrl,
|
|
11
|
-
findLastTokenCount,
|
|
12
|
-
formatDoctorReport,
|
|
13
|
-
mergeDetailBatches,
|
|
14
|
-
pruneClaudeState,
|
|
15
|
-
} from './lib.mjs'
|
|
16
|
-
|
|
17
|
-
test('codexDelta leaves post-token-count tool calls for the next batch', () => {
|
|
18
|
-
const toolCalls = [
|
|
19
|
-
{ ts: '2026-06-18T12:01:00.000Z', group: 'exec_command' },
|
|
20
|
-
{ ts: '2026-06-18T12:03:00.000Z', group: 'apply_patch' },
|
|
21
|
-
]
|
|
22
|
-
const first = codexDelta(
|
|
23
|
-
{
|
|
24
|
-
totals: { input_tokens: 100, cached_input_tokens: 0, output_tokens: 0, reasoning_output_tokens: 0 },
|
|
25
|
-
totalsTs: '2026-06-18T12:02:00.000Z',
|
|
26
|
-
maxTs: '2026-06-18T12:03:00.000Z',
|
|
27
|
-
model: 'gpt-5',
|
|
28
|
-
fileDate: '2026-06-18',
|
|
29
|
-
toolCalls,
|
|
30
|
-
},
|
|
31
|
-
null,
|
|
32
|
-
)
|
|
33
|
-
assert.deepEqual(
|
|
34
|
-
first.toolDeltas.map((d) => d.tool),
|
|
35
|
-
['exec_command'],
|
|
36
|
-
)
|
|
37
|
-
assert.equal(first.state.lastToolTs, '2026-06-18T12:02:00.000Z')
|
|
38
|
-
|
|
39
|
-
const second = codexDelta(
|
|
40
|
-
{
|
|
41
|
-
totals: { input_tokens: 200, cached_input_tokens: 0, output_tokens: 0, reasoning_output_tokens: 0 },
|
|
42
|
-
totalsTs: '2026-06-18T12:04:00.000Z',
|
|
43
|
-
maxTs: '2026-06-18T12:04:00.000Z',
|
|
44
|
-
model: 'gpt-5',
|
|
45
|
-
fileDate: '2026-06-18',
|
|
46
|
-
toolCalls,
|
|
47
|
-
},
|
|
48
|
-
first.state,
|
|
49
|
-
)
|
|
50
|
-
assert.deepEqual(
|
|
51
|
-
second.toolDeltas.map((d) => d.tool),
|
|
52
|
-
['apply_patch'],
|
|
53
|
-
)
|
|
54
|
-
})
|
|
55
|
-
|
|
56
|
-
test('pruneClaudeState keeps newest timestamped ids and normalizes legacy entries', () => {
|
|
57
|
-
const pruned = pruneClaudeState(
|
|
58
|
-
{
|
|
59
|
-
old: { ts: '2026-06-18T10:00:00.000Z' },
|
|
60
|
-
legacy: 1,
|
|
61
|
-
newest: { ts: '2026-06-18T12:00:00.000Z' },
|
|
62
|
-
middle: { ts: '2026-06-18T11:00:00.000Z' },
|
|
63
|
-
},
|
|
64
|
-
2,
|
|
65
|
-
)
|
|
66
|
-
assert.deepEqual(Object.keys(pruned), ['middle', 'newest'])
|
|
67
|
-
assert.deepEqual(pruned.middle, { ts: '2026-06-18T11:00:00.000Z' })
|
|
68
|
-
})
|
|
69
|
-
|
|
70
|
-
test('deriveHourlyUrl fails closed for nonstandard ingest URLs unless overridden', () => {
|
|
71
|
-
assert.equal(
|
|
72
|
-
deriveHourlyUrl('https://x.test/functions/v1/ingest/'),
|
|
73
|
-
'https://x.test/functions/v1/ingest-hourly',
|
|
74
|
-
)
|
|
75
|
-
assert.equal(deriveHourlyUrl('https://x.test/custom'), null)
|
|
76
|
-
assert.equal(deriveHourlyUrl('https://x.test/custom', 'https://x.test/detail'), 'https://x.test/detail')
|
|
77
|
-
})
|
|
78
|
-
|
|
79
|
-
test('mergeDetailBatches deduplicates additive rows by bucket', () => {
|
|
80
|
-
const merged = mergeDetailBatches(
|
|
81
|
-
{
|
|
82
|
-
hours: [
|
|
83
|
-
{
|
|
84
|
-
provider: 'openai',
|
|
85
|
-
model: 'gpt-5',
|
|
86
|
-
bucketHour: '2026-06-18T12:00:00.000Z',
|
|
87
|
-
uncachedInputTokens: 1,
|
|
88
|
-
cacheReadInputTokens: 2,
|
|
89
|
-
cacheCreationInputTokens: 3,
|
|
90
|
-
outputTokens: 4,
|
|
91
|
-
numRequests: 1,
|
|
92
|
-
},
|
|
93
|
-
],
|
|
94
|
-
tools: [{ tool: 'exec_command', bucketDate: '2026-06-18', calls: 1, tokens: 10 }],
|
|
95
|
-
},
|
|
96
|
-
{
|
|
97
|
-
hours: [
|
|
98
|
-
{
|
|
99
|
-
provider: 'openai',
|
|
100
|
-
model: 'gpt-5',
|
|
101
|
-
bucketHour: '2026-06-18T12:00:00.000Z',
|
|
102
|
-
uncachedInputTokens: 5,
|
|
103
|
-
cacheReadInputTokens: 0,
|
|
104
|
-
cacheCreationInputTokens: 0,
|
|
105
|
-
outputTokens: 6,
|
|
106
|
-
numRequests: 1,
|
|
107
|
-
},
|
|
108
|
-
],
|
|
109
|
-
tools: [{ tool: 'exec_command', bucketDate: '2026-06-18', calls: 2, tokens: 20 }],
|
|
110
|
-
},
|
|
111
|
-
)
|
|
112
|
-
assert.equal(merged.hours.length, 1)
|
|
113
|
-
assert.equal(merged.hours[0].uncachedInputTokens, 6)
|
|
114
|
-
assert.equal(merged.hours[0].outputTokens, 10)
|
|
115
|
-
assert.equal(merged.hours[0].numRequests, 2)
|
|
116
|
-
assert.deepEqual(merged.tools, [{ tool: 'exec_command', bucketDate: '2026-06-18', calls: 3, tokens: 30 }])
|
|
117
|
-
})
|
|
118
|
-
|
|
119
|
-
test('claude parsing and aggregations exclude prompt text', () => {
|
|
120
|
-
const event = claudeEventFromLine({
|
|
121
|
-
uuid: 'c1',
|
|
122
|
-
timestamp: '2026-06-18T12:34:56.000Z',
|
|
123
|
-
message: {
|
|
124
|
-
role: 'assistant',
|
|
125
|
-
model: 'claude-sonnet',
|
|
126
|
-
usage: {
|
|
127
|
-
input_tokens: 100,
|
|
128
|
-
cache_read_input_tokens: 20,
|
|
129
|
-
cache_creation_input_tokens: 5,
|
|
130
|
-
output_tokens: 30,
|
|
131
|
-
},
|
|
132
|
-
content: [
|
|
133
|
-
{ type: 'text', text: 'response text must not be copied' },
|
|
134
|
-
{ type: 'tool_use', name: 'mcp__supabase__query' },
|
|
135
|
-
{ type: 'tool_use', name: 'exec_command' },
|
|
136
|
-
],
|
|
137
|
-
},
|
|
138
|
-
})
|
|
139
|
-
assert.equal(event.id, 'c1')
|
|
140
|
-
assert.equal(event.provider, 'anthropic')
|
|
141
|
-
assert.deepEqual(event.tools, ['mcp__supabase__query', 'exec_command'])
|
|
142
|
-
assert.equal(JSON.stringify(event).includes('response text'), false)
|
|
143
|
-
|
|
144
|
-
assert.equal(aggregateDaily([event], '2026-06-18')[0].uncachedInputTokens, 100)
|
|
145
|
-
assert.equal(aggregateHourly([event], new Date('2026-06-18T13:00:00.000Z').getTime()).length, 1)
|
|
146
|
-
assert.deepEqual(
|
|
147
|
-
aggregateTools([event], [], '2026-06-18').map((d) => d.tool).sort(),
|
|
148
|
-
['exec_command', 'mcp:supabase'],
|
|
149
|
-
)
|
|
150
|
-
})
|
|
151
|
-
|
|
152
|
-
test('findLastTokenCount and doctor report cover nested Codex usage and privacy copy', () => {
|
|
153
|
-
const totals = findLastTokenCount({
|
|
154
|
-
payload: [
|
|
155
|
-
{ type: 'token_count', info: { total_token_usage: { input_tokens: 1 } } },
|
|
156
|
-
{ nested: { type: 'token_count', info: { total_token_usage: { input_tokens: 2 } } } },
|
|
157
|
-
],
|
|
158
|
-
})
|
|
159
|
-
assert.deepEqual(totals, { input_tokens: 2 })
|
|
160
|
-
|
|
161
|
-
const report = formatDoctorReport({
|
|
162
|
-
configPath: '/tmp/config.json',
|
|
163
|
-
configFound: true,
|
|
164
|
-
token: 'mm_live_abcdefghijklmnop',
|
|
165
|
-
ingestUrl: 'https://x.test/functions/v1/ingest',
|
|
166
|
-
lookbackDays: 14,
|
|
167
|
-
nowMs: 1000,
|
|
168
|
-
claude: { dir: '/tmp/claude', found: false },
|
|
169
|
-
codex: { dir: '/tmp/codex', found: true, recentCount: 1, lastWriteMs: 1000 },
|
|
170
|
-
})
|
|
171
|
-
assert.match(report, /mm_live_abcd\.\.\./)
|
|
172
|
-
assert.match(report, /never sent: prompts/)
|
|
173
|
-
})
|