modelmeter-collect 0.1.0 → 0.4.0

package/collect.mjs

@@ -86,6 +86,9 @@ function recentFiles(dir, limit = Infinity) {
 }
 
 const events = []
+// Codex tool attribution is computed per session (its token accounting is periodic,
+// not per-turn), so it is collected here and folded into the tool aggregation later.
+const codexToolDeltas = []
 
 // --- Claude Code: assistant turns carry message.usage; dedup by message uuid.
 function scanClaude(files) {
@@ -110,10 +113,15 @@ function scanClaude(files) {
       if (!id || state.claude[id]) continue
       state.claude[id] = 1
       const u = msg.usage
+      const toolNames = Array.isArray(msg.content)
+        ? msg.content.filter((b) => b && b.type === 'tool_use').map((b) => b.name).filter(Boolean)
+        : []
       events.push({
         provider: 'anthropic',
         model: msg.model || 'claude-unknown',
         occurredOn: (o.timestamp || '').slice(0, 10) || undefined,
+        occurredAt: o.timestamp || undefined,
+        tools: toolNames,
         uncachedInputTokens: u.input_tokens || 0,
         cacheReadInputTokens: u.cache_read_input_tokens || 0,
         cacheCreationInputTokens: u.cache_creation_input_tokens || 0,
@@ -138,14 +146,30 @@ function findLastTokenCount(obj) {
   }
   return last
 }
+// Codex tool names are plain (exec_command, apply_patch, ...); MCP calls carry an
+// invocation with a server name. Returns a group key, or null if not a tool call.
+function codexToolFromEvent(payload, ptype) {
+  if (ptype === 'function_call' || ptype === 'custom_tool_call') {
+    return typeof payload.name === 'string' && payload.name ? payload.name : null
+  }
+  if (ptype === 'mcp_tool_call_end' || ptype === 'mcp_tool_call_begin') {
+    const inv = payload.invocation || {}
+    const server = inv.server || inv.server_name
+    return server ? `mcp:${server}` : 'mcp'
+  }
+  return null
+}
+
 function scanCodex(files) {
   for (const file of files) {
     const m = file.match(/rollout-(\d{4}-\d{2}-\d{2})T[\d-]+-([0-9a-f-]+)\.jsonl$/)
     if (!m) continue
-    const date = m[1]
     const sessionId = m[2]
     let totals = null
+    let totalsTs = null // timestamp of the latest token_count event, for the 5-hour window
+    let maxTs = ''
     let model = 'gpt-5'
+    const toolCalls = [] // { ts, group }
     let text = ''
     try {
       text = readFileSync(file, 'utf8')
@@ -160,34 +184,63 @@ function scanCodex(files) {
       } catch {
         continue
       }
-      const t = findLastTokenCount(o)
-      if (t) totals = t
+      const ts = typeof o.timestamp === 'string' ? o.timestamp : null
+      if (ts && ts > maxTs) maxTs = ts
+      const p = o.payload || o
+      const ptype = p.type || o.type
       if (typeof o.model === 'string') model = o.model
-      else if (typeof o.payload?.model === 'string') model = o.payload.model
+      else if (typeof p.model === 'string') model = p.model
+      const tc = findLastTokenCount(o)
+      if (tc) {
+        totals = tc
+        if (ts) totalsTs = ts
+      }
+      const g = codexToolFromEvent(p, ptype)
+      if (g) toolCalls.push({ ts: ts || totalsTs || '', group: g })
     }
     if (!totals) continue
-    const prev = state.codex[sessionId] || {
-      input_tokens: 0,
-      cached_input_tokens: 0,
-      output_tokens: 0,
-      reasoning_output_tokens: 0,
-    }
-    const dInput = Math.max(0, (totals.input_tokens || 0) - prev.input_tokens)
-    const dCached = Math.max(0, (totals.cached_input_tokens || 0) - prev.cached_input_tokens)
-    const dOut = Math.max(0, (totals.output_tokens || 0) - prev.output_tokens)
-    const dReason = Math.max(0, (totals.reasoning_output_tokens || 0) - prev.reasoning_output_tokens)
+    const stored = state.codex[sessionId]
+    const prevTotal =
+      (stored && stored.total) ||
+      (stored && stored.input_tokens != null ? stored : null) || {
+        input_tokens: 0,
+        cached_input_tokens: 0,
+        output_tokens: 0,
+        reasoning_output_tokens: 0,
+      }
+    const prevLastTs = (stored && stored.lastTs) || ''
+    const dInput = Math.max(0, (totals.input_tokens || 0) - prevTotal.input_tokens)
+    const dCached = Math.max(0, (totals.cached_input_tokens || 0) - prevTotal.cached_input_tokens)
+    const dOut = Math.max(0, (totals.output_tokens || 0) - prevTotal.output_tokens)
+    const dReason = Math.max(0, (totals.reasoning_output_tokens || 0) - prevTotal.reasoning_output_tokens)
+    const newTotalTokens = Math.max(0, dInput - dCached) + dCached + (dOut + dReason)
     if (dInput + dCached + dOut + dReason > 0) {
+      const occurredOn = (totalsTs || '').slice(0, 10) || m[1]
       events.push({
         provider: 'openai',
         model,
-        occurredOn: date,
+        occurredOn,
+        occurredAt: totalsTs || undefined, // enables the 5-hour window for Codex
         uncachedInputTokens: Math.max(0, dInput - dCached),
         cacheReadInputTokens: dCached,
         cacheCreationInputTokens: 0,
         outputTokens: dOut + dReason, // reasoning tokens bill as output
         numRequests: 1,
       })
-      state.codex[sessionId] = totals
+      // Tool calls new since the last run; even-split this run's new tokens across them.
+      const newCalls = toolCalls.filter((c) => c.ts && c.ts > prevLastTs)
+      if (newCalls.length > 0 && newTotalTokens > 0) {
+        const share = Math.round(newTotalTokens / newCalls.length)
+        for (const c of newCalls) {
+          codexToolDeltas.push({
+            tool: c.group,
+            bucketDate: (c.ts || '').slice(0, 10) || occurredOn,
+            calls: 1,
+            tokens: share,
+          })
+        }
+      }
+      state.codex[sessionId] = { total: totals, lastTs: maxTs }
     }
   }
 }
@@ -225,6 +278,84 @@ for (const e of events) {
 }
 const payload = [...byKey.values()]
 
+// Recent hourly buckets feed the 5-hour rolling window via a separate, additive
+// endpoint. Only events with a real timestamp in the last 8 hours qualify, so a
+// backfill scan never pollutes the recent window. (Codex deltas lack per-event
+// timestamps, so the 5-hour window is Claude Code for now.)
+const HOUR_MS = 3_600_000
+const recentCutoff = Date.now() - 8 * HOUR_MS
+const byHour = new Map()
+for (const e of events) {
+  if (!e.occurredAt) continue
+  const t = new Date(e.occurredAt).getTime()
+  if (Number.isNaN(t) || t < recentCutoff) continue
+  const d = new Date(t)
+  d.setMinutes(0, 0, 0)
+  const hourIso = d.toISOString()
+  const key = `${e.provider}|${e.model}|${hourIso}`
+  const cur = byHour.get(key) || {
+    provider: e.provider,
+    model: e.model,
+    bucketHour: hourIso,
+    uncachedInputTokens: 0,
+    cacheReadInputTokens: 0,
+    cacheCreationInputTokens: 0,
+    outputTokens: 0,
+    numRequests: 0,
+  }
+  cur.uncachedInputTokens += e.uncachedInputTokens || 0
+  cur.cacheReadInputTokens += e.cacheReadInputTokens || 0
+  cur.cacheCreationInputTokens += e.cacheCreationInputTokens || 0
+  cur.outputTokens += e.outputTokens || 0
+  cur.numRequests += e.numRequests || 1
+  byHour.set(key, cur)
+}
+const hourly = [...byHour.values()]
+const HOURLY_URL = INGEST_URL.replace(/\/ingest$/, '/ingest-hourly')
+
+// Per-tool / per-MCP attribution. Group MCP tools by server (mcp__server__tool ->
+// mcp:server) and keep built-ins by name. Calls are exact; tokens are an even
+// split of each turn's usage across the distinct tool groups it called.
+function toolGroup(name) {
+  if (typeof name !== 'string' || !name) return 'unknown'
+  if (name.startsWith('mcp__')) {
+    const parts = name.split('__')
+    return parts[1] ? `mcp:${parts[1]}` : 'mcp:unknown'
+  }
+  return name
+}
+const byTool = new Map()
+for (const e of events) {
+  if (!Array.isArray(e.tools) || e.tools.length === 0) continue
+  const date = e.occurredOn || new Date().toISOString().slice(0, 10)
+  const callsByGroup = new Map()
+  for (const name of e.tools) {
+    const g = toolGroup(name)
+    callsByGroup.set(g, (callsByGroup.get(g) || 0) + 1)
+  }
+  const eventTokens =
+    (e.uncachedInputTokens || 0) +
+    (e.cacheReadInputTokens || 0) +
+    (e.cacheCreationInputTokens || 0) +
+    (e.outputTokens || 0)
+  const tokenShare = Math.round(eventTokens / callsByGroup.size)
+  for (const [g, calls] of callsByGroup) {
+    const key = `${g}|${date}`
+    const cur = byTool.get(key) || { tool: g, bucketDate: date, calls: 0, tokens: 0 }
+    cur.calls += calls
+    cur.tokens += tokenShare
+    byTool.set(key, cur)
+  }
+}
+for (const d of codexToolDeltas) {
+  const key = `${d.tool}|${d.bucketDate}`
+  const cur = byTool.get(key) || { tool: d.tool, bucketDate: d.bucketDate, calls: 0, tokens: 0 }
+  cur.calls += d.calls
+  cur.tokens += d.tokens
+  byTool.set(key, cur)
+}
+const toolsPayload = [...byTool.values()]
+
 if (payload.length === 0) {
   process.exit(0)
 }
@@ -233,10 +364,13 @@ if (process.env.MODELMETER_DRYRUN) {
   const tally = {}
   for (const e of events) tally[e.provider] = (tally[e.provider] || 0) + 1
   console.log(`DRY RUN: ${events.length} raw events -> ${payload.length} daily rows`, tally)
+  console.log(`         + ${hourly.length} recent hourly rows, ${toolsPayload.length} tool rows -> ${HOURLY_URL}`)
   console.log(JSON.stringify(payload, null, 2))
+  if (toolsPayload.length) console.log('tools:', JSON.stringify(toolsPayload, null, 2))
   process.exit(0)
 }
 
+let committed = false
 try {
   const res = await fetch(INGEST_URL, {
     method: 'POST',
@@ -246,6 +380,7 @@ try {
   if (res.ok) {
     if (!existsSync(MM_DIR)) mkdirSync(MM_DIR, { recursive: true })
     writeFileSync(STATE_PATH, JSON.stringify(state))
+    committed = true
     console.error(`modelmeter: reported ${payload.length} usage rows`)
   } else {
     console.error(`modelmeter: ingest returned ${res.status}`)
@@ -253,4 +388,18 @@ try {
 } catch (err) {
   console.error(`modelmeter: ${err.message}`)
 }
+
+// Additive + best-effort: only after the daily batch is committed (state written),
+// so a retry cannot double-count into the hourly window.
+if (committed && (hourly.length > 0 || toolsPayload.length > 0)) {
+  try {
+    await fetch(HOURLY_URL, {
+      method: 'POST',
+      headers: { Authorization: `Bearer ${TOKEN}`, 'Content-Type': 'application/json' },
+      body: JSON.stringify({ source: 'collector', hours: hourly, tools: toolsPayload }),
+    })
+  } catch {
+    // detail (hourly + per-tool) is best-effort; never block the collector on it
+  }
+}
 process.exit(0)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "modelmeter-collect",
-  "version": "0.1.0",
+  "version": "0.4.0",
   "description": "Report LLM token usage from local Claude Code / Codex logs to ModelMeter. Token counts only, never prompts or keys.",
   "type": "module",
   "bin": {