modelmeter-collect 0.8.0 → 0.9.0

package/collect.mjs

@@ -19,7 +19,8 @@ import {
   chmodSync,
 } from 'node:fs'
 import { homedir } from 'node:os'
-import { join } from 'node:path'
+import { join, dirname } from 'node:path'
+import { createHash } from 'node:crypto'
 import {
   findLastTokenCount,
   codexToolFromEvent,
@@ -29,8 +30,9 @@ import {
   aggregateDaily,
   aggregateHourly,
   aggregateTools,
-  pruneClaudeState,
-  mergeDetailBatches,
+  claudeSessionSummary,
+  codexSessionSummary,
+  sessionSendRow,
 } from './lib.mjs'
 
 const HOME = homedir()
@@ -64,6 +66,11 @@ const TOKEN = process.env.MODELMETER_TOKEN || cfg.token
 const INGEST_URL = process.env.MODELMETER_INGEST_URL || cfg.ingestUrl
 if (!TOKEN || !INGEST_URL) process.exit(0) // not configured: do nothing, never block
 
+// Repo attribution is opt-in for the human-readable label only. A repo hash is always
+// sent (opaque). The label (the repo folder name, never the full path) is sent only
+// when explicitly enabled, so a repo name never leaves the machine by default.
+const REPO_LABELS = process.env.MODELMETER_REPO_LABELS === '1' || cfg.repoLabels === true
+
 // POST JSON with a hard timeout so a stuck network path can never hang a Stop
 // hook or pile up scheduled collectors. Callers handle the thrown abort/error.
 async function postJson(url, body) {
@@ -81,29 +88,13 @@ async function postJson(url, body) {
   }
 }
 
-let state = { claude: {}, codex: {}, pendingDetail: { hours: [], tools: [] } }
+let state = { claude: {}, codex: {} }
 try {
-  state = {
-    claude: {},
-    codex: {},
-    pendingDetail: { hours: [], tools: [] },
-    ...JSON.parse(readFileSync(STATE_PATH, 'utf8')),
-  }
+  state = { claude: {}, codex: {}, ...JSON.parse(readFileSync(STATE_PATH, 'utf8')) }
 } catch {
   // first run
 }
 
-function saveState() {
-  if (!existsSync(MM_DIR)) mkdirSync(MM_DIR, { recursive: true })
-  state.claude = pruneClaudeState(state.claude, CLAUDE_STATE_CAP)
-  writeFileSync(STATE_PATH, JSON.stringify(state))
-  try {
-    chmodSync(STATE_PATH, 0o600) // usage metadata is not secret, but keep it owner-only
-  } catch {
-    // best effort on platforms without POSIX perms
-  }
-}
-
 // --- invocation detection -------------------------------------------------
 let hookInput = null
 if (!process.stdin.isTTY) {
@@ -177,7 +168,7 @@ function scanClaude(files) {
       }
       const ev = claudeEventFromLine(o)
       if (!ev || !ev.id || state.claude[ev.id]) continue
-      state.claude[ev.id] = { ts: ev.occurredAt || ev.occurredOn || '' }
+      state.claude[ev.id] = 1
       events.push(ev)
     }
   }
@@ -234,13 +225,66 @@ function scanCodex(files) {
   }
 }
 
+let claudeFiles = []
+let codexFiles = []
 if (hookInput?.transcript_path) {
-  scanClaude([hookInput.transcript_path]) // Claude Code hook: just this session
+  claudeFiles = [hookInput.transcript_path] // Claude Code hook: just this session
 } else if (codexNotify) {
-  scanCodex(recentFiles(join(HOME, '.codex', 'sessions'), 2)) // Codex notify: newest session(s)
+  codexFiles = recentFiles(join(HOME, '.codex', 'sessions'), 2) // Codex notify: newest session(s)
 } else {
-  scanClaude(recentFiles(join(HOME, '.claude', 'projects')))
-  scanCodex(recentFiles(join(HOME, '.codex', 'sessions')))
+  claudeFiles = recentFiles(join(HOME, '.claude', 'projects'))
+  codexFiles = recentFiles(join(HOME, '.codex', 'sessions'))
+}
+scanClaude(claudeFiles)
+scanCodex(codexFiles)
+
+// Per-session summaries (context bloat, comparisons) for the detail endpoint. Hash
+// the session-file basename (the session id), never the path, so no repo/project
+// label leaks here. Recomputed in full each run; the backend overwrites by hash.
+const SESSION_CAP = 500
+function sessionHashFor(file) {
+  const base = file.split('/').pop() || file
+  return createHash('sha256').update(base).digest('hex').slice(0, 40)
+}
+// Resolve a cwd to its git-root for a stable repo identity (so subdirs of one repo do
+// not fragment). Falls back to the cwd if no .git is found (e.g. the repo is gone).
+function repoRoot(cwd) {
+  if (!cwd) return ''
+  let dir = cwd
+  for (let i = 0; i < 12; i++) {
+    try {
+      if (existsSync(join(dir, '.git'))) return dir
+    } catch {
+      // ignore
+    }
+    const parent = dirname(dir)
+    if (parent === dir) break
+    dir = parent
+  }
+  return cwd
+}
+const sessions = []
+for (const [files, summarize] of [
+  [claudeFiles, claudeSessionSummary],
+  [codexFiles, codexSessionSummary],
+]) {
+  for (const file of files.slice(0, SESSION_CAP)) {
+    let text = ''
+    try {
+      text = readFileSync(file, 'utf8')
+    } catch {
+      continue
+    }
+    const summary = summarize(text)
+    const row = sessionSendRow(summary, sessionHashFor(file))
+    if (!row) continue
+    const root = repoRoot(summary.cwd)
+    if (root) {
+      row.repoHash = createHash('sha256').update(root).digest('hex').slice(0, 40)
+      if (REPO_LABELS) row.repoLabel = (root.split('/').pop() || root).slice(0, 80)
+    }
+    sessions.push(row)
+  }
 }
 
 // Collapse to one row per (provider, model, day) so the request stays small.
@@ -255,11 +299,10 @@ const HOURLY_URL = deriveHourlyUrl(INGEST_URL, process.env.MODELMETER_HOURLY_ING
 // Per-tool / per-MCP attribution. Claude even-splits each turn's tokens across the
 // tools it called; Codex contributes precomputed deltas. Calls are exact.
 const toolsPayload = aggregateTools(events, codexToolDeltas, today)
-const currentDetail = { hours: hourly, tools: toolsPayload }
-const detailBatch = mergeDetailBatches(state.pendingDetail, currentDetail)
-const hasDetail = detailBatch.hours.length > 0 || detailBatch.tools.length > 0
 
-if (payload.length === 0 && !hasDetail) {
+const haveDaily = payload.length > 0
+const haveDetail = hourly.length > 0 || toolsPayload.length > 0 || sessions.length > 0
+if (!haveDaily && !haveDetail) {
   process.exit(0)
 }
 
@@ -267,23 +310,34 @@ if (process.env.MODELMETER_DRYRUN) {
   const tally = {}
   for (const e of events) tally[e.provider] = (tally[e.provider] || 0) + 1
   console.log(`DRY RUN: ${events.length} raw events -> ${payload.length} daily rows`, tally)
-  console.log(`         + ${hourly.length} recent hourly rows, ${toolsPayload.length} tool rows -> ${HOURLY_URL}`)
-  if (state.pendingDetail?.hours?.length || state.pendingDetail?.tools?.length) {
-    console.log(
-      `         + pending retry rows: ${state.pendingDetail.hours?.length || 0} hourly, ${state.pendingDetail.tools?.length || 0} tool`,
-    )
-  }
+  console.log(`         + ${hourly.length} recent hourly rows, ${toolsPayload.length} tool rows, ${sessions.length} session summaries -> ${HOURLY_URL}`)
   console.log(JSON.stringify(payload, null, 2))
   if (toolsPayload.length) console.log('tools:', JSON.stringify(toolsPayload, null, 2))
+  if (sessions.length) console.log('sessions:', JSON.stringify(sessions.slice(0, 3), null, 2))
   process.exit(0)
 }
 
-let committed = false
-if (payload.length > 0) {
+// With no new daily events there is nothing to commit, but session summaries (and any
+// timestamped detail) are recomputed in full each run and must still be sent, so treat
+// that case as already committed.
+let committed = !haveDaily
+if (haveDaily) {
   try {
     const res = await postJson(INGEST_URL, { source: 'collector', events: payload })
     if (res.ok) {
-      saveState()
+      if (!existsSync(MM_DIR)) mkdirSync(MM_DIR, { recursive: true })
+      const claudeIds = Object.keys(state.claude)
+      if (claudeIds.length > CLAUDE_STATE_CAP) {
+        const next = {}
+        for (const id of claudeIds.slice(-CLAUDE_STATE_CAP)) next[id] = 1
+        state.claude = next
+      }
+      writeFileSync(STATE_PATH, JSON.stringify(state))
+      try {
+        chmodSync(STATE_PATH, 0o600) // usage metadata is not secret, but keep it owner-only
+      } catch {
+        // best effort on platforms without POSIX perms
+      }
       committed = true
       console.error(`modelmeter: reported ${payload.length} usage rows`)
     } else {
@@ -292,32 +346,24 @@ if (payload.length > 0) {
   } catch (err) {
     console.error(`modelmeter: ${err.message}`)
   }
-} else {
-  committed = true // retrying previously committed detail rows
 }
 
-// Additive detail rows are sent only after daily usage is committed. If the detail
-// endpoint fails, keep the merged batch in state and retry on the next run.
-if (committed && hasDetail) {
-  if (!HOURLY_URL) {
-    state.pendingDetail = detailBatch
-    saveState()
-    console.error('modelmeter: detail ingest URL could not be derived; set MODELMETER_HOURLY_INGEST_URL')
-    process.exit(0)
-  }
-  let detailSent = false
+// Detail rows go only after the daily batch is committed (or when there was none), so a
+// daily retry cannot race them. The endpoint applies them in one transaction; retry once
+// to cover a transient failure, then give up (the window self-heals as new data flows).
+if (committed && haveDetail) {
   for (let attempt = 0; attempt < 2; attempt++) {
     try {
-      const res = await postJson(HOURLY_URL, { source: 'collector', ...detailBatch })
-      if (res.ok) {
-        detailSent = true
-        break
-      }
+      const res = await postJson(HOURLY_URL, {
+        source: 'collector',
+        hours: hourly,
+        tools: toolsPayload,
+        sessions,
+      })
+      if (res.ok) break
     } catch {
       // fall through to one retry, then give up
     }
   }
-  state.pendingDetail = detailSent ? { hours: [], tools: [] } : detailBatch
-  saveState()
 }
 process.exit(0)

package/lib.mjs

@@ -356,7 +356,9 @@ export function claudeSessionSummary(text) {
     }
   }
   if (requests === 0) return null
-  return { provider: 'anthropic', model, uncached, cacheRead, cacheCreate, output, requests, firstTs, lastTs, cwd, contextSeq, tools }
+  // Claude folds extended-thinking into output_tokens with no separate field, so
+  // reasoning is 0 here (the breakdown is only available for Codex).
+  return { provider: 'anthropic', model, uncached, cacheRead, cacheCreate, output, reasoning: 0, requests, firstTs, lastTs, cwd, contextSeq, tools }
 }
 
 // Summarize one Codex session from its final cumulative token_count. Codex totals are
@@ -401,7 +403,11 @@ export function codexSessionSummary(text) {
   if (!totals) return null
   const cacheRead = totals.cached_input_tokens || 0
   const uncached = Math.max(0, (totals.input_tokens || 0) - cacheRead)
-  const output = (totals.output_tokens || 0) + (totals.reasoning_output_tokens || 0)
+  // Reasoning tokens bill as output (the priciest tier) but are reported separately and
+  // are otherwise hidden, so keep the breakdown: output is the total billed output and
+  // reasoning is the share of it spent thinking.
+  const reasoning = totals.reasoning_output_tokens || 0
+  const output = (totals.output_tokens || 0) + reasoning
   return {
     provider: 'openai',
     model,
@@ -409,6 +415,7 @@ export function codexSessionSummary(text) {
     cacheRead,
     cacheCreate: 0,
     output,
+    reasoning,
     requests: Math.max(1, requests),
     firstTs,
     lastTs,
@@ -439,6 +446,7 @@ export function sessionSendRow(summary, sessionHash) {
     cacheRead: summary.cacheRead,
     cacheCreation: summary.cacheCreate,
     output: summary.output,
+    reasoning: summary.reasoning || 0,
     maxInputTurn: bloat ? bloat.max : 0,
     first5Avg: bloat ? bloat.first5 : 0,
     last5Avg: bloat ? bloat.last5 : 0,
@@ -464,6 +472,7 @@ export function buildLocalRecommendations(summaries) {
   let cacheRead = 0
   let cacheCreate = 0
   let output = 0
+  let reasoning = 0
   let requests = 0
   const tools = {}
   for (const s of list) {
@@ -471,6 +480,7 @@ export function buildLocalRecommendations(summaries) {
     cacheRead += s.cacheRead
     cacheCreate += s.cacheCreate
     output += s.output
+    reasoning += s.reasoning || 0
     requests += s.requests
     for (const [g, v] of Object.entries(s.tools || {})) {
       const cur = tools[g] || { tokens: 0, calls: 0 }
@@ -547,7 +557,19 @@ export function buildLocalRecommendations(summaries) {
     })
   }
 
-  // 4. Context bloat (Claude sessions carry a per-turn sequence).
+  // 4. Reasoning share (Codex). Reasoning bills as output, the priciest tier, but is
+  // hidden by default, so a large reasoning share is invisible cost.
+  if (output > 100_000 && reasoning / output > 0.4) {
+    recs.push({
+      kind: 'reasoning',
+      level: 'warn',
+      text: `Reasoning is ${pct(
+        reasoning / output,
+      )} of your output tokens, the priciest tier, and hidden by default. Lower the reasoning effort for routine tasks and reserve high effort for genuinely hard problems.`,
+    })
+  }
+
+  // 5. Context bloat (Claude sessions carry a per-turn sequence).
   let worst = null
   for (const s of list) {
     const b = sessionBloat(s.contextSeq)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "modelmeter-collect",
-  "version": "0.8.0",
+  "version": "0.9.0",
   "description": "Report LLM token usage from local Claude Code / Codex logs to ModelMeter. Token counts only, never prompts or keys.",
   "type": "module",
   "bin": {