modelmeter-collect 0.5.0 → 0.8.0

package/README.md

@@ -31,13 +31,19 @@ Prefer env vars? Set `MODELMETER_TOKEN` and `MODELMETER_INGEST_URL` and skip `in
 ## Check your setup
 
 ```bash
-npx modelmeter-collect doctor            # which logs were found, last activity, config
-npx modelmeter-collect doctor --payload  # + the exact JSON that would be sent
+npx modelmeter-collect doctor                  # which logs were found, last activity, config
+npx modelmeter-collect doctor --recommendations # local optimization tips from your logs
+npx modelmeter-collect doctor --payload        # + the exact JSON that would be sent
 ```
 
 `doctor` confirms it found your Claude Code and Codex logs and shows precisely what leaves
 your machine: model names, token counts, and tool/MCP names only. Never prompts or keys.
 
+`doctor --recommendations` scores your recent sessions locally (nothing is sent) and prints
+optimization tips: cache reuse, a dominant MCP server, output verbosity, and context bloat
+(a session whose per-turn context kept growing). All computed from token counts, never your
+prompt text.
+
 ## Keep it live (per prompt)
 
 **Claude Code** — add a `Stop` hook (fires after every response). It passes the session
@@ -81,6 +87,7 @@ does the same job.)
 | --- | --- | --- |
 | `MODELMETER_TOKEN` | from config file | Your `mm_live_...` ingest token |
 | `MODELMETER_INGEST_URL` | from config file | The ingest endpoint |
+| `MODELMETER_HOURLY_INGEST_URL` | derived from ingest URL | Detail endpoint for recent hourly/tool rows |
 | `MODELMETER_LOOKBACK_DAYS` | `14` | How many days of logs to scan |
 | `MODELMETER_DRYRUN` | unset | When set, print the payload instead of sending |
 

package/cli.mjs

@@ -18,7 +18,12 @@ import {
 import { homedir } from 'node:os'
 import { join, dirname } from 'node:path'
 import { fileURLToPath } from 'node:url'
-import { formatDoctorReport } from './lib.mjs'
+import {
+  formatDoctorReport,
+  claudeSessionSummary,
+  codexSessionSummary,
+  buildLocalRecommendations,
+} from './lib.mjs'
 
 const HOME = homedir()
 const MM_DIR = join(HOME, '.modelmeter')
@@ -64,8 +69,9 @@ Commands:
   init    Save your ingest token to ~/.modelmeter/config.json (chmod 600).
           Pass the token as an argument or via MODELMETER_TOKEN.
   doctor  Check your setup: which logs were found, last activity, config
-          status, and exactly what would be sent. Add --payload for the raw
-          JSON (token counts only, never transcript text).
+          status, and exactly what would be sent. Add --recommendations for
+          local optimization tips (cache, MCP, output, context bloat), or
+          --payload for the raw JSON (token counts only, never transcript text).
   (none)  Scan Claude Code + Codex logs and report token counts. Deduped,
           so it is safe to run repeatedly. MODELMETER_DRYRUN=1 previews only.
 
@@ -142,6 +148,73 @@ function discoverLogs(dir, cutoffMs) {
   return { dir, found: true, recentCount, lastWriteMs }
 }
 
+// Recent .jsonl session file paths under a logs directory, newest first, capped.
+function recentSessionFiles(dir, cutoffMs, limit) {
+  try {
+    statSync(dir)
+  } catch {
+    return []
+  }
+  const out = []
+  const stack = [dir]
+  while (stack.length) {
+    const d = stack.pop()
+    let entries = []
+    try {
+      entries = readdirSync(d, { withFileTypes: true })
+    } catch {
+      continue
+    }
+    for (const e of entries) {
+      const p = join(d, e.name)
+      if (e.isDirectory()) stack.push(p)
+      else if (e.isFile() && p.endsWith('.jsonl')) {
+        let m = 0
+        try {
+          m = statSync(p).mtimeMs
+        } catch {
+          continue
+        }
+        if (m >= cutoffMs) out.push({ p, m })
+      }
+    }
+  }
+  return out
+    .sort((a, b) => b.m - a.m)
+    .slice(0, limit)
+    .map((x) => x.p)
+}
+
+function printRecommendations(cutoffMs) {
+  const summaries = []
+  const sources = [
+    [join(HOME, '.claude', 'projects'), claudeSessionSummary],
+    [join(HOME, '.codex', 'sessions'), codexSessionSummary],
+  ]
+  for (const [dir, summarize] of sources) {
+    for (const file of recentSessionFiles(dir, cutoffMs, 200)) {
+      let text = ''
+      try {
+        text = readFileSync(file, 'utf8')
+      } catch {
+        continue
+      }
+      const summary = summarize(text)
+      if (summary) summaries.push(summary)
+    }
+  }
+  console.log('\nLocal recommendations (computed from your logs, nothing sent):')
+  const recs = buildLocalRecommendations(summaries)
+  if (recs.length === 0) {
+    console.log('  Nothing flagged. Your recent usage looks efficient.')
+    return
+  }
+  for (const r of recs) {
+    const mark = r.level === 'warn' ? '!' : r.level === 'ok' ? '+' : '-'
+    console.log(`  ${mark} ${r.text}`)
+  }
+}
+
 if (cmd === 'doctor') {
   const cfg = readConfig()
   const lookbackDays = 14
@@ -159,14 +232,19 @@ if (cmd === 'doctor') {
       codex: discoverLogs(join(HOME, '.codex', 'sessions'), cutoffMs),
     }),
   )
+  if (args.includes('--recommendations')) {
+    printRecommendations(cutoffMs)
+    process.exit(0)
+  }
   if (args.includes('--payload')) {
     console.log('\nNext batch (dry run, nothing is sent):')
     process.env.MODELMETER_DRYRUN = '1'
     await runCollector() // prints the exact payload (counts only), then exits
   } else {
     console.log(
-      '\nRun `npx modelmeter-collect doctor --payload` to preview the exact JSON that would be sent.',
+      '\nRun `npx modelmeter-collect doctor --recommendations` for local optimization tips, or',
     )
+    console.log('`doctor --payload` to preview the exact JSON that would be sent.')
     process.exit(0)
   }
 }

package/collect.mjs

@@ -29,6 +29,8 @@ import {
   aggregateDaily,
   aggregateHourly,
   aggregateTools,
+  pruneClaudeState,
+  mergeDetailBatches,
 } from './lib.mjs'
 
 const HOME = homedir()
@@ -79,13 +81,29 @@ async function postJson(url, body) {
   }
 }
 
-let state = { claude: {}, codex: {} }
+let state = { claude: {}, codex: {}, pendingDetail: { hours: [], tools: [] } }
 try {
-  state = { claude: {}, codex: {}, ...JSON.parse(readFileSync(STATE_PATH, 'utf8')) }
+  state = {
+    claude: {},
+    codex: {},
+    pendingDetail: { hours: [], tools: [] },
+    ...JSON.parse(readFileSync(STATE_PATH, 'utf8')),
+  }
 } catch {
   // first run
 }
 
+function saveState() {
+  if (!existsSync(MM_DIR)) mkdirSync(MM_DIR, { recursive: true })
+  state.claude = pruneClaudeState(state.claude, CLAUDE_STATE_CAP)
+  writeFileSync(STATE_PATH, JSON.stringify(state))
+  try {
+    chmodSync(STATE_PATH, 0o600) // usage metadata is not secret, but keep it owner-only
+  } catch {
+    // best effort on platforms without POSIX perms
+  }
+}
+
 // --- invocation detection -------------------------------------------------
 let hookInput = null
 if (!process.stdin.isTTY) {
@@ -159,7 +177,7 @@ function scanClaude(files) {
       }
       const ev = claudeEventFromLine(o)
       if (!ev || !ev.id || state.claude[ev.id]) continue
-      state.claude[ev.id] = 1
+      state.claude[ev.id] = { ts: ev.occurredAt || ev.occurredOn || '' }
       events.push(ev)
     }
   }
@@ -237,8 +255,11 @@ const HOURLY_URL = deriveHourlyUrl(INGEST_URL, process.env.MODELMETER_HOURLY_ING
 // Per-tool / per-MCP attribution. Claude even-splits each turn's tokens across the
 // tools it called; Codex contributes precomputed deltas. Calls are exact.
 const toolsPayload = aggregateTools(events, codexToolDeltas, today)
+const currentDetail = { hours: hourly, tools: toolsPayload }
+const detailBatch = mergeDetailBatches(state.pendingDetail, currentDetail)
+const hasDetail = detailBatch.hours.length > 0 || detailBatch.tools.length > 0
 
-if (payload.length === 0) {
+if (payload.length === 0 && !hasDetail) {
   process.exit(0)
 }
 
@@ -247,53 +268,56 @@ if (process.env.MODELMETER_DRYRUN) {
   for (const e of events) tally[e.provider] = (tally[e.provider] || 0) + 1
   console.log(`DRY RUN: ${events.length} raw events -> ${payload.length} daily rows`, tally)
   console.log(`         + ${hourly.length} recent hourly rows, ${toolsPayload.length} tool rows -> ${HOURLY_URL}`)
+  if (state.pendingDetail?.hours?.length || state.pendingDetail?.tools?.length) {
+    console.log(
+      `         + pending retry rows: ${state.pendingDetail.hours?.length || 0} hourly, ${state.pendingDetail.tools?.length || 0} tool`,
+    )
+  }
   console.log(JSON.stringify(payload, null, 2))
   if (toolsPayload.length) console.log('tools:', JSON.stringify(toolsPayload, null, 2))
   process.exit(0)
 }
 
 let committed = false
-try {
-  const res = await postJson(INGEST_URL, { source: 'collector', events: payload })
-  if (res.ok) {
-    if (!existsSync(MM_DIR)) mkdirSync(MM_DIR, { recursive: true })
-    const claudeIds = Object.keys(state.claude)
-    if (claudeIds.length > CLAUDE_STATE_CAP) {
-      const next = {}
-      for (const id of claudeIds.slice(-CLAUDE_STATE_CAP)) next[id] = 1
-      state.claude = next
-    }
-    writeFileSync(STATE_PATH, JSON.stringify(state))
-    try {
-      chmodSync(STATE_PATH, 0o600) // usage metadata is not secret, but keep it owner-only
-    } catch {
-      // best effort on platforms without POSIX perms
+if (payload.length > 0) {
+  try {
+    const res = await postJson(INGEST_URL, { source: 'collector', events: payload })
+    if (res.ok) {
+      saveState()
+      committed = true
+      console.error(`modelmeter: reported ${payload.length} usage rows`)
+    } else {
+      console.error(`modelmeter: ingest returned ${res.status}`)
     }
-    committed = true
-    console.error(`modelmeter: reported ${payload.length} usage rows`)
-  } else {
-    console.error(`modelmeter: ingest returned ${res.status}`)
+  } catch (err) {
+    console.error(`modelmeter: ${err.message}`)
   }
-} catch (err) {
-  console.error(`modelmeter: ${err.message}`)
+} else {
+  committed = true // retrying previously committed detail rows
 }
 
-// Additive + best-effort: only after the daily batch is committed (state written),
-// so a retry cannot double-count into the hourly window. The daily state is already
-// committed, so these detail rows will not be resent; retry once to cover a transient
-// failure, then give up (the window self-heals as new data flows).
-if (committed && (hourly.length > 0 || toolsPayload.length > 0)) {
+// Additive detail rows are sent only after daily usage is committed. If the detail
+// endpoint fails, keep the merged batch in state and retry on the next run.
+if (committed && hasDetail) {
+  if (!HOURLY_URL) {
+    state.pendingDetail = detailBatch
+    saveState()
+    console.error('modelmeter: detail ingest URL could not be derived; set MODELMETER_HOURLY_INGEST_URL')
+    process.exit(0)
+  }
+  let detailSent = false
   for (let attempt = 0; attempt < 2; attempt++) {
     try {
-      const res = await postJson(HOURLY_URL, {
-        source: 'collector',
-        hours: hourly,
-        tools: toolsPayload,
-      })
-      if (res.ok) break
+      const res = await postJson(HOURLY_URL, { source: 'collector', ...detailBatch })
+      if (res.ok) {
+        detailSent = true
+        break
+      }
     } catch {
       // fall through to one retry, then give up
     }
   }
+  state.pendingDetail = detailSent ? { hours: [], tools: [] } : detailBatch
+  saveState()
 }
 process.exit(0)

package/lib.mjs

@@ -42,6 +42,25 @@ export function findLastTokenCount(obj) {
   return last
 }
 
+// Depth-first find of the first string `cwd` in a parsed line (Claude carries it at
+// top level, Codex nests it under payload). Used only to derive a repo identity; the
+// collector hashes it and never sends the raw path.
+export function findCwd(obj) {
+  const stack = [obj]
+  let guard = 0
+  while (stack.length && guard < 5000) {
+    guard++
+    const d = stack.pop()
+    if (Array.isArray(d)) {
+      for (const v of d) stack.push(v)
+    } else if (d && typeof d === 'object') {
+      if (typeof d.cwd === 'string' && d.cwd) return d.cwd
+      for (const v of Object.values(d)) stack.push(v)
+    }
+  }
+  return ''
+}
+
 // Detail endpoint from the ingest URL, tolerating a trailing slash, or an override.
 export function deriveHourlyUrl(ingestUrl, override) {
   if (override) return override
@@ -270,3 +289,281 @@ export function aggregateTools(events, codexToolDeltas = [], today) {
   for (const d of codexToolDeltas) add(d.tool, d.bucketDate, d.calls, d.tokens)
   return [...byTool.values()]
 }
+
+// --- Local recommendations: session summaries + scoring for `doctor
+// --recommendations`. Computed entirely from local logs, no network. ---
+
+function pct(x) {
+  return `${Math.round(x * 100)}%`
+}
+
+function fmtTok(value) {
+  if (value >= 1_000_000_000) return `${(value / 1_000_000_000).toFixed(1)}B`
+  if (value >= 1_000_000) return `${(value / 1_000_000).toFixed(1)}M`
+  if (value >= 1_000) return `${(value / 1_000).toFixed(1)}K`
+  return String(Math.round(value))
+}
+
+// Summarize one Claude session file into per-session totals, the per-turn context
+// size sequence (for bloat detection), and tool token attribution.
+export function claudeSessionSummary(text) {
+  let model = 'claude-unknown'
+  let uncached = 0
+  let cacheRead = 0
+  let cacheCreate = 0
+  let output = 0
+  let requests = 0
+  let firstTs = ''
+  let lastTs = ''
+  let cwd = ''
+  const contextSeq = []
+  const tools = {}
+  for (const line of String(text).split('\n')) {
+    if (!line.trim()) continue
+    let o
+    try {
+      o = JSON.parse(line)
+    } catch {
+      continue
+    }
+    if (!cwd) cwd = findCwd(o)
+    const ev = claudeEventFromLine(o)
+    if (!ev) continue
+    model = ev.model || model
+    if (ev.occurredAt) {
+      if (!firstTs || ev.occurredAt < firstTs) firstTs = ev.occurredAt
+      if (ev.occurredAt > lastTs) lastTs = ev.occurredAt
+    }
+    uncached += ev.uncachedInputTokens
+    cacheRead += ev.cacheReadInputTokens
+    cacheCreate += ev.cacheCreationInputTokens
+    output += ev.outputTokens
+    requests += 1
+    contextSeq.push(ev.uncachedInputTokens + ev.cacheReadInputTokens + ev.cacheCreationInputTokens)
+    const turnTokens =
+      ev.uncachedInputTokens + ev.cacheReadInputTokens + ev.cacheCreationInputTokens + ev.outputTokens
+    const groups = new Map()
+    for (const name of ev.tools) {
+      const g = toolGroup(name)
+      groups.set(g, (groups.get(g) || 0) + 1)
+    }
+    const share = groups.size > 0 ? Math.round(turnTokens / groups.size) : 0
+    for (const [g, calls] of groups) {
+      const cur = tools[g] || { tokens: 0, calls: 0 }
+      cur.tokens += share
+      cur.calls += calls
+      tools[g] = cur
+    }
+  }
+  if (requests === 0) return null
+  return { provider: 'anthropic', model, uncached, cacheRead, cacheCreate, output, requests, firstTs, lastTs, cwd, contextSeq, tools }
+}
+
+// Summarize one Codex session from its final cumulative token_count. Codex totals are
+// cumulative, so there is no reliable per-turn context sequence (bloat is Claude-only).
+export function codexSessionSummary(text) {
+  let model = 'gpt-5'
+  let totals = null
+  let requests = 0
+  let firstTs = ''
+  let lastTs = ''
+  let cwd = ''
+  const tools = {}
+  for (const line of String(text).split('\n')) {
+    if (!line.trim()) continue
+    let o
+    try {
+      o = JSON.parse(line)
+    } catch {
+      continue
+    }
+    if (!cwd) cwd = findCwd(o)
+    const p = o.payload || o
+    const ptype = p.type || o.type
+    if (typeof o.model === 'string') model = o.model
+    else if (typeof p.model === 'string') model = p.model
+    if (typeof o.timestamp === 'string') {
+      if (!firstTs || o.timestamp < firstTs) firstTs = o.timestamp
+      if (o.timestamp > lastTs) lastTs = o.timestamp
+    }
+    const tc = findLastTokenCount(o)
+    if (tc) {
+      totals = tc
+      requests += 1
+    }
+    const g = codexToolFromEvent(p, ptype)
+    if (g) {
+      const cur = tools[g] || { tokens: 0, calls: 0 }
+      cur.calls += 1
+      tools[g] = cur
+    }
+  }
+  if (!totals) return null
+  const cacheRead = totals.cached_input_tokens || 0
+  const uncached = Math.max(0, (totals.input_tokens || 0) - cacheRead)
+  const output = (totals.output_tokens || 0) + (totals.reasoning_output_tokens || 0)
+  return {
+    provider: 'openai',
+    model,
+    uncached,
+    cacheRead,
+    cacheCreate: 0,
+    output,
+    requests: Math.max(1, requests),
+    firstTs,
+    lastTs,
+    cwd,
+    contextSeq: [],
+    tools,
+  }
+}
+
+// Shape a session summary into the row the collector sends to ingest-hourly. Derives
+// the bloat metrics from the context sequence and the bucket date from the timestamps.
+// Returns null if there is no date to bucket on. The hash is supplied by the caller
+// (the collector hashes the session-file basename, never the path).
+export function sessionSendRow(summary, sessionHash) {
+  if (!summary || !sessionHash) return null
+  const bucketDate = (summary.lastTs || summary.firstTs || '').slice(0, 10)
+  if (!bucketDate) return null
+  const bloat = sessionBloat(summary.contextSeq)
+  return {
+    sessionHash,
+    provider: summary.provider,
+    model: summary.model,
+    bucketDate,
+    firstTs: summary.firstTs || '',
+    lastTs: summary.lastTs || '',
+    requests: summary.requests,
+    uncached: summary.uncached,
+    cacheRead: summary.cacheRead,
+    cacheCreation: summary.cacheCreate,
+    output: summary.output,
+    maxInputTurn: bloat ? bloat.max : 0,
+    first5Avg: bloat ? bloat.first5 : 0,
+    last5Avg: bloat ? bloat.last5 : 0,
+    toolCounts: summary.tools,
+  }
+}
+
+// First-5 vs last-5 average turn size + max, for context-bloat detection.
+export function sessionBloat(seq) {
+  if (!Array.isArray(seq) || seq.length < 10) return null
+  const avg = (arr) => Math.round(arr.reduce((a, b) => a + b, 0) / arr.length)
+  return { first5: avg(seq.slice(0, 5)), last5: avg(seq.slice(-5)), max: Math.max(...seq) }
+}
+
+// Turn session summaries into local recommendations. Pure, so the whole engine is
+// fixture-tested. Returns [{ kind, level, text }] in cache -> mcp -> output -> bloat order.
+export function buildLocalRecommendations(summaries) {
+  const recs = []
+  const list = (summaries || []).filter(Boolean)
+  if (list.length === 0) return recs
+
+  let uncached = 0
+  let cacheRead = 0
+  let cacheCreate = 0
+  let output = 0
+  let requests = 0
+  const tools = {}
+  for (const s of list) {
+    uncached += s.uncached
+    cacheRead += s.cacheRead
+    cacheCreate += s.cacheCreate
+    output += s.output
+    requests += s.requests
+    for (const [g, v] of Object.entries(s.tools || {})) {
+      const cur = tools[g] || { tokens: 0, calls: 0 }
+      cur.tokens += v.tokens
+      cur.calls += v.calls
+      tools[g] = cur
+    }
+  }
+  const inputTotal = uncached + cacheRead + cacheCreate
+  const total = inputTotal + output
+
+  // 1. Cache effectiveness.
+  if (inputTotal > 0) {
+    const readRatio = cacheRead / inputTotal
+    const createRatio = cacheCreate / inputTotal
+    const uncachedRatio = uncached / inputTotal
+    if (createRatio > 0.3 && readRatio < createRatio) {
+      recs.push({
+        kind: 'cache',
+        level: 'warn',
+        text: `High cache creation, low reuse: ${pct(createRatio)} of input is cache writes vs ${pct(
+          readRatio,
+        )} reads. Keep your prompt prefix byte-for-byte stable so it gets reused.`,
+      })
+    } else if (uncachedRatio > 0.5) {
+      recs.push({
+        kind: 'cache',
+        level: 'warn',
+        text: `Repeated uncached context: ${pct(
+          uncachedRatio,
+        )} of input pays full price. Move stable content (system prompt, tools, examples) into a cached prefix.`,
+      })
+    } else if (readRatio >= 0.6) {
+      recs.push({
+        kind: 'cache',
+        level: 'ok',
+        text: `Good cache reuse: ${pct(readRatio)} of input is cached reads.`,
+      })
+    }
+  }
+
+  // 2. MCP / tool ranking.
+  const toolArr = Object.entries(tools)
+    .map(([tool, v]) => ({ tool, ...v }))
+    .sort((a, b) => b.tokens - a.tokens)
+  const toolTotal = toolArr.reduce((n, t) => n + t.tokens, 0)
+  const topMcp = toolArr.find((t) => t.tool.startsWith('mcp:'))
+  if (topMcp && toolTotal > 0 && topMcp.tokens / toolTotal >= 0.25) {
+    recs.push({
+      kind: 'mcp',
+      level: 'warn',
+      text: `${topMcp.tool} is ${pct(
+        topMcp.tokens / toolTotal,
+      )} of tool-attributed usage. Disable it when you are not actively using it.`,
+    })
+  }
+
+  // 3. Output verbosity.
+  if (total > 0 && output / total > 0.4) {
+    recs.push({
+      kind: 'output',
+      level: 'warn',
+      text: `Output is ${pct(
+        output / total,
+      )} of usage. Ask for patch-only responses or short summaries; output is the priciest token tier.`,
+    })
+  } else if (requests > 0 && Math.round(output / requests) > 5000) {
+    recs.push({
+      kind: 'output',
+      level: 'info',
+      text: `Responses average ${fmtTok(
+        Math.round(output / requests),
+      )} output tokens. A max_tokens cap or terser prompt trims the priciest tier.`,
+    })
+  }
+
+  // 4. Context bloat (Claude sessions carry a per-turn sequence).
+  let worst = null
+  for (const s of list) {
+    const b = sessionBloat(s.contextSeq)
+    if (b && b.last5 > b.first5 * 2 && b.last5 > 30_000 && (!worst || b.last5 > worst.last5)) {
+      worst = b
+    }
+  }
+  if (worst) {
+    recs.push({
+      kind: 'bloat',
+      level: 'warn',
+      text: `A recent session's context grew from ${fmtTok(worst.first5)} to ${fmtTok(
+        worst.last5,
+      )} tokens per turn. Start a fresh session or ask the model to summarize state.`,
+    })
+  }
+
+  return recs
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "modelmeter-collect",
-  "version": "0.5.0",
+  "version": "0.8.0",
   "description": "Report LLM token usage from local Claude Code / Codex logs to ModelMeter. Token counts only, never prompts or keys.",
   "type": "module",
   "bin": {