@neuralmindlabs/tokn 0.1.0

package/packages/tokn/src/node.ts

@@ -0,0 +1,50 @@
+import { readFileSync, readdirSync } from 'node:fs';
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+import { parseClaudeSession } from './parsers/claude.ts';
+import { parseCodexSession } from './parsers/codex.ts';
+import { eventsToDays, type SourceContribution } from './aggregate.ts';
+
+export const CLAUDE_ROOT = join(homedir(), '.claude', 'projects');
+export const CODEX_ROOT = join(homedir(), '.codex', 'sessions');
+
+function* jsonlFiles(dir: string): Generator<string> {
+  let entries;
+  try {
+    entries = readdirSync(dir, { withFileTypes: true });
+  } catch {
+    return;
+  }
+  for (const entry of entries) {
+    const p = join(dir, entry.name);
+    if (entry.isDirectory()) yield* jsonlFiles(p);
+    else if (entry.name.endsWith('.jsonl')) yield p;
+  }
+}
+
+/** Scan local Claude Code + Codex logs synchronously (CLI / Node use). */
+export function scanLocalSources(): SourceContribution[] {
+  const sources: SourceContribution[] = [];
+  const dedupe = new Set<string>();
+  for (const file of [...jsonlFiles(CLAUDE_ROOT)].sort()) {
+    try {
+      const parsed = parseClaudeSession(readFileSync(file, 'utf8'), dedupe);
+      if (parsed.events.length > 0) {
+        sources.push({ tool: 'claude', project: parsed.project, days: eventsToDays(parsed.events) });
+      }
+    } catch {
+      /* unreadable file — skip */
+    }
+  }
+  for (const file of jsonlFiles(CODEX_ROOT)) {
+    try {
+      const parsed = parseCodexSession(readFileSync(file, 'utf8'));
+      if (parsed.events.length > 0) {
+        sources.push({ tool: 'codex', project: parsed.project, days: eventsToDays(parsed.events) });
+      }
+    } catch {
+      /* unreadable file — skip */
+    }
+  }
+  return sources;
+}

package/packages/tokn/src/parsers/claude.ts

@@ -0,0 +1,53 @@
+import type { UsageEvent } from '../types.ts';
+
+export interface ParsedSession {
+  events: UsageEvent[];
+  project?: string;
+}
+
+/**
+ * Parse one Claude Code session transcript (~/.claude/projects/** /*.jsonl).
+ * Assistant messages carry message.usage; the same API response can appear in
+ * multiple lines/files, so callers pass a shared dedupe set keyed by
+ * message.id + requestId.
+ */
+export function parseClaudeSession(content: string, dedupe: Set<string>): ParsedSession {
+  const events: UsageEvent[] = [];
+  let project: string | undefined;
+
+  for (const line of content.split('\n')) {
+    if (!line.trim()) continue;
+    let entry: any;
+    try {
+      entry = JSON.parse(line);
+    } catch {
+      continue;
+    }
+    if (!project && typeof entry.cwd === 'string') project = entry.cwd;
+    if (entry.type !== 'assistant') continue;
+    const msg = entry.message;
+    const usage = msg?.usage;
+    if (!usage || !entry.timestamp) continue;
+    const model = msg.model;
+    if (!model || model === '<synthetic>') continue;
+
+    if (msg.id && entry.requestId) {
+      const key = `${msg.id}:${entry.requestId}`;
+      if (dedupe.has(key)) continue;
+      dedupe.add(key);
+    }
+
+    events.push({
+      tool: 'claude',
+      timestamp: entry.timestamp,
+      model,
+      project,
+      inputTokens: usage.input_tokens ?? 0,
+      outputTokens: usage.output_tokens ?? 0,
+      cacheWriteTokens: usage.cache_creation_input_tokens ?? 0,
+      cacheWrite1hTokens: usage.cache_creation?.ephemeral_1h_input_tokens ?? 0,
+      cacheReadTokens: usage.cache_read_input_tokens ?? 0,
+    });
+  }
+  return { events, project };
+}

package/packages/tokn/src/parsers/codex.ts

@@ -0,0 +1,74 @@
+import type { UsageEvent } from '../types.ts';
+import type { ParsedSession } from './claude.ts';
+
+interface TokenTotals {
+  input_tokens: number;
+  cached_input_tokens: number;
+  output_tokens: number;
+}
+
+/**
+ * Parse one Codex rollout file (~/.codex/sessions/YYYY/MM/DD/rollout-*.jsonl).
+ *
+ * token_count events carry a session-cumulative total_token_usage. We take
+ * consecutive deltas of the cumulative counter rather than summing
+ * last_token_usage, which stays correct if an event is emitted twice for the
+ * same API call. A counter that shrinks means it was reset (e.g. compaction);
+ * the new value is taken as a fresh delta.
+ */
+export function parseCodexSession(content: string): ParsedSession {
+  const events: UsageEvent[] = [];
+  let project: string | undefined;
+  let model = 'unknown';
+  let prev: TokenTotals | null = null;
+
+  for (const line of content.split('\n')) {
+    if (!line.trim()) continue;
+    let entry: any;
+    try {
+      entry = JSON.parse(line);
+    } catch {
+      continue;
+    }
+    const payload = entry.payload;
+    if (!payload) continue;
+
+    if (entry.type === 'session_meta') {
+      if (typeof payload.cwd === 'string') project ??= payload.cwd;
+      continue;
+    }
+    if (entry.type === 'turn_context') {
+      if (typeof payload.model === 'string') model = payload.model;
+      if (typeof payload.cwd === 'string') project ??= payload.cwd;
+      continue;
+    }
+    if (entry.type !== 'event_msg' || payload.type !== 'token_count') continue;
+    const total: TokenTotals | undefined = payload.info?.total_token_usage;
+    if (!total || !entry.timestamp) continue;
+
+    let d: TokenTotals;
+    if (prev === null || total.input_tokens < prev.input_tokens) {
+      d = total;
+    } else {
+      d = {
+        input_tokens: total.input_tokens - prev.input_tokens,
+        cached_input_tokens: Math.max(0, (total.cached_input_tokens ?? 0) - (prev.cached_input_tokens ?? 0)),
+        output_tokens: Math.max(0, total.output_tokens - prev.output_tokens),
+      };
+    }
+    prev = total;
+    if (d.input_tokens === 0 && d.output_tokens === 0) continue;
+
+    events.push({
+      tool: 'codex',
+      timestamp: entry.timestamp,
+      model,
+      project,
+      inputTokens: d.input_tokens,
+      outputTokens: d.output_tokens,
+      cacheWriteTokens: 0,
+      cacheReadTokens: d.cached_input_tokens ?? 0,
+    });
+  }
+  return { events, project };
+}

package/packages/tokn/src/pricing/index.ts

@@ -0,0 +1,106 @@
+import vendored from './prices.ts';
+import type { UsageEvent } from '../types.ts';
+
+export interface PriceEntry {
+  provider: string;
+  input_cost_per_token?: number;
+  output_cost_per_token?: number;
+  cache_creation_input_token_cost?: number;
+  cache_read_input_token_cost?: number;
+}
+
+let table: Record<string, PriceEntry> = vendored as Record<string, PriceEntry>;
+let lastFetchedAt = 0;
+
+/** Replace the vendored table with a freshly fetched one (same LiteLLM shape). */
+export function setPriceTable(t: Record<string, PriceEntry>) {
+  table = t;
+}
+
+/** Fetch up-to-date pricing from NeuralMind Labs API, throttled to at most once per 10 minutes. */
+export async function fetchPrices(
+  opts: { fetchImpl?: typeof globalThis.fetch; force?: boolean } = {},
+): Promise<boolean> {
+  const { fetchImpl = globalThis.fetch, force = false } = opts;
+  const now = Date.now();
+  if (!force && now - lastFetchedAt < 10 * 60 * 1000) {
+    return true;
+  }
+
+  try {
+    const res = await fetchImpl('https://www.neuralmindlabs.com/api/models');
+    if (!res.ok) return false;
+    const body = await res.json();
+    if (!body || !Array.isArray(body.data)) return false;
+
+    const newTable: Record<string, PriceEntry> = { ...table };
+    for (const m of body.data) {
+      if (!m.id) continue;
+      const input = (m.inputPricePer1M ?? 0) / 1000000;
+      const output = (m.outputPricePer1M ?? 0) / 1000000;
+      const provider = m.provider ? m.provider.toLowerCase() : 'openai';
+
+      const entry: PriceEntry = {
+        provider,
+        input_cost_per_token: input,
+        output_cost_per_token: output,
+      };
+
+      if (provider === 'anthropic') {
+        entry.cache_creation_input_token_cost = input * 1.25;
+        entry.cache_read_input_token_cost = input * 0.1;
+      } else if (provider === 'openai') {
+        entry.cache_read_input_token_cost = input * 0.5;
+      }
+
+      newTable[m.id] = entry;
+
+      const parts = m.id.split('/');
+      if (parts.length > 1) {
+        const bare = parts.slice(1).join('/');
+        newTable[bare] = entry;
+      }
+    }
+    table = newTable;
+    lastFetchedAt = now;
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+export function findPrice(model: string): PriceEntry | undefined {
+  if (table[model]) return table[model];
+  const noDate = model.replace(/-\d{8}$/, '');
+  if (table[noDate]) return table[noDate];
+  const noLatest = model.replace(/-latest$/, '');
+  if (table[noLatest]) return table[noLatest];
+  return undefined;
+}
+
+/**
+ * Cost of one API call. Anthropic reports input/cache-write/cache-read
+ * disjointly; OpenAI's input_tokens includes cached_input_tokens, which are
+ * billed at the cache-read rate instead of the input rate.
+ */
+export function eventCostUSD(e: UsageEvent): number | undefined {
+  const p = findPrice(e.model);
+  if (!p) return undefined;
+  const inRate = p.input_cost_per_token ?? 0;
+  const outRate = p.output_cost_per_token ?? 0;
+  const writeRate = p.cache_creation_input_token_cost ?? inRate * 1.25;
+  const readRate = p.cache_read_input_token_cost ?? inRate * 0.1;
+  if (p.provider === 'openai') {
+    const uncached = Math.max(0, e.inputTokens - e.cacheReadTokens);
+    return uncached * inRate + e.cacheReadTokens * readRate + e.outputTokens * outRate;
+  }
+  const write1h = e.cacheWrite1hTokens ?? 0;
+  const write5m = e.cacheWriteTokens - write1h;
+  return (
+    e.inputTokens * inRate +
+    write5m * writeRate +
+    write1h * inRate * 2 +
+    e.cacheReadTokens * readRate +
+    e.outputTokens * outRate
+  );
+}