@vibescore/tracker 0.0.6 → 0.0.8

package/src/lib/rollout.js

@@ -5,6 +5,10 @@ const readline = require('node:readline');
 
 const { ensureDir } = require('./fs');
 
+const DEFAULT_SOURCE = 'codex';
+const DEFAULT_MODEL = 'unknown';
+const BUCKET_SEPARATOR = '|';
+
 async function listRolloutFiles(sessionsDir) {
   const out = [];
   const years = await safeReadDir(sessionsDir);
@@ -33,7 +37,14 @@ async function listRolloutFiles(sessionsDir) {
   return out;
 }
 
-async function parseRolloutIncremental({ rolloutFiles, cursors, queuePath, onProgress }) {
+async function listClaudeProjectFiles(projectsDir) {
+  const out = [];
+  await walkClaudeProjects(projectsDir, out);
+  out.sort((a, b) => a.localeCompare(b));
+  return out;
+}
+
+async function parseRolloutIncremental({ rolloutFiles, cursors, queuePath, onProgress, source }) {
   await ensureDir(path.dirname(queuePath));
   let filesProcessed = 0;
   let eventsAggregated = 0;
@@ -42,13 +53,18 @@ async function parseRolloutIncremental({ rolloutFiles, cursors, queuePath, onPro
   const totalFiles = Array.isArray(rolloutFiles) ? rolloutFiles.length : 0;
   const hourlyState = normalizeHourlyState(cursors?.hourly);
   const touchedBuckets = new Set();
+  const defaultSource = normalizeSourceInput(source) || DEFAULT_SOURCE;
 
   if (!cursors.files || typeof cursors.files !== 'object') {
     cursors.files = {};
   }
 
   for (let idx = 0; idx < rolloutFiles.length; idx++) {
-    const filePath = rolloutFiles[idx];
+    const entry = rolloutFiles[idx];
+    const filePath = typeof entry === 'string' ? entry : entry?.path;
+    if (!filePath) continue;
+    const fileSource =
+      typeof entry === 'string' ? defaultSource : normalizeSourceInput(entry?.source) || defaultSource;
     const st = await fs.stat(filePath).catch(() => null);
     if (!st || !st.isFile()) continue;
 
@@ -65,7 +81,8 @@ async function parseRolloutIncremental({ rolloutFiles, cursors, queuePath, onPro
       lastTotal,
       lastModel,
       hourlyState,
-      touchedBuckets
+      touchedBuckets,
+      source: fileSource
     });
 
     cursors.files[key] = {
@@ -98,7 +115,81 @@ async function parseRolloutIncremental({ rolloutFiles, cursors, queuePath, onPro
   return { filesProcessed, eventsAggregated, bucketsQueued };
 }
 
-async function parseRolloutFile({ filePath, startOffset, lastTotal, lastModel, hourlyState, touchedBuckets }) {
+async function parseClaudeIncremental({ projectFiles, cursors, queuePath, onProgress, source }) {
+  await ensureDir(path.dirname(queuePath));
+  let filesProcessed = 0;
+  let eventsAggregated = 0;
+
+  const cb = typeof onProgress === 'function' ? onProgress : null;
+  const files = Array.isArray(projectFiles) ? projectFiles : [];
+  const totalFiles = files.length;
+  const hourlyState = normalizeHourlyState(cursors?.hourly);
+  const touchedBuckets = new Set();
+  const defaultSource = normalizeSourceInput(source) || 'claude';
+
+  if (!cursors.files || typeof cursors.files !== 'object') {
+    cursors.files = {};
+  }
+
+  for (let idx = 0; idx < files.length; idx++) {
+    const entry = files[idx];
+    const filePath = typeof entry === 'string' ? entry : entry?.path;
+    if (!filePath) continue;
+    const fileSource =
+      typeof entry === 'string' ? defaultSource : normalizeSourceInput(entry?.source) || defaultSource;
+    const st = await fs.stat(filePath).catch(() => null);
+    if (!st || !st.isFile()) continue;
+
+    const key = filePath;
+    const prev = cursors.files[key] || null;
+    const inode = st.ino || 0;
+    const startOffset = prev && prev.inode === inode ? prev.offset || 0 : 0;
+
+    const result = await parseClaudeFile({
+      filePath,
+      startOffset,
+      hourlyState,
+      touchedBuckets,
+      source: fileSource
+    });
+
+    cursors.files[key] = {
+      inode,
+      offset: result.endOffset,
+      updatedAt: new Date().toISOString()
+    };
+
+    filesProcessed += 1;
+    eventsAggregated += result.eventsAggregated;
+
+    if (cb) {
+      cb({
+        index: idx + 1,
+        total: totalFiles,
+        filePath,
+        filesProcessed,
+        eventsAggregated,
+        bucketsQueued: touchedBuckets.size
+      });
+    }
+  }
+
+  const bucketsQueued = await enqueueTouchedBuckets({ queuePath, hourlyState, touchedBuckets });
+  hourlyState.updatedAt = new Date().toISOString();
+  cursors.hourly = hourlyState;
+
+  return { filesProcessed, eventsAggregated, bucketsQueued };
+}
+
+async function parseRolloutFile({
+  filePath,
+  startOffset,
+  lastTotal,
+  lastModel,
+  hourlyState,
+  touchedBuckets,
+  source
+}) {
   const st = await fs.stat(filePath);
   const endOffset = st.size;
   if (startOffset >= endOffset) {
@@ -130,13 +221,13 @@ async function parseRolloutFile({ filePath, startOffset, lastTotal, lastModel, h
       continue;
     }
 
-    const payload = obj?.payload;
-    if (!payload || payload.type !== 'token_count') continue;
+    const token = extractTokenCount(obj);
+    if (!token) continue;
 
-    const info = payload.info;
+    const info = token.info;
     if (!info || typeof info !== 'object') continue;
 
-    const tokenTimestamp = typeof obj.timestamp === 'string' ? obj.timestamp : null;
+    const tokenTimestamp = typeof token.timestamp === 'string' ? token.timestamp : null;
     if (!tokenTimestamp) continue;
 
     const lastUsage = info.last_token_usage;
@@ -152,27 +243,78 @@ async function parseRolloutFile({ filePath, startOffset, lastTotal, lastModel, h
     const bucketStart = toUtcHalfHourStart(tokenTimestamp);
     if (!bucketStart) continue;
 
-    const bucket = getHourlyBucket(hourlyState, bucketStart);
+    const bucket = getHourlyBucket(hourlyState, source, model, bucketStart);
     addTotals(bucket.totals, delta);
-    touchedBuckets.add(bucketStart);
+    touchedBuckets.add(bucketKey(source, model, bucketStart));
     eventsAggregated += 1;
   }
 
   return { endOffset, lastTotal: totals, lastModel: model, eventsAggregated };
 }
 
+async function parseClaudeFile({ filePath, startOffset, hourlyState, touchedBuckets, source }) {
+  const st = await fs.stat(filePath).catch(() => null);
+  if (!st || !st.isFile()) return { endOffset: startOffset, eventsAggregated: 0 };
+
+  const endOffset = st.size;
+  if (startOffset >= endOffset) return { endOffset, eventsAggregated: 0 };
+
+  const stream = fssync.createReadStream(filePath, { encoding: 'utf8', start: startOffset });
+  const rl = readline.createInterface({ input: stream, crlfDelay: Infinity });
+
+  let eventsAggregated = 0;
+  for await (const line of rl) {
+    if (!line || !line.includes('\"usage\"')) continue;
+    let obj;
+    try {
+      obj = JSON.parse(line);
+    } catch (_e) {
+      continue;
+    }
+
+    const usage = obj?.message?.usage || obj?.usage;
+    if (!usage || typeof usage !== 'object') continue;
+
+    const model = normalizeModelInput(obj?.message?.model || obj?.model) || DEFAULT_MODEL;
+    const tokenTimestamp = typeof obj?.timestamp === 'string' ? obj.timestamp : null;
+    if (!tokenTimestamp) continue;
+
+    const delta = normalizeClaudeUsage(usage);
+    if (!delta || isAllZeroUsage(delta)) continue;
+
+    const bucketStart = toUtcHalfHourStart(tokenTimestamp);
+    if (!bucketStart) continue;
+
+    const bucket = getHourlyBucket(hourlyState, source, model, bucketStart);
+    addTotals(bucket.totals, delta);
+    touchedBuckets.add(bucketKey(source, model, bucketStart));
+    eventsAggregated += 1;
+  }
+
+  rl.close();
+  stream.close?.();
+  return { endOffset, eventsAggregated };
+}
+
 async function enqueueTouchedBuckets({ queuePath, hourlyState, touchedBuckets }) {
   if (!touchedBuckets || touchedBuckets.size === 0) return 0;
 
   const toAppend = [];
   for (const bucketStart of touchedBuckets) {
-    const bucket = hourlyState.buckets[bucketStart];
+    const parsedKey = parseBucketKey(bucketStart);
+    const source = parsedKey.source || DEFAULT_SOURCE;
+    const model = parsedKey.model || DEFAULT_MODEL;
+    const hourStart = parsedKey.hourStart;
+    const bucket =
+      hourlyState.buckets[bucketKey(source, model, hourStart)] || hourlyState.buckets[bucketStart];
     if (!bucket || !bucket.totals) continue;
     const key = totalsKey(bucket.totals);
     if (bucket.queuedKey === key) continue;
     toAppend.push(
       JSON.stringify({
-        hour_start: bucketStart,
+        source,
+        model,
+        hour_start: hourStart,
         input_tokens: bucket.totals.input_tokens,
         cached_input_tokens: bucket.totals.cached_input_tokens,
         output_tokens: bucket.totals.output_tokens,
@@ -192,20 +334,39 @@ async function enqueueTouchedBuckets({ queuePath, hourlyState, touchedBuckets })
 
 function normalizeHourlyState(raw) {
   const state = raw && typeof raw === 'object' ? raw : {};
-  const buckets = state.buckets && typeof state.buckets === 'object' ? state.buckets : {};
+  const version = Number(state.version || 1);
+  if (!Number.isFinite(version) || version < 2) {
+    return {
+      version: 2,
+      buckets: {},
+      updatedAt: null
+    };
+  }
+  const rawBuckets = state.buckets && typeof state.buckets === 'object' ? state.buckets : {};
+  const buckets = {};
+  for (const [key, value] of Object.entries(rawBuckets)) {
+    const parsed = parseBucketKey(key);
+    const hourStart = parsed.hourStart;
+    if (!hourStart) continue;
+    const normalizedKey = bucketKey(parsed.source, parsed.model, hourStart);
+    buckets[normalizedKey] = value;
+  }
   return {
-    version: 1,
+    version: 2,
     buckets,
     updatedAt: typeof state.updatedAt === 'string' ? state.updatedAt : null
   };
 }
 
-function getHourlyBucket(state, hourStart) {
+function getHourlyBucket(state, source, model, hourStart) {
   const buckets = state.buckets;
-  let bucket = buckets[hourStart];
+  const normalizedSource = normalizeSourceInput(source) || DEFAULT_SOURCE;
+  const normalizedModel = normalizeModelInput(model) || DEFAULT_MODEL;
+  const key = bucketKey(normalizedSource, normalizedModel, hourStart);
+  let bucket = buckets[key];
   if (!bucket || typeof bucket !== 'object') {
     bucket = { totals: initTotals(), queuedKey: null };
-    buckets[hourStart] = bucket;
+    buckets[key] = bucket;
     return bucket;
   }
 
@@ -267,6 +428,52 @@ function toUtcHalfHourStart(ts) {
   return bucketStart.toISOString();
 }
 
+function bucketKey(source, model, hourStart) {
+  const safeSource = normalizeSourceInput(source) || DEFAULT_SOURCE;
+  const safeModel = normalizeModelInput(model) || DEFAULT_MODEL;
+  return `${safeSource}${BUCKET_SEPARATOR}${safeModel}${BUCKET_SEPARATOR}${hourStart}`;
+}
+
+function parseBucketKey(key) {
+  if (typeof key !== 'string') return { source: DEFAULT_SOURCE, model: DEFAULT_MODEL, hourStart: '' };
+  const first = key.indexOf(BUCKET_SEPARATOR);
+  if (first <= 0) return { source: DEFAULT_SOURCE, model: DEFAULT_MODEL, hourStart: key };
+  const second = key.indexOf(BUCKET_SEPARATOR, first + 1);
+  if (second <= 0) {
+    return { source: key.slice(0, first), model: DEFAULT_MODEL, hourStart: key.slice(first + 1) };
+  }
+  return {
+    source: key.slice(0, first),
+    model: key.slice(first + 1, second),
+    hourStart: key.slice(second + 1)
+  };
+}
+
+function normalizeSourceInput(value) {
+  if (typeof value !== 'string') return null;
+  const trimmed = value.trim().toLowerCase();
+  return trimmed.length > 0 ? trimmed : null;
+}
+
+function normalizeModelInput(value) {
+  if (typeof value !== 'string') return null;
+  const trimmed = value.trim();
+  return trimmed.length > 0 ? trimmed : null;
+}
+
+function extractTokenCount(obj) {
+  const payload = obj?.payload;
+  if (!payload) return null;
+  if (payload.type === 'token_count') {
+    return { info: payload.info, timestamp: obj?.timestamp || null };
+  }
+  const msg = payload.msg;
+  if (msg && msg.type === 'token_count') {
+    return { info: msg.info, timestamp: obj?.timestamp || null };
+  }
+  return null;
+}
+
 function pickDelta(lastUsage, totalUsage, prevTotals) {
   const hasLast = isNonEmptyObject(lastUsage);
   const hasTotal = isNonEmptyObject(totalUsage);
@@ -315,6 +522,16 @@ function normalizeUsage(u) {
   return out;
 }
 
+function normalizeClaudeUsage(u) {
+  return {
+    input_tokens: toNonNegativeInt(u?.input_tokens),
+    cached_input_tokens: toNonNegativeInt(u?.cache_read_input_tokens),
+    output_tokens: toNonNegativeInt(u?.output_tokens),
+    reasoning_output_tokens: 0,
+    total_tokens: toNonNegativeInt(u?.input_tokens) + toNonNegativeInt(u?.output_tokens)
+  };
+}
+
 function isNonEmptyObject(v) {
   return Boolean(v && typeof v === 'object' && !Array.isArray(v) && Object.keys(v).length > 0);
 }
@@ -359,7 +576,21 @@ async function safeReadDir(dir) {
   }
 }
 
+async function walkClaudeProjects(dir, out) {
+  const entries = await safeReadDir(dir);
+  for (const entry of entries) {
+    const fullPath = path.join(dir, entry.name);
+    if (entry.isDirectory()) {
+      await walkClaudeProjects(fullPath, out);
+      continue;
+    }
+    if (entry.isFile() && entry.name.endsWith('.jsonl')) out.push(fullPath);
+  }
+}
+
 module.exports = {
   listRolloutFiles,
-  parseRolloutIncremental
+  listClaudeProjectFiles,
+  parseRolloutIncremental,
+  parseClaudeIncremental
 };

package/src/lib/uploader.js

@@ -5,6 +5,10 @@ const readline = require('node:readline');
 const { ensureDir, readJson, writeJson } = require('./fs');
 const { ingestHourly } = require('./vibescore-api');
 
+const DEFAULT_SOURCE = 'codex';
+const DEFAULT_MODEL = 'unknown';
+const BUCKET_SEPARATOR = '|';
+
 async function drainQueueToCloud({ baseUrl, deviceToken, queuePath, queueStatePath, maxBatches, batchSize, onProgress }) {
   await ensureDir(require('node:path').dirname(queueStatePath));
 
@@ -70,7 +74,11 @@ async function readBatch(queuePath, startOffset, maxBuckets) {
     }
     const hourStart = typeof bucket?.hour_start === 'string' ? bucket.hour_start : null;
     if (!hourStart) continue;
-    bucketMap.set(hourStart, bucket);
+    const source = normalizeSource(bucket?.source) || DEFAULT_SOURCE;
+    const model = normalizeModel(bucket?.model) || DEFAULT_MODEL;
+    bucket.source = source;
+    bucket.model = model;
+    bucketMap.set(bucketKey(source, model, hourStart), bucket);
     linesRead += 1;
     if (linesRead >= maxBuckets) break;
   }
@@ -89,4 +97,20 @@ async function safeFileSize(p) {
   }
 }
 
+function bucketKey(source, model, hourStart) {
+  return `${source}${BUCKET_SEPARATOR}${model}${BUCKET_SEPARATOR}${hourStart}`;
+}
+
+function normalizeSource(value) {
+  if (typeof value !== 'string') return null;
+  const trimmed = value.trim().toLowerCase();
+  return trimmed.length > 0 ? trimmed : null;
+}
+
+function normalizeModel(value) {
+  if (typeof value !== 'string') return null;
+  const trimmed = value.trim();
+  return trimmed.length > 0 ? trimmed : null;
+}
+
 module.exports = { drainQueueToCloud };