@vibescore/tracker 0.0.1

package/src/lib/fs.js

@@ -0,0 +1,62 @@
+const fs = require('node:fs/promises');
+const path = require('node:path');
+
+async function ensureDir(p) {
+  await fs.mkdir(p, { recursive: true });
+}
+
+async function writeFileAtomic(filePath, content) {
+  const dir = path.dirname(filePath);
+  await ensureDir(dir);
+  const tmp = `${filePath}.tmp.${Date.now()}`;
+  await fs.writeFile(tmp, content, { encoding: 'utf8' });
+  await fs.rename(tmp, filePath);
+}
+
+async function readJson(filePath) {
+  try {
+    const raw = await fs.readFile(filePath, 'utf8');
+    return JSON.parse(raw);
+  } catch (_e) {
+    return null;
+  }
+}
+
+async function writeJson(filePath, obj) {
+  await writeFileAtomic(filePath, JSON.stringify(obj, null, 2) + '\n');
+}
+
+async function chmod600IfPossible(filePath) {
+  try {
+    await fs.chmod(filePath, 0o600);
+  } catch (_e) {}
+}
+
+async function openLock(lockPath, { quietIfLocked }) {
+  try {
+    const handle = await fs.open(lockPath, 'wx');
+    return {
+      async release() {
+        await handle.close().catch(() => {});
+      }
+    };
+  } catch (e) {
+    if (e && e.code === 'EEXIST') {
+      if (!quietIfLocked) {
+        process.stdout.write('Another sync is already running.\n');
+      }
+      return null;
+    }
+    throw e;
+  }
+}
+
+module.exports = {
+  ensureDir,
+  writeFileAtomic,
+  readJson,
+  writeJson,
+  chmod600IfPossible,
+  openLock
+};
+

package/src/lib/insforge-client.js

@@ -0,0 +1,59 @@
+'use strict';
+
+const { createClient } = require('@insforge/sdk');
+
+function getAnonKey() {
+  return process.env.VIBESCORE_INSFORGE_ANON_KEY || process.env.INSFORGE_ANON_KEY || '';
+}
+
+function getHttpTimeoutMs() {
+  const raw = process.env.VIBESCORE_HTTP_TIMEOUT_MS;
+  if (raw == null || raw === '') return 20_000;
+  const n = Number(raw);
+  if (!Number.isFinite(n)) return 20_000;
+  if (n <= 0) return 0;
+  return clampInt(n, 1000, 120_000);
+}
+
+function createTimeoutFetch(baseFetch) {
+  if (!baseFetch) return baseFetch;
+  return async (input, init = {}) => {
+    const timeoutMs = getHttpTimeoutMs();
+    if (!timeoutMs) return baseFetch(input, init);
+    const controller = new AbortController();
+    const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
+    try {
+      return await baseFetch(input, { ...init, signal: controller.signal });
+    } catch (err) {
+      if (controller.signal.aborted) {
+        const timeoutErr = new Error(`Request timeout after ${timeoutMs}ms`);
+        timeoutErr.cause = err;
+        throw timeoutErr;
+      }
+      throw err;
+    } finally {
+      clearTimeout(timeoutId);
+    }
+  };
+}
+
+function createInsforgeClient({ baseUrl, accessToken } = {}) {
+  if (!baseUrl) throw new Error('Missing baseUrl');
+  const anonKey = getAnonKey();
+  return createClient({
+    baseUrl,
+    anonKey: anonKey || undefined,
+    edgeFunctionToken: accessToken || undefined,
+    fetch: createTimeoutFetch(globalThis.fetch)
+  });
+}
+
+function clampInt(value, min, max) {
+  const n = Number(value);
+  if (!Number.isFinite(n)) return min;
+  return Math.min(max, Math.max(min, Math.floor(n)));
+}
+
+module.exports = {
+  createInsforgeClient
+};

package/src/lib/insforge.js

@@ -0,0 +1,17 @@
+const { issueDeviceToken, signInWithPassword } = require('./vibescore-api');
+
+async function issueDeviceTokenWithPassword({ baseUrl, email, password, deviceName }) {
+  const accessToken = await signInWithPassword({ baseUrl, email, password });
+  const issued = await issueDeviceToken({ baseUrl, accessToken, deviceName });
+  return issued;
+}
+
+async function issueDeviceTokenWithAccessToken({ baseUrl, accessToken, deviceName }) {
+  const issued = await issueDeviceToken({ baseUrl, accessToken, deviceName });
+  return issued;
+}
+
+module.exports = {
+  issueDeviceTokenWithPassword,
+  issueDeviceTokenWithAccessToken
+};

package/src/lib/progress.js

@@ -0,0 +1,77 @@
+function createProgress({ stream } = {}) {
+  const out = stream || process.stdout;
+  const enabled = Boolean(out && out.isTTY);
+  const frames = ['|', '/', '-', '\\'];
+  const intervalMs = 90;
+
+  let timer = null;
+  let text = '';
+  let frame = 0;
+  let lastLen = 0;
+
+  function render() {
+    if (!enabled) return;
+    const line = `${frames[frame++ % frames.length]} ${text}`;
+    const pad = lastLen > line.length ? ' '.repeat(lastLen - line.length) : '';
+    lastLen = line.length;
+    out.write(`\r${line}${pad}`);
+  }
+
+  function start(initialText) {
+    if (!enabled) return;
+    text = initialText || '';
+    if (timer) clearInterval(timer);
+    timer = setInterval(render, intervalMs);
+    render();
+  }
+
+  function update(nextText) {
+    text = nextText || '';
+    render();
+  }
+
+  function stop() {
+    if (!enabled) return;
+    if (timer) clearInterval(timer);
+    timer = null;
+    out.write(`\r${' '.repeat(lastLen)}\r`);
+    lastLen = 0;
+  }
+
+  return { enabled, start, update, stop };
+}
+
+function renderBar(progress, width = 20) {
+  const p = Number.isFinite(progress) ? Math.min(1, Math.max(0, progress)) : 0;
+  const filled = Math.round(p * width);
+  const empty = Math.max(0, width - filled);
+  return `[${'='.repeat(filled)}${'-'.repeat(empty)}] ${Math.round(p * 100)}%`;
+}
+
+function formatNumber(n) {
+  const v = Number(n);
+  if (!Number.isFinite(v)) return '0';
+  return Math.trunc(v).toLocaleString('en-US');
+}
+
+function formatBytes(bytes) {
+  const n = Number(bytes);
+  if (!Number.isFinite(n) || n <= 0) return '0 B';
+  const units = ['B', 'KB', 'MB', 'GB', 'TB'];
+  let v = n;
+  let i = 0;
+  while (v >= 1024 && i < units.length - 1) {
+    v /= 1024;
+    i += 1;
+  }
+  const fixed = i === 0 ? String(Math.trunc(v)) : v.toFixed(v >= 10 ? 1 : 2);
+  return `${fixed} ${units[i]}`;
+}
+
+module.exports = {
+  createProgress,
+  renderBar,
+  formatNumber,
+  formatBytes
+};
+

package/src/lib/prompt.js

@@ -0,0 +1,20 @@
+const readline = require('node:readline');
+
+async function prompt(label) {
+  const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
+  const value = await new Promise((resolve) => rl.question(label, resolve));
+  rl.close();
+  return String(value || '').trim();
+}
+
+async function promptHidden(label) {
+  const rl = readline.createInterface({ input: process.stdin, output: process.stdout, terminal: true });
+  const value = await new Promise((resolve) => {
+    rl._writeToOutput = function _writeToOutput() {};
+    rl.question(label, (answer) => resolve(answer));
+  });
+  rl.close();
+  return String(value || '').trim();
+}
+
+module.exports = { prompt, promptHidden };

package/src/lib/rollout.js

@@ -0,0 +1,263 @@
+const fs = require('node:fs/promises');
+const fssync = require('node:fs');
+const path = require('node:path');
+const readline = require('node:readline');
+const crypto = require('node:crypto');
+
+const { ensureDir } = require('./fs');
+
+async function listRolloutFiles(sessionsDir) {
+  const out = [];
+  const years = await safeReadDir(sessionsDir);
+  for (const y of years) {
+    if (!/^[0-9]{4}$/.test(y.name) || !y.isDirectory()) continue;
+    const yearDir = path.join(sessionsDir, y.name);
+    const months = await safeReadDir(yearDir);
+    for (const m of months) {
+      if (!/^[0-9]{2}$/.test(m.name) || !m.isDirectory()) continue;
+      const monthDir = path.join(yearDir, m.name);
+      const days = await safeReadDir(monthDir);
+      for (const d of days) {
+        if (!/^[0-9]{2}$/.test(d.name) || !d.isDirectory()) continue;
+        const dayDir = path.join(monthDir, d.name);
+        const files = await safeReadDir(dayDir);
+        for (const f of files) {
+          if (!f.isFile()) continue;
+          if (!f.name.startsWith('rollout-') || !f.name.endsWith('.jsonl')) continue;
+          out.push(path.join(dayDir, f.name));
+        }
+      }
+    }
+  }
+
+  out.sort((a, b) => a.localeCompare(b));
+  return out;
+}
+
+async function parseRolloutIncremental({ rolloutFiles, cursors, queuePath, onProgress }) {
+  await ensureDir(path.dirname(queuePath));
+  let filesProcessed = 0;
+  let eventsQueued = 0;
+
+  const cb = typeof onProgress === 'function' ? onProgress : null;
+  const totalFiles = Array.isArray(rolloutFiles) ? rolloutFiles.length : 0;
+
+  for (let idx = 0; idx < rolloutFiles.length; idx++) {
+    const filePath = rolloutFiles[idx];
+    const st = await fs.stat(filePath).catch(() => null);
+    if (!st || !st.isFile()) continue;
+
+    const key = filePath;
+    const prev = cursors.files[key] || null;
+    const inode = st.ino || 0;
+    const startOffset = prev && prev.inode === inode ? prev.offset || 0 : 0;
+    const lastTotal = prev && prev.inode === inode ? prev.lastTotal || null : null;
+    const lastModel = prev && prev.inode === inode ? prev.lastModel || null : null;
+
+    const result = await parseRolloutFile({
+      filePath,
+      startOffset,
+      lastTotal,
+      lastModel,
+      queuePath
+    });
+
+    cursors.files[key] = {
+      inode,
+      offset: result.endOffset,
+      lastTotal: result.lastTotal,
+      lastModel: result.lastModel,
+      updatedAt: new Date().toISOString()
+    };
+
+    filesProcessed += 1;
+    eventsQueued += result.eventsQueued;
+
+    if (cb) {
+      cb({
+        index: idx + 1,
+        total: totalFiles,
+        filePath,
+        filesProcessed,
+        eventsQueued
+      });
+    }
+  }
+
+  return { filesProcessed, eventsQueued };
+}
+
+async function parseRolloutFile({ filePath, startOffset, lastTotal, lastModel, queuePath }) {
+  const st = await fs.stat(filePath);
+  const endOffset = st.size;
+  if (startOffset >= endOffset) {
+    return { endOffset, lastTotal, lastModel, eventsQueued: 0 };
+  }
+
+  const stream = fssync.createReadStream(filePath, { encoding: 'utf8', start: startOffset });
+  const rl = readline.createInterface({ input: stream, crlfDelay: Infinity });
+
+  const toAppend = [];
+  let model = typeof lastModel === 'string' ? lastModel : null;
+  let totals = lastTotal && typeof lastTotal === 'object' ? lastTotal : null;
+  let eventsQueued = 0;
+
+  for await (const line of rl) {
+    if (!line) continue;
+    const maybeTokenCount = line.includes('"token_count"');
+    const maybeTurnContext = !maybeTokenCount && line.includes('"turn_context"') && line.includes('"model"');
+    if (!maybeTokenCount && !maybeTurnContext) continue;
+
+    let obj;
+    try {
+      obj = JSON.parse(line);
+    } catch (_e) {
+      continue;
+    }
+
+    if (obj?.type === 'turn_context' && obj?.payload && typeof obj.payload.model === 'string') {
+      model = obj.payload.model;
+      continue;
+    }
+
+    const payload = obj?.payload;
+    if (!payload || payload.type !== 'token_count') continue;
+
+    const info = payload.info;
+    if (!info || typeof info !== 'object') continue;
+
+    const tokenTimestamp = typeof obj.timestamp === 'string' ? obj.timestamp : null;
+    if (!tokenTimestamp) continue;
+
+    const lastUsage = info.last_token_usage;
+    const totalUsage = info.total_token_usage;
+
+    const delta = pickDelta(lastUsage, totalUsage, totals);
+    if (!delta) continue;
+
+    if (totalUsage && typeof totalUsage === 'object') {
+      totals = totalUsage;
+    }
+
+    const event = {
+      event_id: sha256Hex(line),
+      token_timestamp: tokenTimestamp,
+      model: model || null,
+      input_tokens: delta.input_tokens || 0,
+      cached_input_tokens: delta.cached_input_tokens || 0,
+      output_tokens: delta.output_tokens || 0,
+      reasoning_output_tokens: delta.reasoning_output_tokens || 0,
+      total_tokens: delta.total_tokens || 0
+    };
+
+    toAppend.push(JSON.stringify(event));
+    eventsQueued += 1;
+  }
+
+  if (toAppend.length > 0) {
+    await fs.appendFile(queuePath, toAppend.join('\n') + '\n', 'utf8');
+  }
+
+  return { endOffset, lastTotal: totals, lastModel: model, eventsQueued };
+}
+
+function pickDelta(lastUsage, totalUsage, prevTotals) {
+  const hasLast = isNonEmptyObject(lastUsage);
+  const hasTotal = isNonEmptyObject(totalUsage);
+  const hasPrevTotals = isNonEmptyObject(prevTotals);
+
+  // Codex rollout logs sometimes emit duplicate token_count records where total_token_usage does not
+  // change between adjacent entries. Counting last_token_usage in those cases will double-count.
+  if (hasTotal && hasPrevTotals && sameUsage(totalUsage, prevTotals)) {
+    return null;
+  }
+
+  if (!hasLast && hasTotal && hasPrevTotals && totalsReset(totalUsage, prevTotals)) {
+    const normalized = normalizeUsage(totalUsage);
+    return isAllZeroUsage(normalized) ? null : normalized;
+  }
+
+  if (hasLast) {
+    return normalizeUsage(lastUsage);
+  }
+
+  if (hasTotal && hasPrevTotals) {
+    const delta = {};
+    for (const k of ['input_tokens', 'cached_input_tokens', 'output_tokens', 'reasoning_output_tokens', 'total_tokens']) {
+      const a = Number(totalUsage[k]);
+      const b = Number(prevTotals[k]);
+      if (Number.isFinite(a) && Number.isFinite(b)) delta[k] = Math.max(0, a - b);
+    }
+    const normalized = normalizeUsage(delta);
+    return isAllZeroUsage(normalized) ? null : normalized;
+  }
+
+  if (hasTotal) {
+    const normalized = normalizeUsage(totalUsage);
+    return isAllZeroUsage(normalized) ? null : normalized;
+  }
+
+  return null;
+}
+
+function normalizeUsage(u) {
+  const out = {};
+  for (const k of ['input_tokens', 'cached_input_tokens', 'output_tokens', 'reasoning_output_tokens', 'total_tokens']) {
+    const n = Number(u[k] || 0);
+    out[k] = Number.isFinite(n) && n >= 0 ? Math.floor(n) : 0;
+  }
+  return out;
+}
+
+function sha256Hex(s) {
+  return crypto.createHash('sha256').update(s, 'utf8').digest('hex');
+}
+
+function isNonEmptyObject(v) {
+  return Boolean(v && typeof v === 'object' && !Array.isArray(v) && Object.keys(v).length > 0);
+}
+
+function isAllZeroUsage(u) {
+  if (!u || typeof u !== 'object') return true;
+  for (const k of ['input_tokens', 'cached_input_tokens', 'output_tokens', 'reasoning_output_tokens', 'total_tokens']) {
+    if (Number(u[k] || 0) !== 0) return false;
+  }
+  return true;
+}
+
+function sameUsage(a, b) {
+  for (const k of ['input_tokens', 'cached_input_tokens', 'output_tokens', 'reasoning_output_tokens', 'total_tokens']) {
+    if (toNonNegativeInt(a?.[k]) !== toNonNegativeInt(b?.[k])) return false;
+  }
+  return true;
+}
+
+function totalsReset(curr, prev) {
+  const currTotal = curr?.total_tokens;
+  const prevTotal = prev?.total_tokens;
+  if (!isFiniteNumber(currTotal) || !isFiniteNumber(prevTotal)) return false;
+  return currTotal < prevTotal;
+}
+
+function isFiniteNumber(v) {
+  return typeof v === 'number' && Number.isFinite(v);
+}
+
+function toNonNegativeInt(v) {
+  const n = Number(v);
+  if (!Number.isFinite(n) || n < 0) return 0;
+  return Math.floor(n);
+}
+
+async function safeReadDir(dir) {
+  try {
+    return await fs.readdir(dir, { withFileTypes: true });
+  } catch (_e) {
+    return [];
+  }
+}
+
+module.exports = {
+  listRolloutFiles,
+  parseRolloutIncremental
+};

package/src/lib/upload-throttle.js

@@ -0,0 +1,129 @@
+const DEFAULTS = {
+  intervalMs: 10 * 60_000,
+  jitterMsMax: 120_000,
+  backlogBytes: 1_000_000,
+  batchSize: 300,
+  maxBatchesSmall: 2,
+  maxBatchesLarge: 4,
+  backoffInitialMs: 60_000,
+  backoffMaxMs: 30 * 60_000
+};
+
+function normalizeState(raw) {
+  const s = raw && typeof raw === 'object' ? raw : {};
+  return {
+    version: 1,
+    lastSuccessMs: toSafeInt(s.lastSuccessMs),
+    nextAllowedAtMs: toSafeInt(s.nextAllowedAtMs),
+    backoffUntilMs: toSafeInt(s.backoffUntilMs),
+    backoffStep: toSafeInt(s.backoffStep),
+    lastErrorAt: typeof s.lastErrorAt === 'string' ? s.lastErrorAt : null,
+    lastError: typeof s.lastError === 'string' ? s.lastError : null,
+    updatedAt: typeof s.updatedAt === 'string' ? s.updatedAt : null
+  };
+}
+
+function decideAutoUpload({ nowMs, pendingBytes, state, config }) {
+  const cfg = { ...DEFAULTS, ...(config || {}) };
+  const s = normalizeState(state);
+  const pending = Number(pendingBytes || 0);
+
+  if (pending <= 0) {
+    return { allowed: false, reason: 'no-pending', maxBatches: 0, batchSize: cfg.batchSize, blockedUntilMs: 0 };
+  }
+
+  const blockedUntilMs = Math.max(s.nextAllowedAtMs || 0, s.backoffUntilMs || 0);
+  if (blockedUntilMs > 0 && nowMs < blockedUntilMs) {
+    return { allowed: false, reason: 'throttled', maxBatches: 0, batchSize: cfg.batchSize, blockedUntilMs };
+  }
+
+  const maxBatches = pending >= cfg.backlogBytes ? cfg.maxBatchesLarge : cfg.maxBatchesSmall;
+  return { allowed: true, reason: 'allowed', maxBatches, batchSize: cfg.batchSize, blockedUntilMs: 0 };
+}
+
+function recordUploadSuccess({ nowMs, state, config, randInt }) {
+  const cfg = { ...DEFAULTS, ...(config || {}) };
+  const s = normalizeState(state);
+  const jitter = typeof randInt === 'function' ? randInt(0, cfg.jitterMsMax) : randomInt(0, cfg.jitterMsMax);
+  const nextAllowedAtMs = nowMs + cfg.intervalMs + jitter;
+
+  return {
+    ...s,
+    lastSuccessMs: nowMs,
+    nextAllowedAtMs,
+    backoffUntilMs: 0,
+    backoffStep: 0,
+    lastErrorAt: null,
+    lastError: null,
+    updatedAt: new Date(nowMs).toISOString()
+  };
+}
+
+function recordUploadFailure({ nowMs, state, error, config }) {
+  const cfg = { ...DEFAULTS, ...(config || {}) };
+  const s = normalizeState(state);
+
+  const retryAfterMs = toSafeInt(error?.retryAfterMs);
+  const status = toSafeInt(error?.status);
+
+  let backoffMs = 0;
+  if (retryAfterMs > 0) {
+    backoffMs = Math.min(cfg.backoffMaxMs, Math.max(cfg.backoffInitialMs, retryAfterMs));
+  } else {
+    const step = Math.min(10, Math.max(0, s.backoffStep || 0));
+    backoffMs = Math.min(cfg.backoffMaxMs, cfg.backoffInitialMs * Math.pow(2, step));
+  }
+
+  const backoffUntilMs = nowMs + backoffMs;
+  const nextAllowedAtMs = Math.max(s.nextAllowedAtMs || 0, backoffUntilMs);
+
+  return {
+    ...s,
+    nextAllowedAtMs,
+    backoffUntilMs,
+    backoffStep: Math.min(20, (s.backoffStep || 0) + 1),
+    lastErrorAt: new Date(nowMs).toISOString(),
+    lastError: truncate(String(error?.message || 'upload failed'), 200),
+    updatedAt: new Date(nowMs).toISOString()
+  };
+}
+
+function parseRetryAfterMs(headerValue, nowMs = Date.now()) {
+  if (typeof headerValue !== 'string' || headerValue.trim().length === 0) return null;
+  const v = headerValue.trim();
+  const seconds = Number(v);
+  if (Number.isFinite(seconds) && seconds >= 0) return Math.floor(seconds * 1000);
+  const d = new Date(v);
+  if (Number.isNaN(d.getTime())) return null;
+  const delta = d.getTime() - nowMs;
+  return delta > 0 ? delta : 0;
+}
+
+function randomInt(min, maxInclusive) {
+  const lo = Math.floor(min);
+  const hi = Math.floor(maxInclusive);
+  if (hi <= lo) return lo;
+  return lo + Math.floor(Math.random() * (hi - lo + 1));
+}
+
+function toSafeInt(v) {
+  const n = Number(v);
+  if (!Number.isFinite(n)) return 0;
+  if (n <= 0) return 0;
+  return Math.floor(n);
+}
+
+function truncate(s, maxLen) {
+  if (typeof s !== 'string') return '';
+  if (s.length <= maxLen) return s;
+  return s.slice(0, maxLen - 1) + '…';
+}
+
+module.exports = {
+  DEFAULTS,
+  normalizeState,
+  decideAutoUpload,
+  recordUploadSuccess,
+  recordUploadFailure,
+  parseRetryAfterMs
+};