@vimoxshah/tokenflow 1.1.0 → 1.1.2

package/src/core/watch-lock.js

@@ -0,0 +1,160 @@
+/**
+ * The watcher's single-instance lock.
+ *
+ * A pidfile alone cannot answer "is my watcher running?". PID numbers restart
+ * at boot and the kernel reuses them, so a pidfile that outlives a reboot
+ * eventually names somebody else's process. That is not hypothetical: a lock
+ * left at pid 810 was inherited by `/usr/libexec/mobilerepaird` after a
+ * restart, `kill(810, 0)` kept succeeding, and every `tokenflow watch` — from
+ * the launch agent and from the menu bar's play button alike — refused to
+ * start with "a watcher is already running" for days. The data silently went
+ * stale behind a lock held by a phantom.
+ *
+ * So the lock records an IDENTITY, not just a number:
+ *
+ *   pid        the process to signal
+ *   boot       epoch ms of the boot the pid was issued by
+ *   startedAt  when the watcher took the lock (human-readable diagnostics)
+ *
+ * A lock is live only when the pid is alive AND its boot stamp matches this
+ * boot. A pidfile from an earlier boot is stale by construction, whoever holds
+ * that number now.
+ *
+ * Legacy bare-number pidfiles carry no boot stamp, so they fall back to asking
+ * the OS who owns the number: a command line without "tokenflow" in it is
+ * somebody else's process and the lock is stale.
+ *
+ * This module owns the lock so that both the watcher and the read-only status
+ * surfaces can consult it without importing each other.
+ */
+import fs from 'node:fs';
+import os from 'node:os';
+import { execFileSync } from 'node:child_process';
+import { paths } from './config.js';
+
+export const LOCK_VERSION = 2;
+
+/**
+ * Boot stamps are derived from uptime, which the OS reports in whole seconds,
+ * so two readings inside one boot can differ by a second or two. Anything
+ * inside this window is the same boot; a reboot moves the stamp by at least
+ * the previous session's uptime.
+ */
+const BOOT_TOLERANCE_MS = 30000;
+
+/** Epoch ms of the last boot. Stable to ~1s for the life of the boot. */
+export function bootTimeMs() {
+  return Date.now() - os.uptime() * 1000;
+}
+
+/**
+ * @typedef {{pid:number, boot:number|null, startedAt:string|null, legacy:boolean}} WatchLock
+ */
+
+/** Read the lock file. Accepts both the JSON form and the legacy bare number. */
+export function readLock() {
+  let raw;
+  try {
+    raw = fs.readFileSync(paths().watchPid, 'utf8').trim();
+  } catch {
+    return null;
+  }
+  if (!raw) return null;
+  if (raw.startsWith('{')) {
+    try {
+      const o = JSON.parse(raw);
+      const pid = Number(o.pid);
+      if (!Number.isFinite(pid) || pid <= 0) return null;
+      const boot = Number(o.boot);
+      return {
+        pid,
+        boot: Number.isFinite(boot) ? boot : null,
+        startedAt: typeof o.startedAt === 'string' ? o.startedAt : null,
+        legacy: false,
+      };
+    } catch {
+      return null;
+    }
+  }
+  const pid = Number(raw);
+  if (!Number.isFinite(pid) || pid <= 0) return null;
+  return { pid, boot: null, startedAt: null, legacy: true };
+}
+
+/** Just the pid, for callers that only want to signal it. */
+export function readLockPid() {
+  return readLock()?.pid ?? null;
+}
+
+/** Does SOME process hold this pid? (EPERM = alive, owned by someone else.) */
+export function processAlive(pid) {
+  if (!Number.isFinite(pid) || pid <= 0) return false;
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch (err) {
+    return err.code !== 'ESRCH';
+  }
+}
+
+/**
+ * The command line behind a pid, or null when the OS will not say.
+ * Cheap enough for lock checks; never called in a loop.
+ */
+export function processCommand(pid) {
+  try {
+    if (process.platform === 'linux') {
+      const raw = fs.readFileSync(`/proc/${pid}/cmdline`, 'utf8');
+      return raw.replace(/\0/g, ' ').trim() || null;
+    }
+    const out = execFileSync('ps', ['-p', String(pid), '-o', 'command='], {
+      encoding: 'utf8',
+      timeout: 2000,
+      stdio: ['ignore', 'pipe', 'ignore'],
+    });
+    return out.trim() || null;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Is this lock held by a live watcher of OURS?
+ *
+ * @param {WatchLock|null} lock
+ */
+export function lockIsLive(lock) {
+  if (!lock || !processAlive(lock.pid)) return false;
+  if (lock.boot !== null) {
+    return Math.abs(lock.boot - bootTimeMs()) <= BOOT_TOLERANCE_MS;
+  }
+  // No boot stamp to check: ask who owns the number instead. When the OS
+  // will not say, keep the lock — refusing to start is safer than two
+  // watchers racing on one store.
+  const cmd = processCommand(lock.pid);
+  if (cmd === null) return true;
+  return /tokenflow/i.test(cmd);
+}
+
+/** Write this process in as the lock holder. */
+export function writeLock() {
+  fs.writeFileSync(
+    paths().watchPid,
+    `${JSON.stringify({
+      v: LOCK_VERSION,
+      pid: process.pid,
+      boot: Math.round(bootTimeMs()),
+      startedAt: new Date().toISOString(),
+    })}\n`,
+  );
+}
+
+/** Remove the lock file. */
+export function clearLock() {
+  try {
+    fs.unlinkSync(paths().watchPid);
+    return true;
+  } catch {
+    return false; // already gone — that is fine
+  }
+}

package/src/core/watch.js

@@ -8,8 +8,11 @@
  *
  * Design constraints this file takes seriously:
  *
- * Single instance.  A pidfile guards against two watchers racing on the same
- *   store; a stale pidfile from a crashed run is detected and replaced.
+ * Single instance.  An identity-bearing lock guards against two watchers
+ *   racing on the same store. It records the boot the pid was issued by, so a
+ *   lock that outlived a reboot is stale by construction rather than being
+ *   trusted because the kernel handed its number to something else
+ *   (see watch-lock.js).
  * Failure isolation. One bad provider cannot stop the loop: refresh errors
  *   are recorded into the status file and back off exponentially instead.
  * Sleep/wake. The loop reschedules from wall-clock reality every tick, so a
@@ -18,12 +21,14 @@
  * Nothing leaves the machine. Refresh reads local logs; notifications go to
  *   the local OS; the status file stays in $TOKENFLOW_HOME.
  */
-import fs from 'node:fs';
-import { loadConfig, paths, ensureDirs } from './config.js';
+import { loadConfig, ensureDirs } from './config.js';
 import { refresh } from './ingest.js';
 import { listProviders } from './registry.js';
 import { buildLiveStatus, writeLiveStatus, readLiveStatus } from './live-status.js';
 import { notify as osNotify } from './notify.js';
+import {
+  bootTimeMs, clearLock, lockIsLive, processAlive, readLock, readLockPid, writeLock,
+} from './watch-lock.js';
 
 const MAX_BACKOFF_MS = 15 * 60 * 1000;
 
@@ -100,25 +105,9 @@ function humanize(ms) {
 
 // ------------------------------------------------------------- instance ----
 
-function readLockPid() {
-  try {
-    const pid = Number(fs.readFileSync(paths().watchPid, 'utf8').trim());
-    return Number.isFinite(pid) && pid > 0 ? pid : null;
-  } catch {
-    return null;
-  }
-}
-
-function processAlive(pid) {
-  try {
-    process.kill(pid, 0);
-    return true;
-  } catch (err) {
-    // ESRCH = no such process; EPERM = alive but owned by someone else.
-    return err.code !== 'ESRCH';
-  }
-}
-export { processAlive };
+// The lock itself lives in watch-lock.js so read-only status surfaces can ask
+// "is a watcher running?" without importing the daemon.
+export { processAlive, readLock, lockIsLive, bootTimeMs };
 
 /** Set while THIS process runs a watcher loop — guards same-process double start. */
 let ownedHere = false;
@@ -133,40 +122,37 @@ export function acquireWatchLock() {
     throw new Error('a watcher is already running in this process');
   }
   ensureDirs();
-  const existing = readLockPid();
-  if (existing && processAlive(existing)) {
-    throw Object.assign(new Error(`a watcher is already running (pid ${existing})`), {
+  const existing = readLock();
+  if (existing && lockIsLive(existing)) {
+    throw Object.assign(new Error(`a watcher is already running (pid ${existing.pid})`), {
       hint: '`tokenflow watch --status` shows it; `tokenflow watch --stop` stops it.',
     });
   }
-  fs.writeFileSync(paths().watchPid, String(process.pid));
+  writeLock();
   ownedHere = true;
 }
 
 export function releaseWatchLock() {
-  const p = paths().watchPid;
-  const pid = readLockPid();
-  if (pid === process.pid) {
-    try { fs.unlinkSync(p); } catch { /* already gone — that is fine */ }
-  }
+  if (readLockPid() === process.pid) clearLock();
   ownedHere = false;
 }
 
 /** Is a watcher running right now (and does it own the lock)? */
 export function watchIsRunning() {
-  const pid = readLockPid();
-  return !!(pid && processAlive(pid));
+  return lockIsLive(readLock());
 }
 
 export function stopWatch() {
-  const pid = readLockPid();
-  if (!pid || !processAlive(pid)) {
-    try { fs.unlinkSync(paths().watchPid); } catch { /* nothing to clean */ }
-    return { stopped: false, reason: 'not running' };
+  const lock = readLock();
+  if (!lock || !lockIsLive(lock)) {
+    // Clearing here is the manual escape hatch for a lock the identity check
+    // cannot judge (an unreadable /proc, a pid the OS will not describe).
+    const had = lock ? clearLock() : false;
+    return { stopped: false, reason: had ? 'not running (cleared a stale lock)' : 'not running' };
   }
   try {
-    process.kill(pid, 'SIGTERM');
-    return { stopped: true, pid };
+    process.kill(lock.pid, 'SIGTERM');
+    return { stopped: true, pid: lock.pid };
   } catch (err) {
     return { stopped: false, reason: err.message };
   }
@@ -204,6 +190,9 @@ export async function runCycle(opt = {}) {
       ? {
         ...(prev?.watcher || {}),
         pid: process.pid,
+        // The boot this pid was issued by: a reader can tell a live watcher
+        // from a pid number the kernel has since handed to somebody else.
+        boot: Math.round(bootTimeMs()),
         mode: 'daemon',
         intervalSeconds: opt.daemon.intervalSeconds ?? null,
         startedAt: prev?.watcher?.startedAt || new Date().toISOString(),

package/src/providers/hermes/index.js

@@ -60,6 +60,26 @@
  * already been emitted, and emit only the DELTA when a row grew. The base
  * record carries the group's first_seen; a delta carries the last_seen at
  * which the new usage was observed.
+ *
+ * ### The tail key MUST be the table's whole primary key
+ *
+ * session_model_usage is keyed on SIX columns:
+ *
+ *   (session_id, model, billing_provider, billing_base_url, billing_mode, task)
+ *
+ * An earlier version of this adapter keyed its tails on four of them, leaving
+ * out billing_base_url and billing_mode. Two real rows — the same session and
+ * model, billed once with mode "" and once with mode "chat_completions" —
+ * therefore shared one tail. Each pass, each row computed its delta against
+ * the OTHER row's totals and then overwrote the tail, so the pair ping-ponged
+ * forever: every refresh cycle emitted the difference between them again, with
+ * a fresh emission index that made each one look like a new request. Five
+ * colliding sessions produced 37 BILLION phantom tokens in a single day and
+ * the totals grew with every cycle, not with usage.
+ *
+ * Two rules keep that from recurring: the key below is the full primary key,
+ * and the tail stores a HIGH-WATER MARK rather than the row's latest numbers,
+ * so a total that comes back lower can never manufacture a delta.
  */
 import fs from 'node:fs';
 import path from 'node:path';
@@ -118,6 +138,7 @@ export default createProvider({
 
       const rows = db.prepare(
         `SELECT u.session_id AS session_id, u.model AS model, u.billing_provider AS billing_provider,
+                u.billing_base_url AS billing_base_url, u.billing_mode AS billing_mode,
                 u.task AS task, u.api_call_count AS api_call_count,
                 u.input_tokens AS input_tokens, u.output_tokens AS output_tokens,
                 u.cache_read_tokens AS cache_read_tokens, u.cache_write_tokens AS cache_write_tokens,
@@ -159,7 +180,13 @@ export default createProvider({
         // dashboard can show; skip it rather than emit an empty request.
         if (FIELDS.every((k) => !base[k]) && measuredCost === null) { skipped++; continue; }
 
-        const key = [r.session_id, r.model ?? '', r.billing_provider ?? '', r.task ?? ''].join('|');
+        // The table's whole primary key — see "The tail key MUST be the
+        // table's whole primary key" above. Dropping a column here silently
+        // merges distinct rows and makes their deltas oscillate.
+        const key = [
+          r.session_id, r.model ?? '', r.billing_provider ?? '',
+          r.billing_base_url ?? '', r.billing_mode ?? '', r.task ?? '',
+        ].join('|');
         const prev = tails[key];
         const delta = {};
         let any = false;
@@ -177,10 +204,18 @@ export default createProvider({
         // row (not against its current total), so cost never double counts.
         const costDelta = prev && measuredCost !== null ? Math.max(0, round6(measuredCost - (prev.c || 0))) : measuredCost;
 
-        // Remember what this row accounts for. The emission index keeps each
-        // delta's record id distinct and stable across runs.
+        // Remember what this row accounts for, as a HIGH-WATER MARK. These
+        // totals only ever grow in the source table, so a lower reading is an
+        // anomaly (a source-side reset, a re-count, or two rows this adapter
+        // cannot tell apart) — and keeping the maximum means such a reading
+        // can never be turned into usage that did not happen.
         const emissionIndex = prev ? (prev.n || 0) : 0;
-        tails[key] = { ls: lastSeen ?? (prev?.ls || 0), f: { ...base }, c: measuredCost, n: emissionIndex + 1 };
+        const highWater = {};
+        for (const k of FIELDS) highWater[k] = Math.max(prev ? (prev.f[k] ?? 0) : 0, base[k] ?? 0);
+        const costHighWater = measuredCost === null && (prev?.c ?? null) === null
+          ? null
+          : Math.max(prev?.c ?? 0, measuredCost ?? 0);
+        tails[key] = { ls: lastSeen ?? (prev?.ls || 0), f: highWater, c: costHighWater, n: emissionIndex + 1 };
         records++;
 
         const model = str(r.model);
@@ -239,6 +274,8 @@ export default createProvider({
             hermes_task: str(r.task),
             api_calls: intOrNull(r.api_call_count),
             billing_provider: str(r.billing_provider),
+            billing_base_url: str(r.billing_base_url),
+            billing_mode: str(r.billing_mode),
             cost_status: str(r.cost_status),
             session_open: r.ended_at === null || r.ended_at === undefined ? true : undefined,
             ...(prev ? { continuation_of: key } : {}),

package/src/server/server.js

@@ -232,6 +232,35 @@ export async function startServer({ port = 7799, host = '127.0.0.1', open = fals
   return { server, url: addr, token: authToken, close: () => new Promise((r) => server.close(r)) };
 }
 
+/**
+ * Ask whoever is listening on host:port whether they are a TokenFlow server.
+ *
+ * Used before opening a browser window and before starting a second server:
+ * "the port is busy" and "the dashboard is already up" look identical from
+ * the outside, and only one of them is good news.
+ *
+ * @returns {Promise<object|null>} the /api/ping payload, or null when nothing
+ *   there answers as TokenFlow (closed port, stranger, timeout, garbage).
+ */
+export function pingServer({ host = '127.0.0.1', port = 7799, timeoutMs = 1500 } = {}) {
+  return new Promise((resolve) => {
+    const req = http.get({ host, port, path: '/api/ping', timeout: timeoutMs }, (res) => {
+      if (res.statusCode !== 200) { res.resume(); return resolve(null); }
+      let body = '';
+      res.setEncoding('utf8');
+      res.on('data', (c) => { body += c.length > 4096 ? '' : c; });
+      res.on('end', () => {
+        try {
+          const o = JSON.parse(body);
+          resolve(o && o.app === 'tokenflow' ? o : null);
+        } catch { resolve(null); }
+      });
+    });
+    req.on('timeout', () => { req.destroy(); resolve(null); });
+    req.on('error', () => resolve(null));
+  });
+}
+
 function send(res, code, type, body) {
   res.writeHead(code, { 'content-type': type, 'cache-control': 'no-store' });
   res.end(body);