mixdog 0.9.172 → 0.9.174

package/src/runtime/shared/llm/usage-ledger.mjs

@@ -11,9 +11,32 @@ import { createHash, randomUUID } from 'node:crypto';
 import { resolvePluginData } from '../plugin-paths.mjs';
 import { usageRollupDayKey, isConversationUsageSource } from './usage-rollup.mjs';
 import { priceUsage } from './cost.mjs';
-import { normalizeUsageMeasurement, normalizeLegacyUsageDay } from './usage-measurement.mjs';
+import { rollupUsage } from './usage-ledger-rollup.mjs';
+import { QUOTA_SCHEMA, readQuotaHistory, readQuotaWindows, recordQuotaSamples } from './usage-ledger-quota.mjs';
+import { SHARE_ENV } from 'node:worker_threads';
+import { createWorkerRequestClient } from '../worker-requests.mjs';
 
+// One worker (usage-ledger-worker.mjs) runs the ledger's heavy SQLite work —
+// rollups, queued live writes, unpriced-refresh queries and repair writes — on
+// its own connections. SHARE_ENV keeps its local-time day boundaries (TZ)
+// identical to this thread's. The main thread already reports node:sqlite's
+// experimental warning; the worker must not print it again.
+// The worker retires after 30 s with nothing queued or in flight (like the
+// session save worker) and restarts on the next request. Writes are posted
+// one batch at a time, so a retirement can only fall between batches.
+const configuredIdleMs = Number(process.env.MIXDOG_USAGE_LEDGER_WORKER_IDLE_MS);
+const requestLedgerWorker = createWorkerRequestClient(new URL('./usage-ledger-worker.mjs', import.meta.url), {
+  env: SHARE_ENV,
+  execArgv: ['--disable-warning=ExperimentalWarning'],
+  idleExitMs: Number.isFinite(configuredIdleMs) && configuredIdleMs > 0 ? configuredIdleMs : 30_000,
+});
+
+/** Whether the usage-ledger worker thread is currently running. */
+export function usageLedgerWorkerRunning() {
+  return requestLedgerWorker.running();
+}
 const stores = new Map();
+const ROLLUP_CACHE_LIMIT = 8;
 const number = (value) => (Number.isFinite(Number(value)) && Number(value) >= 0 ? Number(value) : 0);
 const text = (value) => (typeof value === 'string' ? value.slice(0, 300) : '');
 
@@ -40,6 +63,11 @@ const COMPACT_SCHEMA = `
         cost_usd REAL, duration_ms REAL NOT NULL
     ) WITHOUT ROWID;
     CREATE INDEX IF NOT EXISTS usage_events_time ON usage_events(ts);
+    -- Every column a period read sums, so a time range is read from this
+    -- index alone: one primary-key search per record in this WITHOUT ROWID
+    -- table made a week of real records take ~400 ms instead of ~100 ms.
+    CREATE INDEX IF NOT EXISTS usage_events_time_usage ON usage_events(
+        ts,route,day,session,input,output,cache_read,cache_write,cost_usd,duration_ms);
 `;
 
 // Preserve the old read surface. Only storage changes: every field, including
@@ -87,6 +115,9 @@ function compactEventWriter(db) {
       row.rates ?? null,
       String(row.origin),
       row.rank,
+      // The provider account arrived later: a record without one keeps the
+      // route it always had.
+      ...(row.account ? [String(row.account)] : []),
     ]);
     insertRoute.run(signature);
     const route = findRoute.get(signature).id;
@@ -241,6 +272,7 @@ export function makeUsageRecord(args) {
     costSource: usageCostSource({ kind, costUsd, subscription, reported }),
     rates,
     responseId: text(args.responseId),
+    account: text(args.account),
     origin: args.origin || 'live',
     durationMs: number(args.durationMs),
   };
@@ -249,11 +281,28 @@ export function makeUsageRecord(args) {
 }
 
 export class UsageLedger {
-  constructor(path) {
+  /**
+   * `existing: true` attaches another connection to a ledger that an owning
+   * UsageLedger already opened, migrated and initialized (the ledger worker's
+   * per-request writer): no directory, migration or schema writes.
+   */
+  constructor(path, { existing = false } = {}) {
     this.path = path;
-    if (path !== ':memory:') mkdirSync(dirname(path), { recursive: true, mode: 0o700 });
+    if (!existing && path !== ':memory:') mkdirSync(dirname(path), { recursive: true, mode: 0o700 });
     this.db = new DatabaseSync(path);
     this.db.exec('PRAGMA busy_timeout=5000; PRAGMA journal_mode=WAL; PRAGMA synchronous=FULL; PRAGMA foreign_keys=ON;');
+    this.rollupStamp = null;
+    this.rollups = new Map();
+    this.pendingRollups = new Map();
+    this.writeQueue = [];
+    this.writing = null;
+    this.quotaWrites = new Set();
+    this.captureBegun = false;
+    if (existing) {
+      this.migration = null;
+      this.prepareStatements();
+      return;
+    }
     const version = this.db.prepare('PRAGMA user_version').get().user_version;
     if (version !== 0 && version !== 1 && version !== 2) {
       this.db.close();
@@ -288,8 +337,13 @@ export class UsageLedger {
                 PRIMARY KEY(day,rank,session_id)
             );
             CREATE TABLE IF NOT EXISTS legacy_days (day TEXT PRIMARY KEY, document TEXT NOT NULL);
+            ${QUOTA_SCHEMA}
             PRAGMA user_version=2;
         `);
+    this.prepareStatements();
+  }
+
+  prepareStatements() {
     this.insert = compactEventWriter(this.db);
     this.daily = this.db.prepare(`INSERT INTO daily VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
             ON CONFLICT(day,rank,provider,model,kind,cost_source,conversation) DO UPDATE SET
@@ -299,6 +353,16 @@ export class UsageLedger {
             imported=MAX(imported,excluded.imported)`);
     this.session = this.db.prepare(`INSERT INTO day_sessions VALUES (?,?,?,?)
             ON CONFLICT(day,rank,session_id) DO UPDATE SET tokens=tokens+excluded.tokens`);
+    // Moves on every commit: data_version for other connections (other
+    // processes included), total_changes() for this one.
+    this.changeStamp = this.db.prepare(
+      'SELECT total_changes() AS local,(SELECT data_version FROM pragma_data_version) AS shared'
+    );
+  }
+
+  /** Run one ledger-worker operation against this ledger's file. */
+  workerRequest(op, payload = {}) {
+    return requestLedgerWorker({ ...payload, op, path: this.path });
   }
 
   close() {
@@ -313,8 +377,12 @@ export class UsageLedger {
       .run(key, String(value));
   }
 
+  // liveSince is written once and never removed, so after this connection has
+  // ensured it exists, later sends skip the per-send write statement.
   beginCapture(ts) {
+    if (this.captureBegun) return;
     this.db.prepare('INSERT OR IGNORE INTO metadata VALUES (?,?)').run('liveSince', String(ts));
+    this.captureBegun = true;
   }
 
   indexRecord(row, rank) {
@@ -393,6 +461,88 @@ export class UsageLedger {
     }
   }
 
+  /**
+   * record([row]) without blocking the event loop: rows queue in call order and
+   * every row queued while a write is in flight commits in the next single
+   * transaction on the ledger worker's own connection (same insert-or-ignore
+   * idempotency, BEGIN IMMEDIATE and busy timeout as record()). A batch that
+   * fails is retried row by row, so one bad row only fails itself. Resolves
+   * once the row is durable. An in-memory ledger has no second connection and
+   * records in-thread.
+   */
+  recordQueued(row) {
+    if (this.path === ':memory:') {
+      return Promise.resolve().then(() => {
+        this.record([row]);
+      });
+    }
+    return new Promise((resolve, reject) => {
+      this.writeQueue.push({ row, resolve, reject });
+      this.pumpWrites();
+    });
+  }
+
+  pumpWrites() {
+    if (this.writing || this.writeQueue.length === 0) return;
+    const batch = this.writeQueue.splice(0);
+    this.writing = this.workerRequest('record', { rows: batch.map((entry) => entry.row) })
+      .then(
+        (outcomes) => {
+          batch.forEach((entry, index) => {
+            const outcome = outcomes[index];
+            if (outcome.error)
+              entry.reject(Object.assign(new Error(outcome.error.message), { code: outcome.error.code }));
+            else entry.resolve();
+          });
+        },
+        (error) => {
+          for (const entry of batch) entry.reject(error);
+        }
+      )
+      .finally(() => {
+        this.writing = null;
+        this.pumpWrites();
+      });
+  }
+
+  /** Resolves once every row queued so far is committed (or has failed). */
+  async settleWrites() {
+    while (this.writing || this.writeQueue.length) {
+      this.pumpWrites();
+      await this.writing;
+    }
+    // Quota readings post on their own; a failed one already reported itself.
+    await Promise.allSettled(this.quotaWrites);
+  }
+
+  /** Readings of subscription quota windows (usage-ledger-quota.mjs). */
+  recordQuota(samples) {
+    return recordQuotaSamples(this.db, samples);
+  }
+
+  /** recordQuota() on the ledger worker's own connection, off the event loop. */
+  recordQuotaQueued(samples) {
+    const write =
+      this.path === ':memory:'
+        ? Promise.resolve().then(() => this.recordQuota(samples))
+        : this.workerRequest('recordQuota', { samples });
+    const tracked = write.finally(() => this.quotaWrites.delete(tracked));
+    this.quotaWrites.add(tracked);
+    return tracked;
+  }
+
+  /** The quota history of one limit window, read on the ledger worker. */
+  quotaHistoryAsync(options) {
+    if (this.path === ':memory:') return Promise.resolve().then(() => readQuotaHistory(this.db, options));
+    return this.workerRequest('quotaHistory', { options });
+  }
+
+  /** One page of a limit window's history, read on the ledger worker. */
+  quotaWindowsAsync(options) {
+    if (this.path === ':memory:') return Promise.resolve().then(() => readQuotaWindows(this.db, options));
+    return this.workerRequest('quotaWindows', { options });
+  }
+
   preserveLegacyDays(days) {
     const insert = this.db.prepare('INSERT OR IGNORE INTO legacy_days VALUES (?,?)');
     this.db.exec('BEGIN IMMEDIATE');
@@ -407,223 +557,75 @@ export class UsageLedger {
     }
   }
 
-  /** Read cached amounts; group retained attribution separately for distinct sessions. */
-  rollup({ hourlyDay = null, fromDay = '0000-01-01', toDay = '9999-12-31', fromMs = null, toMs = null } = {}) {
-    const days = {};
-    const hourly = hourlyDay ? { rows: [], unallocated: [] } : null;
-    const fromTs = fromMs ?? (fromDay === '0000-01-01' ? 0 : new Date(`${fromDay}T00:00:00`).getTime());
-    const end = toDay === '9999-12-31' ? null : new Date(`${toDay}T00:00:00`);
-    if (end) end.setDate(end.getDate() + 1);
-    let toTs = toMs + 1;
-    if (toMs == null) toTs = end ? end.getTime() : Number.MAX_SAFE_INTEGER;
-    const empty = () => ({
-      turns: 0,
-      input: 0,
-      output: 0,
-      cacheRead: 0,
-      cacheWrite: 0,
-      costUsd: 0,
-      costBilled: 0,
-      costEstimated: 0,
-      costKnownTurns: 0,
-      durationMs: 0,
-      durationTurns: 0,
-      unmeasuredTurns: 0,
-      sessions: {},
-      sessionsComplete: true,
-    });
-    const add = (target, row) => {
-      const usage = normalizeUsageMeasurement(row.provider, {
-        turns: row.turns,
-        input: row.input,
-        output: row.output,
-        cacheRead: row.cache_read,
-        cacheWrite: row.cache_write,
-        costUsd: row.cost_usd,
-        costKnownTurns: row.cost_source !== 'unpriced' ? row.turns : 0,
-        costBilled: row.cost_source === 'provider' ? row.cost_usd : 0,
-        costEstimated: row.cost_source === 'provider' ? 0 : row.cost_usd,
-      });
-      for (const field of [
-        'turns',
-        'input',
-        'output',
-        'cacheRead',
-        'cacheWrite',
-        'costUsd',
-        'costKnownTurns',
-        'costBilled',
-        'costEstimated',
-        'unmeasuredTurns',
-      ]) {
-        target[field] += usage[field] || 0;
-      }
-      target.durationMs += row.duration_ms;
-      if (row.duration_ms > 0) target.durationTurns += row.turns;
-    };
-    // A detail and its terminal summary can overlap on the SAME route.
-    // Another provider/model on that day is independent and must survive.
-    const best = `SELECT day,provider,model,MIN(rank) AS rank FROM daily
-            WHERE day BETWEEN ? AND ? GROUP BY day,provider,model`;
-    if (hourly) {
-      const start = new Date(`${hourlyDay}T00:00:00`);
-      const end = new Date(start);
-      end.setDate(end.getDate() + 1);
-      hourly.rows = this.db
-        .prepare(`
-                SELECT e.ts,e.provider,e.source_type,e.input,e.output,e.cache_read,e.cache_write,e.cost_usd
-                FROM events e JOIN (${best}) b USING(day,provider,model,rank)
-                WHERE e.ts>=? AND e.ts<? ORDER BY e.ts
-            `)
-        .all(fromDay, toDay, fromMs == null ? start.getTime() : fromTs, toMs == null ? end.getTime() : toTs)
-        .map((row) => {
-          const usage = normalizeUsageMeasurement(row.provider, {
-            turns: 1,
-            input: row.input,
-            cacheRead: row.cache_read,
-            cacheWrite: row.cache_write,
-            costUsd: row.cost_usd,
-          });
-          return {
-            ...row,
-            input: usage.input,
-            cache_read: usage.cacheRead,
-            cache_write: usage.cacheWrite,
-            cost_usd: usage.costUsd,
-            unmeasuredTurns: usage.unmeasuredTurns || 0,
-          };
-        });
-    }
-    // Partial calendar days must be rebuilt from retained timestamps;
-    // cached whole-day totals would include usage outside a rolling window.
-    const amounts =
-      fromMs == null
-        ? this.db.prepare(`SELECT d.* FROM daily d JOIN (${best}) b USING(day,provider,model,rank)`).all(fromDay, toDay)
-        : this.db
-            .prepare(`
-                SELECT e.day,e.rank,e.provider,e.model,e.kind,e.cost_source,e.source_type,
-                    COUNT(*) AS turns,SUM(e.input) AS input,SUM(e.output) AS output,
-                    SUM(e.cache_read) AS cache_read,SUM(e.cache_write) AS cache_write,
-                    SUM(e.cost_usd) AS cost_usd,SUM(e.duration_ms) AS duration_ms,
-                    MAX(e.origin!='live') AS imported
-                FROM events e JOIN (${best}) b USING(day,provider,model,rank)
-                WHERE e.ts>=? AND e.ts<?
-                GROUP BY e.day,e.rank,e.provider,e.model,e.kind,e.cost_source,e.source_type
-            `)
-            .all(fromDay, toDay, fromTs, toTs)
-            .map((row) => ({
-              ...row,
-              conversation: isConversationUsageSource(row.source_type),
-            }));
-    for (const row of amounts) {
-      days[row.day] ||= { ...empty(), models: {}, sessions: {}, conversation: empty() };
-      const day = days[row.day];
-      const key = `${row.provider}/${row.model}`;
-      day.models[key] ||= {
-        ...empty(),
-        provider: row.provider,
-        model: row.model,
-        kind: row.kind,
-        conversation: empty(),
-      };
-      const route = day.models[key];
-      add(day, row);
-      add(route, row);
-      if (row.conversation) {
-        add(day.conversation, row);
-        add(route.conversation, row);
-      }
-      if (row.imported) day.importedPartial = true;
+  /**
+   * Read cached amounts; group retained attribution separately for distinct
+   * sessions. Window resolution, the bucket algebra, the three queries and
+   * the legacy merge live in usage-ledger-rollup.mjs.
+   *
+   * A rollup is a pure function of the stored rows and its query, so an
+   * unchanged ledger answers a repeated query from memory. The result is
+   * shared between callers and must be treated as read-only.
+   */
+  rollup(options = {}) {
+    this.syncRollupStamp();
+    const key = JSON.stringify(options);
+    const result = this.rollups.get(key) ?? rollupUsage(this.db, options);
+    this.rememberRollup(key, result);
+    return result;
+  }
+
+  /**
+   * rollup() with the query run on the shared rollup worker's own read-only
+   * connection, so a changed ledger never costs the event loop the full
+   * rollup. Shares rollup()'s cache: an unchanged ledger answers from memory
+   * and concurrent identical queries share one worker request. An in-memory
+   * ledger cannot be opened by a second connection and rolls up in-thread.
+   */
+  async rollupAsync(options = {}) {
+    if (this.path === ':memory:') return this.rollup(options);
+    const stamp = this.syncRollupStamp();
+    const key = JSON.stringify(options);
+    const cached = this.rollups.get(key);
+    if (cached) {
+      this.rememberRollup(key, cached);
+      return cached;
     }
-    // day_sessions cannot attribute an id to a route. Use the retained
-    // originals, without modifying them or guessing from account/pool ids.
-    // Aggregate compact integer keys BEFORE decoding attribution. The old
-    // view decoded JSON and joined every historical request on each open.
-    const sessions = this.db
-      .prepare(`
-            WITH grouped AS (
-                SELECT day,route,session,SUM(input) AS input,SUM(output) AS output,
-                    SUM(cache_read) AS cacheRead,SUM(cache_write) AS cacheWrite
-                FROM usage_events WHERE ts>=? AND ts<?
-                GROUP BY day,route,session
-            ), attributed AS (
-                SELECT printf('%04d-%02d-%02d',e.day/10000,(e.day/100)%100,e.day%100) AS day,
-                    json_extract(r.signature,'$[0]') AS provider,
-                    json_extract(r.signature,'$[1]') AS model,
-                    json_extract(r.signature,'$[3]') AS source_type,
-                    json_extract(r.signature,'$[6]') AS origin,
-                    json_extract(r.signature,'$[7]') AS rank,
-                    s.value AS session_id,e.input,e.output,e.cacheRead,e.cacheWrite
-                FROM grouped e JOIN usage_routes r ON r.id=e.route
-                JOIN usage_sessions s ON s.id=e.session
-            )
-            SELECT e.* FROM attributed e JOIN (${best}) b USING(day,provider,model,rank)
-        `)
-      .all(fromTs, toTs, fromDay, toDay);
-    for (const row of sessions) {
-      const classified = row.origin !== 'trace' || Boolean(row.source_type);
-      if (classified && !isConversationUsageSource(row.source_type)) continue;
-      const day = days[row.day];
-      const route = day.models[`${row.provider}/${row.model}`];
-      for (const target of [day, route, day.conversation, route.conversation]) {
-        // Old raw traces used session_id for account/socket scopes too.
-        // Without source attribution they cannot establish a session count.
-        if (!classified || !row.session_id || ['no-session', '(none)'].includes(row.session_id)) {
-          target.sessionsComplete = false;
-        } else {
-          const usage = normalizeUsageMeasurement(row.provider, row);
-          const tokens = usage.input + usage.output + usage.cacheRead + usage.cacheWrite;
-          target.sessions[row.session_id] = (target.sessions[row.session_id] || 0) + tokens;
-        }
-      }
+    let pending = this.pendingRollups.get(key);
+    if (!pending) {
+      pending = this.workerRequest('rollup', { options })
+        .then((result) => {
+          // A newer stamp cleared the cache meanwhile; never file this
+          // answer under it. The worker may have read commits newer than
+          // `stamp`, which only means the next stamp change discards it.
+          if (this.rollupStamp === stamp) this.rememberRollup(key, result);
+          return result;
+        })
+        .finally(() => {
+          if (this.pendingRollups.get(key) === pending) this.pendingRollups.delete(key);
+        });
+      this.pendingRollups.set(key, pending);
     }
-    // Timeless legacy totals cannot establish membership in a 24-hour window.
-    // They remain available in the calendar/history views.
-    if (fromMs != null) return hourly ? { days, hourly } : { days };
-    for (const row of this.db
-      .prepare('SELECT day,document FROM legacy_days WHERE day BETWEEN ? AND ?')
-      .all(fromDay, toDay)) {
-      const legacy = normalizeLegacyUsageDay(JSON.parse(row.document));
-      const day = days[row.day];
-      if (!day) {
-        days[row.day] = { ...legacy, importedPartial: true };
-        if (row.day === hourlyDay) hourly.unallocated.push(...Object.values(legacy.models || {}));
-        continue;
-      }
-      for (const [key, route] of Object.entries(legacy.models || {})) {
-        if (day.models[key]) continue;
-        // Preserve a disjoint historical route, not an inferred balance
-        // of an overlapping one. Whole-day session ids cannot be assigned
-        // to just this route, so its session count remains unknown.
-        day.models[key] = { ...route, sessionsComplete: false };
-        if (row.day === hourlyDay) hourly.unallocated.push(route);
-        const merge = (target, source) => {
-          if (!source) return;
-          for (const field of [
-            'turns',
-            'input',
-            'output',
-            'cacheRead',
-            'cacheWrite',
-            'costUsd',
-            'durationMs',
-            'durationTurns',
-            'unmeasuredTurns',
-          ])
-            target[field] += number(source[field]);
-          if (source.costKnownTurns != null) target.costKnownTurns += number(source.costKnownTurns);
-          else if (number(source.costUsd) > 0 || route.kind === 'local') target.costKnownTurns += number(source.turns);
-          target.costBilled += number(source.costBilled);
-          target.costEstimated += number(source.costEstimated ?? source.costUsd);
-          target.sessionsComplete = false;
-        };
-        merge(day, route);
-        merge(day.conversation, route.conversation);
-        day.importedPartial = true;
-        if (legacy.restored) day.restored = true;
-      }
+    return pending;
+  }
+
+  /** Drop cached and in-flight rollups once any connection has committed. */
+  syncRollupStamp() {
+    const { local, shared } = this.changeStamp.get();
+    const stamp = `${local}:${shared}`;
+    if (this.rollupStamp !== stamp) {
+      this.rollups.clear();
+      this.pendingRollups.clear();
+      this.rollupStamp = stamp;
     }
-    return hourly ? { days, hourly } : { days };
+    return stamp;
+  }
+
+  rememberRollup(key, result) {
+    this.rollups.delete(key);
+    this.rollups.set(key, result);
+    // Rolling windows (the hour view) change key on every call; keep the
+    // most recently used queries only.
+    if (this.rollups.size > ROLLUP_CACHE_LIMIT) this.rollups.delete(this.rollups.keys().next().value);
   }
 }
 

package/src/runtime/shared/llm/usage-log.mjs

@@ -1,14 +1,21 @@
 /**
- * Unified LLM usage logger.
- *
- * Phase 2: Routes via appendAgentTrace (HTTP buffer to memory-service)
- * instead of writing directly to the local trace jsonl.
- *
- * Signature unchanged — callers are unaffected.
+ * Unified LLM usage logger. Routes via appendAgentTrace (HTTP buffer to
+ * memory-service) rather than writing the local trace jsonl directly.
  */
 
 import { appendAgentTrace } from '../../agent/orchestrator/agent-trace.mjs';
 
+const _missingProviderWarned = new Set();
+function warnMissingProviderOnce(key) {
+  if (_missingProviderWarned.has(key)) return;
+  _missingProviderWarned.add(key);
+  try {
+    process.stderr.write(`[usage-log] provider missing on usage entry (model=${key}). audit the caller.\n`);
+  } catch {
+    /* logging only */
+  }
+}
+
 /**
  * Append a usage entry to the trace store.
  *
@@ -22,17 +29,6 @@ import { appendAgentTrace } from '../../agent/orchestrator/agent-trace.mjs';
  *   inputTokens, outputTokens, cacheReadTokens, cacheWriteTokens,
  *   prefixHash, costUsd
  */
-const _missingProviderWarned = new Set();
-function warnMissingProviderOnce(key) {
-  if (_missingProviderWarned.has(key)) return;
-  _missingProviderWarned.add(key);
-  try {
-    process.stderr.write(`[usage-log] provider missing on usage entry (model=${key}). audit the caller.\n`);
-  } catch {
-    /* logging only */
-  }
-}
-
 export function logLlmCall(entry, opts = {}) {
   try {
     if (!entry.provider) warnMissingProviderOnce(entry.model || '?');