mixdog 0.9.174 → 1.0.0

package/src/vendor/statusline/src/gateway/route-meta.mjs

@@ -1,8 +1,12 @@
-import { existsSync, readFileSync } from 'fs';
-import { join } from 'path';
+import {
+  appendFileSync, closeSync, existsSync, mkdirSync, openSync, readFileSync, readSync,
+  renameSync, statSync, fstatSync, writeFileSync,
+} from 'fs';
+import { appendFile, mkdir, readFile, rename, writeFile } from 'fs/promises';
+import { dirname, join } from 'path';
 import { resolvePluginData } from '../../../../runtime/shared/plugin-paths.mjs';
 import { readSection } from '../../../../runtime/shared/config.mjs';
-import { updateJsonAtomic, updateJsonAtomicSync } from '../../../../runtime/shared/atomic-file.mjs';
+import { withFileLock, withFileLockSync } from '../../../../runtime/shared/file-lock.mjs';
 import { computeCostUsd, isInclusiveProvider } from '../../../../runtime/shared/llm/cost.mjs';
 import { getModelMetadataSync } from '../../../../runtime/agent/orchestrator/providers/model-catalog.mjs';
 import {
@@ -11,7 +15,12 @@ import {
 } from '../../../../runtime/agent/orchestrator/session/context-utils.mjs';
 import { CLAUDE_CURRENT_MODE } from './claude-current.mjs';
 
-const GATEWAY_USAGE_FILE = 'gateway-usage.local.json';
+// Legacy store: one JSON document rewritten on every flush. Current store: an
+// append-only JSONL (one event per line) compacted off the hot path.
+const LEGACY_USAGE_FILE = 'gateway-usage.local.json';
+const GATEWAY_USAGE_FILE = 'gateway-usage.local.jsonl';
+const LEGACY_BACKUP_SUFFIX = '.migrated.bak';
+const COMPACT_MIN_BYTES = 8 * 1024 * 1024;
 // Background runners push several hundred turns a day through here, so a
 // thousand rows covered barely a day of history. The daily rollup carries the
 // long range, but the events are what a fresh range request re-reads in full
@@ -469,11 +478,16 @@ function promptFootprintTokens(provider, usage) {
   return isInclusiveProvider(provider) ? input : input + cacheRead + cacheWrite;
 }
 
-function usageCostUsd(routeInfo, usage) {
+function usageCostUsd(routeInfo, providerOut) {
+  const usage = providerOut?.usage || {};
   if (Number.isFinite(Number(usage?.costUsd))) return round(Number(usage.costUsd), 6);
   return computeCostUsd({
     provider: routeInfo?.provider,
     model: routeInfo?.model,
+    pricingModel: providerOut?.pricingModel,
+    kind: routeInfo?.providerKind || providerKind(routeInfo?.provider),
+    requestServiceTier: providerOut?.requestServiceTier,
+    serviceTier: cleanString(providerOut?.serviceTier) || cleanString(usage?.raw?.service_tier) || undefined,
     inputTokens: num(usage?.inputTokens, 0),
     outputTokens: num(usage?.outputTokens, 0),
     cacheReadTokens: num(usage?.cachedTokens ?? usage?.cacheReadTokens, 0),
@@ -507,7 +521,7 @@ export function summarizeGatewayUsage(routeInfo, providerOut, compact = null, du
       contextUsedPct = Math.min(100, round(promptTokens * 100 / boundaryTokens, 2));
     }
   }
-  const costUsd = usageCostUsd(routeInfo, u);
+  const costUsd = usageCostUsd(routeInfo, providerOut);
   const usageCompact = compact && typeof compact === 'object' ? { ...compact } : compact;
   return {
     at: Date.now(),
@@ -536,9 +550,123 @@ function usageStorePath() {
   return join(resolvePluginData(), GATEWAY_USAGE_FILE);
 }
 
-const USAGE_FLUSH_OPTS = Object.freeze({ compact: true, fsync: false, fsyncDir: false });
+function usageLockPath(file) {
+  return `${file}.lock`;
+}
+
+function parseJsonlEvents(text) {
+  const out = [];
+  for (const line of text.split('\n')) {
+    if (!line.trim()) continue;
+    try {
+      const e = JSON.parse(line);
+      if (e && typeof e === 'object') out.push(e);
+    } catch {
+      // A torn append (crash mid-write) is skipped, never fatal.
+    }
+  }
+  return out;
+}
+
+// Idempotent, crash-safe legacy JSON -> JSONL migration, once per path per
+// process. Serialized by the store lock; each step is an atomic rename:
+//   1. events -> <store>.migrating.<pid>, renamed onto the JSONL store
+//   2. legacy file renamed to a backup
+// A crash between the steps leaves both files; the next run sees the JSONL
+// store already present and only finishes step 2. Appenders always migrate
+// first, so an existing JSONL store is never older than the legacy file.
+const migratedPaths = new Set();
+function ensureUsageStoreMigrated() {
+  const store = usageStorePath();
+  if (migratedPaths.has(store)) return;
+  const legacy = join(dirname(store), LEGACY_USAGE_FILE);
+  if (existsSync(legacy)) {
+    try {
+      withFileLockSync(usageLockPath(store), () => {
+        if (!existsSync(legacy)) return;
+        if (!existsSync(store)) {
+          const raw = readJsonFile(legacy);
+          const events = (Array.isArray(raw?.events) ? raw.events : []).slice(-MAX_USAGE_EVENTS);
+          const tmp = `${store}.migrating.${process.pid}`;
+          writeFileSync(tmp, events.map(e => JSON.stringify(e) + '\n').join(''));
+          renameSync(tmp, store);
+        }
+        let backup = legacy + LEGACY_BACKUP_SUFFIX;
+        if (existsSync(backup)) backup = `${legacy}.${Date.now()}${LEGACY_BACKUP_SUFFIX}`;
+        renameSync(legacy, backup);
+      });
+    } catch {
+      return; // retry on the next call; readers fall back to whatever exists
+    }
+  }
+  migratedPaths.add(store);
+}
+
+// Incrementally updated in-memory view of the JSONL store. A refresh stats the
+// file and reads only the bytes appended since the last refresh; a replaced
+// (compacted) or truncated file triggers one full reload.
+let usageView = { path: null, ino: null, offset: 0, events: [] };
+
+function refreshUsageView() {
+  const path = usageStorePath();
+  let fd;
+  try {
+    fd = openSync(path, 'r');
+    const st = fstatSync(fd);
+    let view = usageView;
+    if (view.path !== path || view.ino !== st.ino || st.size < view.offset) {
+      view = usageView = { path, ino: st.ino, offset: 0, events: [] };
+    }
+    if (st.size > view.offset) {
+      const buf = Buffer.allocUnsafe(st.size - view.offset);
+      const n = readSync(fd, buf, 0, buf.length, view.offset);
+      const end = buf.subarray(0, n).lastIndexOf(0x0a) + 1; // complete lines only
+      if (end > 0) {
+        view.offset += end;
+        for (const e of parseJsonlEvents(buf.toString('utf8', 0, end))) view.events.push(e);
+        if (view.events.length > MAX_USAGE_EVENTS * 2) view.events = view.events.slice(-MAX_USAGE_EVENTS);
+      }
+    }
+    return usageView.events;
+  } catch {
+    usageView = { path: null, ino: null, offset: 0, events: [] };
+    return usageView.events;
+  } finally {
+    if (fd !== undefined) closeSync(fd);
+  }
+}
+
+// Bounded compaction: keep the newest MAX_USAGE_EVENTS lines. Runs in the
+// background under the store lock (appenders take it too, so no append is lost).
+let compactRunning = false;
+async function compactUsageStore(store) {
+  if (compactRunning) return;
+  compactRunning = true;
+  try {
+    await withFileLock(usageLockPath(store), async () => {
+      const lines = (await readFile(store, 'utf8')).split('\n').filter(l => l.trim());
+      if (lines.length <= MAX_USAGE_EVENTS * 1.5) return;
+      const tmp = `${store}.compact.${process.pid}`;
+      await writeFile(tmp, lines.slice(-MAX_USAGE_EVENTS).join('\n') + '\n');
+      await rename(tmp, store);
+    });
+  } catch {
+    // Compaction is best effort; the next flush retries.
+  } finally {
+    compactRunning = false;
+  }
+}
+
+function maybeScheduleCompaction(store) {
+  try {
+    if (statSync(store).size < COMPACT_MIN_BYTES) return;
+  } catch {
+    return;
+  }
+  const t = setTimeout(() => { void compactUsageStore(store); }, 1000);
+  t.unref?.();
+}
 
-// Take the pending batch and build its read-modify-write mutator.
 function takePendingUsageBatch() {
   if (usageFlushTimer) {
     clearTimeout(usageFlushTimer);
@@ -547,25 +675,22 @@ function takePendingUsageBatch() {
   if (!pendingUsageEvents.length) return null;
   const eventsToWrite = pendingUsageEvents;
   pendingUsageEvents = [];
-  const cutoff = Date.now() - USAGE_EVENT_TTL_MS;
-  return (curRaw) => {
-    const cur = curRaw && typeof curRaw === 'object' ? curRaw : {};
-    const events = Array.isArray(cur.events) ? cur.events : [];
-    const kept = events
-      .filter(e => num(e?.ts, 0) >= cutoff)
-      .slice(-MAX_USAGE_EVENTS + eventsToWrite.length);
-    kept.push(...eventsToWrite);
-    return { version: 1, updatedAt: Date.now(), events: kept.slice(-MAX_USAGE_EVENTS) };
-  };
+  // Leading newline terminates any torn line left by a crashed writer.
+  return '\n' + eventsToWrite.map(e => JSON.stringify(e)).join('\n') + '\n';
 }
 
-// Normal flushes run on the event loop's schedule but never block it: the
-// cross-process lock wait and the file I/O are asynchronous. Batches keep
-// their order because withFileLock queues same-path callers in call order.
+// Normal flushes never block the loop: the lock wait and the append are
+// asynchronous, and only the new events are serialized. Batches keep their
+// order because withFileLock queues same-path callers in call order.
 function flushGatewayUsageEvents() {
-  const mutate = takePendingUsageBatch();
-  if (!mutate) return usageFlushTail;
-  const flush = updateJsonAtomic(usageStorePath(), mutate, USAGE_FLUSH_OPTS).catch(() => {
+  const data = takePendingUsageBatch();
+  if (!data) return usageFlushTail;
+  ensureUsageStoreMigrated();
+  const store = usageStorePath();
+  const flush = withFileLock(usageLockPath(store), async () => {
+    await mkdir(dirname(store), { recursive: true });
+    await appendFile(store, data);
+  }).then(() => maybeScheduleCompaction(store)).catch(() => {
     // Local telemetry must never affect the routed model call.
   });
   usageFlushTail = Promise.all([usageFlushTail, flush]).then(() => {});
@@ -574,10 +699,13 @@ function flushGatewayUsageEvents() {
 
 // Process exit cannot await, so the last batch is written synchronously.
 function flushGatewayUsageEventsSync() {
-  const mutate = takePendingUsageBatch();
-  if (!mutate) return;
+  const data = takePendingUsageBatch();
+  if (!data) return;
   try {
-    updateJsonAtomicSync(usageStorePath(), mutate, USAGE_FLUSH_OPTS);
+    ensureUsageStoreMigrated();
+    const store = usageStorePath();
+    mkdirSync(dirname(store), { recursive: true });
+    withFileLockSync(usageLockPath(store), () => appendFileSync(store, data));
   } catch {
     // Local telemetry must never affect the routed model call.
   }
@@ -692,8 +820,8 @@ export function recordGatewayUsageEvent(summary) {
 }
 
 export function loadUsageEvents() {
-  const raw = readJsonFile(usageStorePath());
-  const events = Array.isArray(raw?.events) ? raw.events : [];
+  ensureUsageStoreMigrated();
+  const events = refreshUsageView().slice(-MAX_USAGE_EVENTS);
   const cutoff = Date.now() - USAGE_EVENT_TTL_MS;
   return events.filter(e => num(e?.ts, 0) >= cutoff);
 }

package/scripts/fixtures/spawn-lease-child-harness.mjs

@@ -1,8 +0,0 @@
-// Child-side gate harness: runs child-spawn-gate acquire() in shard mode so a
-// forked test parent can play the pool's role in the lease protocol.
-const { acquire } = await import(new URL('../../src/runtime/shared/child-spawn-gate.mjs', import.meta.url));
-const release = await acquire(null, 'search', { ownerKey: 'harness' });
-process.send({ type: 'harness-event', event: 'granted' });
-release();
-process.send({ type: 'harness-event', event: 'released' });
-setTimeout(() => process.exit(0), 200);

package/scripts/fixtures/spawn-lease-fixture-worker.mjs

@@ -1,37 +0,0 @@
-// Minimal shard stand-in for the machine spawn-budget protocol test. Replies
-// ok to any requestId message (prewarm/shutdown) and drives the lease
-// protocol on a fixed clock; grants are recorded as marker files so the test
-// can observe ordering without extra IPC surface.
-import { writeFileSync } from 'node:fs';
-import { join } from 'node:path';
-
-const outDir = process.env.SPAWN_LEASE_FIXTURE_DIR || process.cwd();
-const record = (name) => {
-  try {
-    writeFileSync(join(outDir, name), String(Date.now()));
-  } catch {
-    /* test observes absence */
-  }
-};
-
-process.on('message', (message) => {
-  if (!message || typeof message !== 'object') return;
-  if (message.type === 'spawn-lease-result') {
-    record(`${message.leaseId}-${message.ok === true ? 'granted' : 'rejected'}`);
-    return;
-  }
-  if (message.requestId) {
-    process.send({ type: 'response', requestId: message.requestId, ok: true, value: { ready: true } });
-    if (message.type === 'shutdown') setImmediate(() => process.exit(0));
-  }
-});
-
-process.send({ type: 'spawn-lease', leaseId: 'lease-a', lane: 'search', ownerKey: 'fixture-a' });
-setTimeout(() => {
-  process.send({ type: 'spawn-lease', leaseId: 'lease-b', lane: 'search', ownerKey: 'fixture-b' });
-}, 150);
-// Release A well after the test has observed that B stays queued behind the
-// machine cap of 1.
-setTimeout(() => {
-  process.send({ type: 'spawn-release', leaseId: 'lease-a' });
-}, 1_800);

package/src/runtime/agent/orchestrator/activity-bus.mjs

@@ -1,31 +0,0 @@
-/**
- * activity-bus — tiny leaf module so orchestrator-layer code can signal
- * "session is actively doing something" without importing the channels
- * Scheduler instance (which would create a module cycle).
- *
- * channels/index.mjs registers a listener at boot that forwards into
- * scheduler.noteActivity(). Producers signal activity via setListener's
- * forwarded pings.
- *
- * Boot race: a ping that fires before channels boot registers the listener
- * is buffered (most-recent wins) and replayed on setListener. Only one
- * timestamp is buffered — "most recent activity was at T" is all
- * scheduler.mjs needs. All failures are swallowed — a ping is never
- * load-bearing.
- */
-
-let _listener = null;
-let _pendingPingAt = null;
-
-export function setListener(fn) {
-  _listener = typeof fn === 'function' ? fn : null;
-  if (_listener && _pendingPingAt != null) {
-    const ts = _pendingPingAt;
-    _pendingPingAt = null;
-    try {
-      _listener(ts);
-    } catch {
-      /* best-effort */
-    }
-  }
-}

package/src/runtime/shared/child-spawn-remote.mjs

@@ -1,210 +0,0 @@
-// Runtime-worker client for the MACHINE-WIDE child-spawn budget.
-//
-// child-spawn-gate's semaphore was written for the single-process daemon; the
-// session-runtime split creates another process-local copy, which would
-// multiply the intended machine-wide cap. In the runtime worker the gate
-// therefore forwards each
-// acquire as a lease request over the existing runtime IPC channel; the
-// daemon-side host grants leases from its OWN child-spawn-gate instance,
-// which becomes the single machine-wide authority (daemon-hosted maintenance
-// agents already use that instance locally). Transport loss degrades to the
-// local per-process lane — bounded, never unbounded.
-import { safeIpcSend } from './safe-ipc-send.mjs';
-
-const _pending = new Map(); // leaseId -> { grant, fail }
-let _sequence = 0;
-let _listening = false;
-// "Lost" means the parent IPC link is actually gone — never merely slow. A
-// single timed-out or failed lease must NOT latch this process onto the
-// process-local lane: with several runtime shards that silently multiplies the
-// machine-wide child-spawn cap, which is exactly the Windows spawn storm this
-// gate exists to prevent.
-let _channelLost = false;
-// Bounded patience for a busy/recovering daemon. Each expiry re-arms while the
-// link is alive, so IPC latency keeps shards on the machine-wide budget.
-const LEASE_WAIT_EXTENSIONS = Math.max(0, Math.floor(Number(process.env.MIXDOG_SPAWN_LEASE_WAIT_EXTENSIONS) || 3));
-const LEASE_WARN_THROTTLE_MS = 30_000;
-let _lastLeaseWarnAt = 0;
-
-function _warnSlowLease(budgetMs, extensionsLeft) {
-  const now = Date.now();
-  if (now - _lastLeaseWarnAt < LEASE_WARN_THROTTLE_MS) return;
-  _lastLeaseWarnAt = now;
-  try {
-    process.stderr.write(
-      `[child-spawn-remote] machine spawn lease pending >${budgetMs}ms; staying on the daemon budget` +
-        ` (${extensionsLeft} extension(s) left)\n`
-    );
-  } catch {
-    /* diagnostics only */
-  }
-}
-
-function _fallbackError(reason) {
-  const error = new Error(`machine spawn budget unavailable (${reason}); using the local lane`);
-  error.code = 'ELEASEFALLBACK';
-  return error;
-}
-
-function _leaseTimeoutError(reason) {
-  const error = new Error(`machine spawn budget did not answer (${reason}); spawn was not started`);
-  error.code = 'ESPAWNLEASETIMEOUT';
-  return error;
-}
-
-function _abortError(signal) {
-  return signal.reason ?? Object.assign(new Error('child-spawn-gate: aborted while queued'), { code: 'ABORT_ERR' });
-}
-
-function _ensureListener() {
-  if (_listening) return;
-  _listening = true;
-  process.on('message', (message) => {
-    if (!message || typeof message !== 'object' || message.type !== 'spawn-lease-result') return;
-    const pending = _pending.get(String(message.leaseId || ''));
-    if (!pending) return;
-    _pending.delete(String(message.leaseId || ''));
-    if (message.ok === true) {
-      pending.grant();
-      return;
-    }
-    const error = new Error(String(message.error || 'spawn lease rejected'));
-    if (message.code) error.code = String(message.code);
-    if (message.statusCode) error.statusCode = Number(message.statusCode);
-    pending.fail(error);
-  });
-  process.on('disconnect', () => {
-    _channelLost = true;
-    for (const [leaseId, pending] of [..._pending]) {
-      _pending.delete(leaseId);
-      pending.fail(_fallbackError('pool channel disconnected'));
-    }
-  });
-}
-
-/**
- * PID-accurate shard detection. MIXDOG_SESSION_RUNTIME_WORKER is inherited by
- * every grandchild (tools, helpers, spawned nodes), so the plain flag cannot
- * decide who owns a runtime IPC channel: only the process whose OWN pid matches
- * the pid-scoped marker is a runtime shard. A grandchild therefore uses its own
- * bounded local lane, and a real shard is never misread into it.
- */
-export function isSessionRuntimeWorkerProcess(env = process.env, pid = process.pid) {
-  return String(env.MIXDOG_SESSION_RUNTIME_WORKER_PID || '') === String(pid);
-}
-
-export function remoteSpawnLeasesEnabled(env = process.env) {
-  return (
-    !_channelLost &&
-    isSessionRuntimeWorkerProcess(env) &&
-    env.MIXDOG_DISABLE_MACHINE_SPAWN_BUDGET !== '1' &&
-    typeof process.send === 'function' &&
-    process.connected === true
-  );
-}
-
-/**
- * Request one machine-wide spawn lease from the pool. Resolves to a release
- * function; rejects with code ELEASEFALLBACK when the pool channel cannot
- * answer (the caller then uses its bounded local lane), or with the pool's
- * real admission error (wait timeout / queue full) which must surface as-is.
- */
-export function acquireRemoteSpawnLease({ lane, ownerKey, signal, waitTimeoutMs }) {
-  _ensureListener();
-  if (signal?.aborted) return Promise.reject(_abortError(signal));
-  const leaseId = `lease-${process.pid}-${++_sequence}`;
-  return new Promise((resolve, reject) => {
-    let settled = false;
-    let safety = null;
-    let onAbort = null;
-    const cleanup = () => {
-      if (safety) {
-        clearTimeout(safety);
-        safety = null;
-      }
-      if (onAbort && signal) {
-        try {
-          signal.removeEventListener('abort', onAbort);
-        } catch {
-          /* teardown */
-        }
-        onAbort = null;
-      }
-      _pending.delete(leaseId);
-    };
-    const sendRelease = () => {
-      safeIpcSend(process, { type: 'spawn-release', leaseId }, { onError: () => {} });
-    };
-    const fail = (error) => {
-      if (settled) return;
-      settled = true;
-      cleanup();
-      reject(error);
-    };
-    const grant = () => {
-      if (settled) {
-        // Raced an abort/fallback: the slot is held by nobody — return it.
-        sendRelease();
-        return;
-      }
-      settled = true;
-      cleanup();
-      let released = false;
-      resolve(() => {
-        if (released) return;
-        released = true;
-        sendRelease();
-      });
-    };
-    _pending.set(leaseId, { grant, fail });
-    if (signal) {
-      onAbort = () => {
-        // Cancels a queued lease (the pool aborts its waiter) or returns a
-        // racing grant; either way the pool-side record settles exactly once.
-        sendRelease();
-        fail(_abortError(signal));
-      };
-      signal.addEventListener('abort', onAbort, { once: true });
-    }
-    // The pool enforces the real waitTimeoutMs; this guard only covers a
-    // parent that never answers (bug/kill) so a spawn can never hang forever.
-    const budget = Math.max(1_000, Math.floor(Number(waitTimeoutMs) || 0)) + 5_000;
-    let extensionsLeft = LEASE_WAIT_EXTENSIONS;
-    const armSafety = () => {
-      safety = setTimeout(() => {
-        // A busy or recovering daemon is latency, not a broken budget
-        // authority: while the link is alive, keep waiting instead of dropping
-        // to a per-process lane that multiplies the machine-wide cap.
-        if (extensionsLeft > 0 && !_channelLost && process.connected === true) {
-          extensionsLeft -= 1;
-          _warnSlowLease(budget, extensionsLeft);
-          armSafety();
-          return;
-        }
-        // A still-connected authority may be wedged, but bypassing it would
-        // multiply the machine cap by the shard count. Fail this spawn instead;
-        // only an actually lost IPC channel is allowed to use the bounded
-        // process-local fallback lane.
-        fail(
-          process.connected === true && !_channelLost
-            ? _leaseTimeoutError(`no lease response in ${budget}ms`)
-            : _fallbackError(`no lease response in ${budget}ms`)
-        );
-      }, budget);
-      safety.unref?.();
-    };
-    armSafety();
-    const sent = safeIpcSend(
-      process,
-      {
-        type: 'spawn-lease',
-        leaseId,
-        lane,
-        ownerKey: String(ownerKey || ''),
-        waitTimeoutMs,
-      },
-      { onError: () => fail(_fallbackError('pool channel send failed')) }
-    );
-    if (!sent) fail(_fallbackError('pool channel unavailable'));
-  });
-}