mixdog 1.2.6 → 1.3.1

package/src/runtime/shared/usage-readout-totals.mjs

@@ -0,0 +1,58 @@
+// Session-cumulative figures behind the composer's usage readout, kept on the
+// session record (`session.usageReadout`) so they survive restart, reload and
+// compaction. One fold per model call of the session itself; calls of other
+// sessions (delegated sub-agents have their own records) never enter.
+//
+// - cache hit = cacheRead / (input + cacheRead + cacheWrite) over every call
+//   whose prompt was measured. A call that could not measure its prompt
+//   (promptKnown === false) has unknown zeros, so it adds nothing.
+// - speed = output tokens over request ms (provider request start → end of the
+//   response, the wait for the first token included: the same wall time as the
+//   usage ledger's `duration_ms`) of every call with positive output and
+//   positive duration: the same eligibility as the usage ledger and the usage
+//   dialog, so the readout and the dialog agree. Records saved before this
+//   definition carry `speedOutputTokens` / `speedMs` (first visible token →
+//   end); they are kept as they are and read only while no request-timed call
+//   has landed.
+// - total tokens = input + cacheRead + cacheWrite + output, from the same sums.
+
+function num(value) {
+  const n = Number(value);
+  return Number.isFinite(n) && n > 0 ? n : 0;
+}
+
+/** The zeroed totals record. */
+export function emptyUsageReadout() {
+  return { input: 0, cacheRead: 0, cacheWrite: 0, output: 0, requestOutputTokens: 0, requestMs: 0 };
+}
+
+/**
+ * Fold one `onUsageDelta` event (and the call's request ms, 0 when it was
+ * not timed) into the totals; returns a new record, never mutates `totals`.
+ * Providers differ on whether input counts cache; the prompt total does not,
+ * so fresh input is the prompt less its cache read/write.
+ */
+export function foldUsageReadout(totals, delta = {}, requestMs = 0) {
+  const base = totals || emptyUsageReadout();
+  const next = { ...base };
+  if (delta.promptKnown !== false) {
+    const cacheRead = num(delta.deltaCachedRead);
+    const cacheWrite = num(delta.deltaCacheWrite);
+    const prompt = num(delta.deltaPrompt) || num(delta.deltaInput) + cacheRead + cacheWrite;
+    next.input = num(base.input) + Math.max(0, prompt - cacheRead - cacheWrite);
+    next.cacheRead = num(base.cacheRead) + cacheRead;
+    next.cacheWrite = num(base.cacheWrite) + cacheWrite;
+  }
+  const output = num(delta.deltaOutput);
+  // Every call's output enters the session's total tokens, timed or not. A
+  // record saved before the field existed has no output sum to extend, and a
+  // partial one would pass for the whole: it stays without until the session
+  // starts over.
+  if (Number.isFinite(base.output)) next.output = base.output + output;
+  const ms = num(requestMs);
+  if (ms > 0 && output > 0) {
+    next.requestOutputTokens = num(base.requestOutputTokens) + output;
+    next.requestMs = num(base.requestMs) + ms;
+  }
+  return next;
+}

package/src/runtime/shared/user-data-guard.mjs

@@ -61,25 +61,92 @@ function scopeDirFor(dataDir) {
 let lastBackupMs = 0;
 let backupSeq = 0;
 
-// Claims `<iso-ms>-<10-digit seq>-<reason>` with an exclusive mkdir, so names
-// sort by timestamp, then sequence, and an existing backup is never reused
-// (EEXIST retries with the next sequence). The clamp keeps a backwards clock
-// from reordering this process's backups.
+// `<iso-ms>-<10-digit seq>-<reason>`; an older build's name has no sequence.
+const BACKUP_NAME_RE = /^(\d{4}-\d{2}-\d{2})T(\d{2})-(\d{2})-(\d{2})-(\d{3})Z-(?:(\d{10})-)?/;
+
+// The newest backup order among `names` (time, then sequence). An older
+// build's name sorts after every sequenced name of its millisecond, so its
+// sequence counts as unbounded. Names that are no backup's are ignored.
+function newestBackupOrder(names) {
+  let ms = 0;
+  let seq = -1;
+  for (const name of names) {
+    const match = BACKUP_NAME_RE.exec(name);
+    if (!match) continue;
+    const time = Date.parse(`${match[1]}T${match[2]}:${match[3]}:${match[4]}.${match[5]}Z`);
+    if (!Number.isFinite(time)) continue;
+    const order = match[6] === undefined ? Number.POSITIVE_INFINITY : Number(match[6]);
+    if (time > ms || (time === ms && order > seq)) {
+      ms = time;
+      seq = order;
+    }
+  }
+  return { ms, seq };
+}
+
+// A name that sorts after every backup in `names`, whichever process wrote
+// them and whatever its clock said: the backup a restore picks as the newest
+// is then the one written last, and pruning never takes it.
+function nextBackupName(names, reason) {
+  const newest = newestBackupOrder(names);
+  lastBackupMs = Math.max(Date.now(), lastBackupMs, newest.ms);
+  if (lastBackupMs === newest.ms) {
+    if (newest.seq === Number.POSITIVE_INFINITY) lastBackupMs += 1;
+    else backupSeq = Math.max(backupSeq, newest.seq + 1);
+  }
+  const time = new Date(lastBackupMs).toISOString().replace(/[:.]/g, '-');
+  return `${time}-${String(backupSeq++).padStart(10, '0')}-${safeReason(reason)}`;
+}
+
+// The scope's backup directory names. A reservation lists the scope first and
+// a listing failure throws: an order guessed without it could sort before the
+// newest backup.
+function scopeBackupNames(entries) {
+  return entries.filter((entry) => entry.isDirectory()).map((entry) => entry.name);
+}
+
+// Claims the next name with an exclusive mkdir (EEXIST retries after it).
+// Returns the claimed dir and the scope's names including it, which pruning
+// reuses instead of listing the scope again.
 function reserveBackupDir(scopeDir, reason) {
   mkdirSync(scopeDir, { recursive: true });
+  const names = scopeBackupNames(readdirSync(scopeDir, { withFileTypes: true }));
+  for (;;) {
+    const name = nextBackupName(names, reason);
+    names.push(name);
+    try {
+      mkdirSync(join(scopeDir, name));
+      return { backupDir: join(scopeDir, name), names };
+    } catch (error) {
+      if (error?.code !== 'EEXIST') throw error;
+    }
+  }
+}
+
+async function reserveBackupDirAsync(scopeDir, reason) {
+  await mkdirP(scopeDir, { recursive: true });
+  const names = scopeBackupNames(await readdirP(scopeDir, { withFileTypes: true }));
   for (;;) {
-    lastBackupMs = Math.max(Date.now(), lastBackupMs);
-    const time = new Date(lastBackupMs).toISOString().replace(/[:.]/g, '-');
-    const backupDir = join(scopeDir, `${time}-${String(backupSeq++).padStart(10, '0')}-${safeReason(reason)}`);
+    const name = nextBackupName(names, reason);
+    names.push(name);
     try {
-      mkdirSync(backupDir);
-      return backupDir;
+      await mkdirP(join(scopeDir, name));
+      return { backupDir: join(scopeDir, name), names };
     } catch (error) {
       if (error?.code !== 'EEXIST') throw error;
     }
   }
 }
 
+// The names past the newest `keep`, never the just-written backup.
+function backupsToPrune(names, own, keep = 40) {
+  return [...names]
+    .sort()
+    .reverse()
+    .slice(keep)
+    .filter((name) => name !== own);
+}
+
 function safeReason(reason) {
   return (
     String(reason || 'snapshot')
@@ -197,14 +264,30 @@ function isStructurallyCompleteMixdogConfigBackup(parsed) {
   return true;
 }
 
-/** Backup directory names in `dir`, newest first; empty when unreadable. */
-function backupDirsNewestFirst(dir) {
+// A backup that is absent, not a backup, or not JSON is no backup; any other
+// failure (EIO, EACCES, EMFILE…) is transient and must not pass for one.
+function isAbsentBackupError(error) {
+  return error instanceof SyntaxError || error?.code === 'ENOENT' || error?.code === 'ENOTDIR';
+}
+
+/** Backup directory names in `dir`, newest first; empty when absent, throws on other failures. */
+function listBackupDirsNewestFirst(dir) {
   try {
     return readdirSync(dir, { withFileTypes: true })
       .filter((entry) => entry.isDirectory())
       .map((entry) => entry.name)
       .sort()
       .reverse();
+  } catch (error) {
+    if (isAbsentBackupError(error)) return [];
+    throw error;
+  }
+}
+
+/** Backup directory names in `dir`, newest first; empty when unreadable. */
+function backupDirsNewestFirst(dir) {
+  try {
+    return listBackupDirsNewestFirst(dir);
   } catch {
     return [];
   }
@@ -213,7 +296,8 @@ function backupDirsNewestFirst(dir) {
 /**
  * A backup is restorable only when its manifest names this data dir and this
  * incarnation of it. Legacy unscoped backups carry no manifest, so they stay
- * on disk but never match.
+ * on disk but never match. A transient read failure throws: skipping the
+ * backup would restore an older one instead.
  */
 function readRestorableConfig(backupDir, dataDirKey, instanceId) {
   try {
@@ -222,20 +306,21 @@ function readRestorableConfig(backupDir, dataDirKey, instanceId) {
     if (manifest.instanceId !== instanceId) return null;
     const parsed = JSON.parse(readFileSync(join(backupDir, 'mixdog-config.json'), 'utf8'));
     return isStructurallyCompleteMixdogConfigBackup(parsed) ? parsed : null;
-  } catch {
-    return null;
+  } catch (error) {
+    if (isAbsentBackupError(error)) return null;
+    throw error;
   }
 }
 
 function findRestorableConfig(dataDir, instanceId) {
   const key = normalizeDataDir(dataDir);
   const scopeDir = scopeDirFor(dataDir);
-  for (const name of backupDirsNewestFirst(scopeDir)) {
+  for (const name of listBackupDirsNewestFirst(scopeDir)) {
     const parsed = readRestorableConfig(join(scopeDir, name), key, instanceId);
     if (parsed) return parsed;
   }
   const root = getBackupRoot();
-  for (const name of backupDirsNewestFirst(root)) {
+  for (const name of listBackupDirsNewestFirst(root)) {
     if (name.startsWith(SCOPE_PREFIX)) continue;
     const parsed = readRestorableConfig(join(root, name), key, instanceId);
     if (parsed) return parsed;
@@ -245,13 +330,15 @@ function findRestorableConfig(dataDir, instanceId) {
 
 // Per-process restore lookup, keyed by data dir. A hit is reused while the
 // backup root and the marker's instanceId are unchanged; this process's own
-// backups invalidate it.
+// backups invalidate it, and a `fresh` lookup (an RMW write, which another
+// process's newer backup may precede) always rescans.
 const restoreLookupCache = new Map();
 
 /**
  * Newest structurally complete mixdog-config.json backed up from THIS data
  * dir (skips degenerate single-section snapshots from a prior failed RMW).
  * Never another data dir's config, and nothing for an uninitialized one.
+ * Throws on a transient read failure, which is never cached.
  */
 export function loadLatestMixdogConfigFromBackup(dataDir, options) {
   const instanceId = readInitInstanceId(dataDir);
@@ -259,7 +346,8 @@ export function loadLatestMixdogConfigFromBackup(dataDir, options) {
   const key = normalizeDataDir(dataDir);
   const root = getBackupRoot();
   let hit = restoreLookupCache.get(key);
-  if (!hit || hit.root !== root || hit.instanceId !== instanceId) {
+  if (options?.fresh || !hit || hit.root !== root || hit.instanceId !== instanceId) {
+    restoreLookupCache.delete(key);
     hit = { root, instanceId, config: findRestorableConfig(dataDir, instanceId) };
     restoreLookupCache.set(key, hit);
   }
@@ -297,8 +385,8 @@ function copyTree(src, dst, copied) {
 }
 
 /** Prune one data dir's scope only; other scopes and legacy entries stay. */
-function pruneBackups(scopeDir, keep = 40) {
-  for (const name of backupDirsNewestFirst(scopeDir).slice(keep)) {
+function pruneBackups(scopeDir, names, own) {
+  for (const name of backupsToPrune(names, own)) {
     try {
       rmSync(join(scopeDir, name), { recursive: true, force: true });
     } catch {}
@@ -364,23 +452,26 @@ function finishBackup(dataDir, backupDir, reason) {
   legacyEvidenceCache.delete(normalizeDataDir(dataDir));
 }
 
-export function backupUserData(dataDir, reason = 'snapshot') {
+/** `dirs`: more data-dir folders this one snapshot also carries (a one-time
+ *  migration's, e.g. agents/ and bots/); routine snapshots keep to
+ *  USER_DATA_DIRS so the ring stays small. */
+export function backupUserData(dataDir, reason = 'snapshot', { dirs = [] } = {}) {
   if (skipBackups()) return { dir: null, copied: [] };
   if (!dataDir || !existsSync(dataDir)) return { dir: null, copied: [] };
   const scopeDir = scopeDirFor(dataDir);
-  const backupDir = reserveBackupDir(scopeDir, reason);
+  const { backupDir, names } = reserveBackupDir(scopeDir, reason);
   const copied = [];
   for (const rel of USER_DATA_FILES) {
     const src = join(dataDir, rel);
     if (existsSync(src)) copyTree(src, join(backupDir, rel), copied);
   }
-  for (const rel of USER_DATA_DIRS) {
+  for (const rel of [...USER_DATA_DIRS, ...dirs]) {
     const src = join(dataDir, rel);
     if (existsSync(src)) copyTree(src, join(backupDir, rel), copied);
   }
   if (copied.length > 0) {
     finishBackup(dataDir, backupDir, reason);
-    pruneBackups(scopeDir);
+    pruneBackups(scopeDir, names, basename(backupDir));
     if (process.env.MIXDOG_SETUP_QUIET !== '1') {
       process.stderr.write(`[user-data-backup] ${reason}: copied ${copied.length} file(s) to ${backupDir}\n`);
     }
@@ -408,18 +499,8 @@ async function copyTreeAsync(src, dst, copied) {
   copied.push(dst);
 }
 
-async function pruneBackupsAsync(scopeDir, keep = 40) {
-  let entries = [];
-  try {
-    entries = (await readdirP(scopeDir, { withFileTypes: true }))
-      .filter((entry) => entry.isDirectory())
-      .map((entry) => entry.name)
-      .sort()
-      .reverse();
-  } catch {
-    return;
-  }
-  for (const name of entries.slice(keep)) {
+async function pruneBackupsAsync(scopeDir, names, own) {
+  for (const name of backupsToPrune(names, own)) {
     try {
       await rmP(join(scopeDir, name), { recursive: true, force: true });
     } catch {}
@@ -430,7 +511,7 @@ export async function backupUserDataAsync(dataDir, reason = 'snapshot') {
   if (skipBackups()) return { dir: null, copied: [] };
   if (!dataDir || !existsSync(dataDir)) return { dir: null, copied: [] };
   const scopeDir = scopeDirFor(dataDir);
-  const backupDir = reserveBackupDir(scopeDir, reason);
+  const { backupDir, names } = await reserveBackupDirAsync(scopeDir, reason);
   const copied = [];
   for (const rel of USER_DATA_FILES) {
     const src = join(dataDir, rel);
@@ -442,7 +523,7 @@ export async function backupUserDataAsync(dataDir, reason = 'snapshot') {
   }
   if (copied.length > 0) {
     finishBackup(dataDir, backupDir, reason);
-    await pruneBackupsAsync(scopeDir);
+    await pruneBackupsAsync(scopeDir, names, basename(backupDir));
     if (process.env.MIXDOG_SETUP_QUIET !== '1') {
       process.stderr.write(`[user-data-backup] ${reason}: copied ${copied.length} file(s) to ${backupDir}\n`);
     }