mixdog 0.7.11 → 0.7.12

package/src/channels/lib/webhook.mjs

@@ -4,7 +4,7 @@ import { join } from "path";
 import { spawn, spawnSync, execSync } from "child_process";
 import { DATA_DIR, isInQuietWindow } from "./config.mjs";
 import { getWebhookAuthtoken } from "../../shared/config.mjs";
-import { appendFileSync, readFileSync, readdirSync, mkdirSync, writeFileSync, unlinkSync, statSync, existsSync, watch as fsWatch } from "fs";
+import { appendFileSync, readFileSync, readdirSync, mkdirSync, writeFileSync, unlinkSync, statSync, existsSync, renameSync, watch as fsWatch } from "fs";
 import { randomUUID } from "crypto";
 const WEBHOOKS_DIR = join(DATA_DIR, "webhooks");
 const WEBHOOK_LOG = join(DATA_DIR, "webhook.log");
@@ -144,39 +144,160 @@ function loadEndpointConfig(name) {
 // then {status:"done"|"failed"|"dedup"}. Earlier fields (payloadPreview,
 // headersSummary) are kept on the first line only; later status updates
 // reference the same `id` and are merged latest-wins at read time.
+const DELIVERY_INDEX_MAX_IDS = 2000;
+const DELIVERY_LOG_MAX_LINES = 10_000;
+/** @type {Map<string, Map<string, object>>} */
+const _deliveryIndexByEndpoint = new Map();
+/** @type {Set<string>} */
+const _deliveryIndexWarmed = new Set();
+/** @type {Map<string, number>} */
+const _deliveryLogLineCountByEndpoint = new Map();
+/** @type {Map<string, number>} distinct-id count at last warm/compaction; drives the redundancy-based compaction trigger. */
+const _deliveryKeptCountByEndpoint = new Map();
 function _deliveriesPath(name) {
   return join(WEBHOOKS_DIR, name, "deliveries.jsonl");
 }
-function appendDelivery(name, entry) {
-  try {
-    const dir = join(WEBHOOKS_DIR, name);
-    if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
-    const line = JSON.stringify({ ts: new Date().toISOString(), ...entry }) + "\n";
-    appendFileSync(_deliveriesPath(name), line);
-    return true;
-  } catch (err) {
-    logWebhook(`${name}: deliveries append failed: ${err?.message ?? err}`);
-    return false;
+function _mergeDeliveryRows(prior, entry) {
+  return prior ? { ...prior, ...entry } : entry;
+}
+function _isBlockingDeliveryStatus(status) {
+  return status === "received" || status === "processing" || status === "done";
+}
+function _deliveryIndexFor(name) {
+  let map = _deliveryIndexByEndpoint.get(name);
+  if (!map) {
+    map = new Map();
+    _deliveryIndexByEndpoint.set(name, map);
   }
+  return map;
 }
-function readDeliveries(name) {
+function _pruneDeliveryIndexMap(byId) {
+  if (byId.size <= DELIVERY_INDEX_MAX_IDS) return;
+  const rows = [...byId.entries()];
+  const blocking = rows.filter(([, e]) => _isBlockingDeliveryStatus(e.status));
+  const nonBlocking = rows.filter(([, e]) => !_isBlockingDeliveryStatus(e.status));
+  nonBlocking.sort((a, b) => String(b[1].ts || "").localeCompare(String(a[1].ts || "")));
+  const keepIds = new Set(blocking.map(([id]) => id));
+  for (const [id] of nonBlocking) {
+    if (keepIds.size >= DELIVERY_INDEX_MAX_IDS) break;
+    keepIds.add(id);
+  }
+  for (const id of byId.keys()) {
+    if (!keepIds.has(id)) byId.delete(id);
+  }
+}
+function _deliveryLogLineCount(name) {
+  return _deliveryLogLineCountByEndpoint.get(name) ?? 0;
+}
+function _setDeliveryLogLineCount(name, n) {
+  _deliveryLogLineCountByEndpoint.set(name, Math.max(0, n));
+}
+function _bumpDeliveryLogLineCount(name, delta = 1) {
+  _setDeliveryLogLineCount(name, _deliveryLogLineCount(name) + delta);
+}
+function _readDeliveriesFileMerged(name) {
   const p = _deliveriesPath(name);
-  if (!existsSync(p)) return [];
   const byId = new Map();
+  let lineCount = 0;
+  if (!existsSync(p)) return { byId, lineCount };
   try {
     const raw = readFileSync(p, "utf8");
     for (const line of raw.split("\n")) {
       if (!line) continue;
+      lineCount++;
       try {
         const entry = JSON.parse(line);
-        if (!entry || !entry.id) continue;
-        const prior = byId.get(entry.id);
-        const merged = prior ? { ...prior, ...entry } : entry;
-        byId.set(entry.id, merged);
+        if (!entry?.id) continue;
+        byId.set(entry.id, _mergeDeliveryRows(byId.get(entry.id), entry));
       } catch {}
     }
   } catch {}
-  return [...byId.values()];
+  return { byId, lineCount };
+}
+function _ingestDeliveriesFileIntoIndex(name) {
+  const { byId: merged, lineCount } = _readDeliveriesFileMerged(name);
+  const byId = _deliveryIndexFor(name);
+  byId.clear();
+  for (const [id, row] of merged) byId.set(id, row);
+  _pruneDeliveryIndexMap(byId);
+  _setDeliveryLogLineCount(name, lineCount);
+  _deliveryKeptCountByEndpoint.set(name, merged.size);
+}
+function _ensureDeliveryIndex(name) {
+  if (_deliveryIndexWarmed.has(name)) return;
+  _deliveryIndexWarmed.add(name);
+  _ingestDeliveriesFileIntoIndex(name);
+}
+function _applyDeliveryEntryToIndex(name, entry) {
+  if (!entry?.id) return;
+  const byId = _deliveryIndexFor(name);
+  byId.set(entry.id, _mergeDeliveryRows(byId.get(entry.id), entry));
+  _pruneDeliveryIndexMap(byId);
+}
+function _compactDeliveriesLogIfNeeded(name) {
+  // Redundancy-based trigger: compact only when the log holds meaningfully more
+  // lines than distinct ids (i.e. there are status-update rows to collapse).
+  // The threshold scales with the distinct-id count so an endpoint with many
+  // legitimate blocking ids does NOT re-compact on every append (which would
+  // re-read the whole log permanently once distinct > DELIVERY_LOG_MAX_LINES).
+  const kept = _deliveryKeptCountByEndpoint.get(name) ?? _deliveryIndexFor(name).size;
+  const threshold = Math.max(DELIVERY_LOG_MAX_LINES, kept * 2);
+  if (_deliveryLogLineCount(name) <= threshold) return;
+  const { byId: merged } = _readDeliveriesFileMerged(name);
+  const rows = [...merged.values()];
+  const blocking = rows.filter((e) => _isBlockingDeliveryStatus(e.status));
+  const nonBlocking = rows.filter((e) => !_isBlockingDeliveryStatus(e.status));
+  nonBlocking.sort((a, b) => String(b.ts || "").localeCompare(String(a.ts || "")));
+  const keep = new Map();
+  for (const e of blocking) keep.set(e.id, e);
+  for (const e of nonBlocking) {
+    if (keep.size >= DELIVERY_INDEX_MAX_IDS) break;
+    if (!keep.has(e.id)) keep.set(e.id, e);
+  }
+  const lines = [...keep.values()]
+    .sort((a, b) => String(a.ts || "").localeCompare(String(b.ts || "")))
+    .map((e) => JSON.stringify(e) + "\n")
+    .join("");
+  const p = _deliveriesPath(name);
+  const tmp = `${p}.compact-${process.pid}-${Date.now()}.tmp`;
+  try {
+    writeFileSync(tmp, lines);
+    renameSync(tmp, p);
+    // Refresh index + counters by RE-READING the post-rename file (not the
+    // pre-rename `keep` snapshot). The webhook daemon is the single writer and
+    // append+compact run synchronously in one process, so no append can
+    // interleave between the fresh read and the rename; re-reading keeps the
+    // warmed state exactly matching on-disk content.
+    _ingestDeliveriesFileIntoIndex(name);
+  } catch (err) {
+    logWebhook(`${name}: deliveries compact failed: ${err?.message ?? err}`);
+    try {
+      if (existsSync(tmp)) unlinkSync(tmp);
+    } catch {}
+  }
+}
+function appendDelivery(name, entry) {
+  try {
+    const dir = join(WEBHOOKS_DIR, name);
+    if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
+    const full = { ts: new Date().toISOString(), ...entry };
+    const line = JSON.stringify(full) + "\n";
+    appendFileSync(_deliveriesPath(name), line);
+    const wasWarmed = _deliveryIndexWarmed.has(name);
+    _ensureDeliveryIndex(name);
+    if (wasWarmed) _bumpDeliveryLogLineCount(name, 1);
+    _applyDeliveryEntryToIndex(name, full);
+    _compactDeliveriesLogIfNeeded(name);
+    return true;
+  } catch (err) {
+    logWebhook(`${name}: deliveries append failed: ${err?.message ?? err}`);
+    return false;
+  }
+}
+function readDeliveries(name) {
+  _ensureDeliveryIndex(name);
+  const byId = _deliveryIndexByEndpoint.get(name);
+  return byId ? [...byId.values()] : [];
 }
 // Dedup gate against a still-active claim or a successful prior delivery.
 // Only rows with status "received" (non-terminal claim) or "done"
@@ -186,12 +307,13 @@ function readDeliveries(name) {
 // scoping, every prior row would permanently dedup the id and stop
 // legit redelivery.
 function deliveryExists(name, id) {
-  const list = readDeliveries(name);
   // "processing" must also dedup: a delegate dispatch in flight (up to
   // DISPATCH_TIMEOUT_MS = 10 min) would otherwise be duplicated by a
   // retried delivery of the same id while the first handler is still
   // running. Block on any non-terminal status.
-  return list.some((e) => e.id === id && (e.status === "received" || e.status === "processing" || e.status === "done"));
+  _ensureDeliveryIndex(name);
+  const entry = _deliveryIndexFor(name).get(id);
+  return Boolean(entry && _isBlockingDeliveryStatus(entry.status));
 }
 function extractDeliveryId(headers) {
   return headers["x-github-delivery"]

package/src/memory/lib/memory-cycle1.mjs

@@ -545,9 +545,13 @@ async function _runCycle1Impl(db, config = {}, options = {}, dataDir = null) {
     }
   }
 
-  // Fire every window at once: windowCount is bounded by fetchLimit / batchSize
-  // (<= session_cap), so there is no need to throttle the fan-out.
-  const sem = createSemaphore(Math.max(1, windows.length))
+  // Cap fan-out concurrency so a large batch (or a manual run) doesn't fire all
+  // window LLM calls at once and spike the provider / collide with the global
+  // agent-IPC limit. Small batches (<= cap) still run fully parallel.
+  const cycle1Concurrency = Math.max(1, Number(
+    config.cycle1_concurrency ?? config.concurrency ?? options.concurrency ?? options.maxConcurrent ?? 4,
+  ))
+  const sem = createSemaphore(Math.min(Math.max(1, windows.length), cycle1Concurrency))
   const settled = await Promise.allSettled(
     windows.map((rows, idx) => sem(() => {
       throwIfAborted(signal)

package/src/memory/lib/memory-recall-store.mjs

@@ -504,6 +504,32 @@ LEFT JOIN exact  x ON x.id = c.id`
   const rootIdsForReturn = []
   const seen = new Set()
 
+  // Batch-resolve member-chunk roots in ONE query (was an N+1: a per-row SELECT
+  // inside the loop below). Collect the distinct in-scope chunk_root ids, fetch
+  // all matching roots at once, then resolve each member from rootById.
+  const memberRootIds = []
+  const memberRootSeen = new Set()
+  for (const { id } of filtered) {
+    const r0 = byId.get(id)
+    if (!r0 || r0.is_root === 1) continue
+    if (r0.chunk_root != null && r0.chunk_root !== r0.id) {
+      const rid = Number(r0.chunk_root)
+      if (!memberRootSeen.has(rid)) { memberRootSeen.add(rid); memberRootIds.push(rid) }
+    }
+  }
+  const rootById = new Map()
+  if (memberRootIds.length > 0) {
+    const { clause: rootScopeClause, params: rootScopeParams } = buildScopeClause(2)
+    const { rows: rootRows } = await recallReadQuery(
+      db,
+      `SELECT id, ts, role, content, session_id, source_turn, chunk_root, is_root,
+              element, category, summary, project_id, status, score, last_seen_at
+       FROM entries WHERE id = ANY($1::bigint[]) AND is_root = 1 ${rootScopeClause}`,
+      [memberRootIds, ...rootScopeParams],
+    )
+    for (const rr of rootRows) rootById.set(Number(rr.id), rr)
+  }
+
   for (const { id, rrf, retrievalScore } of filtered) {
     const row = byId.get(id)
     if (!row) continue
@@ -511,16 +537,7 @@ LEFT JOIN exact  x ON x.id = c.id`
     if (row.is_root === 1) {
       targetRow = row
     } else if (row.chunk_root != null && row.chunk_root !== row.id) {
-      // $1 = chunk_root id, scope param (if any) = $2
-      const { clause: rootScopeClause, params: rootScopeParams } = buildScopeClause(2)
-      const { rows: rootRows } = await recallReadQuery(
-        db,
-        `SELECT id, ts, role, content, session_id, source_turn, chunk_root, is_root,
-                element, category, summary, project_id, status, score, last_seen_at
-         FROM entries WHERE id = $1 AND is_root = 1 ${rootScopeClause}`,
-        [row.chunk_root, ...rootScopeParams],
-      )
-      const r = rootRows[0]
+      const r = rootById.get(Number(row.chunk_root))
       if (!r) continue
       memberHitRootIds.add(r.id)
       targetRow = r

package/src/search/lib/backends/openai-oauth.mjs

@@ -5,7 +5,11 @@
  * Calls Codex WebSocket endpoint via sendViaWebSocket with web_search server
  * tool. Model is config-driven (search.models.openai default 'gpt-5.4-mini').
  */
-import { OpenAIOAuthProvider, ensureLatestCodexModel } from '../../../agent/orchestrator/providers/openai-oauth.mjs'
+import {
+  OpenAIOAuthProvider,
+  ensureLatestCodexModel,
+  codexModelSupportsServiceTier,
+} from '../../../agent/orchestrator/providers/openai-oauth.mjs'
 import {
   OPENAI_SEARCH_SYSTEM_INSTRUCTIONS,
   buildOpenAISearchPrompt,
@@ -66,7 +70,7 @@ export async function searchViaOpenAIOAuth({
     parallel_tool_calls: false,
     tools: [buildOpenAIWebSearchTool({ site, type, locale, contextSize })],
   }
-  if (fast === true) body.service_tier = 'priority'
+  if (fast === true && codexModelSupportsServiceTier(useModel, 'priority')) body.service_tier = 'priority'
   // Route through provider.send() (not sendViaWebSocket directly) so the search
   // request inherits the 401/403 force-refresh retry + HTTP/SSE fallback. A
   // stale token or unhealthy WebSocket then recovers instead of hard-failing.

package/src/search/lib/cache.mjs

@@ -5,6 +5,52 @@ const DEFAULT_CACHE_STATE = {
   entries: {},
 }
 
+// Size bounds on top of TTL expiry so cache.local.json can't grow unbounded.
+const MAX_CACHE_ENTRIES = 500
+const MAX_CACHE_BYTES = 8 * 1024 * 1024
+
+// Approximate serialized size of ONE entry (never the whole map), so insert and
+// evict stay cheap. A running total on the state (`__approxBytes`) lets the byte
+// cap be checked without re-serializing every entry on each set.
+function approxEntryBytes(key, entry) {
+  try { return String(key).length + JSON.stringify(entry).length } catch { return String(key).length }
+}
+function cacheApproxBytes(state) {
+  if (typeof state.__approxBytes === 'number') return state.__approxBytes
+  let total = 0
+  for (const [k, e] of Object.entries(state.entries)) total += approxEntryBytes(k, e)
+  state.__approxBytes = total
+  return total
+}
+function removeCacheEntry(state, key) {
+  const e = state.entries[key]
+  if (e === undefined) return
+  if (typeof state.__approxBytes === 'number') state.__approxBytes -= approxEntryBytes(key, e)
+  delete state.entries[key]
+}
+function evictOldestCacheEntry(state) {
+  let oldestKey = null
+  let oldestAt = Infinity
+  for (const [k, e] of Object.entries(state.entries)) {
+    const at = e?.cachedAt ?? 0
+    if (at < oldestAt) { oldestAt = at; oldestKey = k }
+  }
+  if (oldestKey == null) return false
+  removeCacheEntry(state, oldestKey)
+  return true
+}
+// Best-effort: evict oldest entries until under both the count and byte caps.
+function enforceCacheSizeBounds(state) {
+  try {
+    while (Object.keys(state.entries).length > MAX_CACHE_ENTRIES) {
+      if (!evictOldestCacheEntry(state)) break
+    }
+    while (cacheApproxBytes(state) > MAX_CACHE_BYTES && Object.keys(state.entries).length > 0) {
+      if (!evictOldestCacheEntry(state)) break
+    }
+  } catch { /* size bounding is best-effort */ }
+}
+
 const FLUSH_DELAY_MS = 5000
 
 let cacheDirty = false
@@ -75,6 +121,8 @@ export function loadCacheState() {
   _instance = state
   activeCacheState = state
   pruneExpiredEntries(state)
+  cacheApproxBytes(state)
+  enforceCacheSizeBounds(state)
   return state
 }
 
@@ -90,7 +138,7 @@ export function getCachedEntry(state, key) {
   const entry = state.entries[key]
   if (!entry) return null
   if (entry.expiresAt && entry.expiresAt <= nowMs()) {
-    delete state.entries[key]
+    removeCacheEntry(state, key)
     scheduleCacheFlush(state)
     return null
   }
@@ -99,11 +147,11 @@ export function getCachedEntry(state, key) {
 
 export function setCachedEntry(state, key, payload, ttlMs) {
   const cachedAt = nowMs()
-  state.entries[key] = {
-    cachedAt,
-    expiresAt: cachedAt + ttlMs,
-    payload,
-  }
+  if (state.entries[key] !== undefined) removeCacheEntry(state, key)
+  const entry = { cachedAt, expiresAt: cachedAt + ttlMs, payload }
+  state.entries[key] = entry
+  if (typeof state.__approxBytes === 'number') state.__approxBytes += approxEntryBytes(key, entry)
+  enforceCacheSizeBounds(state)
   scheduleCacheFlush(state)
   return state.entries[key]
 }
@@ -121,7 +169,7 @@ function pruneExpiredEntries(state) {
   let dirty = false
   for (const [key, entry] of Object.entries(state.entries)) {
     if (entry?.expiresAt && entry.expiresAt <= current) {
-      delete state.entries[key]
+      removeCacheEntry(state, key)
       dirty = true
     }
   }

package/scripts/test-config-rmw-restore.mjs

@@ -1,122 +0,0 @@
-/**
- * Repro: malformed mixdog-config.json + writeSection('search', …) must not
- * wipe channels/memory/agent; restores from backup or throws.
- */
-import assert from 'node:assert/strict';
-import {
-  mkdtempSync,
-  mkdirSync,
-  writeFileSync,
-  readFileSync,
-  rmSync,
-} from 'fs';
-import { tmpdir } from 'os';
-import { join, dirname } from 'path';
-import { fileURLToPath } from 'url';
-
-const __dirname = dirname(fileURLToPath(import.meta.url));
-
-async function loadConfigModule(dataDir, backupRoot) {
-  process.env.CLAUDE_PLUGIN_DATA = dataDir;
-  process.env.MIXDOG_USER_DATA_BACKUP_ROOT = backupRoot;
-  process.env.MIXDOG_SKIP_USER_DATA_BACKUP = '1';
-  const url = new URL(`../src/shared/config.mjs?run=${Date.now()}`, import.meta.url).href;
-  return import(url);
-}
-
-function writeConfig(dataDir, obj) {
-  writeFileSync(
-    join(dataDir, 'mixdog-config.json'),
-    JSON.stringify(obj, null, 2) + '\n',
-    'utf8',
-  );
-}
-
-async function main() {
-  const dataDir = mkdtempSync(join(tmpdir(), 'mixdog-config-rmw-'));
-  const backupRoot = mkdtempSync(join(tmpdir(), 'mixdog-config-backup-'));
-
-  const prior = {
-    channels: { guild: '111' },
-    memory: { enabled: true },
-    agent: { presets: { default: { model: 'x' } } },
-  };
-  writeConfig(dataDir, prior);
-
-  process.env.CLAUDE_PLUGIN_DATA = dataDir;
-  process.env.MIXDOG_USER_DATA_BACKUP_ROOT = backupRoot;
-  const guardUrl = new URL(`../src/shared/user-data-guard.mjs?t=${Date.now()}`, import.meta.url).href;
-  const { backupUserData, markUserDataInitialized } = await import(guardUrl);
-  const snap = backupUserData(dataDir, 'test-fixture');
-  assert.ok(snap.dir, 'backup fixture should copy mixdog-config.json');
-
-  writeFileSync(join(dataDir, 'mixdog-config.json'), '{ not valid json\n', 'utf8');
-
-  const { writeSection } = await loadConfigModule(dataDir, backupRoot);
-  writeSection('search', { provider: 'brave' });
-
-  const onDisk = JSON.parse(readFileSync(join(dataDir, 'mixdog-config.json'), 'utf8'));
-  assert.deepEqual(onDisk.channels, prior.channels);
-  assert.deepEqual(onDisk.memory, prior.memory);
-  assert.deepEqual(onDisk.agent, prior.agent);
-  assert.deepEqual(onDisk.search, { provider: 'brave' });
-
-  const freshDir = mkdtempSync(join(tmpdir(), 'mixdog-config-fresh-'));
-  const { writeSection: writeFresh } = await loadConfigModule(freshDir, backupRoot);
-  writeFresh('search', { only: true });
-  const freshDisk = JSON.parse(readFileSync(join(freshDir, 'mixdog-config.json'), 'utf8'));
-  assert.deepEqual(freshDisk, { search: { only: true } });
-
-  const noBackupDir = mkdtempSync(join(tmpdir(), 'mixdog-config-noback-'));
-  markUserDataInitialized(noBackupDir);
-  writeFileSync(join(noBackupDir, 'mixdog-config.json'), '[]', 'utf8');
-  const { writeSection: writeNoBackup } = await loadConfigModule(
-    noBackupDir,
-    mkdtempSync(join(tmpdir(), 'empty-backup-')),
-  );
-  let threw = false;
-  try {
-    writeNoBackup('search', { x: 1 });
-  } catch (err) {
-    threw = true;
-    assert.match(String(err.message), /refusing section write/);
-  }
-  assert.equal(threw, true, 'malformed config with init marker and no backup must throw');
-
-  const pickRoot = mkdtempSync(join(tmpdir(), 'mixdog-config-pick-'));
-  const fullCfg = {
-    channels: { guild: '222' },
-    memory: { enabled: false },
-    agent: { presets: {} },
-  };
-  const oldDir = join(pickRoot, '2026-06-03T19-00-00-000Z-old-full');
-  const newDir = join(pickRoot, '2026-06-03T21-00-00-000Z-new-degenerate');
-  mkdirSync(oldDir, { recursive: true });
-  mkdirSync(newDir, { recursive: true });
-  writeFileSync(join(oldDir, 'mixdog-config.json'), JSON.stringify(fullCfg) + '\n', 'utf8');
-  writeFileSync(
-    join(newDir, 'mixdog-config.json'),
-    JSON.stringify({ search: { provider: 'tavily' } }) + '\n',
-    'utf8',
-  );
-  process.env.MIXDOG_USER_DATA_BACKUP_ROOT = pickRoot;
-  const pickUrl = new URL(`../src/shared/user-data-guard.mjs?pick=${Date.now()}`, import.meta.url).href;
-  const { loadLatestMixdogConfigFromBackup } = await import(pickUrl);
-  const picked = loadLatestMixdogConfigFromBackup(null);
-  assert.deepEqual(picked?.channels, fullCfg.channels);
-  assert.deepEqual(picked?.agent, fullCfg.agent);
-  assert.equal(picked?.search, undefined, 'must not restore newest search-only snapshot');
-  rmSync(pickRoot, { recursive: true, force: true });
-
-  rmSync(dataDir, { recursive: true, force: true });
-  rmSync(backupRoot, { recursive: true, force: true });
-  rmSync(freshDir, { recursive: true, force: true });
-  rmSync(noBackupDir, { recursive: true, force: true });
-
-  console.log('test-config-rmw-restore: ok');
-}
-
-main().catch((err) => {
-  console.error(err);
-  process.exit(1);
-});