mixdog 0.9.53 → 0.9.55

package/src/runtime/agent/orchestrator/stall-policy.mjs

@@ -246,21 +246,14 @@ export const PROVIDER_WS_FIRST_MEANINGFUL_TIMEOUT_MS = resolveTimeoutMs(
     { minMs: 10_000, maxMs: STALL_WARN_MS },
 );
 
-// WS-specific semantic idle window. Do NOT reuse
-// PROVIDER_SEMANTIC_IDLE_TIMEOUT_MS here: that shared SSE default was raised to
-// ~285s to avoid false-aborting Anthropic effort-mode streams that legitimately
-// produce no deltas while thinking. The WS transport is different: a silent
-// OpenAI/xAI websocket turn can sit with only connection/metadata activity while
-// the TUI lead-turn watchdog is also 300s. If the provider waits ~285-300s
-// before surfacing the stall, the UI watchdog often wins and the user only sees
-// "Turn timed out after 300s" instead of a provider retry/fallback.
-//
-// Keep WS hangs bounded well below the TUI watchdog while still allowing normal
-// long reasoning streams that emit reasoning/text/tool deltas (those reset this
-// timer). Env-overridable for experiments.
+// WS semantic idle uses the same default ceiling as OpenAI HTTP/SSE semantic
+// idle. This is semantic progress only (not the WS inter-chunk byte timer), so
+// reasoning/text/tool deltas reset it while metadata cannot. Keep the default
+// at PROVIDER_MAX_BEFORE_WARN_MS (~285s), strictly below the 300s worker
+// watchdog, rather than rounding it to 300s.
 export const PROVIDER_WS_SEMANTIC_IDLE_TIMEOUT_MS = resolveTimeoutMs(
     ['MIXDOG_PROVIDER_WS_SEMANTIC_IDLE_TIMEOUT_MS', 'MIXDOG_PROVIDER_WS_OUTPUT_IDLE_TIMEOUT_MS'],
-    120_000,
+    PROVIDER_MAX_BEFORE_WARN_MS,
     { minMs: 10_000, maxMs: PROVIDER_MAX_BEFORE_WARN_MS },
 );
 

package/src/runtime/memory/lib/trace-store.mjs

@@ -658,6 +658,8 @@ const TRACE_COLS = [
   'cached_tokens', 'cache_write_tokens', 'duration_ms',
   'error_message', 'payload', 'parent_span_id', 'entry_id',
 ]
+const PG_MAX_BIND_PARAMETERS = 65_535
+const TRACE_INSERT_MAX_ROWS = Math.floor(PG_MAX_BIND_PARAMETERS / TRACE_COLS.length)
 
 // ---------------------------------------------------------------------------
 // Cross-request trace_events write queue (100ms / 500-row flush window)
@@ -665,6 +667,9 @@ const TRACE_COLS = [
 
 const TRACE_QUEUE_FLUSH_MS  = 100
 const TRACE_QUEUE_MAX_ROWS  = 500
+// Twenty normal flush batches leaves ample room for ordinary DB jitter while
+// bounding producer memory across repeated flush failures.
+export const TRACE_QUEUE_MAX_PENDING_EVENTS = 10_000
 
 // Per-db queue state (keyed by db object identity via WeakMap).
 const _traceQueues = new WeakMap()
@@ -672,12 +677,54 @@ const _traceQueues = new WeakMap()
 function _getQueue(db) {
   let q = _traceQueues.get(db)
   if (!q) {
-    q = { pending: [], timer: null, flushPromise: null }
+    q = { pending: [], timer: null, flushPromise: null, droppedEvents: 0 }
     _traceQueues.set(db, q)
   }
   return q
 }
 
+function _pendingEventCount(q) {
+  return q.pending.reduce((n, w) => n + w.events.length, 0)
+}
+
+function _trimPendingQueue(q) {
+  let overflow = _pendingEventCount(q) - TRACE_QUEUE_MAX_PENDING_EVENTS
+  while (overflow > 0 && q.pending.length > 0) {
+    const oldest = q.pending[0]
+    if (oldest.events.length <= overflow) {
+      q.pending.shift()
+      q.droppedEvents += oldest.events.length
+      overflow -= oldest.events.length
+    } else {
+      oldest.events.splice(0, overflow)
+      q.droppedEvents += overflow
+      overflow = 0
+    }
+  }
+}
+
+function _dropWrittenEvents(wrappers, count) {
+  while (count > 0 && wrappers.length > 0) {
+    const wrapper = wrappers[0]
+    if (wrapper.events.length <= count) {
+      wrappers.shift()
+      count -= wrapper.events.length
+    } else {
+      wrapper.events.splice(0, count)
+      count = 0
+    }
+  }
+}
+
+/** Observable queue depth and overflow count for a trace database. */
+export function getTraceQueueStats(db) {
+  const q = _traceQueues.get(db)
+  return {
+    pendingEvents: q ? _pendingEventCount(q) : 0,
+    droppedEvents: q?.droppedEvents ?? 0,
+  }
+}
+
 async function _flushQueue(db, q) {
   // When a flush is already running, callers reuse its promise — but enqueues
   // that arrive during that flush schedule a fresh timer that fires here and
@@ -691,10 +738,12 @@ async function _flushQueue(db, q) {
   if (wrappers.length === 0) return { inserted: 0 }
   const MAX_RETRIES = 3
   q.flushPromise = (async () => {
+    let writtenEvents = 0
     try {
       const allEvents = wrappers.flatMap(w => w.events)
-      return await _insertTraceEventsDirect(db, allEvents)
+      return await _insertTraceEventBatches(db, allEvents, count => { writtenEvents += count })
     } catch (err) {
+      _dropWrittenEvents(wrappers, writtenEvents)
       for (const w of wrappers) w.attempts += 1
       const keep = wrappers.filter(w => w.attempts < MAX_RETRIES)
       const dropped = wrappers.filter(w => w.attempts >= MAX_RETRIES)
@@ -702,6 +751,7 @@ async function _flushQueue(db, q) {
         __mixdogMemoryLog(`[trace-queue] dropped ${dropped.reduce((n, w) => n + w.events.length, 0)} events after ${MAX_RETRIES} retries: ${err?.message}\n`)
       }
       q.pending.unshift(...keep)
+      _trimPendingQueue(q)
       __mixdogMemoryLog(`[trace-queue] flush error: ${err?.message}\n`)
       throw err
     } finally {
@@ -737,7 +787,7 @@ async function drainTraceQueue(db) {
   try {
     if (q.timer) { clearTimeout(q.timer); q.timer = null }
     if (q.flushPromise) await q.flushPromise.catch(() => {})
-    if (q.pending.length > 0) await _insertTraceEventsDirect(db, q.pending.splice(0).flatMap(w => w.events))
+    if (q.pending.length > 0) await _insertTraceEventBatches(db, q.pending.splice(0).flatMap(w => w.events))
   } catch (e) {
     __mixdogMemoryLog(`[trace-queue] drain failed: ${e?.message}\n`)
   }
@@ -813,7 +863,8 @@ export function enqueueTraceEvents(db, events) {
   q.pending.push({ events: [...events], attempts: 0 })
   // Row cap counts pending EVENTS, not wrappers — a single wrapper with
   // >TRACE_QUEUE_MAX_ROWS events would otherwise sit until the timer.
-  const pendingEvents = q.pending.reduce((n, w) => n + w.events.length, 0)
+  _trimPendingQueue(q)
+  const pendingEvents = _pendingEventCount(q)
   if (pendingEvents >= TRACE_QUEUE_MAX_ROWS) {
     // Flush immediately when row cap reached — don't wait for timer.
     if (q.timer) { clearTimeout(q.timer); q.timer = null }
@@ -831,6 +882,17 @@ async function _insertTraceEventsDirect(db, events) {
   return insertTraceEvents(db, events)
 }
 
+async function _insertTraceEventBatches(db, events, onBatchInserted) {
+  let inserted = 0
+  for (let start = 0; start < events.length; start += TRACE_INSERT_MAX_ROWS) {
+    const batch = events.slice(start, start + TRACE_INSERT_MAX_ROWS)
+    const result = await _insertTraceEventsDirect(db, batch)
+    inserted += result.inserted
+    onBatchInserted?.(batch.length)
+  }
+  return { inserted }
+}
+
 export async function insertTraceEvents(db, events) {
   if (!Array.isArray(events) || events.length === 0) return { inserted: 0 }
 

package/src/runtime/shared/buffered-appender.mjs

@@ -16,14 +16,22 @@ import { appendFile } from 'node:fs/promises';
 
 const BUFFER_FLUSH_BYTES = 32 * 1024;
 const DEBOUNCE_MS = 50;
+// Per path: comfortably above normal transcript/log bursts while bounding
+// producer memory when a filesystem append is stalled.
+export const BUFFERED_APPEND_MAX_BYTES = 8 * 1024 * 1024;
+let totalDroppedBufferedBytes = 0;
+const droppedBytesByPath = new Map();
 
-// path -> { chunks: string[], bytes: number, timer, flushing, inFlight, failCount }
+// path -> { chunks: string[], bytes: number, timer, flushing, inFlight, failCount, droppedBytes }
 const queues = new Map();
 
 function getQueue(path) {
     let q = queues.get(path);
     if (!q) {
-        q = { chunks: [], bytes: 0, timer: null, flushing: false, inFlight: null, failCount: 0 };
+        q = {
+            chunks: [], bytes: 0, timer: null, flushing: false, inFlight: null,
+            failCount: 0, droppedBytes: droppedBytesByPath.get(path) ?? 0, path,
+        };
         queues.set(path, q);
     }
     return q;
@@ -45,6 +53,63 @@ function deleteIfIdle(path, q) {
     }
 }
 
+function recordDroppedBytes(q, bytes) {
+    q.droppedBytes += bytes;
+    totalDroppedBufferedBytes += bytes;
+    droppedBytesByPath.set(q.path, q.droppedBytes);
+}
+
+function keepNewestUtf8(text, maxBytes, totalBytes) {
+    // At most maxBytes UTF-16 code units are encoded (plus one code unit when
+    // the slice would split a surrogate pair), bounding this work even when a
+    // producer supplies an arbitrarily large string.
+    let windowStart = Math.max(0, text.length - maxBytes);
+    if (windowStart > 0
+        && /[\uDC00-\uDFFF]/.test(text[windowStart])
+        && /[\uD800-\uDBFF]/.test(text[windowStart - 1])) {
+        windowStart -= 1;
+    }
+    const bytes = Buffer.from(text.slice(windowStart), 'utf8');
+    let start = Math.max(0, bytes.length - maxBytes);
+    while (start < bytes.length && (bytes[start] & 0xC0) === 0x80) start += 1;
+    return {
+        text: bytes.subarray(start).toString('utf8'),
+        bytes: bytes.length - start,
+        droppedBytes: totalBytes - (bytes.length - start),
+    };
+}
+
+function capIncomingText(text) {
+    // A single native byteLength pass keeps queue accounting and flush timing
+    // exact; only oversized input uses bounded tail-window encoding below.
+    const bytes = Buffer.byteLength(text, 'utf8');
+    if (bytes <= BUFFERED_APPEND_MAX_BYTES) return { text, bytes, droppedBytes: 0 };
+    return keepNewestUtf8(text, BUFFERED_APPEND_MAX_BYTES, bytes);
+}
+
+function trimBufferedQueue(q) {
+    while (q.bytes > BUFFERED_APPEND_MAX_BYTES && q.chunks.length > 0) {
+        const oldest = q.chunks[0];
+        const oldestBytes = Buffer.byteLength(oldest, 'utf8');
+        const overflowBytes = q.bytes - BUFFERED_APPEND_MAX_BYTES;
+        if (oldestBytes <= overflowBytes) {
+            q.chunks.shift();
+            q.bytes -= oldestBytes;
+            recordDroppedBytes(q, oldestBytes);
+            continue;
+        }
+
+        const retained = keepNewestUtf8(oldest, oldestBytes - overflowBytes, oldestBytes);
+        const removedBytes = oldestBytes - retained.bytes;
+        q.chunks[0] = retained.text;
+        q.bytes -= removedBytes;
+        recordDroppedBytes(q, removedBytes);
+        // A character-boundary trim can remove up to three extra bytes rather
+        // than split a code point or exceed the hard byte cap.
+        if (q.bytes > BUFFERED_APPEND_MAX_BYTES) continue;
+    }
+}
+
 function _flush(path, q) {
     if (q.flushing) return;
     if (q.chunks.length === 0) return;
@@ -69,6 +134,7 @@ function _flush(path, q) {
             if (q.failCount <= 3) {
                 q.chunks.unshift(data);
                 q.bytes += Buffer.byteLength(data, 'utf8');
+                trimBufferedQueue(q);
             }
         })
         .finally(() => {
@@ -89,8 +155,11 @@ function _flush(path, q) {
 export function appendBuffered(path, text) {
     if (!path || !text) return;
     const q = getQueue(path);
-    q.chunks.push(text);
-    q.bytes += Buffer.byteLength(text, 'utf8');
+    const incoming = capIncomingText(text);
+    q.chunks.push(incoming.text);
+    q.bytes += incoming.bytes;
+    if (incoming.droppedBytes > 0) recordDroppedBytes(q, incoming.droppedBytes);
+    trimBufferedQueue(q);
     if (q.bytes >= BUFFER_FLUSH_BYTES) {
         if (q.timer) { clearTimeout(q.timer); q.timer = null; }
         _flush(path, q);
@@ -99,6 +168,16 @@ export function appendBuffered(path, text) {
     }
 }
 
+/** Observable overflow counters for buffered append queues. */
+export function getBufferedAppenderStats(path) {
+    const q = queues.get(path);
+    return {
+        bufferedBytes: q?.bytes ?? 0,
+        droppedBytes: q?.droppedBytes ?? droppedBytesByPath.get(path) ?? 0,
+        totalDroppedBytes: totalDroppedBufferedBytes,
+    };
+}
+
 /**
  * Synchronously drain any remaining buffered bytes for every known path.
  * Intended for process exit hooks only — best-effort, never throws.

package/src/runtime/shared/memory-snapshot.mjs

@@ -1,28 +1,27 @@
 import { execFile } from 'node:child_process';
 import {
-  closeSync,
-  existsSync,
-  fsyncSync,
-  mkdirSync,
-  openSync,
-  renameSync,
-  statSync,
-  unlinkSync,
-  writeFileSync,
-  writeSync,
-} from 'node:fs';
+  mkdir,
+  open,
+  rename,
+  stat,
+  unlink,
+  writeFile,
+} from 'node:fs/promises';
 import { freemem, totalmem } from 'node:os';
 import { join } from 'node:path';
 import { promisify } from 'node:util';
-import { withFileLockSync } from './atomic-file.mjs';
+import { withFileLock } from './atomic-file.mjs';
 import { resolvePluginData } from './plugin-paths.mjs';
 
 const execFileAsync = promisify(execFile);
 export const MEMORY_PRESSURE_SNAPSHOT_MAX_BYTES = 64 * 1024;
 const SNAPSHOT_NAME = 'memory-pressure.jsonl';
 const SNAPSHOT_INTERVAL_MS = 10 * 60 * 1000;
+const SNAPSHOT_INTERVAL_JITTER_MS = 60 * 1000;
+const PERIODIC_SNAPSHOT_INTERVAL_MS = SNAPSHOT_INTERVAL_MS
+  + Math.floor(Math.random() * ((2 * SNAPSHOT_INTERVAL_JITTER_MS) + 1))
+  - SNAPSHOT_INTERVAL_JITTER_MS;
 const SNAPSHOT_RATE_LIMIT_MS = 60 * 1000;
-const SNAPSHOT_LOCK_TIMEOUT_MS = 2000;
 const SNAPSHOT_LOCK_STALE_MS = 10000;
 
 let periodicTimer = null;
@@ -59,42 +58,50 @@ function systemMemory() {
   }
 }
 
-function rotateSnapshot(paths) {
-  const size = Number(statSync(paths.snapshot).size) || 0;
-  try { unlinkSync(paths.previousSnapshot); } catch {}
+async function pathExists(path) {
+  try {
+    await stat(path);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+async function rotateSnapshot(paths) {
+  const size = Number((await stat(paths.snapshot)).size) || 0;
+  try { await unlink(paths.previousSnapshot); } catch {}
   if (size <= MEMORY_PRESSURE_SNAPSHOT_MAX_BYTES) {
-    renameSync(paths.snapshot, paths.previousSnapshot);
+    await rename(paths.snapshot, paths.previousSnapshot);
     return;
   }
-  writeFileSync(paths.previousSnapshot, '', { mode: 0o600 });
-  unlinkSync(paths.snapshot);
+  await writeFile(paths.previousSnapshot, '', { mode: 0o600 });
+  await unlink(paths.snapshot);
 }
 
-function appendSnapshot(entry) {
+async function appendSnapshot(entry) {
   const line = `${JSON.stringify(entry)}\n`;
   if (Buffer.byteLength(line) > MEMORY_PRESSURE_SNAPSHOT_MAX_BYTES) return false;
   const paths = snapshotPaths();
   try {
-    mkdirSync(paths.directory, { recursive: true, mode: 0o700 });
-    return withFileLockSync(paths.lock, () => {
-      if (existsSync(paths.previousSnapshot)
-        && statSync(paths.previousSnapshot).size > MEMORY_PRESSURE_SNAPSHOT_MAX_BYTES) {
-        writeFileSync(paths.previousSnapshot, '', { mode: 0o600 });
+    await mkdir(paths.directory, { recursive: true, mode: 0o700 });
+    return await withFileLock(paths.lock, async () => {
+      if (await pathExists(paths.previousSnapshot)
+        && (await stat(paths.previousSnapshot)).size > MEMORY_PRESSURE_SNAPSHOT_MAX_BYTES) {
+        await writeFile(paths.previousSnapshot, '', { mode: 0o600 });
       }
-      if (existsSync(paths.snapshot)
-        && statSync(paths.snapshot).size + Buffer.byteLength(line) > MEMORY_PRESSURE_SNAPSHOT_MAX_BYTES) {
-        rotateSnapshot(paths);
+      if (await pathExists(paths.snapshot)
+        && (await stat(paths.snapshot)).size + Buffer.byteLength(line) > MEMORY_PRESSURE_SNAPSHOT_MAX_BYTES) {
+        await rotateSnapshot(paths);
       }
-      const fd = openSync(paths.snapshot, 'a', 0o600);
+      const fd = await open(paths.snapshot, 'a', 0o600);
       try {
-        writeSync(fd, line, null, 'utf8');
-        fsyncSync(fd);
+        await fd.write(line, null, 'utf8');
       } finally {
-        closeSync(fd);
+        await fd.close();
       }
-      return statSync(paths.snapshot).size <= MEMORY_PRESSURE_SNAPSHOT_MAX_BYTES;
+      return (await stat(paths.snapshot)).size <= MEMORY_PRESSURE_SNAPSHOT_MAX_BYTES;
     }, {
-      timeoutMs: SNAPSHOT_LOCK_TIMEOUT_MS,
+      timeoutMs: 0,
       staleMs: SNAPSHOT_LOCK_STALE_MS,
     });
   } catch {
@@ -197,7 +204,7 @@ export async function captureMemoryPressureSnapshot(reason = 'memory-pressure')
   return appendSnapshot(entry);
 }
 
-function capturePeriodicMemorySnapshot() {
+async function capturePeriodicMemorySnapshot() {
   if (!enabled()) return false;
   return appendSnapshot(baseSnapshot('periodic'));
 }
@@ -224,7 +231,9 @@ export function requestMemoryPressureSnapshot(reason) {
 
 export function armMemoryPressureSampling() {
   if (!enabled() || periodicTimer) return false;
-  periodicTimer = setInterval(() => { capturePeriodicMemorySnapshot(); }, SNAPSHOT_INTERVAL_MS);
+  periodicTimer = setInterval(() => {
+    void capturePeriodicMemorySnapshot().catch(() => {});
+  }, PERIODIC_SNAPSHOT_INTERVAL_MS);
   periodicTimer.unref?.();
   return true;
 }