@oar-sh/oar 0.9.8 → 0.9.10

package/server/services/stale-attempt-log-limiter.mjs

@@ -0,0 +1,96 @@
+'use strict';
+
+// One STALE ATTEMPT warning per superseded attempt, not one per write. A
+// worker whose turn was requeued under it keeps streaming for the old
+// attempt until it learns of the refusal, several writes a second (live:
+// 696 lines in four minutes after one recovery). The first refusal of an
+// attempt is logged in full; the rest are counted and summarized in one
+// line once the attempt has been quiet for `quietMs`.
+
+export const STALE_ATTEMPT_SUMMARY_QUIET_MS = 60_000;
+const MAX_TRACKED_ATTEMPTS = 500;
+
+function short(value) {
+  return String(value || '').trim().slice(0, 8) || 'none';
+}
+
+/**
+ * `warn(line)` writes one line (without timestamp). `now`, `setTimer` and
+ * `clearTimer` are injectable for tests.
+ */
+export function createStaleAttemptLogLimiter({
+  warn = (line) => console.warn(line),
+  now = () => Date.now(),
+  setTimer = (fn, ms) => {
+    const timer = setTimeout(fn, ms);
+    timer.unref?.();
+    return timer;
+  },
+  clearTimer = (timer) => clearTimeout(timer),
+  quietMs = STALE_ATTEMPT_SUMMARY_QUIET_MS,
+  maxTracked = MAX_TRACKED_ATTEMPTS,
+} = {}) {
+  // `${messageId}:${attemptId}` → { messageId, attemptId, count, firstAt, lastAt, routes, current, status, timer }
+  const entries = new Map();
+
+  function summarize(key) {
+    const entry = entries.get(key);
+    if (!entry) return;
+    entries.delete(key);
+    if (entry.timer) clearTimer(entry.timer);
+    const more = entry.count - 1;
+    if (more <= 0) return;
+    const seconds = Math.max(1, Math.round((entry.lastAt - entry.firstAt) / 1000));
+    warn(`STALE ATTEMPT ${short(entry.messageId)} attempt=${short(entry.attemptId)}: ${more} more write${more === 1 ? '' : 's'} refused over ${seconds}s (routes ${[...entry.routes].join(',')}; current=${short(entry.current)} status=${entry.status || 'unknown'})`);
+  }
+
+  function arm(key, entry) {
+    if (entry.timer) clearTimer(entry.timer);
+    entry.timer = setTimer(() => {
+      entry.timer = null;
+      summarize(key);
+    }, quietMs);
+  }
+
+  /**
+   * A write of `attemptId` for `messageId` was refused because the row's
+   * attempt is `currentAttemptId`. Returns `{ logged, count }`.
+   */
+  function note({ messageId, attemptId, currentAttemptId = '', route = 'unknown', status = '' } = {}) {
+    const key = `${String(messageId || '')}:${String(attemptId || '')}`;
+    const at = now();
+    const existing = entries.get(key);
+    if (existing) {
+      existing.count += 1;
+      existing.lastAt = at;
+      existing.routes.add(String(route || 'unknown'));
+      existing.current = currentAttemptId;
+      existing.status = String(status || existing.status || '');
+      arm(key, existing);
+      return { logged: false, count: existing.count };
+    }
+    if (entries.size >= maxTracked) summarize(entries.keys().next().value);
+    const entry = {
+      messageId,
+      attemptId,
+      count: 1,
+      firstAt: at,
+      lastAt: at,
+      routes: new Set([String(route || 'unknown')]),
+      current: currentAttemptId,
+      status: String(status || ''),
+      timer: null,
+    };
+    entries.set(key, entry);
+    arm(key, entry);
+    warn(`STALE ATTEMPT ${short(messageId)} route=${route} claimed=${short(attemptId)} current=${short(currentAttemptId)} status=${status || 'unknown'} (further writes of this attempt are counted and summarized once it goes quiet)`);
+    return { logged: true, count: 1 };
+  }
+
+  /** Writes every pending summary now (shutdown, tests). */
+  function flush() {
+    for (const key of [...entries.keys()]) summarize(key);
+  }
+
+  return { note, flush, size: () => entries.size };
+}

package/server/services/stale-recovery-clock-guard.mjs

@@ -0,0 +1,57 @@
+'use strict';
+
+// Stale recovery after a wall-clock jump. The stale window is measured from
+// the worker's last heartbeat: after the host slept for hours, every
+// heartbeat is hours old the moment the relay's timer fires again, and the
+// recovery requeued live turns (live: three messages recovered after a 6 h
+// sleep, delivered a second time, and the live worker's replies refused as
+// stale attempts, 696 warnings in 4 minutes).
+//
+// The guard notices the jump itself: the gap between two ticks of the
+// recovery timer far larger than its interval (a sleep, a stopped VM, a
+// blocked event loop, which stalls heartbeats just the same). After one, no
+// recovery runs for a grace period, long enough for a live worker to
+// heartbeat again; a dead worker's rows are recovered after it.
+
+export const CLOCK_JUMP_FACTOR = 4;
+
+/**
+ * `intervalMs` is the recovery timer's interval; a gap above
+ * `jumpThresholdMs` (default: four intervals) is a jump; `graceMs` is how
+ * long recovery then waits. `now()` is injectable for tests.
+ */
+export function createStaleRecoveryClockGuard({
+  now = () => Date.now(),
+  intervalMs,
+  jumpThresholdMs = null,
+  graceMs,
+  log = () => {},
+} = {}) {
+  const interval = Math.max(1, Number(intervalMs) || 1);
+  const threshold = Number(jumpThresholdMs) > 0 ? Number(jumpThresholdMs) : interval * CLOCK_JUMP_FACTOR;
+  const grace = Math.max(0, Number(graceMs) || 0);
+  let lastTickMs = null;
+  let graceUntilMs = 0;
+  let inGrace = false;
+
+  /**
+   * One tick of the recovery timer. `{ skip, jumped, gapMs,
+   * graceRemainingMs }`: `skip` says recovery must not run now.
+   */
+  function tick() {
+    const at = now();
+    const gapMs = lastTickMs === null ? 0 : Math.max(0, at - lastTickMs);
+    lastTickMs = at;
+    const jumped = gapMs > threshold;
+    if (jumped) {
+      graceUntilMs = Math.max(graceUntilMs, at + grace);
+      log(`Clock jump: ${Math.round(gapMs / 1000)}s passed between stale-recovery ticks (interval ${Math.round(interval / 1000)}s); stale recovery paused for ${Math.round(grace / 1000)}s so live workers can report`);
+    }
+    const skip = at < graceUntilMs;
+    if (inGrace && !skip) log('Stale recovery resumed after the clock jump');
+    inGrace = skip;
+    return { skip, jumped, gapMs, graceRemainingMs: Math.max(0, graceUntilMs - at) };
+  }
+
+  return { tick, thresholdMs: threshold, graceMs: grace };
+}

package/server/services/stale-subagent-run-sweep.mjs

@@ -0,0 +1,109 @@
+'use strict';
+
+// Sub-agent runs left `running` for good. A run's end is written by its
+// worker (or by the worker-death and terminal-turn reconciliations); a run
+// whose worker is gone without either leaves its row `running` forever, and
+// such rows count as running work everywhere: the activity tracker, stalled
+// and all_idle for a subscription to all sessions, the suspend-when-idle
+// watchdog (live: rows two months old still `running`).
+//
+// The sweep closes them: a `running` row older than `minAgeMs` whose
+// conversation has no live worker becomes `cancelled`. The age bound keeps a
+// run that just started, with its worker still registering, out of it; the
+// liveness check keeps a long background agent of a live worker in. It runs
+// shortly after boot (the workers of the previous runtime have reconnected
+// by then) and periodically.
+
+export const STALE_SUBAGENT_RUN_MIN_AGE_MS = 10 * 60_000;
+export const STALE_SUBAGENT_RUN_SWEEP_INTERVAL_MS = 10 * 60_000;
+export const STALE_SUBAGENT_RUN_BOOT_DELAY_MS = 60_000;
+
+function toText(value) {
+  return String(value ?? '').trim();
+}
+
+/**
+ * `isConversationWorkerAlive(conversationId)` says whether a live worker owns
+ * the conversation. `now()` is injectable for tests.
+ */
+export function createStaleSubagentRunSweep({
+  db,
+  isConversationWorkerAlive,
+  now = () => Date.now(),
+  minAgeMs = STALE_SUBAGENT_RUN_MIN_AGE_MS,
+  log = (line) => console.log(line),
+  warn = (line) => console.warn(line),
+} = {}) {
+  if (!db) throw new Error('createStaleSubagentRunSweep requires db');
+  if (typeof isConversationWorkerAlive !== 'function') throw new Error('createStaleSubagentRunSweep requires isConversationWorkerAlive');
+
+  let statements = null;
+  function prepare() {
+    if (statements) return statements;
+    try {
+      statements = {
+        listOld: db.prepare(`
+          SELECT id, conversation_id, started_at, updated_at
+          FROM subagent_runs
+          WHERE status = 'running' AND COALESCE(updated_at, started_at) < ?
+          ORDER BY conversation_id, started_at
+        `),
+        close: db.prepare(`
+          UPDATE subagent_runs
+          SET status = 'cancelled', updated_at = ?, completed_at = COALESCE(completed_at, ?)
+          WHERE id = ? AND status = 'running'
+        `),
+      };
+    } catch {
+      statements = null;
+    }
+    return statements;
+  }
+
+  /** One pass. Returns `{ checked, closed, conversations }`. */
+  function sweep() {
+    const prepared = prepare();
+    if (!prepared) return { checked: 0, closed: 0, conversations: 0, skipped: 'unsupported' };
+    const at = now();
+    const cutoff = new Date(at - minAgeMs).toISOString();
+    let rows = [];
+    try {
+      rows = prepared.listOld.all(cutoff);
+    } catch (error) {
+      warn(`STALE SUBAGENT RUNS sweep failed: ${error?.message || error}`);
+      return { checked: 0, closed: 0, conversations: 0, error: String(error?.message || error) };
+    }
+    if (!rows.length) return { checked: 0, closed: 0, conversations: 0 };
+    const aliveByConversation = new Map();
+    const alive = (conversationId) => {
+      if (!aliveByConversation.has(conversationId)) {
+        let live = false;
+        try { live = isConversationWorkerAlive(conversationId) === true; } catch { live = false; }
+        aliveByConversation.set(conversationId, live);
+      }
+      return aliveByConversation.get(conversationId);
+    };
+    const nowIso = new Date(at).toISOString();
+    const closedConversations = new Set();
+    let closed = 0;
+    let oldest = null;
+    db.transaction(() => {
+      for (const row of rows) {
+        const conversationId = toText(row.conversation_id);
+        if (alive(conversationId)) continue;
+        if (prepared.close.run(nowIso, nowIso, row.id).changes > 0) {
+          closed += 1;
+          closedConversations.add(conversationId);
+          const since = toText(row.updated_at) || toText(row.started_at);
+          if (since && (!oldest || since < oldest)) oldest = since;
+        }
+      }
+    })();
+    if (closed) {
+      log(`STALE SUBAGENT RUNS closed ${closed} run${closed === 1 ? '' : 's'} still marked running in ${closedConversations.size} conversation${closedConversations.size === 1 ? '' : 's'} without a live worker (oldest ${oldest})`);
+    }
+    return { checked: rows.length, closed, conversations: closedConversations.size };
+  }
+
+  return { sweep };
+}

package/server/services/subagent-record-retention.mjs

@@ -0,0 +1,164 @@
+'use strict';
+
+// Retention for sub-agent records. After the configured number of days a
+// finished run's tool results lose their text and its tool calls keep only
+// their one-line summary; both keep their row, marked
+// `{ pruned: true, prunedAt, originalChars, retentionDays }` so the viewer
+// can say what happened. Thoughts, reply text, notes, child rows and the
+// runs stay. The sweep runs at relay start and once a day, a few thousand
+// rows per transaction with the event loop free in between, and logs one
+// line. No VACUUM: SQLite reuses the freed pages.
+
+import { subagentToolCallSummary } from '../../shared/subagent-timeline.mjs';
+
+export const SUBAGENT_RETENTION_BATCH_SIZE = 2000;
+const DAY_MS = 24 * 60 * 60 * 1000;
+const METADATA_MAX_CHARS = 4096;
+
+function parseMetadata(raw) {
+  if (!raw) return {};
+  try {
+    const value = JSON.parse(raw);
+    return value && typeof value === 'object' && !Array.isArray(value) ? value : {};
+  } catch {
+    return {};
+  }
+}
+
+/** What a stripped entry keeps: text and metadata, from its stored row. */
+export function prunedTimelineEntry(row, { prunedAt, retentionDays }) {
+  const original = String(row?.text ?? '');
+  const text = row?.kind === 'tool_call' ? subagentToolCallSummary(original) : '';
+  const marker = { pruned: true, prunedAt, originalChars: original.length, retentionDays };
+  let metadataJson = JSON.stringify({ ...parseMetadata(row?.metadata_json), ...marker });
+  if (metadataJson.length > METADATA_MAX_CHARS) metadataJson = JSON.stringify(marker);
+  return { text, metadataJson, freedChars: Math.max(0, original.length - text.length) };
+}
+
+function columnExists(db, table, column) {
+  try {
+    return db.prepare(`PRAGMA table_info(${table})`).all().some((entry) => entry.name === column);
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * `getRetentionDays()` reads the setting (0 = off). `now()` and
+ * `yieldToLoop()` are injectable for tests.
+ */
+export function createSubagentRecordRetention({
+  db,
+  getRetentionDays,
+  now = () => Date.now(),
+  batchSize = SUBAGENT_RETENTION_BATCH_SIZE,
+  yieldToLoop = () => new Promise((resolve) => setImmediate(resolve)),
+  isStopped = () => false,
+  log = (line) => console.log(line),
+  warn = (line) => console.warn(line),
+} = {}) {
+  if (!db) throw new Error('createSubagentRecordRetention requires db');
+  if (typeof getRetentionDays !== 'function') throw new Error('createSubagentRecordRetention requires getRetentionDays');
+
+  let statements = null;
+  let running = null;
+  let scheduled = null;
+
+  function prepare() {
+    if (statements) return statements;
+    if (!columnExists(db, 'subagent_timeline', 'pruned_at')) return null;
+    statements = {
+      // The first condition repeats the partial index's WHERE word for word.
+      // A run still running keeps its whole record, however old.
+      candidates: db.prepare(`
+        SELECT t.id, t.kind, t.text, t.metadata_json
+        FROM subagent_timeline t
+        WHERE t.pruned_at IS NULL AND t.kind IN ('tool_call', 'tool_result')
+          AND t.updated_at < @cutoff
+          AND NOT EXISTS (
+            SELECT 1 FROM subagent_runs r
+            WHERE r.id = t.run_id AND r.status IN ('running', 'pending', 'processing')
+          )
+        ORDER BY t.updated_at
+        LIMIT @limit
+      `),
+      prune: db.prepare(`
+        UPDATE subagent_timeline
+        SET text = @text, metadata_json = @metadataJson, pruned_at = @prunedAt
+        WHERE id = @id AND pruned_at IS NULL
+      `),
+    };
+    return statements;
+  }
+
+  function pruneBatch(cutoff, prunedAt, retentionDays) {
+    const { candidates, prune } = statements;
+    return db.transaction(() => {
+      const rows = candidates.all({ cutoff, limit: batchSize });
+      let pruned = 0;
+      let freedChars = 0;
+      for (const row of rows) {
+        const next = prunedTimelineEntry(row, { prunedAt, retentionDays });
+        pruned += prune.run({ id: row.id, text: next.text, metadataJson: next.metadataJson, prunedAt }).changes;
+        freedChars += next.freedChars;
+      }
+      return { seen: rows.length, pruned, freedChars };
+    })();
+  }
+
+  async function runSweep() {
+    const retentionDays = Number(getRetentionDays()) || 0;
+    if (retentionDays <= 0) return { retentionDays: 0, pruned: 0, freedChars: 0, batches: 0, skipped: 'off' };
+    if (!prepare()) return { retentionDays, pruned: 0, freedChars: 0, batches: 0, skipped: 'unsupported' };
+    const startedAt = now();
+    const cutoff = new Date(startedAt - retentionDays * DAY_MS).toISOString();
+    const prunedAt = new Date(startedAt).toISOString();
+    let pruned = 0;
+    let freedChars = 0;
+    let batches = 0;
+    for (;;) {
+      if (isStopped()) break;
+      const batch = pruneBatch(cutoff, prunedAt, retentionDays);
+      if (!batch.seen) break;
+      batches += 1;
+      pruned += batch.pruned;
+      freedChars += batch.freedChars;
+      if (batch.seen < batchSize) break;
+      await yieldToLoop();
+    }
+    const elapsedMs = Math.max(0, now() - startedAt);
+    log(pruned
+      ? `SUBAGENT RETENTION stripped ${pruned} tool entr${pruned === 1 ? 'y' : 'ies'} (${freedChars} chars) older than ${retentionDays} d in ${batches} batch${batches === 1 ? '' : 'es'}, ${elapsedMs} ms`
+      : `SUBAGENT RETENTION nothing older than ${retentionDays} d to strip`);
+    return { retentionDays, pruned, freedChars, batches, cutoff };
+  }
+
+  /** One sweep at a time; a call while one runs gets that one. */
+  function sweep() {
+    if (running) return running;
+    running = runSweep()
+      .catch((error) => {
+        warn(`SUBAGENT RETENTION sweep failed: ${error?.message || error}`);
+        return { error: String(error?.message || error) };
+      })
+      .finally(() => { running = null; });
+    return running;
+  }
+
+  /** A sweep in `delayMs` (the relay's start, a changed setting); later calls move it. */
+  function scheduleSweep(delayMs = 0) {
+    if (scheduled) clearTimeout(scheduled);
+    scheduled = setTimeout(() => {
+      scheduled = null;
+      if (!isStopped()) void sweep();
+    }, Math.max(0, delayMs));
+    if (typeof scheduled.unref === 'function') scheduled.unref();
+  }
+
+  function stop() {
+    if (scheduled) clearTimeout(scheduled);
+    scheduled = null;
+  }
+
+  return { sweep, scheduleSweep, stop, DAY_MS };
+}