@shumkov/orchestra 0.2.0

package/lib/process-guard.js

@@ -0,0 +1,297 @@
+// provenance: polygram@0.17.11 lib/process-guard.js (git 746bca6) — verbatim: env prefix WATER_, bridge name water-bridge, vendor path (SHARED-LIB.md).
+/**
+ * rc.50: process-guard helpers — orphan-detection PID file + safety
+ * handlers for uncaughtException / unhandledRejection that don't
+ * re-enter on broken stdout.
+ *
+ * Background — the rc.50 incident:
+ *   PID 6335 (rc.48) was orphaned when its tmux pane was destroyed
+ *   during `launchctl kickstart -k`. polygram's existing SIGHUP
+ *   handler should have drained cleanly, but during the drain
+ *   `console.error` inside the uncaughtException handler itself
+ *   threw EIO (stdout was wired to a now-destroyed pty). That fired
+ *   the same handler, which logged again, which threw EIO again — a
+ *   tight re-entrant loop that hijacked the event loop and prevented
+ *   shutdown from completing. The orphan ran for 3+ hours writing
+ *   3.59M+ uncaught-exception rows to the DB at ~12k/sec, and
+ *   polled the same Telegram bot token in parallel with the new
+ *   daemon.
+ *
+ * This module provides three primitives. polygram.js wires them
+ * together at boot.
+ */
+
+'use strict';
+
+const fs = require('fs');
+
+/**
+ * Boot-time orphan detection. Writes our PID to `pidPath`. If the
+ * file already exists with a different live PID, kill it before
+ * proceeding (SIGTERM, then SIGKILL after `sigtermWaitMs`). Without
+ * this, two daemons can end up sharing the same Telegram bot token
+ * and SQLite DB — the cascade that made the rc.50 incident
+ * production-visible.
+ *
+ * @returns {{ priorPid: number|null, priorAction: string }}
+ */
+function claimPidFile(pidPath, { logger = console, sigtermWaitMs = 2000 } = {}) {
+  const ownPid = process.pid;
+  let priorPid = null;
+  let priorAction = 'no-prior';
+
+  if (fs.existsSync(pidPath)) {
+    const raw = (() => {
+      try { return fs.readFileSync(pidPath, 'utf8').trim(); }
+      catch { return ''; }
+    })();
+    const parsed = /^\d+$/.test(raw) ? parseInt(raw, 10) : null;
+    if (!parsed) {
+      priorAction = 'malformed-overwritten';
+    } else if (parsed === ownPid) {
+      // Re-entrant call from same process — write but don't kill self.
+      priorPid = parsed;
+      priorAction = 'self-skip';
+    } else {
+      priorPid = parsed;
+      const alive = isAlive(parsed);
+      if (!alive) {
+        priorAction = 'stale-overwritten';
+      } else {
+        logger.log?.(`[orphan-guard] prior daemon PID ${parsed} still alive — sending SIGTERM`);
+        try { process.kill(parsed, 'SIGTERM'); } catch {}
+        const start = Date.now();
+        while (Date.now() - start < sigtermWaitMs && isAlive(parsed)) {
+          // Busy-wait. Boot is single-threaded; we have nothing else to do
+          // until the orphan is gone, and we don't want to bind the bot
+          // token while it's still polling. sigtermWaitMs is configurable
+          // (default 2s; tests override to 100ms).
+          sleepSync(50);
+        }
+        if (isAlive(parsed)) {
+          logger.log?.(`[orphan-guard] PID ${parsed} ignored SIGTERM — escalating to SIGKILL`);
+          try { process.kill(parsed, 'SIGKILL'); } catch {}
+          // Poll for actual death — SIGKILL is delivered async, the
+          // kernel may take a tick to reap (esp. for detached children).
+          const killStart = Date.now();
+          while (Date.now() - killStart < 1000 && isAlive(parsed)) {
+            sleepSync(20);
+          }
+          priorAction = 'sigkill-killed';
+        } else {
+          priorAction = 'sigterm-killed';
+        }
+      }
+    }
+  }
+
+  fs.writeFileSync(pidPath, String(ownPid) + '\n', { mode: 0o600 });
+  return { priorPid, priorAction };
+}
+
+/**
+ * Delete the PID file on clean shutdown. Only deletes if the file
+ * still contains OUR PID — protects against the race where a new
+ * daemon already claimed the file and rewrote it before we got here.
+ */
+function releasePidFile(pidPath) {
+  if (!fs.existsSync(pidPath)) return;
+  try {
+    const content = fs.readFileSync(pidPath, 'utf8').trim();
+    if (content === String(process.pid)) {
+      fs.unlinkSync(pidPath);
+    }
+    // Else: another daemon owns it now. Leaving alone is correct.
+  } catch {}
+}
+
+/**
+ * Build an uncaughtException handler that:
+ *   1. Wraps `logger.error` AND `logEvent` in try/catch — neither
+ *      can re-throw out of the handler. (Pre-rc.50 the bare
+ *      console.error threw EIO and re-fired this same handler in
+ *      an event-loop-hijacking loop.)
+ *   2. Tracks repetitions of the same exception message in a sliding
+ *      window. If the same message fires `eioThreshold` times within
+ *      `eioWindowMs`, calls `panicExit(2)` so launchd restarts us
+ *      cleanly. Without the circuit breaker, a stuck-stdout EIO
+ *      cascade just keeps writing rows forever.
+ *
+ * @param {object} opts
+ * @param {object} opts.logger - { error(msg) } sink for human-readable logs.
+ * @param {function(string, object)} opts.logEvent - DB persist sink.
+ * @param {string} opts.botName
+ * @param {number} [opts.eioThreshold=100]
+ * @param {number} [opts.eioWindowMs=5000]
+ * @param {function(number)} [opts.panicExit=process.exit]
+ * @param {function(): number} [opts.now=Date.now]
+ * @returns {function(Error)}
+ */
+function _makeUncaughtHandler({
+  logger,
+  logEvent,
+  botName,
+  eioThreshold = 100,
+  eioWindowMs = 5000,
+  panicExit = (code) => process.exit(code),
+  now = Date.now,
+} = {}) {
+  // Per-message sliding-window timestamps. Map<message, number[]>.
+  const recent = new Map();
+  let panicked = false;
+
+  return function uncaughtHandler(err) {
+    if (panicked) return; // bail — we're on our way out
+    const msg = String(err?.message || err || 'unknown').slice(0, 500);
+    const stack = err?.stack?.split('\n').slice(0, 5).join('\n') || '';
+
+    // 1. Log defensively. Stdout may be broken (the original incident);
+    //    must not re-throw out of this handler.
+    try {
+      logger?.error?.(`[polygram] uncaughtException: ${msg}\n${stack}`);
+    } catch { /* swallow — broken stdout */ }
+
+    // 2. Persist defensively. DB might be closing during shutdown.
+    try {
+      logEvent?.('uncaught-exception', { message: msg, bot_name: botName });
+    } catch { /* swallow */ }
+
+    // 3. Storm circuit breaker: same message N times in window → exit.
+    const t = now();
+    let timestamps = recent.get(msg);
+    if (!timestamps) { timestamps = []; recent.set(msg, timestamps); }
+    timestamps.push(t);
+    // Drop expired.
+    while (timestamps.length && t - timestamps[0] > eioWindowMs) timestamps.shift();
+    if (timestamps.length >= eioThreshold) {
+      panicked = true;
+      try {
+        logger?.error?.(`[polygram] uncaughtException circuit breaker: ${timestamps.length}× "${msg}" in ${eioWindowMs}ms — exit(2)`);
+      } catch {}
+      try {
+        logEvent?.('panic-exit', { message: msg, count: timestamps.length, window_ms: eioWindowMs, bot_name: botName });
+      } catch {}
+      panicExit(2);
+    }
+  };
+}
+
+// Build a parallel handler for unhandledRejection: same defensive
+// posture, separate counter (rejections and exceptions can come
+// from different code paths and shouldn't share a budget).
+function _makeUnhandledRejectionHandler(opts) {
+  const inner = _makeUncaughtHandler({
+    ...opts,
+    // Override the 'kind' written to events table.
+    logEvent: opts.logEvent
+      ? (kind, detail) => opts.logEvent(kind === 'panic-exit' ? 'panic-exit' : 'unhandled-rejection', detail)
+      : undefined,
+  });
+  return (reason /* , promise */) => {
+    const err = reason instanceof Error ? reason : new Error(String(reason));
+    inner(err);
+  };
+}
+
+/**
+ * Swallow EPIPE/EIO on the process's own stdout/stderr so a broken pipe during
+ * shutdown can't become an uncaughtException.
+ *
+ * rc.50 stopped the re-entrant LOOP (the handler no longer re-throws), but the
+ * write errors themselves still surface: when `launchctl kickstart` destroys the
+ * tmux pane, in-flight `console.log`/`console.error` calls hit a now-dead pty.
+ * Because stdout/stderr are TTYs, those writes throw EIO **synchronously** — each
+ * unguarded throw becomes an uncaughtException. Observed live on the rc.29→rc.30
+ * restart (2026-06-08): 100 `write EIO` rows then a circuit-breaker panic-exit on
+ * every deploy, interrupting the graceful drain.
+ *
+ * This guards BOTH delivery paths:
+ *   (a) sync: wraps `write()` to drop EPIPE/EIO throws (TTY case — the real one);
+ *   (b) async: attaches an `error` listener for the pipe case (errors arrive as
+ *       events). Genuine, non-EPIPE/EIO errors still surface unchanged.
+ *
+ * @returns {{ uninstall: function() }}
+ */
+function guardStdio({ streams = [process.stdout, process.stderr] } = {}) {
+  const guarded = streams.filter(Boolean);
+  const isBrokenPipe = (err) => err && (err.code === 'EPIPE' || err.code === 'EIO');
+  const onError = (err) => { if (isBrokenPipe(err)) return; throw err; };
+  const restores = [];
+
+  for (const s of guarded) {
+    s.on?.('error', onError);
+    restores.push(() => s.off?.('error', onError));
+
+    if (typeof s.write === 'function' && !s.__polygramStdioGuarded) {
+      const origWrite = s.write;
+      s.write = function guardedWrite(...args) {
+        try {
+          return origWrite.apply(this, args);
+        } catch (err) {
+          if (!isBrokenPipe(err)) throw err;
+          // Pane is gone — nothing to write to. Invoke the write callback (the
+          // last arg, if any) so callers awaiting it don't hang, and report
+          // backpressure (false) instead of throwing to uncaughtException.
+          const cb = args[args.length - 1];
+          if (typeof cb === 'function') { try { cb(err); } catch { /* ignore */ } }
+          return false;
+        }
+      };
+      s.__polygramStdioGuarded = true;
+      restores.push(() => { s.write = origWrite; delete s.__polygramStdioGuarded; });
+    }
+  }
+
+  return { uninstall() { for (const r of restores) r(); } };
+}
+
+/**
+ * Convenience: install both handlers in one call (plus the stdio guard, so the
+ * shutdown broken-pipe writes never reach the uncaughtException handler).
+ * @returns {{ uninstall: function() }}
+ */
+function installSafetyHandlers(opts) {
+  const onException = _makeUncaughtHandler(opts);
+  const onRejection = _makeUnhandledRejectionHandler(opts);
+  process.on('uncaughtException', onException);
+  process.on('unhandledRejection', onRejection);
+  const stdio = guardStdio();
+  return {
+    uninstall() {
+      process.off('uncaughtException', onException);
+      process.off('unhandledRejection', onRejection);
+      stdio.uninstall();
+    },
+  };
+}
+
+// ─── helpers ─────────────────────────────────────────────────────────
+
+function isAlive(pid) {
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch (err) {
+    // ESRCH = no such process. EPERM = exists but we lack rights
+    // (treat as alive — same UID typically; we will fail to kill it
+    // but at least we know it's there).
+    if (err.code === 'EPERM') return true;
+    return false;
+  }
+}
+
+function sleepSync(ms) {
+  // Atomics-based busy-wait. 50ms granularity is fine for boot
+  // orphan-killing; we're not in a hot path.
+  const buf = new Int32Array(new SharedArrayBuffer(4));
+  Atomics.wait(buf, 0, 0, ms);
+}
+
+module.exports = {
+  claimPidFile,
+  releasePidFile,
+  installSafetyHandlers,
+  guardStdio,
+  _makeUncaughtHandler,
+  _makeUnhandledRejectionHandler,
+};

package/lib/questions/store.js

@@ -0,0 +1,107 @@
+// provenance: polygram@0.17.11 lib/questions/store.js (git 746bca6) — verbatim: env prefix WATER_, bridge name water-bridge, vendor path (SHARED-LIB.md).
+/**
+ * pending_questions store — persistence for the 0.12 interactive-question flow.
+ *
+ * Mirrors lib/approvals/store.js: per-row 128-bit callback token, status
+ * lifecycle, audit-kept rows (never deleted; 'pending' at boot → 'expired').
+ * One OPEN question per session at a time. The answer routes back to claude on
+ * tool_call_id (a `question_answer` bridge message).
+ */
+
+'use strict';
+
+const { newToken, tokensEqual } = require('../approvals/store');
+
+// A question waits for the user — the turn no longer times out while an `ask` is open
+// (cli-process defers its ceilings during a question wait, docs/progress-is-not-turn-end-spec.md),
+// so this is only the long SAFETY BACKSTOP: a forgotten/abandoned question eventually
+// expires {timedout} instead of pinning the session forever. Generous (a full day) so a
+// real user answering hours later is never cut off; tune via the `questionTimeoutMs` config
+// if a chat needs shorter/longer.
+const DEFAULT_TIMEOUT_MS = 24 * 60 * 60 * 1000;
+
+function createQuestionStore(rawDb, now = () => Date.now()) {
+  const insertStmt = rawDb.prepare(`
+    INSERT INTO pending_questions (
+      bot_name, session_key, chat_id, thread_id, turn_id, tool_call_id,
+      callback_token, questions_json, state_json, created_ts, timeout_ts
+    ) VALUES (
+      @bot_name, @session_key, @chat_id, @thread_id, @turn_id, @tool_call_id,
+      @callback_token, @questions_json, @state_json, @created_ts, @timeout_ts
+    )
+  `);
+  const getByIdStmt        = rawDb.prepare(`SELECT * FROM pending_questions WHERE id = ?`);
+  const getOpenForSessStmt = rawDb.prepare(`
+    SELECT * FROM pending_questions WHERE session_key = ? AND status = 'pending'
+     ORDER BY created_ts DESC LIMIT 1`);
+  const getByToolCallStmt  = rawDb.prepare(`SELECT * FROM pending_questions WHERE tool_call_id = ? LIMIT 1`);
+  const setMsgIdsStmt      = rawDb.prepare(`UPDATE pending_questions SET message_ids_json = ? WHERE id = ?`);
+  const updateStateStmt    = rawDb.prepare(`
+    UPDATE pending_questions SET state_json = @state_json, awaiting_other = @awaiting_other
+     WHERE id = @id AND status = 'pending'`);
+  const claimStmt          = rawDb.prepare(`
+    UPDATE pending_questions SET from_id = @from_id
+     WHERE id = @id AND from_id IS NULL AND status = 'pending'`);
+  const resolveStmt        = rawDb.prepare(`
+    UPDATE pending_questions SET status = @status, answered_ts = @answered_ts
+     WHERE id = @id AND status = 'pending'`);
+  const listTimedOutStmt   = rawDb.prepare(`SELECT * FROM pending_questions WHERE status = 'pending' AND timeout_ts < ?`);
+  const listOpenStmt       = rawDb.prepare(`SELECT * FROM pending_questions WHERE bot_name = ? AND status = 'pending'`);
+
+  return {
+    issue({ bot_name, session_key, chat_id, thread_id = null, turn_id = null, tool_call_id, questions, state, timeoutMs = DEFAULT_TIMEOUT_MS }) {
+      if (!bot_name || !session_key || !chat_id || !tool_call_id) {
+        throw new Error('issue: bot_name, session_key, chat_id, tool_call_id required');
+      }
+      const created_ts = now();
+      const res = insertStmt.run({
+        bot_name,
+        session_key,
+        chat_id: String(chat_id),
+        thread_id: thread_id != null ? String(thread_id) : null,
+        turn_id,
+        tool_call_id,
+        callback_token: newToken(),
+        questions_json: JSON.stringify(questions ?? []),
+        state_json: JSON.stringify(state ?? {}),
+        created_ts,
+        timeout_ts: created_ts + timeoutMs,
+      });
+      return getByIdStmt.get(res.lastInsertRowid);
+    },
+
+    getById(id) { return getByIdStmt.get(id); },
+    getOpenForSession(session_key) { return getOpenForSessStmt.get(session_key); },
+    getByToolCallId(tool_call_id) { return getByToolCallStmt.get(tool_call_id); },
+    setMessageIds(id, ids) { return setMsgIdsStmt.run(JSON.stringify(ids ?? []), id).changes; },
+
+    updateState(id, state, awaitingOther = false) {
+      return updateStateStmt.run({ id, state_json: JSON.stringify(state ?? {}), awaiting_other: awaitingOther ? 1 : 0 }).changes;
+    },
+
+    /**
+     * Authorize a responder. Claim-on-first-tap: if no from_id is recorded yet,
+     * the first interacting user claims the question; thereafter only that user
+     * may answer. Returns { ok, claimed }.
+     */
+    claimOrCheck(id, from_id) {
+      if (from_id == null) return { ok: false, claimed: false };
+      const claimed = claimStmt.run({ id, from_id }).changes > 0;
+      if (claimed) return { ok: true, claimed: true };
+      const row = getByIdStmt.get(id);
+      return { ok: row && Number(row.from_id) === Number(from_id), claimed: false };
+    },
+
+    resolve(id, status) {
+      if (!['answered', 'cancelled', 'timeout', 'expired'].includes(status)) {
+        throw new Error(`bad status: ${status}`);
+      }
+      return resolveStmt.run({ id, status, answered_ts: now() }).changes;
+    },
+
+    sweepTimedOut() { return listTimedOutStmt.all(now()); },
+    listOpen(bot_name) { return listOpenStmt.all(bot_name); },
+  };
+}
+
+module.exports = { createQuestionStore, tokensEqual, DEFAULT_TIMEOUT_MS };