@threadbase-sh/streamer 1.64.0 → 1.66.0

package/dist/index.js

@@ -4926,7 +4926,7 @@ function redactPath(path) {
 function worstStatus(checks) {
   const rank = { ok: 0, unknown: 1, degraded: 2, failed: 3 };
   return checks.reduce(
-    (worst, c) => rank[c.status] > rank[worst] ? c.status : worst,
+    (worst2, c) => rank[c.status] > rank[worst2] ? c.status : worst2,
     "ok"
   );
 }
@@ -5946,7 +5946,12 @@ var createMiscRoutes = (deps) => {
       // Whether to encrypt to this server. Additive, same contract as `push`:
       // absent means an older server, which a client must read as "unknown" and
       // resolve as today's plaintext path — never as a reason to fail.
-      e2ee: describeE2eeCapability(deps.featureFlagsConfig().values.e2ee)
+      e2ee: describeE2eeCapability(deps.featureFlagsConfig().values.e2ee),
+      // This build samples cheap host signals and pushes `host_pressure` when
+      // the box is starved. Additive capability flag only — live readings stay
+      // off this polled endpoint. Absent means an older server that never
+      // samples. Informational: pressure never holds, kills, or refuses sessions.
+      hostPressure: true
     });
   });
   app.get("/api/profiles", (c) => c.json([]));
@@ -10620,6 +10625,19 @@ var SessionHandlers = class {
       sessions: this.sessionStore.list(this.deps.ptyAttachedIds())
     });
   }
+  /**
+   * Same empty-unused check stop uses (promptCount === 0 and no cache row,
+   * including boundConversationId / resumedFromConversationId). Safe after
+   * putOnHold: the runner may already have dropped the live session.
+   */
+  forgetIfEmptyUnused(sessionId) {
+    const live = this.ptyManager.getSession(sessionId);
+    const stored = this.sessionStore.getManaged(sessionId);
+    const session = live ?? stored;
+    if (!session) return;
+    if (!this.shouldForgetEmptySession(session)) return;
+    this.forgetEmptyStoppedSession(sessionId);
+  }
   async handleAdopt(sessionId, res) {
     const discovered = await discoverClaudeProcesses();
     this.sessionStore.setDiscovered(discovered);
@@ -12814,6 +12832,7 @@ function createLiveSessionOptions(deps) {
       void deps.liveActivityNotifier()?.onStatusChange(session, previousStatus);
       void deps.waitingInputNotifier()?.onStatusChange(session, previousStatus);
       deps.sessionStatusBus.emit(`status:${session.id}`, session.status, session);
+      deps.maybeFireHoldWhenIdle?.(session);
     }
   };
 }
@@ -12837,6 +12856,7 @@ function createApiDeps(deps) {
     wsHub: deps.wsHub,
     cache: () => deps.cache(),
     cacheMonitor: () => deps.cacheMonitor(),
+    hostPressureMonitor: () => deps.hostPressureMonitor(),
     pushRepo: () => deps.pushRepo(),
     liveActivityPushEnabled: () => deps.liveActivityPushEnabled(),
     devicesRepo: () => deps.devicesRepo(),
@@ -12887,6 +12907,8 @@ function createApiDeps(deps) {
       }
       const alertMsg = deps.cacheMonitor()?.wsMessage();
       if (alertMsg) deps.wsHub.unicast(ws, alertMsg);
+      const pressureMsg = deps.hostPressureMonitor()?.wsMessage();
+      if (pressureMsg) deps.wsHub.unicast(ws, pressureMsg);
     },
     handleWsMessage: async (ws, raw, principal) => {
       const deny = (type, required) => {
@@ -12962,7 +12984,20 @@ function createApiDeps(deps) {
             deny(msg.type, "session:control");
             return;
           }
-          deps.startGraceTimer(msg.sessionId, deps.ptyGracePeriodMs);
+          const when = msg.when;
+          if (when === void 0 || when === "grace") {
+            deps.startGraceTimer(msg.sessionId, deps.ptyGracePeriodMs);
+            return;
+          }
+          if (when === "waiting_input") {
+            deps.armHoldWhenIdle(msg.sessionId);
+            return;
+          }
+          deps.log().warn(`[pty.hold_when_unknown] hold_session when=${String(when)}`, {
+            event: "pty.hold_when_unknown",
+            sessionId: msg.sessionId,
+            when
+          });
         }
       } catch {
       }
@@ -13499,6 +13534,221 @@ function pruneAgentConversations(cache) {
   return { scanned: rows.length, pruned, missing };
 }
 
+// src/services/host-pressure/hostPressure.ts
+import { cpus, freemem, loadavg, totalmem } from "os";
+import { monitorEventLoopDelay } from "perf_hooks";
+var HOST_PRESSURE_SAMPLE_MS = 5e3;
+var HOST_PRESSURE_BARS = {
+  memFreeRatio: {
+    enterElevated: 0.15,
+    leaveElevated: 0.17,
+    enterCritical: 0.08,
+    leaveCritical: 0.15
+  },
+  eventLoopP99Ms: {
+    enterElevated: 100,
+    leaveElevated: 80,
+    enterCritical: 250,
+    leaveCritical: 100
+  },
+  loadPerCpu: {
+    enterElevated: 1.25,
+    leaveElevated: 1.05,
+    enterCritical: 2,
+    leaveCritical: 1.25
+  },
+  // win32 has no loadavg. Busy ratio from os.cpus()[].times deltas is 0–1, so
+  // it cannot reuse loadPerCpu's 2.0 critical bar. Reason on the wire stays `load`.
+  cpuBusy: {
+    enterElevated: 0.85,
+    leaveElevated: 0.7,
+    enterCritical: 0.97,
+    leaveCritical: 0.85
+  },
+  liveAgentsPair: 4
+};
+var REASON_ORDER = ["memory", "event_loop", "load", "agents"];
+var RANK = { ok: 0, elevated: 1, critical: 2 };
+function worst(levels) {
+  return levels.reduce(
+    (acc, level) => RANK[level] > RANK[acc] ? level : acc,
+    "ok"
+  );
+}
+function schmittLowIsWorse(value, previous, enterElevated, leaveElevated, enterCritical, leaveCritical) {
+  if (previous === "critical") {
+    if (value < leaveCritical) return "critical";
+    if (value < leaveElevated) return "elevated";
+    return "ok";
+  }
+  if (previous === "elevated") {
+    if (value < enterCritical) return "critical";
+    if (value < leaveElevated) return "elevated";
+    return "ok";
+  }
+  if (value < enterCritical) return "critical";
+  if (value < enterElevated) return "elevated";
+  return "ok";
+}
+function schmittHighIsWorse(value, previous, enterElevated, leaveElevated, enterCritical, leaveCritical) {
+  if (previous === "critical") {
+    if (value > leaveCritical) return "critical";
+    if (value > leaveElevated) return "elevated";
+    return "ok";
+  }
+  if (previous === "elevated") {
+    if (value > enterCritical) return "critical";
+    if (value > leaveElevated) return "elevated";
+    return "ok";
+  }
+  if (value > enterCritical) return "critical";
+  if (value > enterElevated) return "elevated";
+  return "ok";
+}
+function timesTotal(times) {
+  return times.user + times.nice + times.sys + times.idle + times.irq;
+}
+function hostPressureOs(platform3) {
+  if (platform3 === "darwin" || platform3 === "linux" || platform3 === "win32") return platform3;
+  return void 0;
+}
+function cpuBusyRatio(previous, next) {
+  if (!previous || previous.length === 0 || next.length === 0 || previous.length !== next.length) {
+    return 0;
+  }
+  let idle = 0;
+  let total = 0;
+  for (let i = 0; i < next.length; i++) {
+    const dt = timesTotal(next[i]) - timesTotal(previous[i]);
+    if (dt <= 0) continue;
+    total += dt;
+    idle += Math.max(0, next[i].idle - previous[i].idle);
+  }
+  if (total <= 0) return 0;
+  return 1 - idle / total;
+}
+function classifyHostPressure(sample, previous, platform3) {
+  const memRaw = schmittLowIsWorse(
+    sample.memFreeRatio,
+    previous,
+    HOST_PRESSURE_BARS.memFreeRatio.enterElevated,
+    HOST_PRESSURE_BARS.memFreeRatio.leaveElevated,
+    HOST_PRESSURE_BARS.memFreeRatio.enterCritical,
+    HOST_PRESSURE_BARS.memFreeRatio.leaveCritical
+  );
+  const mem = platform3 === "darwin" && memRaw === "critical" ? "elevated" : memRaw;
+  const eventLoop = schmittHighIsWorse(
+    sample.eventLoopP99Ms,
+    previous,
+    HOST_PRESSURE_BARS.eventLoopP99Ms.enterElevated,
+    HOST_PRESSURE_BARS.eventLoopP99Ms.leaveElevated,
+    HOST_PRESSURE_BARS.eventLoopP99Ms.enterCritical,
+    HOST_PRESSURE_BARS.eventLoopP99Ms.leaveCritical
+  );
+  const ncpu = sample.ncpu > 0 ? sample.ncpu : 1;
+  const windows = platform3 === "win32";
+  const loadValue = windows ? sample.cpuBusyRatio ?? 0 : sample.load1 / ncpu;
+  const loadBars = windows ? HOST_PRESSURE_BARS.cpuBusy : HOST_PRESSURE_BARS.loadPerCpu;
+  const load = schmittHighIsWorse(
+    loadValue,
+    previous,
+    loadBars.enterElevated,
+    loadBars.leaveElevated,
+    loadBars.enterCritical,
+    loadBars.leaveCritical
+  );
+  const resourceLevel = worst([mem, eventLoop, load]);
+  const agentsPair = sample.liveAgents >= HOST_PRESSURE_BARS.liveAgentsPair && resourceLevel !== "ok";
+  const level = resourceLevel;
+  const firing = [];
+  if (mem !== "ok") firing.push("memory");
+  if (eventLoop !== "ok") firing.push("event_loop");
+  if (load !== "ok") firing.push("load");
+  if (agentsPair) firing.push("agents");
+  const reasons = REASON_ORDER.filter((reason) => firing.includes(reason));
+  return { level, reasons };
+}
+var HostPressureMonitor = class {
+  constructor(opts) {
+    this.opts = opts;
+  }
+  opts;
+  timer = null;
+  level = "ok";
+  lastWarning = null;
+  start() {
+    this.opts.histogram?.enable();
+    if (this.timer) return;
+    const intervalMs = this.opts.intervalMs ?? HOST_PRESSURE_SAMPLE_MS;
+    this.timer = setInterval(() => this.tick(), intervalMs);
+    this.timer.unref?.();
+  }
+  tick() {
+    const sample = this.opts.readSample();
+    const platform3 = this.opts.platform ?? process.platform;
+    const classified = classifyHostPressure(sample, this.level, platform3);
+    if (classified.level === this.level) return;
+    this.level = classified.level;
+    const updatedAt = (this.opts.now ?? (() => /* @__PURE__ */ new Date()))().toISOString();
+    if (classified.level === "ok") {
+      this.lastWarning = null;
+      this.opts.wsHub.broadcast({ type: "host_pressure_cleared", updatedAt });
+      return;
+    }
+    const os2 = hostPressureOs(platform3);
+    const message = {
+      type: "host_pressure",
+      level: classified.level,
+      reasons: classified.reasons,
+      liveAgents: sample.liveAgents,
+      updatedAt,
+      ...os2 ? { os: os2 } : {}
+    };
+    this.lastWarning = message;
+    this.opts.wsHub.broadcast(message);
+  }
+  wsMessage() {
+    return this.lastWarning;
+  }
+  dispose() {
+    if (this.timer) {
+      clearInterval(this.timer);
+      this.timer = null;
+    }
+    this.opts.histogram?.disable();
+  }
+};
+function createHostPressureMonitor(wsHub, liveAgents) {
+  const histogram = monitorEventLoopDelay({ resolution: 20 });
+  const windows = process.platform === "win32";
+  let prevCpuTimes = null;
+  const monitor = new HostPressureMonitor({
+    wsHub,
+    histogram,
+    readSample: () => {
+      const eventLoopP99Ms = histogram.percentile(99) / 1e6;
+      histogram.reset();
+      const total = totalmem();
+      const cpuList = cpus();
+      const sample = {
+        liveAgents: liveAgents(),
+        memFreeRatio: total > 0 ? freemem() / total : 1,
+        eventLoopP99Ms,
+        load1: loadavg()[0],
+        ncpu: cpuList.length
+      };
+      if (windows) {
+        const cpuTimes = cpuList.map((cpu) => cpu.times);
+        sample.cpuBusyRatio = cpuBusyRatio(prevCpuTimes, cpuTimes);
+        prevCpuTimes = cpuTimes;
+      }
+      return sample;
+    }
+  });
+  monitor.start();
+  return monitor;
+}
+
 // src/services/push/expoPushSender.ts
 var log5 = getLogger("expo-push");
 var EXPO_PUSH_ENDPOINT = "https://exp.host/--/api/v2/push/send";
@@ -15518,6 +15768,10 @@ var StreamerServer = class {
   // Consecutive grace-timer defers for a still-`running` session (see
   // GRACE_MAX_DEFERS). Reset when a subscriber reconnects or the PTY settles.
   ptyGraceDeferCounts = /* @__PURE__ */ new Map();
+  // Session ids that should be putOnHold at the next waiting_input/idle.
+  // Same lifetime as ptyGraceTimers: in-memory, dropped on close/restart.
+  // Last writer wins against the grace timer — never both armed.
+  holdWhenIdle = /* @__PURE__ */ new Set();
   // Map of sessionId → set of subscribed WS clients
   sessionSubscribers = /* @__PURE__ */ new Map();
   // sessionId → wall-clock ms of the last PTY chunk. Written from onOutput for
@@ -15543,6 +15797,7 @@ var StreamerServer = class {
   wsToClientId = /* @__PURE__ */ new Map();
   cache = null;
   cacheMonitor = null;
+  hostPressureMonitor = null;
   projectsRepo = null;
   conversationsRepo = null;
   sessionsRepo = null;
@@ -15745,7 +16000,8 @@ var StreamerServer = class {
         waitingInputNotifier: () => this.waitingInputNotifier,
         ptyAttachedIds: () => this.ptyAttachedIds(),
         cancelPendingQuestion: (sessionId) => this.cancelPendingQuestion(sessionId),
-        rememberSelfPtyEnded: (conversationId) => this.rememberSelfPtyEnded(conversationId)
+        rememberSelfPtyEnded: (conversationId) => this.rememberSelfPtyEnded(conversationId),
+        maybeFireHoldWhenIdle: (session) => this.maybeFireHoldWhenIdle(session)
       })
     );
     this.sessionWatchers = new SessionWatchers({
@@ -15892,6 +16148,7 @@ var StreamerServer = class {
       // reset-and-rescan, and tests swap methods on the server instance.
       cache: () => this.cache,
       cacheMonitor: () => this.cacheMonitor,
+      hostPressureMonitor: () => this.hostPressureMonitor,
       pushRepo: () => this.pushRepo,
       liveActivityPushEnabled: () => this.liveActivityNotifier !== null,
       devicesRepo: () => this.devicesRepo,
@@ -15908,6 +16165,7 @@ var StreamerServer = class {
       currentWarmupState: () => this.currentWarmupState(),
       addSessionSubscriber: (sessionId, ws) => this.addSessionSubscriber(sessionId, ws),
       startGraceTimer: (sessionId, delayMs) => this.startGraceTimer(sessionId, delayMs),
+      armHoldWhenIdle: (sessionId) => this.armHoldWhenIdle(sessionId),
       handleSessionsCount: (res) => this.handleSessionsCount(res),
       applyLiveSessionSetting: (id, req, res, setting) => this.applyLiveSessionSetting(id, req, res, setting),
       handlePairStart: (res) => this.handlePairStart(res),
@@ -16027,6 +16285,13 @@ var StreamerServer = class {
       this.ptyGraceTimers.delete(sessionId);
     }
     this.ptyGraceDeferCounts.delete(sessionId);
+    if (this.holdWhenIdle.delete(sessionId)) {
+      this.log.info(
+        `[hold-when-idle] cancelled ${sessionId} (subscribe)`,
+        { sessionId, event: "pty.hold_when_idle_cancel", reason: "subscribe" },
+        "pino"
+      );
+    }
   }
   /**
    * Bring up Live Activity push, if credentials are present (Feature 12).
@@ -16134,6 +16399,7 @@ var StreamerServer = class {
     return reaped;
   }
   startGraceTimer(sessionId, delayMs) {
+    this.holdWhenIdle.delete(sessionId);
     const existing = this.ptyGraceTimers.get(sessionId);
     if (existing) clearTimeout(existing);
     const timer = setTimeout(() => {
@@ -16175,6 +16441,63 @@ var StreamerServer = class {
     }, delayMs);
     this.ptyGraceTimers.set(sessionId, timer);
   }
+  clearGrace(sessionId) {
+    const existing = this.ptyGraceTimers.get(sessionId);
+    if (existing) {
+      clearTimeout(existing);
+      this.ptyGraceTimers.delete(sessionId);
+    }
+    this.ptyGraceDeferCounts.delete(sessionId);
+  }
+  /**
+   * Arm the in-app "Kill on idle" latch: hold now if already settled, otherwise
+   * on the next running → waiting_input (or idle). No grace delay, no defer cap.
+   * A subscribed leaving socket must not block an immediate hold.
+   */
+  armHoldWhenIdle(sessionId) {
+    if (!this.ptyManager.hasSession(sessionId)) return;
+    this.clearGrace(sessionId);
+    const status = this.ptyManager.getSession(sessionId)?.status ?? this.sessionStore.getManaged(sessionId)?.status;
+    if (status === "waiting_input" || status === "idle") {
+      this.holdWhenIdle.delete(sessionId);
+      this.ptyManager.putOnHold(sessionId);
+      this.forgetIfEmptyUnused(sessionId);
+      return;
+    }
+    this.holdWhenIdle.add(sessionId);
+    this.log.info(
+      `[hold-when-idle] armed ${sessionId}`,
+      { sessionId, event: "pty.hold_when_idle_armed" },
+      "pino"
+    );
+  }
+  /**
+   * Fire the Kill-on-idle latch from the shared onStatusChange funnel.
+   * Delete first so the ensuing idle transition cannot re-enter.
+   */
+  maybeFireHoldWhenIdle(session) {
+    if (session.status !== "waiting_input" && session.status !== "idle") return;
+    if (!this.holdWhenIdle.has(session.id)) return;
+    this.holdWhenIdle.delete(session.id);
+    if (this.hasSessionSubscriber(session.id)) {
+      this.log.info(
+        `[hold-when-idle] cancelled ${session.id} (subscriber)`,
+        { sessionId: session.id, event: "pty.hold_when_idle_cancel", reason: "subscriber" },
+        "pino"
+      );
+      return;
+    }
+    this.log.info(
+      `[hold-when-idle] holding ${session.id}`,
+      { sessionId: session.id, event: "pty.hold_when_idle_fire" },
+      "pino"
+    );
+    this.ptyManager.putOnHold(session.id);
+    this.forgetIfEmptyUnused(session.id);
+  }
+  forgetIfEmptyUnused(sessionId) {
+    this.sessionHandlers.forgetIfEmptyUnused(sessionId);
+  }
   get port() {
     const addr = this.httpServer.address();
     return typeof addr === "object" && addr ? addr.port : 0;
@@ -16300,6 +16623,10 @@ var StreamerServer = class {
       this.idleReaperTimer = setInterval(() => this.reapIdleSessions(), IDLE_REAP_SWEEP_MS);
       this.idleReaperTimer.unref?.();
     }
+    this.hostPressureMonitor = createHostPressureMonitor(
+      this.wsHub,
+      () => this.ptyAttachedIds().size
+    );
     const warmUp = new Promise((resolveWarm) => {
       {
         this.log.info(`Streamer server listening on port ${port}`, {
@@ -16607,10 +16934,13 @@ var StreamerServer = class {
   async close() {
     for (const timer of this.ptyGraceTimers.values()) clearTimeout(timer);
     this.ptyGraceTimers.clear();
+    this.holdWhenIdle.clear();
     if (this.idleReaperTimer) {
       clearInterval(this.idleReaperTimer);
       this.idleReaperTimer = null;
     }
+    this.hostPressureMonitor?.dispose();
+    this.hostPressureMonitor = null;
     this.lastAgentChunkAt.clear();
     this.terminalSeq.clear();
     if (this.ptyManager.isRemote()) this.ptyManager.dispose();