@threadbase-sh/streamer 1.65.0 → 1.67.0

package/dist/index.js

@@ -4926,7 +4926,7 @@ function redactPath(path) {
 function worstStatus(checks) {
   const rank = { ok: 0, unknown: 1, degraded: 2, failed: 3 };
   return checks.reduce(
-    (worst, c) => rank[c.status] > rank[worst] ? c.status : worst,
+    (worst2, c) => rank[c.status] > rank[worst2] ? c.status : worst2,
     "ok"
   );
 }
@@ -5946,7 +5946,12 @@ var createMiscRoutes = (deps) => {
       // Whether to encrypt to this server. Additive, same contract as `push`:
       // absent means an older server, which a client must read as "unknown" and
       // resolve as today's plaintext path — never as a reason to fail.
-      e2ee: describeE2eeCapability(deps.featureFlagsConfig().values.e2ee)
+      e2ee: describeE2eeCapability(deps.featureFlagsConfig().values.e2ee),
+      // This build samples cheap host signals and pushes `host_pressure` when
+      // the box is starved. Additive capability flag only — live readings stay
+      // off this polled endpoint. Absent means an older server that never
+      // samples. Informational: pressure never holds, kills, or refuses sessions.
+      hostPressure: true
     });
   });
   app.get("/api/profiles", (c) => c.json([]));
@@ -12851,6 +12856,7 @@ function createApiDeps(deps) {
     wsHub: deps.wsHub,
     cache: () => deps.cache(),
     cacheMonitor: () => deps.cacheMonitor(),
+    hostPressureMonitor: () => deps.hostPressureMonitor(),
     pushRepo: () => deps.pushRepo(),
     liveActivityPushEnabled: () => deps.liveActivityPushEnabled(),
     devicesRepo: () => deps.devicesRepo(),
@@ -12901,6 +12907,8 @@ function createApiDeps(deps) {
       }
       const alertMsg = deps.cacheMonitor()?.wsMessage();
       if (alertMsg) deps.wsHub.unicast(ws, alertMsg);
+      const pressureMsg = deps.hostPressureMonitor()?.wsMessage();
+      if (pressureMsg) deps.wsHub.unicast(ws, pressureMsg);
     },
     handleWsMessage: async (ws, raw, principal) => {
       const deny = (type, required) => {
@@ -12971,6 +12979,13 @@ function createApiDeps(deps) {
             );
           }
         }
+        if (msg.type === "unsubscribe_session" && typeof msg.sessionId === "string") {
+          if (!wsAllows(principal, "history:read")) {
+            deny(msg.type, "history:read");
+            return;
+          }
+          deps.removeSessionSubscriber(msg.sessionId, ws);
+        }
         if (msg.type === "hold_session" && typeof msg.sessionId === "string") {
           if (!wsAllows(principal, "session:control")) {
             deny(msg.type, "session:control");
@@ -13526,6 +13541,221 @@ function pruneAgentConversations(cache) {
   return { scanned: rows.length, pruned, missing };
 }
 
+// src/services/host-pressure/hostPressure.ts
+import { cpus, freemem, loadavg, totalmem } from "os";
+import { monitorEventLoopDelay } from "perf_hooks";
+var HOST_PRESSURE_SAMPLE_MS = 5e3;
+var HOST_PRESSURE_BARS = {
+  memFreeRatio: {
+    enterElevated: 0.15,
+    leaveElevated: 0.17,
+    enterCritical: 0.08,
+    leaveCritical: 0.15
+  },
+  eventLoopP99Ms: {
+    enterElevated: 100,
+    leaveElevated: 80,
+    enterCritical: 250,
+    leaveCritical: 100
+  },
+  loadPerCpu: {
+    enterElevated: 1.25,
+    leaveElevated: 1.05,
+    enterCritical: 2,
+    leaveCritical: 1.25
+  },
+  // win32 has no loadavg. Busy ratio from os.cpus()[].times deltas is 0–1, so
+  // it cannot reuse loadPerCpu's 2.0 critical bar. Reason on the wire stays `load`.
+  cpuBusy: {
+    enterElevated: 0.85,
+    leaveElevated: 0.7,
+    enterCritical: 0.97,
+    leaveCritical: 0.85
+  },
+  liveAgentsPair: 4
+};
+var REASON_ORDER = ["memory", "event_loop", "load", "agents"];
+var RANK = { ok: 0, elevated: 1, critical: 2 };
+function worst(levels) {
+  return levels.reduce(
+    (acc, level) => RANK[level] > RANK[acc] ? level : acc,
+    "ok"
+  );
+}
+function schmittLowIsWorse(value, previous, enterElevated, leaveElevated, enterCritical, leaveCritical) {
+  if (previous === "critical") {
+    if (value < leaveCritical) return "critical";
+    if (value < leaveElevated) return "elevated";
+    return "ok";
+  }
+  if (previous === "elevated") {
+    if (value < enterCritical) return "critical";
+    if (value < leaveElevated) return "elevated";
+    return "ok";
+  }
+  if (value < enterCritical) return "critical";
+  if (value < enterElevated) return "elevated";
+  return "ok";
+}
+function schmittHighIsWorse(value, previous, enterElevated, leaveElevated, enterCritical, leaveCritical) {
+  if (previous === "critical") {
+    if (value > leaveCritical) return "critical";
+    if (value > leaveElevated) return "elevated";
+    return "ok";
+  }
+  if (previous === "elevated") {
+    if (value > enterCritical) return "critical";
+    if (value > leaveElevated) return "elevated";
+    return "ok";
+  }
+  if (value > enterCritical) return "critical";
+  if (value > enterElevated) return "elevated";
+  return "ok";
+}
+function timesTotal(times) {
+  return times.user + times.nice + times.sys + times.idle + times.irq;
+}
+function hostPressureOs(platform3) {
+  if (platform3 === "darwin" || platform3 === "linux" || platform3 === "win32") return platform3;
+  return void 0;
+}
+function cpuBusyRatio(previous, next) {
+  if (!previous || previous.length === 0 || next.length === 0 || previous.length !== next.length) {
+    return 0;
+  }
+  let idle = 0;
+  let total = 0;
+  for (let i = 0; i < next.length; i++) {
+    const dt = timesTotal(next[i]) - timesTotal(previous[i]);
+    if (dt <= 0) continue;
+    total += dt;
+    idle += Math.max(0, next[i].idle - previous[i].idle);
+  }
+  if (total <= 0) return 0;
+  return 1 - idle / total;
+}
+function classifyHostPressure(sample, previous, platform3) {
+  const memRaw = schmittLowIsWorse(
+    sample.memFreeRatio,
+    previous,
+    HOST_PRESSURE_BARS.memFreeRatio.enterElevated,
+    HOST_PRESSURE_BARS.memFreeRatio.leaveElevated,
+    HOST_PRESSURE_BARS.memFreeRatio.enterCritical,
+    HOST_PRESSURE_BARS.memFreeRatio.leaveCritical
+  );
+  const mem = platform3 === "darwin" && memRaw === "critical" ? "elevated" : memRaw;
+  const eventLoop = schmittHighIsWorse(
+    sample.eventLoopP99Ms,
+    previous,
+    HOST_PRESSURE_BARS.eventLoopP99Ms.enterElevated,
+    HOST_PRESSURE_BARS.eventLoopP99Ms.leaveElevated,
+    HOST_PRESSURE_BARS.eventLoopP99Ms.enterCritical,
+    HOST_PRESSURE_BARS.eventLoopP99Ms.leaveCritical
+  );
+  const ncpu = sample.ncpu > 0 ? sample.ncpu : 1;
+  const windows = platform3 === "win32";
+  const loadValue = windows ? sample.cpuBusyRatio ?? 0 : sample.load1 / ncpu;
+  const loadBars = windows ? HOST_PRESSURE_BARS.cpuBusy : HOST_PRESSURE_BARS.loadPerCpu;
+  const load = schmittHighIsWorse(
+    loadValue,
+    previous,
+    loadBars.enterElevated,
+    loadBars.leaveElevated,
+    loadBars.enterCritical,
+    loadBars.leaveCritical
+  );
+  const resourceLevel = worst([mem, eventLoop, load]);
+  const agentsPair = sample.liveAgents >= HOST_PRESSURE_BARS.liveAgentsPair && resourceLevel !== "ok";
+  const level = resourceLevel;
+  const firing = [];
+  if (mem !== "ok") firing.push("memory");
+  if (eventLoop !== "ok") firing.push("event_loop");
+  if (load !== "ok") firing.push("load");
+  if (agentsPair) firing.push("agents");
+  const reasons = REASON_ORDER.filter((reason) => firing.includes(reason));
+  return { level, reasons };
+}
+var HostPressureMonitor = class {
+  constructor(opts) {
+    this.opts = opts;
+  }
+  opts;
+  timer = null;
+  level = "ok";
+  lastWarning = null;
+  start() {
+    this.opts.histogram?.enable();
+    if (this.timer) return;
+    const intervalMs = this.opts.intervalMs ?? HOST_PRESSURE_SAMPLE_MS;
+    this.timer = setInterval(() => this.tick(), intervalMs);
+    this.timer.unref?.();
+  }
+  tick() {
+    const sample = this.opts.readSample();
+    const platform3 = this.opts.platform ?? process.platform;
+    const classified = classifyHostPressure(sample, this.level, platform3);
+    if (classified.level === this.level) return;
+    this.level = classified.level;
+    const updatedAt = (this.opts.now ?? (() => /* @__PURE__ */ new Date()))().toISOString();
+    if (classified.level === "ok") {
+      this.lastWarning = null;
+      this.opts.wsHub.broadcast({ type: "host_pressure_cleared", updatedAt });
+      return;
+    }
+    const os2 = hostPressureOs(platform3);
+    const message = {
+      type: "host_pressure",
+      level: classified.level,
+      reasons: classified.reasons,
+      liveAgents: sample.liveAgents,
+      updatedAt,
+      ...os2 ? { os: os2 } : {}
+    };
+    this.lastWarning = message;
+    this.opts.wsHub.broadcast(message);
+  }
+  wsMessage() {
+    return this.lastWarning;
+  }
+  dispose() {
+    if (this.timer) {
+      clearInterval(this.timer);
+      this.timer = null;
+    }
+    this.opts.histogram?.disable();
+  }
+};
+function createHostPressureMonitor(wsHub, liveAgents) {
+  const histogram = monitorEventLoopDelay({ resolution: 20 });
+  const windows = process.platform === "win32";
+  let prevCpuTimes = null;
+  const monitor = new HostPressureMonitor({
+    wsHub,
+    histogram,
+    readSample: () => {
+      const eventLoopP99Ms = histogram.percentile(99) / 1e6;
+      histogram.reset();
+      const total = totalmem();
+      const cpuList = cpus();
+      const sample = {
+        liveAgents: liveAgents(),
+        memFreeRatio: total > 0 ? freemem() / total : 1,
+        eventLoopP99Ms,
+        load1: loadavg()[0],
+        ncpu: cpuList.length
+      };
+      if (windows) {
+        const cpuTimes = cpuList.map((cpu) => cpu.times);
+        sample.cpuBusyRatio = cpuBusyRatio(prevCpuTimes, cpuTimes);
+        prevCpuTimes = cpuTimes;
+      }
+      return sample;
+    }
+  });
+  monitor.start();
+  return monitor;
+}
+
 // src/services/push/expoPushSender.ts
 var log5 = getLogger("expo-push");
 var EXPO_PUSH_ENDPOINT = "https://exp.host/--/api/v2/push/send";
@@ -15574,6 +15804,7 @@ var StreamerServer = class {
   wsToClientId = /* @__PURE__ */ new Map();
   cache = null;
   cacheMonitor = null;
+  hostPressureMonitor = null;
   projectsRepo = null;
   conversationsRepo = null;
   sessionsRepo = null;
@@ -15924,6 +16155,7 @@ var StreamerServer = class {
       // reset-and-rescan, and tests swap methods on the server instance.
       cache: () => this.cache,
       cacheMonitor: () => this.cacheMonitor,
+      hostPressureMonitor: () => this.hostPressureMonitor,
       pushRepo: () => this.pushRepo,
       liveActivityPushEnabled: () => this.liveActivityNotifier !== null,
       devicesRepo: () => this.devicesRepo,
@@ -15939,6 +16171,7 @@ var StreamerServer = class {
       withReconciledLifecycle: (sessions) => this.withReconciledLifecycle(sessions),
       currentWarmupState: () => this.currentWarmupState(),
       addSessionSubscriber: (sessionId, ws) => this.addSessionSubscriber(sessionId, ws),
+      removeSessionSubscriber: (sessionId, ws) => this.removeSessionSubscriber(sessionId, ws),
       startGraceTimer: (sessionId, delayMs) => this.startGraceTimer(sessionId, delayMs),
       armHoldWhenIdle: (sessionId) => this.armHoldWhenIdle(sessionId),
       handleSessionsCount: (res) => this.handleSessionsCount(res),
@@ -16068,6 +16301,12 @@ var StreamerServer = class {
       );
     }
   }
+  removeSessionSubscriber(sessionId, ws) {
+    const subs = this.sessionSubscribers.get(sessionId);
+    if (!subs) return;
+    subs.delete(ws);
+    if (subs.size === 0) this.sessionSubscribers.delete(sessionId);
+  }
   /**
    * Bring up Live Activity push, if credentials are present (Feature 12).
    *
@@ -16398,6 +16637,10 @@ var StreamerServer = class {
       this.idleReaperTimer = setInterval(() => this.reapIdleSessions(), IDLE_REAP_SWEEP_MS);
       this.idleReaperTimer.unref?.();
     }
+    this.hostPressureMonitor = createHostPressureMonitor(
+      this.wsHub,
+      () => this.ptyAttachedIds().size
+    );
     const warmUp = new Promise((resolveWarm) => {
       {
         this.log.info(`Streamer server listening on port ${port}`, {
@@ -16710,6 +16953,8 @@ var StreamerServer = class {
       clearInterval(this.idleReaperTimer);
       this.idleReaperTimer = null;
     }
+    this.hostPressureMonitor?.dispose();
+    this.hostPressureMonitor = null;
     this.lastAgentChunkAt.clear();
     this.terminalSeq.clear();
     if (this.ptyManager.isRemote()) this.ptyManager.dispose();