ocuclaw 1.3.2 → 1.3.4

package/dist/runtime/relay-worker-transport.js

@@ -16,11 +16,19 @@ import { createWorkerMessageSendQueue } from "./relay-worker-queue.js";
 import { createRelayWorkerHealthMonitor } from "./relay-worker-health.js";
 import { createApprovalReplayCache } from "./relay-worker-approval-replay-cache.js";
 import { createRelayClientNudgeController } from "./relay-client-nudge-controller.js";
+import { constantTimeEqual } from "../domain/constant-time-equal.js";
 
 const WebSocket = WebSocketModule.default || WebSocketModule.WebSocket || WebSocketModule;
 const WebSocketServer = WebSocketModule.WebSocketServer || WebSocketModule.Server || WebSocket.Server;
 const SEND_BUFFER_HIGH_WATER_BYTES = 262_144;
 
+const SEND_BUFFER_HIGH_WATER_SHED_MS = 30_000;
+
+const LIVENESS_MAX_MISSED_PINGS = 2;
+
+const CONTROL_QUEUE_MAX_DEFAULT = 1000;
+const TRANSACTIONAL_QUEUE_MAX_DEFAULT = 1000;
+
 function normalizeLogger(logger) {
   if (!logger || typeof logger !== "object") return console;
   return {
@@ -52,12 +60,19 @@ export function createRelayWorkerTransport(options = {}) {
   const logger = normalizeLogger(options.logger);
   const postToMain = typeof options.postToMain === "function" ? options.postToMain : () => {};
   const now = typeof options.now === "function" ? options.now : () => Date.now();
+  const listenRetryDelayMs =
+    Number.isFinite(options.listenRetryDelayMs) && options.listenRetryDelayMs >= 0
+      ? Math.floor(options.listenRetryDelayMs)
+      : 200;
+  const listenRetryMaxAttempts =
+    Number.isFinite(options.listenRetryMaxAttempts) && options.listenRetryMaxAttempts >= 0
+      ? Math.floor(options.listenRetryMaxAttempts)
+      : 5;
   let manifest = null;
   let httpServer = null;
   let wss = null;
   let nextClientId = 1;
-  // Invalid-token rejects are the one connection log an internet-facing
-  // misconfiguration could flood, so they are rate-limited per remote address.
+
   const TOKEN_REJECT_LOG_WINDOW_MS = 60000;
   const TOKEN_REJECT_LOG_MAX_ADDRESSES = 100;
   const tokenRejectLogState = new Map();
@@ -88,10 +103,13 @@ export function createRelayWorkerTransport(options = {}) {
   }
   let expireTimer = null;
   let healthTimer = null;
+  let livenessTimer = null;
   let loopDelayMonitor = null;
   const clients = new Map();
   const protocolState = new Map();
   const outboundQueues = new Map();
+
+  const sendBufferOverWaterSince = new Map();
   const sockets = new Set();
   const pendingHttp = new Map();
   const cache = {
@@ -223,14 +241,21 @@ export function createRelayWorkerTransport(options = {}) {
     return !!normalizeRequestId(parsed && parsed.requestId);
   }
 
+  function controlQueueMax() {
+    const v = manifest && manifest.queue ? manifest.queue.controlQueueMax : undefined;
+    return Number.isFinite(v) && v > 0 ? Math.floor(v) : CONTROL_QUEUE_MAX_DEFAULT;
+  }
+
+  function transactionalQueueMax() {
+    const v = manifest && manifest.queue ? manifest.queue.transactionalQueueMax : undefined;
+    return Number.isFinite(v) && v > 0 ? Math.floor(v) : TRANSACTIONAL_QUEUE_MAX_DEFAULT;
+  }
+
   function enqueueFrame(clientId, frame, options = {}) {
     const ws = clients.get(clientId);
     if (!ws || ws.readyState !== WebSocket.OPEN) return;
     const parsedFrame = parseFrame(frame);
-    // F11 fan-out: reuse the type the main thread already parsed (threaded via
-    // options.knownType from the broadcast/unicast call site) instead of
-    // re-deriving it per client. parsedFrame is still needed below for
-    // isImportantTransactionalFrame's requestId read, so parseFrame stays.
+
     const type = options.knownType !== undefined ? options.knownType : parseMessageType(parsedFrame);
     const q = ensureOutboundQueue(clientId);
     if (
@@ -247,8 +272,26 @@ export function createRelayWorkerTransport(options = {}) {
       type === APP_PROTOCOL.operationReceived
     ) {
       q.control.push(frame);
+      while (q.control.length > controlQueueMax()) {
+
+        const dropped = q.control.shift();
+        emitDebug("worker_control_frame_dropped", "warn", {
+          clientId,
+          droppedType: parseMessageType(parseFrame(dropped)),
+          queueDepth: q.control.length,
+        });
+      }
     } else if (isImportantTransactionalFrame(type, parsedFrame)) {
       q.transactional.push(frame);
+      while (q.transactional.length > transactionalQueueMax()) {
+
+        const dropped = q.transactional.shift();
+        emitDebug("worker_transactional_frame_dropped", "warn", {
+          clientId,
+          droppedType: parseMessageType(parseFrame(dropped)),
+          queueDepth: q.transactional.length,
+        });
+      }
     } else if (
       type === APP_PROTOCOL.pages ||
       type === APP_PROTOCOL.status ||
@@ -258,7 +301,7 @@ export function createRelayWorkerTransport(options = {}) {
     } else {
       q.bestEffort.push(frame);
       while (q.bestEffort.length > 100) {
-        // Newest-wins eviction: drop the oldest queued best-effort frame.
+
         const dropped = q.bestEffort.shift();
         emitDebug("worker_best_effort_frame_dropped", "warn", {
           clientId,
@@ -600,7 +643,7 @@ export function createRelayWorkerTransport(options = {}) {
           ? parsed.hash
           : null;
       if (!requestedAgentName || !requestedHash) {
-        // Malformed request — silently drop; mirrors the non-worker handler.
+
         return;
       }
       const dataUri =
@@ -918,14 +961,15 @@ export function createRelayWorkerTransport(options = {}) {
         sockets.delete(socket);
       });
     });
-    wss = new WebSocketServer({ noServer: true });
+
+    wss = new WebSocketServer({ noServer: true, maxPayload: manifest.rpc.wsMaxMessageBytes });
     httpServer.on("upgrade", (req, socket, head) => {
       wss.handleUpgrade(req, socket, head, (ws) => wss.emit("connection", ws, req));
     });
     wss.on("connection", (ws, req) => {
       const requestUrl = new URL(req.url || "/", `http://${req.headers.host || "127.0.0.1"}`);
       const remoteAddress = (req.socket && req.socket.remoteAddress) || "unknown";
-      if (requestUrl.searchParams.get("token") !== manifest.relayToken) {
+      if (!constantTimeEqual(requestUrl.searchParams.get("token"), manifest.relayToken)) {
         logTokenReject(remoteAddress);
         ws.close(4001, "invalid_token");
         return;
@@ -936,6 +980,9 @@ export function createRelayWorkerTransport(options = {}) {
         `[ocuclaw] relay client connected clientId=${clientId} remote=${remoteAddress}`,
       );
       clients.set(clientId, ws);
+
+      ws.__ocuMissedPings = 0;
+      ws.on("pong", () => { ws.__ocuMissedPings = 0; });
       protocolState.set(clientId, {
         protocolVersion: null,
         clientKind: "unknown",
@@ -954,6 +1001,7 @@ export function createRelayWorkerTransport(options = {}) {
         clients.delete(clientId);
         protocolState.delete(clientId);
         outboundQueues.delete(clientId);
+        sendBufferOverWaterSince.delete(clientId);
         if (nudgeController) nudgeController.deleteClient(clientId);
         const closeReasonStr =
           reason == null
@@ -978,11 +1026,43 @@ export function createRelayWorkerTransport(options = {}) {
     });
 
     await new Promise((resolve, reject) => {
-      httpServer.once("error", reject);
-      httpServer.listen(manifest.port, manifest.host, () => {
-        httpServer.off("error", reject);
+      let attempt = 0;
+      let settled = false;
+      const onError = (err) => {
+        if (settled) return;
+
+        if (err && err.code === "EADDRINUSE" && attempt < listenRetryMaxAttempts) {
+          attempt += 1;
+          emitDebug("worker_listen_retry", "warn", {
+            attempt,
+            code: err.code,
+            port: manifest.port,
+          });
+          const retryTimer = setTimeout(tryListen, listenRetryDelayMs);
+          if (typeof retryTimer.unref === "function") retryTimer.unref();
+          return;
+        }
+        settled = true;
+        reject(err);
+      };
+      const onListening = () => {
+        if (settled) return;
+        settled = true;
+        httpServer.off("error", onError);
         resolve();
-      });
+      };
+      function tryListen() {
+        if (settled) return;
+        if (!httpServer) {
+
+          settled = true;
+          reject(new Error("relay worker transport closed during listen retry"));
+          return;
+        }
+        httpServer.once("error", onError);
+        httpServer.listen(manifest.port, manifest.host, onListening);
+      }
+      tryListen();
     });
     loopDelayMonitor = monitorEventLoopDelay({ resolution: 50 });
     loopDelayMonitor.enable();
@@ -997,9 +1077,39 @@ export function createRelayWorkerTransport(options = {}) {
     }, Math.max(10, Math.min(1000, manifest.queue.messageSendTtlMs)));
     healthTimer = setInterval(() => {
       health.updateLoopLagP95Ms(sampleLoopLagP95Ms());
-      health.updateSendBufferHighWaterClients(countSendBufferHighWaterClients());
+      const sendBufferHighWaterClients = countSendBufferHighWaterClients();
+      health.updateSendBufferHighWaterClients(sendBufferHighWaterClients);
       health.sample();
+
+      sweepStuckSlowClients();
+
+      postToMain({
+        kind: "worker.backpressure",
+        workerEpoch: manifest.workerEpoch,
+        sendBufferHighWaterClients,
+      });
     }, manifest.health.heartbeatIntervalMs);
+
+    const livenessIntervalMs =
+      Number.isFinite(manifest.health.livenessIntervalMs) && manifest.health.livenessIntervalMs > 0
+        ? Math.floor(manifest.health.livenessIntervalMs)
+        : Math.max(15000, manifest.health.heartbeatIntervalMs * 3);
+    livenessTimer = setInterval(() => {
+      for (const [, ws] of clients) {
+        if (ws.readyState !== WebSocket.OPEN) continue;
+        if ((ws.__ocuMissedPings || 0) >= LIVENESS_MAX_MISSED_PINGS) {
+          ws.terminate();
+          continue;
+        }
+        ws.__ocuMissedPings = (ws.__ocuMissedPings || 0) + 1;
+        try {
+          ws.ping();
+        } catch {
+
+        }
+      }
+    }, livenessIntervalMs);
+    if (typeof livenessTimer.unref === "function") livenessTimer.unref();
     postToMain({ kind: "worker.ready", workerEpoch: manifest.workerEpoch, address: address() });
   }
 
@@ -1025,6 +1135,35 @@ export function createRelayWorkerTransport(options = {}) {
     return count;
   }
 
+  function sweepStuckSlowClients() {
+    const at = now();
+    for (const [clientId, ws] of clients) {
+      if ((protocolState.get(clientId) || {}).clientKind !== "app") continue;
+      const over =
+        ws.readyState === WebSocket.OPEN &&
+        Number.isFinite(ws.bufferedAmount) &&
+        ws.bufferedAmount > SEND_BUFFER_HIGH_WATER_BYTES;
+      if (!over) {
+        sendBufferOverWaterSince.delete(clientId);
+        continue;
+      }
+      const since = sendBufferOverWaterSince.get(clientId);
+      if (since === undefined) {
+        sendBufferOverWaterSince.set(clientId, at);
+        continue;
+      }
+      if (at - since >= SEND_BUFFER_HIGH_WATER_SHED_MS) {
+        emitDebug("worker_client_send_buffer_shed", "warn", {
+          clientId,
+          bufferedAmount: ws.bufferedAmount,
+          overHighWaterMs: at - since,
+        });
+        sendBufferOverWaterSince.delete(clientId);
+        ws.terminate();
+      }
+    }
+  }
+
   function address() {
     return httpServer && typeof httpServer.address === "function" ? httpServer.address() : null;
   }
@@ -1032,8 +1171,10 @@ export function createRelayWorkerTransport(options = {}) {
   function close() {
     if (expireTimer) clearInterval(expireTimer);
     if (healthTimer) clearInterval(healthTimer);
+    if (livenessTimer) clearInterval(livenessTimer);
     expireTimer = null;
     healthTimer = null;
+    livenessTimer = null;
     if (loopDelayMonitor) {
       loopDelayMonitor.disable();
       loopDelayMonitor = null;
@@ -1049,6 +1190,7 @@ export function createRelayWorkerTransport(options = {}) {
     clients.clear();
     protocolState.clear();
     outboundQueues.clear();
+    sendBufferOverWaterSince.clear();
     if (nudgeController) nudgeController.clear();
     nudgeController = null;
     if (approvalReplay) approvalReplay.clear();

package/dist/runtime/session-context-service.js

@@ -1,19 +1,3 @@
-// session-context-service.ts
-//
-// Owns the active-session "context tokens vs context window" snapshot
-// lifecycle. Reads via gateway sessions.describe; reads compaction
-// checkpoint count via sessions.compaction.list; broadcasts on changes.
-//
-// Field mapping (gateway → snapshot):
-//   session.totalTokens   → contextTokens (current cumulative usage)
-//   session.contextTokens → contextWindow (model's configured window)
-// openclaw uses `contextTokens` on the row for window size; `totalTokens`
-// is the running usage. NOTE: the window is NOT resolved on a fresh session —
-// the gateway warms a model's window only after the first turn (sessions.describe
-// returns no window pre-turn). So we cache the observed window per model and
-// reuse it for subsequent cold/new-session reads. The cache is persisted under
-// `stateDir` (keyed by provider/model) so it survives relay restarts.
-
 import * as fs from "node:fs";
 import * as path from "node:path";
 
@@ -42,7 +26,7 @@ function loadModelContextWindowCache(cachePath) {
       }
     }
   } catch {
-    // Missing/corrupt cache file — relearn from the next turn.
+
   }
   return cache;
 }
@@ -55,29 +39,10 @@ function persistModelContextWindowCache(cachePath, cache) {
     for (const [key, value] of cache.entries()) obj[key] = value;
     fs.writeFileSync(cachePath, JSON.stringify(obj), "utf8");
   } catch {
-    // Best-effort persistence; the in-memory cache still works this process.
+
   }
 }
 
-/**
- * @typedef SessionContextSnapshot
- * @property {string} type
- * @property {string} sessionKey
- * @property {number} contextTokens
- * @property {number} contextWindow
- * @property {number} compactionCount
- * @property {boolean} runActive
- * @property {number} snapshotAtMs
- */
-
-/**
- * @param {object} opts
- * @param {{request: (method: string, params: object) => Promise<any>}} opts.gatewayBridge
- * @param {() => string|null} opts.getActiveSessionKey
- * @param {() => boolean} opts.getRunActive
- * @param {() => number} opts.nowMs
- * @param {(frame: SessionContextSnapshot) => void} opts.broadcast
- */
 export function createSessionContextService(opts) {
   const gatewayBridge = opts.gatewayBridge;
   const getActiveSessionKey = opts.getActiveSessionKey;
@@ -87,15 +52,10 @@ export function createSessionContextService(opts) {
   const getActiveModelKey =
     typeof opts.getActiveModelKey === "function" ? opts.getActiveModelKey : () => null;
 
-  /** @type {SessionContextSnapshot|null} */
   let lastSnapshot = null;
 
-  // Per-model context-window cache: the gateway only resolves a model's window
-  // after the first turn warms its discovery, so once we observe a real window
-  // we remember it per model and reuse it for later cold/new-session reads.
-  // Persisted under stateDir so it survives relay restarts.
   const modelContextWindowCachePath = resolveModelContextWindowCachePath(opts.stateDir);
-  /** @type {Map<string, number>} */
+
   const modelContextWindowCache = loadModelContextWindowCache(modelContextWindowCachePath);
 
   async function refreshActiveSessionContext() {
@@ -128,13 +88,13 @@ export function createSessionContextService(opts) {
     const modelKey = getActiveModelKey();
     let contextWindow = describeWindow;
     if (describeWindow > 0) {
-      // Learned naturally (post-turn): remember (and persist) for future cold reads.
+
       if (modelKey && modelContextWindowCache.get(modelKey) !== describeWindow) {
         modelContextWindowCache.set(modelKey, describeWindow);
         persistModelContextWindowCache(modelContextWindowCachePath, modelContextWindowCache);
       }
     } else if (modelKey && modelContextWindowCache.has(modelKey)) {
-      // Cold on a new session: reuse the window learned earlier for this model.
+
       contextWindow = modelContextWindowCache.get(modelKey);
     }
     const checkpoints =