hypha-rpc 0.21.34 → 0.21.35

package/coverage/html/index.html

@@ -86,7 +86,7 @@
             <div class='footer quiet pad2 space-top1 center small'>
                 Code coverage generated by
                 <a href="https://istanbul.js.org/" target="_blank" rel="noopener noreferrer">istanbul</a>
-                at 2026-02-10T16:00:13.893Z
+                at 2026-03-10T23:48:04.149Z
             </div>
         <script src="prettify.js"></script>
         <script>

package/dist/hypha-rpc-websocket.js

@@ -2979,15 +2979,47 @@ class HTTPStreamingRPCConnection {
 
   /**
    * Process msgpack stream with 4-byte length prefix.
+   *
+   * Includes a read timeout to detect dead connections. The Hypha server
+   * sends pings every ~30s, so if no data arrives within READ_TIMEOUT_MS,
+   * the connection is considered dead and the stream is cancelled to
+   * trigger reconnection.
    */
   async _processMsgpackStream(response) {
     const reader = response.body.getReader();
+    // Expose the reader so external code (e.g. daemon heartbeat) can cancel it
+    this._reader = reader;
     // Growing buffer to avoid O(n^2) re-allocation on every chunk
     let buffer = new Uint8Array(4096);
     let bufferLen = 0;
 
+    // Read timeout: server sends pings every ~30s, so 120s = 4 missed pings = dead
+    const READ_TIMEOUT_MS = 120_000;
+
     while (!this._closed) {
-      const { done, value } = await reader.read();
+      let readResult;
+      let timeoutId;
+      try {
+        readResult = await Promise.race([
+          reader.read(),
+          new Promise((_, reject) => {
+            timeoutId = setTimeout(
+              () => reject(new Error("Stream read timeout (no data for 120s)")),
+              READ_TIMEOUT_MS,
+            );
+          }),
+        ]);
+        clearTimeout(timeoutId);
+      } catch (error) {
+        clearTimeout(timeoutId);
+        console.warn(`Stream read error: ${error.message}`);
+        try {
+          reader.cancel();
+        } catch {}
+        break;
+      }
+
+      const { done, value } = readResult;
 
       if (done) break;
 
@@ -3067,6 +3099,8 @@ class HTTPStreamingRPCConnection {
         bufferLen = remaining;
       }
     }
+
+    this._reader = null;
   }
 
   /**
@@ -4002,12 +4036,10 @@ class RPC extends _utils_index_js__WEBPACK_IMPORTED_MODULE_0__.MessageEmitter {
 
           try {
             // Get fresh manager service (one RPC roundtrip, ~50-100ms)
-            const _t0 = Date.now();
             const manager = await this.get_manager_service({
               timeout: 20,
               case_conversion: "camel",
             });
-            console.error(`[DEBUG] get_manager_service took ${Date.now() - _t0}ms`);
 
             // Fire manager_refreshed IMMEDIATELY — before service re-registration.
             // This allows connectToServer's wm proxy to be updated as soon as
@@ -4015,7 +4047,7 @@ class RPC extends _utils_index_js__WEBPACK_IMPORTED_MODULE_0__.MessageEmitter {
             this._fire("manager_refreshed", { manager });
 
             const services = Object.values(this._services);
-            const servicesCount = services.length;
+            let servicesCount = services.length;
             let registeredCount = 0;
             const failedServices = [];
 
@@ -4032,14 +4064,12 @@ class RPC extends _utils_index_js__WEBPACK_IMPORTED_MODULE_0__.MessageEmitter {
                 continue;
               }
               try {
-                const _t1 = Date.now();
                 const serviceInfo = this._extract_service_info(service);
                 await withTimeout(
                   manager.registerService(serviceInfo),
                   serviceRegistrationTimeout,
                   `Timeout registering service ${service.id || "unknown"}`,
                 );
-                console.error(`[DEBUG] registerService(${service.id}) took ${Date.now() - _t1}ms`);
                 registeredCount++;
               } catch (serviceError) {
                 failedServices.push(service.id || "unknown");
@@ -4075,6 +4105,11 @@ class RPC extends _utils_index_js__WEBPACK_IMPORTED_MODULE_0__.MessageEmitter {
               failed: failedServices,
             });
 
+            // Track whether all services were registered so the
+            // reconnection loop can detect partial failures and retry.
+            this._connection._services_registered_ok =
+              failedServices.length === 0;
+
             // Subscribe to client_disconnected events if the manager supports it
             try {
               if (
@@ -4167,6 +4202,9 @@ class RPC extends _utils_index_js__WEBPACK_IMPORTED_MODULE_0__.MessageEmitter {
               error: managerError.toString(),
               total_services: Object.keys(this._services).length,
             });
+            // Mark registration as failed so the reconnection loop
+            // can detect and retry
+            this._connection._services_registered_ok = false;
           }
         } else {
           // console.debug("Connection established", connectionInfo);
@@ -4184,10 +4222,11 @@ class RPC extends _utils_index_js__WEBPACK_IMPORTED_MODULE_0__.MessageEmitter {
       // This ensures no remote function call hangs forever when the connection drops
       if (typeof connection.on_disconnected === "function") {
         connection.on_disconnected((reason) => {
-          // If reconnection is enabled, don't reject pending calls immediately.
-          // The timeout mechanism will handle them if reconnection fails,
-          // allowing calls to succeed after a successful reconnection.
-          if (connection._enable_reconnect) {
+          // If reconnection is enabled AND the connection is not permanently closed,
+          // don't reject pending calls immediately — they may succeed after reconnect.
+          // When _closed is true (max retries exhausted or server refused reconnect),
+          // we must reject immediately so callers are not left hanging forever.
+          if (connection._enable_reconnect && !connection._closed) {
             this._logger.info(
               `Connection lost (${reason}), reconnection enabled - pending calls will be handled by timeout`,
             );
@@ -4541,7 +4580,15 @@ class RPC extends _utils_index_js__WEBPACK_IMPORTED_MODULE_0__.MessageEmitter {
     delete this._rintfCallerIndex[clientId];
     let cleaned = 0;
     for (const sid of serviceIds) {
-      if (this._services[sid]) {
+      const svc = this._services[sid];
+      if (svc) {
+        if (typeof svc._dispose === "function") {
+          try {
+            svc._dispose();
+          } catch (e) {
+            /* ignore errors from dispose handlers */
+          }
+        }
         delete this._services[sid];
         cleaned++;
       }
@@ -10713,6 +10760,19 @@ class WebsocketRPCConnection {
               throw new Error("Connection lost during reconnection settle");
             }
 
+            // Check if service re-registration succeeded.
+            // The on_connected callback sets this flag; if any
+            // services failed to register, retry instead of
+            // declaring success with missing services.
+            if (this._services_registered_ok === false) {
+              this._logger.warn(
+                "Service re-registration failed, retrying...",
+              );
+              throw new Error(
+                "Service re-registration failed after reconnection",
+              );
+            }
+
             this._logger.warn(
               `Successfully reconnected to server ${this._server_url} (services re-registered)`,
             );
@@ -11059,9 +11119,7 @@ async function connectToServer(config) {
     config.ping_interval,
     config.logger,
   );
-  const _tOpen = Date.now();
   const connection_info = await connection.open();
-  console.error(`[DEBUG] connection.open() took ${Date.now() - _tOpen}ms`);
   (0,_utils_index_js__WEBPACK_IMPORTED_MODULE_2__.assert)(
     connection_info,
     "Failed to connect to the server, no connection info obtained. This issue is most likely due to an outdated Hypha server version. Please use `imjoy-rpc` for compatibility, or upgrade the Hypha server to the latest version.",
@@ -11095,7 +11153,6 @@ async function connectToServer(config) {
   }
 
   const workspace = connection_info.workspace;
-  const _tRpc = Date.now();
   const rpc = new _rpc_js__WEBPACK_IMPORTED_MODULE_0__.RPC(connection, {
     client_id: clientId,
     workspace,
@@ -11112,17 +11169,12 @@ async function connectToServer(config) {
     encryption_private_key: config.encryption_private_key || null,
     encryption_public_key: config.encryption_public_key || null,
   });
-  console.error(`[DEBUG] RPC constructor took ${Date.now() - _tRpc}ms`);
-  const _tWait = Date.now();
   await rpc.waitFor("services_registered", config.method_timeout || 120);
-  console.error(`[DEBUG] waitFor('services_registered') took ${Date.now() - _tWait}ms`);
-  const _tWm = Date.now();
   const wm = await rpc.get_manager_service({
     timeout: config.method_timeout,
     case_conversion: "camel",
     kwargs_expansion: config.kwargs_expansion || false,
   });
-  console.error(`[DEBUG] get_manager_service (connectToServer) took ${Date.now() - _tWm}ms`);
   wm.rpc = rpc;
 
   // Auto-refresh workspace manager proxy after reconnection.