hypha-rpc 0.20.61 → 0.20.64

package/dist/hypha-rpc-websocket.js

@@ -356,20 +356,23 @@ class RPC extends _utils_index_js__WEBPACK_IMPORTED_MODULE_0__.MessageEmitter {
         if (!this._silent && this._connection.manager_id) {
           console.debug("Connection established, reporting services...");
           try {
-            const manager = await this.get_manager_service({
-              timeout: 10,
-              case_conversion: "camel",
-            });
+            // Retry getting manager service with exponential backoff
+            const manager = await this._get_manager_with_retry();
             const services = Object.values(this._services);
             const servicesCount = services.length;
             let registeredCount = 0;
+            const failedServices = [];
 
             for (let service of services) {
               try {
                 const serviceInfo = this._extract_service_info(service);
                 await manager.registerService(serviceInfo);
                 registeredCount++;
+                console.debug(
+                  `Successfully registered service: ${service.id || "unknown"}`,
+                );
               } catch (serviceError) {
+                failedServices.push(service.id || "unknown");
                 console.error(
                   `Failed to register service ${service.id || "unknown"}: ${serviceError}`,
                 );
@@ -382,13 +385,25 @@ class RPC extends _utils_index_js__WEBPACK_IMPORTED_MODULE_0__.MessageEmitter {
               );
             } else {
               console.warn(
-                `Only registered ${registeredCount} out of ${servicesCount} services with the server`,
+                `Only registered ${registeredCount} out of ${servicesCount} services with the server. Failed services: ${failedServices.join(", ")}`,
               );
             }
+
+            // Fire event with registration status
+            this._fire("services_registered", {
+              total: servicesCount,
+              registered: registeredCount,
+              failed: failedServices,
+            });
           } catch (managerError) {
             console.error(
               `Failed to get manager service for registering services: ${managerError}`,
             );
+            // Fire event with error status
+            this._fire("services_registration_failed", {
+              error: managerError.toString(),
+              total_services: Object.keys(this._services).length,
+            });
           }
         } else {
           // console.debug("Connection established", connectionInfo);
@@ -428,6 +443,10 @@ class RPC extends _utils_index_js__WEBPACK_IMPORTED_MODULE_0__.MessageEmitter {
   }
 
   async _ping(msg, context) {
+    // Handle the new kwargs format with _rkwargs flag
+    if (typeof context === "object" && context && context._rkwargs) {
+      context = context.context;
+    }
     (0,_utils_index_js__WEBPACK_IMPORTED_MODULE_0__.assert)(msg == "ping");
     return "pong";
   }
@@ -444,6 +463,10 @@ class RPC extends _utils_index_js__WEBPACK_IMPORTED_MODULE_0__.MessageEmitter {
   }
 
   _create_message(key, heartbeat, overwrite, context) {
+    // Handle the new kwargs format with _rkwargs flag
+    if (typeof context === "object" && context && context._rkwargs) {
+      context = context.context;
+    }
     if (heartbeat) {
       if (!this._object_store[key]) {
         throw new Error(`session does not exist anymore: ${key}`);
@@ -463,6 +486,10 @@ class RPC extends _utils_index_js__WEBPACK_IMPORTED_MODULE_0__.MessageEmitter {
   }
 
   _append_message(key, data, heartbeat, context) {
+    // Handle the new kwargs format with _rkwargs flag
+    if (typeof context === "object" && context && context._rkwargs) {
+      context = context.context;
+    }
     if (heartbeat) {
       if (!this._object_store[key]) {
         throw new Error(`session does not exist anymore: ${key}`);
@@ -478,6 +505,10 @@ class RPC extends _utils_index_js__WEBPACK_IMPORTED_MODULE_0__.MessageEmitter {
   }
 
   _set_message(key, index, data, heartbeat, context) {
+    // Handle the new kwargs format with _rkwargs flag
+    if (typeof context === "object" && context && context._rkwargs) {
+      context = context.context;
+    }
     if (heartbeat) {
       if (!this._object_store[key]) {
         throw new Error(`session does not exist anymore: ${key}`);
@@ -493,6 +524,10 @@ class RPC extends _utils_index_js__WEBPACK_IMPORTED_MODULE_0__.MessageEmitter {
   }
 
   _remove_message(key, context) {
+    // Handle the new kwargs format with _rkwargs flag
+    if (typeof context === "object" && context && context._rkwargs) {
+      context = context.context;
+    }
     const cache = this._object_store["message_cache"];
     if (!cache[key]) {
       throw new Error(`Message with key ${key} does not exists.`);
@@ -501,6 +536,10 @@ class RPC extends _utils_index_js__WEBPACK_IMPORTED_MODULE_0__.MessageEmitter {
   }
 
   _process_message(key, heartbeat, context) {
+    // Handle the new kwargs format with _rkwargs flag
+    if (typeof context === "object" && context && context._rkwargs) {
+      context = context.context;
+    }
     if (heartbeat) {
       if (!this._object_store[key]) {
         throw new Error(`session does not exist anymore: ${key}`);
@@ -587,6 +626,35 @@ class RPC extends _utils_index_js__WEBPACK_IMPORTED_MODULE_0__.MessageEmitter {
     await this._connection.disconnect();
   }
 
+  async _get_manager_with_retry(maxRetries = 20) {
+    const baseDelay = 500;
+    const maxDelay = 10000;
+    let lastError = null;
+
+    for (let attempt = 0; attempt < maxRetries; attempt++) {
+      try {
+        const svc = await this.get_remote_service(
+          `*/${this._connection.manager_id}:default`,
+          { timeout: 20, case_conversion: "camel" },
+        );
+        return svc;
+      } catch (e) {
+        lastError = e;
+        console.warn(
+          `Failed to get manager service (attempt ${attempt + 1}/${maxRetries}): ${e.message}`,
+        );
+        if (attempt < maxRetries - 1) {
+          // Exponential backoff with maximum delay
+          const delay = Math.min(baseDelay * Math.pow(2, attempt), maxDelay);
+          await new Promise((resolve) => setTimeout(resolve, delay));
+        }
+      }
+    }
+
+    // If we get here, all retries failed
+    throw lastError;
+  }
+
   async get_manager_service(config) {
     config = config || {};
 
@@ -630,6 +698,10 @@ class RPC extends _utils_index_js__WEBPACK_IMPORTED_MODULE_0__.MessageEmitter {
     return this._services;
   }
   get_local_service(service_id, context) {
+    // Handle the new kwargs format with _rkwargs flag
+    if (typeof context === "object" && context && context._rkwargs) {
+      context = context.context;
+    }
     (0,_utils_index_js__WEBPACK_IMPORTED_MODULE_0__.assert)(service_id);
     const [ws, client_id] = context["to"].split("/");
     (0,_utils_index_js__WEBPACK_IMPORTED_MODULE_0__.assert)(
@@ -1092,9 +1164,9 @@ class RPC extends _utils_index_js__WEBPACK_IMPORTED_MODULE_0__.MessageEmitter {
 
         const taskFn = async () => {
           await message_cache.set(message_id, idx, chunk, !!session_id);
-          console.debug(
-            `Sending chunk ${idx + 1}/${chunk_num} (total=${total_size} bytes)`,
-          );
+          // console.debug(
+          //   `Sending chunk ${idx + 1}/${chunk_num} (total=${total_size} bytes)`,
+          // );
         };
 
         // Push into an array, each one runs under the semaphore
@@ -1125,14 +1197,14 @@ class RPC extends _utils_index_js__WEBPACK_IMPORTED_MODULE_0__.MessageEmitter {
           startByte + this._long_message_chunk_size,
         );
         await message_cache.append(message_id, chunk, !!session_id);
-        console.debug(
-          `Sending chunk ${idx + 1}/${chunk_num} (total=${total_size} bytes)`,
-        );
+        // console.debug(
+        //   `Sending chunk ${idx + 1}/${chunk_num} (total=${total_size} bytes)`,
+        // );
       }
     }
     await message_cache.process(message_id, !!session_id);
     const durationSec = ((Date.now() - start_time) / 1000).toFixed(2);
-    console.debug(`All chunks (${total_size} bytes) sent in ${durationSec} s`);
+    // console.debug(`All chunks (${total_size} bytes) sent in ${durationSec} s`);
   }
 
   emit(main_message, extra_data) {
@@ -1513,34 +1585,30 @@ class RPC extends _utils_index_js__WEBPACK_IMPORTED_MODULE_0__.MessageEmitter {
       } else {
         args = [];
       }
+      let kwargs = {};
+      if (data.with_kwargs) {
+        kwargs = args.pop();
+      }
+
       if (
         this._method_annotations.has(method) &&
         this._method_annotations.get(method).require_context
       ) {
-        // Check if this is a remote service (from external clients)
-        // Remote services start with a session ID or client ID, not "services."
-        const isRemoteService = !data.method.startsWith("services.");
-
-        if (isRemoteService) {
-          // For remote services (external client services), the method.length reflects
-          // the original signature and cannot be modified. We inject context as the last argument
-          // and skip the strict argument validation that fails for external services.
-          args.push(data.ctx);
-        } else {
-          // For local services, use the existing logic with padding and validation
-          // if args.length + 1 is less than the required number of arguments we will pad with undefined
-          // so we make sure the last argument is the context
-          if (args.length + 1 < method.length) {
-            for (let i = args.length; i < method.length - 1; i++) {
-              args.push(undefined);
-            }
-          }
-          args.push(data.ctx);
-          (0,_utils_index_js__WEBPACK_IMPORTED_MODULE_0__.assert)(
-            args.length === method.length,
-            `Runtime Error: Invalid number of arguments for method ${method_name}, expected ${method.length} but got ${args.length}`,
-          );
+        // Always use kwargs approach for consistency
+        kwargs.context = data.ctx;
+        kwargs._rkwargs = true;
+
+        // CRITICAL FIX: Ensure kwargs goes to the correct position
+        // For a function like multiply_context(a, b, kwargs), we need to ensure
+        // that kwargs is always the last parameter, even if fewer args are provided
+        const expectedParamCount = method.length;
+
+        // Pad args array to the expected parameter count (minus 1 for kwargs)
+        while (args.length < expectedParamCount - 1) {
+          args.push(undefined);
         }
+
+        args.push(kwargs);
       }
       // console.debug(`Executing method: ${method_name} (${data.method})`);
       if (data.promise) {
@@ -1560,6 +1628,7 @@ class RPC extends _utils_index_js__WEBPACK_IMPORTED_MODULE_0__.MessageEmitter {
           clearInterval(heartbeat_task);
         }
       } else {
+        // console.log(`DEBUG: About to call method (no promise) ${data.method} with args:`, args);
         method.apply(null, args);
         clearInterval(heartbeat_task);
       }
@@ -5422,8 +5491,19 @@ class WebsocketRPCConnection {
           event.code,
           event.reason,
         );
+
         let retry = 0;
+        const baseDelay = 1000; // Start with 1 second
+        const maxDelay = 60000; // Maximum delay of 60 seconds
+        const maxJitter = 0.1; // Maximum jitter factor
+
         const reconnect = async () => {
+          // Check if we were explicitly closed
+          if (this._closed) {
+            console.info("Connection was closed, stopping reconnection");
+            return;
+          }
+
           try {
             console.warn(
               `Reconnecting to ${this._server_url.split("?")[0]} (attempt #${retry})`,
@@ -5445,6 +5525,10 @@ class WebsocketRPCConnection {
             console.warn(
               `Successfully reconnected to server ${this._server_url} (services re-registered)`,
             );
+            // Emit reconnection success event
+            if (this._handle_connected) {
+              this._handle_connected(this.connection_info);
+            }
           } catch (e) {
             if (`${e}`.includes("ConnectionAbortedError:")) {
               console.warn("Failed to reconnect, connection aborted:", e);
@@ -5455,26 +5539,74 @@ class WebsocketRPCConnection {
               );
               return;
             }
+
+            // Log specific error types for better debugging
+            if (e.name === "NetworkError" || e.message.includes("network")) {
+              console.error(`Network error during reconnection: ${e.message}`);
+            } else if (
+              e.name === "TimeoutError" ||
+              e.message.includes("timeout")
+            ) {
+              console.error(
+                `Connection timeout during reconnection: ${e.message}`,
+              );
+            } else {
+              console.error(
+                `Unexpected error during reconnection: ${e.message}`,
+              );
+            }
+
+            // Calculate exponential backoff with jitter
+            const delay = Math.min(baseDelay * Math.pow(2, retry), maxDelay);
+            // Add jitter to prevent thundering herd
+            const jitter = (Math.random() * 2 - 1) * maxJitter * delay;
+            const finalDelay = Math.max(100, delay + jitter);
+
+            console.debug(
+              `Waiting ${(finalDelay / 1000).toFixed(2)}s before next reconnection attempt`,
+            );
+
             // Track the reconnection timeout to prevent leaks
             const timeoutId = setTimeout(async () => {
               this._reconnect_timeouts.delete(timeoutId);
+
+              // Check if connection was restored externally
               if (
                 this._websocket &&
-                this._websocket.readyState === WebSocket.CONNECTED
+                this._websocket.readyState === WebSocket.OPEN
               ) {
+                console.info("Connection restored externally");
+                return;
+              }
+
+              // Check if we were explicitly closed
+              if (this._closed) {
+                console.info("Connection was closed, stopping reconnection");
                 return;
               }
+
               retry += 1;
               if (retry < MAX_RETRY) {
                 await reconnect();
               } else {
                 console.error(
-                  "Failed to reconnect after",
-                  MAX_RETRY,
-                  "attempts",
+                  `Failed to reconnect after ${MAX_RETRY} attempts, giving up. Exiting process.`,
                 );
+                // Notify about max retry exceeded
+                if (this._handle_disconnected) {
+                  this._handle_disconnected(
+                    "Max reconnection attempts exceeded",
+                  );
+                }
+                // Exit process to prevent stuck event loop
+                if (typeof process !== "undefined" && process.exit) {
+                  console.error(
+                    "Forcing process exit due to unrecoverable connection failure",
+                  );
+                  process.exit(1);
+                }
               }
-            }, 1000);
+            }, finalDelay);
             this._reconnect_timeouts.add(timeoutId);
           }
         };