@openfn/ws-worker 1.14.2 → 1.14.4

package/CHANGELOG.md

@@ -1,5 +1,19 @@
 # ws-worker
 
+## 1.14.4
+
+### Patch Changes
+
+- 5a613e7: Adjust logging
+- 3a33557: Count outstanding claim requests as capacity. This fixes an issue where `work-available` messages can cause a worker to over-claim, particularly during periods of high load on the Lightning database.
+
+## 1.14.3
+
+### Patch Changes
+
+- 032430f: Increase timeout on claim events
+- 0957412: Stop trying to claim after the queue has been closed
+
 ## 1.14.2
 
 ### Patch Changes

package/dist/index.d.ts

@@ -78,6 +78,7 @@ type ServerOptions = {
     sentryEnv?: string;
     socketTimeoutSeconds?: number;
     messageTimeoutSeconds?: number;
+    claimTimeoutSeconds?: number;
     payloadLimitMb?: number;
     collectionsVersion?: string;
     collectionsUrl?: string;
@@ -88,6 +89,7 @@ interface ServerApp extends Koa {
     socket?: any;
     queueChannel?: Channel;
     workflows: Record<string, true | Context>;
+    openClaims: Record<string, number>;
     destroyed: boolean;
     events: EventEmitter;
     server: Server;

package/dist/index.js

@@ -140,14 +140,25 @@ var ClaimError = class extends Error {
     this.abort = true;
   }
 };
+var claimIdGen = 0;
 var claim = (app, logger = mockLogger, options = {}) => {
   return new Promise((resolve, reject) => {
-    const { maxWorkers = 5 } = options;
+    app.openClaims ??= {};
+    const { maxWorkers = 5, demand = 1 } = options;
     const podName = NAME ? `[${NAME}] ` : "";
     const activeWorkers = Object.keys(app.workflows).length;
+    const pendingClaims = Object.values(app.openClaims).reduce(
+      (a, b) => a + b,
+      0
+    );
     if (activeWorkers >= maxWorkers) {
       app.workloop?.stop(`server at capacity (${activeWorkers}/${maxWorkers})`);
       return reject(new ClaimError("Server at capacity"));
+    } else if (activeWorkers + pendingClaims >= maxWorkers) {
+      app.workloop?.stop(
+        `server at capacity (${activeWorkers}/${maxWorkers}, ${pendingClaims} pending)`
+      );
+      return reject(new ClaimError("Server at capacity"));
     }
     if (!app.queueChannel) {
       logger.warn("skipping claim attempt: websocket unavailable");
@@ -159,12 +170,15 @@ var claim = (app, logger = mockLogger, options = {}) => {
       logger.warn("skipping claim attempt: channel closed");
       return reject(e);
     }
+    const claimId = ++claimIdGen;
+    app.openClaims[claimId] = demand;
     logger.debug(`requesting run (capacity ${activeWorkers}/${maxWorkers})`);
     const start = Date.now();
     app.queueChannel.push(CLAIM, {
-      demand: 1,
+      demand,
       worker_name: NAME || null
     }).receive("ok", ({ runs }) => {
+      delete app.openClaims[claimId];
       const duration = Date.now() - start;
       logger.debug(
         `${podName}claimed ${runs.length} runs in ${duration}ms (${runs.length ? runs.map((r) => r.id).join(",") : "-"})`
@@ -192,8 +206,11 @@ var claim = (app, logger = mockLogger, options = {}) => {
         resolve();
       });
     }).receive("error", (e) => {
+      delete app.openClaims[claimId];
       logger.error("Error on claim", e);
+      reject(new Error("claim error"));
     }).receive("timeout", () => {
+      delete app.openClaims[claimId];
       logger.error("TIMEOUT on claim. Runs may be lost.");
       reject(new Error("timeout"));
     });
@@ -1015,9 +1032,8 @@ var joinRunChannel = (socket, token, runId, logger, timeout = 30) => {
     channel.onClose(() => {
       logger.debug(`Leaving ${channelName}`);
     });
-    channel.onError((e) => {
-      logger.debug(`Error in ${channelName}`);
-      logger.debug(e);
+    channel.onError((...args) => {
+      logger.debug(`Critical error in channel ${channelName}`, args);
     });
   });
 };
@@ -1052,7 +1068,16 @@ var generateWorkerToken = async (secret, workerId, logger) => {
 var worker_token_default = generateWorkerToken;
 
 // src/channels/worker-queue.ts
-var connectToWorkerQueue = (endpoint, serverId, secret, timeout = 10, logger, SocketConstructor = PhxSocket) => {
+var DEFAULT_MESSAGE_TIMEOUT_SECONDS = 30;
+var DEFAULT_CLAIM_TIMEOUT_SECONDS = 60 * 60;
+var connectToWorkerQueue = (endpoint, serverId, secret, logger, options) => {
+  const {
+    // Sets the DEFAULT timeout for all messages on the websocket
+    // This can be overridden by different channels (although we tend not to)
+    messageTimeout = DEFAULT_MESSAGE_TIMEOUT_SECONDS,
+    claimTimeout = DEFAULT_CLAIM_TIMEOUT_SECONDS,
+    SocketConstructor = PhxSocket
+  } = options;
   const events = new EventEmitter();
   Sentry5.addBreadcrumb({
     category: "lifecycle",
@@ -1073,7 +1098,7 @@ var connectToWorkerQueue = (endpoint, serverId, secret, timeout = 10, logger, So
     const socket = new SocketConstructor(endpoint, {
       params,
       transport: WebSocket,
-      timeout: timeout * 1e3,
+      timeout: messageTimeout * 1e3,
       reconnectAfterMs: (tries) => Math.max(tries * 1e3)
     });
     let didOpen = false;
@@ -1091,7 +1116,7 @@ var connectToWorkerQueue = (endpoint, serverId, secret, timeout = 10, logger, So
         events.emit("message", ev, load);
         return load;
       };
-      channel.join().receive("ok", () => {
+      channel.join(claimTimeout * 1e3).receive("ok", () => {
         logger.debug("Connected to worker queue socket");
         events.emit("connect", { socket, channel });
       }).receive("error", (e) => {
@@ -1178,13 +1203,12 @@ function connect(app, logger, options = {}) {
       });
     }
   };
-  worker_queue_default(
-    options.lightning,
-    app.id,
-    options.secret,
-    options.socketTimeoutSeconds,
-    logger
-  ).on("connect", onConnect).on("disconnect", onDisconnect).on("error", onError).on("message", onMessage);
+  worker_queue_default(options.lightning, app.id, options.secret, logger, {
+    // TODO: options.socketTimeoutSeconds wins because this is what USED to be used
+    // But it's deprecated and should be removed soon
+    messageTimeout: options.socketTimeoutSeconds ?? options.messageTimeoutSeconds,
+    claimTimeout: options.claimTimeoutSeconds
+  }).on("connect", onConnect).on("disconnect", onDisconnect).on("error", onError).on("message", onMessage);
 }
 async function setupCollections(options, logger) {
   if (options.collectionsUrl) {
@@ -1232,6 +1256,7 @@ function createServer(engine, options = {}) {
       logger.debug(str);
     })
   );
+  app.openClaims = {};
   app.workflows = {};
   app.destroyed = false;
   app.server = app.listen(port);
@@ -1271,7 +1296,6 @@ function createServer(engine, options = {}) {
           collectionsVersion: app.options.collectionsVersion,
           monorepoPath: app.options.monorepoDir
         });
-        logger.debug("converted run body into execution plan:", plan);
         if (plan.workflow.credentials?.collections_token) {
           plan.workflow.credentials.collections_token = token;
         }

package/dist/start.js

@@ -280,14 +280,25 @@ var ClaimError = class extends Error {
     this.abort = true;
   }
 };
+var claimIdGen = 0;
 var claim = (app, logger2 = mockLogger, options = {}) => {
   return new Promise((resolve5, reject) => {
-    const { maxWorkers = 5 } = options;
+    app.openClaims ??= {};
+    const { maxWorkers = 5, demand = 1 } = options;
     const podName = NAME ? `[${NAME}] ` : "";
     const activeWorkers = Object.keys(app.workflows).length;
+    const pendingClaims = Object.values(app.openClaims).reduce(
+      (a, b) => a + b,
+      0
+    );
     if (activeWorkers >= maxWorkers) {
       app.workloop?.stop(`server at capacity (${activeWorkers}/${maxWorkers})`);
       return reject(new ClaimError("Server at capacity"));
+    } else if (activeWorkers + pendingClaims >= maxWorkers) {
+      app.workloop?.stop(
+        `server at capacity (${activeWorkers}/${maxWorkers}, ${pendingClaims} pending)`
+      );
+      return reject(new ClaimError("Server at capacity"));
     }
     if (!app.queueChannel) {
       logger2.warn("skipping claim attempt: websocket unavailable");
@@ -299,12 +310,15 @@ var claim = (app, logger2 = mockLogger, options = {}) => {
       logger2.warn("skipping claim attempt: channel closed");
       return reject(e);
     }
+    const claimId = ++claimIdGen;
+    app.openClaims[claimId] = demand;
     logger2.debug(`requesting run (capacity ${activeWorkers}/${maxWorkers})`);
     const start = Date.now();
     app.queueChannel.push(CLAIM, {
-      demand: 1,
+      demand,
       worker_name: NAME || null
     }).receive("ok", ({ runs }) => {
+      delete app.openClaims[claimId];
       const duration = Date.now() - start;
       logger2.debug(
         `${podName}claimed ${runs.length} runs in ${duration}ms (${runs.length ? runs.map((r) => r.id).join(",") : "-"})`
@@ -332,8 +346,11 @@ var claim = (app, logger2 = mockLogger, options = {}) => {
         resolve5();
       });
     }).receive("error", (e) => {
+      delete app.openClaims[claimId];
       logger2.error("Error on claim", e);
+      reject(new Error("claim error"));
     }).receive("timeout", () => {
+      delete app.openClaims[claimId];
       logger2.error("TIMEOUT on claim. Runs may be lost.");
       reject(new Error("timeout"));
     });
@@ -1155,9 +1172,8 @@ var joinRunChannel = (socket, token, runId, logger2, timeout = 30) => {
     channel.onClose(() => {
       logger2.debug(`Leaving ${channelName}`);
     });
-    channel.onError((e) => {
-      logger2.debug(`Error in ${channelName}`);
-      logger2.debug(e);
+    channel.onError((...args2) => {
+      logger2.debug(`Critical error in channel ${channelName}`, args2);
     });
   });
 };
@@ -1192,7 +1208,16 @@ var generateWorkerToken = async (secret, workerId, logger2) => {
 var worker_token_default = generateWorkerToken;
 
 // src/channels/worker-queue.ts
-var connectToWorkerQueue = (endpoint, serverId, secret, timeout = 10, logger2, SocketConstructor = PhxSocket) => {
+var DEFAULT_MESSAGE_TIMEOUT_SECONDS = 30;
+var DEFAULT_CLAIM_TIMEOUT_SECONDS = 60 * 60;
+var connectToWorkerQueue = (endpoint, serverId, secret, logger2, options) => {
+  const {
+    // Sets the DEFAULT timeout for all messages on the websocket
+    // This can be overridden by different channels (although we tend not to)
+    messageTimeout = DEFAULT_MESSAGE_TIMEOUT_SECONDS,
+    claimTimeout = DEFAULT_CLAIM_TIMEOUT_SECONDS,
+    SocketConstructor = PhxSocket
+  } = options;
   const events = new EventEmitter2();
   Sentry5.addBreadcrumb({
     category: "lifecycle",
@@ -1213,7 +1238,7 @@ var connectToWorkerQueue = (endpoint, serverId, secret, timeout = 10, logger2, S
     const socket = new SocketConstructor(endpoint, {
       params,
       transport: WebSocket,
-      timeout: timeout * 1e3,
+      timeout: messageTimeout * 1e3,
       reconnectAfterMs: (tries) => Math.max(tries * 1e3)
     });
     let didOpen = false;
@@ -1231,7 +1256,7 @@ var connectToWorkerQueue = (endpoint, serverId, secret, timeout = 10, logger2, S
         events.emit("message", ev, load);
         return load;
       };
-      channel.join().receive("ok", () => {
+      channel.join(claimTimeout * 1e3).receive("ok", () => {
         logger2.debug("Connected to worker queue socket");
         events.emit("connect", { socket, channel });
       }).receive("error", (e) => {
@@ -1318,13 +1343,12 @@ function connect(app, logger2, options = {}) {
       });
     }
   };
-  worker_queue_default(
-    options.lightning,
-    app.id,
-    options.secret,
-    options.socketTimeoutSeconds,
-    logger2
-  ).on("connect", onConnect).on("disconnect", onDisconnect).on("error", onError).on("message", onMessage);
+  worker_queue_default(options.lightning, app.id, options.secret, logger2, {
+    // TODO: options.socketTimeoutSeconds wins because this is what USED to be used
+    // But it's deprecated and should be removed soon
+    messageTimeout: options.socketTimeoutSeconds ?? options.messageTimeoutSeconds,
+    claimTimeout: options.claimTimeoutSeconds
+  }).on("connect", onConnect).on("disconnect", onDisconnect).on("error", onError).on("message", onMessage);
 }
 async function setupCollections(options, logger2) {
   if (options.collectionsUrl) {
@@ -1372,6 +1396,7 @@ function createServer(engine, options = {}) {
       logger2.debug(str);
     })
   );
+  app.openClaims = {};
   app.workflows = {};
   app.destroyed = false;
   app.server = app.listen(port);
@@ -1411,7 +1436,6 @@ function createServer(engine, options = {}) {
           collectionsVersion: app.options.collectionsVersion,
           monorepoPath: app.options.monorepoDir
         });
-        logger2.debug("converted run body into execution plan:", plan);
         if (plan.workflow.credentials?.collections_token) {
           plan.workflow.credentials.collections_token = token;
         }
@@ -6352,8 +6376,6 @@ var yargs_default = Yargs;
 // src/util/cli.ts
 var DEFAULT_PORT2 = 2222;
 var DEFAULT_WORKER_CAPACITY = 5;
-var DEFAULT_SOCKET_TIMEOUT_SECONDS = 10;
-var DEFAULT_MESSAGE_TIMEOUT_SECONDS = 30;
 function setArg(argValue, envValue, defaultValue) {
   if (Array.isArray(defaultValue) && !argValue && typeof envValue === "string") {
     return envValue.split(",");
@@ -6368,6 +6390,7 @@ function parseArgs(argv) {
     OPENFN_ADAPTORS_REPO,
     WORKER_BACKOFF,
     WORKER_CAPACITY,
+    WORKER_CLAIM_TIMEOUT_SECONDS,
     WORKER_COLLECTIONS_URL,
     WORKER_COLLECTIONS_VERSION,
     WORKER_LIGHTNING_PUBLIC_KEY,
@@ -6410,9 +6433,12 @@ function parseArgs(argv) {
   }).option("sentry-env", {
     description: "Sentry environment. Defaults to 'dev'. Env: WORKER_SENTRY_ENV"
   }).option("socket-timeout", {
-    description: `Timeout for websockets to Lightning, in seconds. Defaults to 10.Env: WORKER_SOCKET_TIMEOUT_SECONDS`
+    description: `[deprecated] Timeout for websockets to Lightning, in seconds. Defaults to 10.Env: WORKER_SOCKET_TIMEOUT_SECONDS`,
+    hidden: true
   }).option("message-timeout", {
-    description: `Timeout for messages in the run channel in seconds. Defaults to 1. Env: WORKER_MESSAGE_TIMEOUT_SECONDS`
+    description: `Timeout for all messages send to lightning via websocket. Defaults to 30. Env: WORKER_MESSAGE_TIMEOUT_SECONDS`
+  }).option("claim-timeout", {
+    description: `Timeout for claim requests for new Runs. This should be set to a high value or else runs may be lost. Defaults to 3600 (1 hour). Env: WORKER_CLAIM_TIMEOUT_SECONDS`
   }).option("lightning-public-key", {
     description: "Base64-encoded public key. Used to verify run tokens. Env: WORKER_LIGHTNING_PUBLIC_KEY"
   }).option("log", {
@@ -6483,10 +6509,14 @@ function parseArgs(argv) {
       WORKER_MAX_RUN_DURATION_SECONDS,
       300
     ),
+    claimTimeoutSeconds: setArg(
+      args2.claimTimeoutSeconds,
+      WORKER_CLAIM_TIMEOUT_SECONDS,
+      DEFAULT_CLAIM_TIMEOUT_SECONDS
+    ),
     socketTimeoutSeconds: setArg(
       args2.socketTimeoutSeconds,
-      WORKER_SOCKET_TIMEOUT_SECONDS,
-      DEFAULT_SOCKET_TIMEOUT_SECONDS
+      WORKER_SOCKET_TIMEOUT_SECONDS
     ),
     messageTimeoutSeconds: setArg(
       args2.messageTimeoutSeconds,
@@ -6535,8 +6565,15 @@ function engineReady(engine) {
     collectionsUrl: args.collectionsUrl,
     monorepoDir: args.monorepoDir,
     messageTimeoutSeconds: args.messageTimeoutSeconds,
+    claimTimeoutSeconds: args.claimTimeoutSeconds,
+    // deprecated!
     socketTimeoutSeconds: args.socketTimeoutSeconds
   };
+  if ("socketTimeoutSeconds" in args) {
+    logger.warn(
+      "WARNING: deprecated socketTimeoutSeconds value passed.\n\nThis will be respected as the default socket timeout value, but will be removed from future versions of the worker."
+    );
+  }
   if (args.lightningPublicKey) {
     logger.info(
       "Lightning public key found: run tokens from Lightning will be verified by this worker"

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openfn/ws-worker",
-  "version": "1.14.2",
+  "version": "1.14.4",
   "description": "A Websocket Worker to connect Lightning to a Runtime Engine",
   "main": "dist/index.js",
   "type": "module",
@@ -24,8 +24,8 @@
     "phoenix": "1.7.10",
     "ws": "^8.18.0",
     "@openfn/engine-multi": "1.6.8",
-    "@openfn/lexicon": "^1.2.2",
     "@openfn/logger": "1.0.5",
+    "@openfn/lexicon": "^1.2.2",
     "@openfn/runtime": "1.7.1"
   },
   "devDependencies": {
@@ -43,7 +43,7 @@
     "tsup": "^6.2.3",
     "typescript": "^4.6.4",
     "yargs": "^17.6.2",
-    "@openfn/lightning-mock": "2.2.2"
+    "@openfn/lightning-mock": "2.2.3"
   },
   "files": [
     "dist",