@openfn/ws-worker 1.14.3 → 1.14.4

package/CHANGELOG.md

@@ -1,5 +1,12 @@
 # ws-worker
 
+## 1.14.4
+
+### Patch Changes
+
+- 5a613e7: Adjust logging
+- 3a33557: Count outstanding claim requests as capacity. This fixes an issue where `work-available` messages can cause a worker to over-claim, particularly during periods of high load on the Lightning database.
+
 ## 1.14.3
 
 ### Patch Changes

package/dist/index.d.ts

@@ -89,6 +89,7 @@ interface ServerApp extends Koa {
     socket?: any;
     queueChannel?: Channel;
     workflows: Record<string, true | Context>;
+    openClaims: Record<string, number>;
     destroyed: boolean;
     events: EventEmitter;
     server: Server;

package/dist/index.js

@@ -140,14 +140,25 @@ var ClaimError = class extends Error {
     this.abort = true;
   }
 };
+var claimIdGen = 0;
 var claim = (app, logger = mockLogger, options = {}) => {
   return new Promise((resolve, reject) => {
-    const { maxWorkers = 5 } = options;
+    app.openClaims ??= {};
+    const { maxWorkers = 5, demand = 1 } = options;
     const podName = NAME ? `[${NAME}] ` : "";
     const activeWorkers = Object.keys(app.workflows).length;
+    const pendingClaims = Object.values(app.openClaims).reduce(
+      (a, b) => a + b,
+      0
+    );
     if (activeWorkers >= maxWorkers) {
       app.workloop?.stop(`server at capacity (${activeWorkers}/${maxWorkers})`);
       return reject(new ClaimError("Server at capacity"));
+    } else if (activeWorkers + pendingClaims >= maxWorkers) {
+      app.workloop?.stop(
+        `server at capacity (${activeWorkers}/${maxWorkers}, ${pendingClaims} pending)`
+      );
+      return reject(new ClaimError("Server at capacity"));
     }
     if (!app.queueChannel) {
       logger.warn("skipping claim attempt: websocket unavailable");
@@ -159,12 +170,15 @@ var claim = (app, logger = mockLogger, options = {}) => {
       logger.warn("skipping claim attempt: channel closed");
       return reject(e);
     }
+    const claimId = ++claimIdGen;
+    app.openClaims[claimId] = demand;
     logger.debug(`requesting run (capacity ${activeWorkers}/${maxWorkers})`);
     const start = Date.now();
     app.queueChannel.push(CLAIM, {
-      demand: 1,
+      demand,
       worker_name: NAME || null
     }).receive("ok", ({ runs }) => {
+      delete app.openClaims[claimId];
       const duration = Date.now() - start;
       logger.debug(
         `${podName}claimed ${runs.length} runs in ${duration}ms (${runs.length ? runs.map((r) => r.id).join(",") : "-"})`
@@ -192,8 +206,11 @@ var claim = (app, logger = mockLogger, options = {}) => {
         resolve();
       });
     }).receive("error", (e) => {
+      delete app.openClaims[claimId];
       logger.error("Error on claim", e);
+      reject(new Error("claim error"));
     }).receive("timeout", () => {
+      delete app.openClaims[claimId];
       logger.error("TIMEOUT on claim. Runs may be lost.");
       reject(new Error("timeout"));
     });
@@ -1015,9 +1032,8 @@ var joinRunChannel = (socket, token, runId, logger, timeout = 30) => {
     channel.onClose(() => {
       logger.debug(`Leaving ${channelName}`);
     });
-    channel.onError((e) => {
-      logger.debug(`Error in ${channelName}`);
-      logger.debug(e);
+    channel.onError((...args) => {
+      logger.debug(`Critical error in channel ${channelName}`, args);
     });
   });
 };
@@ -1240,6 +1256,7 @@ function createServer(engine, options = {}) {
       logger.debug(str);
     })
   );
+  app.openClaims = {};
   app.workflows = {};
   app.destroyed = false;
   app.server = app.listen(port);
@@ -1279,7 +1296,6 @@ function createServer(engine, options = {}) {
           collectionsVersion: app.options.collectionsVersion,
           monorepoPath: app.options.monorepoDir
         });
-        logger.debug("converted run body into execution plan:", plan);
         if (plan.workflow.credentials?.collections_token) {
           plan.workflow.credentials.collections_token = token;
         }

package/dist/start.js

@@ -280,14 +280,25 @@ var ClaimError = class extends Error {
     this.abort = true;
   }
 };
+var claimIdGen = 0;
 var claim = (app, logger2 = mockLogger, options = {}) => {
   return new Promise((resolve5, reject) => {
-    const { maxWorkers = 5 } = options;
+    app.openClaims ??= {};
+    const { maxWorkers = 5, demand = 1 } = options;
     const podName = NAME ? `[${NAME}] ` : "";
     const activeWorkers = Object.keys(app.workflows).length;
+    const pendingClaims = Object.values(app.openClaims).reduce(
+      (a, b) => a + b,
+      0
+    );
     if (activeWorkers >= maxWorkers) {
       app.workloop?.stop(`server at capacity (${activeWorkers}/${maxWorkers})`);
       return reject(new ClaimError("Server at capacity"));
+    } else if (activeWorkers + pendingClaims >= maxWorkers) {
+      app.workloop?.stop(
+        `server at capacity (${activeWorkers}/${maxWorkers}, ${pendingClaims} pending)`
+      );
+      return reject(new ClaimError("Server at capacity"));
     }
     if (!app.queueChannel) {
       logger2.warn("skipping claim attempt: websocket unavailable");
@@ -299,12 +310,15 @@ var claim = (app, logger2 = mockLogger, options = {}) => {
       logger2.warn("skipping claim attempt: channel closed");
       return reject(e);
     }
+    const claimId = ++claimIdGen;
+    app.openClaims[claimId] = demand;
     logger2.debug(`requesting run (capacity ${activeWorkers}/${maxWorkers})`);
     const start = Date.now();
     app.queueChannel.push(CLAIM, {
-      demand: 1,
+      demand,
       worker_name: NAME || null
     }).receive("ok", ({ runs }) => {
+      delete app.openClaims[claimId];
       const duration = Date.now() - start;
       logger2.debug(
         `${podName}claimed ${runs.length} runs in ${duration}ms (${runs.length ? runs.map((r) => r.id).join(",") : "-"})`
@@ -332,8 +346,11 @@ var claim = (app, logger2 = mockLogger, options = {}) => {
         resolve5();
       });
     }).receive("error", (e) => {
+      delete app.openClaims[claimId];
       logger2.error("Error on claim", e);
+      reject(new Error("claim error"));
     }).receive("timeout", () => {
+      delete app.openClaims[claimId];
       logger2.error("TIMEOUT on claim. Runs may be lost.");
       reject(new Error("timeout"));
     });
@@ -1155,9 +1172,8 @@ var joinRunChannel = (socket, token, runId, logger2, timeout = 30) => {
     channel.onClose(() => {
       logger2.debug(`Leaving ${channelName}`);
     });
-    channel.onError((e) => {
-      logger2.debug(`Error in ${channelName}`);
-      logger2.debug(e);
+    channel.onError((...args2) => {
+      logger2.debug(`Critical error in channel ${channelName}`, args2);
     });
   });
 };
@@ -1380,6 +1396,7 @@ function createServer(engine, options = {}) {
       logger2.debug(str);
     })
   );
+  app.openClaims = {};
   app.workflows = {};
   app.destroyed = false;
   app.server = app.listen(port);
@@ -1419,7 +1436,6 @@ function createServer(engine, options = {}) {
           collectionsVersion: app.options.collectionsVersion,
           monorepoPath: app.options.monorepoDir
         });
-        logger2.debug("converted run body into execution plan:", plan);
         if (plan.workflow.credentials?.collections_token) {
           plan.workflow.credentials.collections_token = token;
         }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openfn/ws-worker",
-  "version": "1.14.3",
+  "version": "1.14.4",
   "description": "A Websocket Worker to connect Lightning to a Runtime Engine",
   "main": "dist/index.js",
   "type": "module",
@@ -25,8 +25,8 @@
     "ws": "^8.18.0",
     "@openfn/engine-multi": "1.6.8",
     "@openfn/logger": "1.0.5",
-    "@openfn/runtime": "1.7.1",
-    "@openfn/lexicon": "^1.2.2"
+    "@openfn/lexicon": "^1.2.2",
+    "@openfn/runtime": "1.7.1"
   },
   "devDependencies": {
     "@types/koa": "^2.13.5",
@@ -43,7 +43,7 @@
     "tsup": "^6.2.3",
     "typescript": "^4.6.4",
     "yargs": "^17.6.2",
-    "@openfn/lightning-mock": "2.2.2"
+    "@openfn/lightning-mock": "2.2.3"
   },
   "files": [
     "dist",