@openfn/ws-worker 1.14.3 → 1.14.5

package/CHANGELOG.md

@@ -1,5 +1,23 @@
 # ws-worker
 
+## 1.14.5
+
+### Patch Changes
+
+- Update dependencies
+- Updated dependencies
+  - @openfn/engine-multi@1.6.9
+  - @openfn/lexicon@1.2.3
+  - @openfn/runtime@1.7.2
+  - @openfn/logger@1.0.6
+
+## 1.14.4
+
+### Patch Changes
+
+- 5a613e7: Adjust logging
+- 3a33557: Count outstanding claim requests as capacity. This fixes an issue where `work-available` messages can cause a worker to over-claim, particularly during periods of high load on the Lightning database.
+
 ## 1.14.3
 
 ### Patch Changes

package/dist/index.d.ts

@@ -89,6 +89,7 @@ interface ServerApp extends Koa {
     socket?: any;
     queueChannel?: Channel;
     workflows: Record<string, true | Context>;
+    openClaims: Record<string, number>;
     destroyed: boolean;
     events: EventEmitter;
     server: Server;

package/dist/index.js

@@ -140,14 +140,25 @@ var ClaimError = class extends Error {
     this.abort = true;
   }
 };
+var claimIdGen = 0;
 var claim = (app, logger = mockLogger, options = {}) => {
   return new Promise((resolve, reject) => {
-    const { maxWorkers = 5 } = options;
+    app.openClaims ??= {};
+    const { maxWorkers = 5, demand = 1 } = options;
     const podName = NAME ? `[${NAME}] ` : "";
     const activeWorkers = Object.keys(app.workflows).length;
+    const pendingClaims = Object.values(app.openClaims).reduce(
+      (a, b) => a + b,
+      0
+    );
     if (activeWorkers >= maxWorkers) {
       app.workloop?.stop(`server at capacity (${activeWorkers}/${maxWorkers})`);
       return reject(new ClaimError("Server at capacity"));
+    } else if (activeWorkers + pendingClaims >= maxWorkers) {
+      app.workloop?.stop(
+        `server at capacity (${activeWorkers}/${maxWorkers}, ${pendingClaims} pending)`
+      );
+      return reject(new ClaimError("Server at capacity"));
     }
     if (!app.queueChannel) {
       logger.warn("skipping claim attempt: websocket unavailable");
@@ -159,12 +170,15 @@ var claim = (app, logger = mockLogger, options = {}) => {
       logger.warn("skipping claim attempt: channel closed");
       return reject(e);
     }
+    const claimId = ++claimIdGen;
+    app.openClaims[claimId] = demand;
     logger.debug(`requesting run (capacity ${activeWorkers}/${maxWorkers})`);
     const start = Date.now();
     app.queueChannel.push(CLAIM, {
-      demand: 1,
+      demand,
       worker_name: NAME || null
     }).receive("ok", ({ runs }) => {
+      delete app.openClaims[claimId];
       const duration = Date.now() - start;
       logger.debug(
         `${podName}claimed ${runs.length} runs in ${duration}ms (${runs.length ? runs.map((r) => r.id).join(",") : "-"})`
@@ -192,8 +206,11 @@ var claim = (app, logger = mockLogger, options = {}) => {
         resolve();
       });
     }).receive("error", (e) => {
+      delete app.openClaims[claimId];
       logger.error("Error on claim", e);
+      reject(new Error("claim error"));
     }).receive("timeout", () => {
+      delete app.openClaims[claimId];
       logger.error("TIMEOUT on claim. Runs may be lost.");
       reject(new Error("timeout"));
     });
@@ -1015,9 +1032,8 @@ var joinRunChannel = (socket, token, runId, logger, timeout = 30) => {
     channel.onClose(() => {
       logger.debug(`Leaving ${channelName}`);
     });
-    channel.onError((e) => {
-      logger.debug(`Error in ${channelName}`);
-      logger.debug(e);
+    channel.onError((...args) => {
+      logger.debug(`Critical error in channel ${channelName}`, args);
     });
   });
 };
@@ -1240,6 +1256,7 @@ function createServer(engine, options = {}) {
       logger.debug(str);
     })
   );
+  app.openClaims = {};
   app.workflows = {};
   app.destroyed = false;
   app.server = app.listen(port);
@@ -1279,7 +1296,6 @@ function createServer(engine, options = {}) {
           collectionsVersion: app.options.collectionsVersion,
           monorepoPath: app.options.monorepoDir
         });
-        logger.debug("converted run body into execution plan:", plan);
         if (plan.workflow.credentials?.collections_token) {
           plan.workflow.credentials.collections_token = token;
         }

package/dist/start.js

@@ -280,14 +280,25 @@ var ClaimError = class extends Error {
     this.abort = true;
   }
 };
+var claimIdGen = 0;
 var claim = (app, logger2 = mockLogger, options = {}) => {
   return new Promise((resolve5, reject) => {
-    const { maxWorkers = 5 } = options;
+    app.openClaims ??= {};
+    const { maxWorkers = 5, demand = 1 } = options;
     const podName = NAME ? `[${NAME}] ` : "";
     const activeWorkers = Object.keys(app.workflows).length;
+    const pendingClaims = Object.values(app.openClaims).reduce(
+      (a, b) => a + b,
+      0
+    );
     if (activeWorkers >= maxWorkers) {
       app.workloop?.stop(`server at capacity (${activeWorkers}/${maxWorkers})`);
       return reject(new ClaimError("Server at capacity"));
+    } else if (activeWorkers + pendingClaims >= maxWorkers) {
+      app.workloop?.stop(
+        `server at capacity (${activeWorkers}/${maxWorkers}, ${pendingClaims} pending)`
+      );
+      return reject(new ClaimError("Server at capacity"));
     }
     if (!app.queueChannel) {
       logger2.warn("skipping claim attempt: websocket unavailable");
@@ -299,12 +310,15 @@ var claim = (app, logger2 = mockLogger, options = {}) => {
       logger2.warn("skipping claim attempt: channel closed");
       return reject(e);
     }
+    const claimId = ++claimIdGen;
+    app.openClaims[claimId] = demand;
     logger2.debug(`requesting run (capacity ${activeWorkers}/${maxWorkers})`);
     const start = Date.now();
     app.queueChannel.push(CLAIM, {
-      demand: 1,
+      demand,
       worker_name: NAME || null
     }).receive("ok", ({ runs }) => {
+      delete app.openClaims[claimId];
       const duration = Date.now() - start;
       logger2.debug(
         `${podName}claimed ${runs.length} runs in ${duration}ms (${runs.length ? runs.map((r) => r.id).join(",") : "-"})`
@@ -332,8 +346,11 @@ var claim = (app, logger2 = mockLogger, options = {}) => {
         resolve5();
       });
     }).receive("error", (e) => {
+      delete app.openClaims[claimId];
       logger2.error("Error on claim", e);
+      reject(new Error("claim error"));
     }).receive("timeout", () => {
+      delete app.openClaims[claimId];
       logger2.error("TIMEOUT on claim. Runs may be lost.");
       reject(new Error("timeout"));
     });
@@ -1155,9 +1172,8 @@ var joinRunChannel = (socket, token, runId, logger2, timeout = 30) => {
     channel.onClose(() => {
       logger2.debug(`Leaving ${channelName}`);
     });
-    channel.onError((e) => {
-      logger2.debug(`Error in ${channelName}`);
-      logger2.debug(e);
+    channel.onError((...args2) => {
+      logger2.debug(`Critical error in channel ${channelName}`, args2);
     });
   });
 };
@@ -1380,6 +1396,7 @@ function createServer(engine, options = {}) {
       logger2.debug(str);
     })
   );
+  app.openClaims = {};
   app.workflows = {};
   app.destroyed = false;
   app.server = app.listen(port);
@@ -1419,7 +1436,6 @@ function createServer(engine, options = {}) {
           collectionsVersion: app.options.collectionsVersion,
           monorepoPath: app.options.monorepoDir
         });
-        logger2.debug("converted run body into execution plan:", plan);
         if (plan.workflow.credentials?.collections_token) {
           plan.workflow.credentials.collections_token = token;
         }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openfn/ws-worker",
-  "version": "1.14.3",
+  "version": "1.14.5",
   "description": "A Websocket Worker to connect Lightning to a Runtime Engine",
   "main": "dist/index.js",
   "type": "module",
@@ -10,40 +10,40 @@
   "author": "Open Function Group <admin@openfn.org>",
   "license": "ISC",
   "dependencies": {
-    "@koa/router": "^12.0.0",
-    "@sentry/node": "^9.5.0",
-    "@types/koa-logger": "^3.1.2",
-    "@types/ws": "^8.5.6",
+    "@koa/router": "^12.0.2",
+    "@sentry/node": "^9.46.0",
+    "@types/koa-logger": "^3.1.5",
+    "@types/ws": "^8.18.1",
     "fast-safe-stringify": "^2.1.1",
     "figures": "^5.0.0",
-    "human-id": "^4.1.0",
+    "human-id": "^4.1.1",
     "jose": "^4.15.9",
-    "koa": "^2.13.4",
-    "koa-bodyparser": "^4.4.0",
+    "koa": "^2.16.2",
+    "koa-bodyparser": "^4.4.1",
     "koa-logger": "^3.2.1",
     "phoenix": "1.7.10",
-    "ws": "^8.18.0",
-    "@openfn/engine-multi": "1.6.8",
-    "@openfn/logger": "1.0.5",
-    "@openfn/runtime": "1.7.1",
-    "@openfn/lexicon": "^1.2.2"
+    "ws": "^8.18.3",
+    "@openfn/engine-multi": "1.6.9",
+    "@openfn/lexicon": "^1.2.3",
+    "@openfn/runtime": "1.7.2",
+    "@openfn/logger": "1.0.6"
   },
   "devDependencies": {
-    "@types/koa": "^2.13.5",
-    "@types/koa-bodyparser": "^4.3.10",
-    "@types/koa__router": "^12.0.1",
-    "@types/node": "^18.15.3",
+    "@types/koa": "^2.15.0",
+    "@types/koa-bodyparser": "^4.3.12",
+    "@types/koa__router": "^12.0.4",
+    "@types/node": "^18.19.127",
     "@types/nodemon": "1.19.3",
-    "@types/phoenix": "^1.6.2",
-    "@types/yargs": "^17.0.12",
+    "@types/phoenix": "^1.6.6",
+    "@types/yargs": "^17.0.33",
     "ava": "5.1.0",
     "nodemon": "3.0.1",
-    "sentry-testkit": "^6.1.0",
-    "tslib": "^2.4.0",
-    "tsup": "^6.2.3",
-    "typescript": "^4.6.4",
-    "yargs": "^17.6.2",
-    "@openfn/lightning-mock": "2.2.2"
+    "sentry-testkit": "^6.2.2",
+    "tslib": "^2.8.1",
+    "tsup": "^6.7.0",
+    "typescript": "^4.9.5",
+    "yargs": "^17.7.2",
+    "@openfn/lightning-mock": "2.2.4"
   },
   "files": [
     "dist",