@openfn/ws-worker 1.14.0 → 1.14.2

package/CHANGELOG.md

@@ -1,5 +1,21 @@
 # ws-worker
 
+## 1.14.2
+
+### Patch Changes
+
+- Updated dependencies [f04acef]
+  - @openfn/runtime@1.7.1
+  - @openfn/engine-multi@1.6.8
+
+## 1.14.1
+
+### Patch Changes
+
+- d765843: Fix an issue where the server can attempt to claim even while it's waiting to shut down
+- 667e3bf: Improve logging of errors returned by lightning
+- d765843: Fix an issue where the --backoff server argument only accepts integer values
+
 ## 1.14.0
 
 ### Minor Changes

package/dist/index.js

@@ -2,7 +2,7 @@
 import { EventEmitter as EventEmitter2 } from "node:events";
 import { promisify } from "node:util";
 import { exec as _exec } from "node:child_process";
-import * as Sentry5 from "@sentry/node";
+import * as Sentry6 from "@sentry/node";
 import Koa from "koa";
 import bodyParser from "koa-bodyparser";
 import koaLogger from "koa-logger";
@@ -82,6 +82,10 @@ var tryWithBackoff = (fn, opts = {}) => {
       await fn();
       resolve();
     } catch (e) {
+      if (e?.abort) {
+        cancelled = true;
+        return reject();
+      }
       if (opts.isCancelled()) {
         return resolve();
       }
@@ -111,6 +115,7 @@ var tryWithBackoff = (fn, opts = {}) => {
 var try_with_backoff_default = tryWithBackoff;
 
 // src/api/claim.ts
+import * as Sentry from "@sentry/node";
 import crypto from "node:crypto";
 import * as jose from "jose";
 import { createMockLogger } from "@openfn/logger";
@@ -128,6 +133,13 @@ var verifyToken = async (token, publicKey) => {
 };
 var { DEPLOYED_POD_NAME, WORKER_NAME } = process.env;
 var NAME = WORKER_NAME || DEPLOYED_POD_NAME;
+var ClaimError = class extends Error {
+  constructor(e) {
+    super(e);
+    // This breaks the parenting backoff loop
+    this.abort = true;
+  }
+};
 var claim = (app, logger = mockLogger, options = {}) => {
   return new Promise((resolve, reject) => {
     const { maxWorkers = 5 } = options;
@@ -135,11 +147,17 @@ var claim = (app, logger = mockLogger, options = {}) => {
     const activeWorkers = Object.keys(app.workflows).length;
     if (activeWorkers >= maxWorkers) {
       app.workloop?.stop(`server at capacity (${activeWorkers}/${maxWorkers})`);
-      return reject(new Error("Server at capacity"));
+      return reject(new ClaimError("Server at capacity"));
     }
     if (!app.queueChannel) {
-      logger.debug("skipping claim attempt: websocket unavailable");
-      return reject(new Error("No websocket available"));
+      logger.warn("skipping claim attempt: websocket unavailable");
+      return reject(new ClaimError("No websocket available"));
+    }
+    if (app.queueChannel.state === "closed") {
+      const e = new ClaimError("queue closed");
+      Sentry.captureException(e);
+      logger.warn("skipping claim attempt: channel closed");
+      return reject(e);
     }
     logger.debug(`requesting run (capacity ${activeWorkers}/${maxWorkers})`);
     const start = Date.now();
@@ -202,6 +220,7 @@ var startWorkloop = (app, logger, minBackoff, maxBackoff, maxWorkers) => {
         if (!cancelled) {
           setTimeout(workLoop, minBackoff);
         }
+      }).catch(() => {
       });
     }
   };
@@ -220,7 +239,7 @@ var startWorkloop = (app, logger, minBackoff, maxBackoff, maxWorkers) => {
 var workloop_default = startWorkloop;
 
 // src/api/execute.ts
-import * as Sentry2 from "@sentry/node";
+import * as Sentry3 from "@sentry/node";
 
 // src/util/convert-lightning-plan.ts
 import crypto2 from "node:crypto";
@@ -435,12 +454,28 @@ var create_run_state_default = (plan, input) => {
 };
 
 // src/util/send-event.ts
-import * as Sentry from "@sentry/node";
+import * as Sentry2 from "@sentry/node";
 
 // src/errors.ts
+function serializeMessage(message) {
+  if (typeof message === "string") {
+    return message;
+  }
+  if (message instanceof Error) {
+    return message.toString();
+  }
+  if (message && typeof message === "object") {
+    try {
+      return JSON.stringify(message);
+    } catch {
+      return String(message);
+    }
+  }
+  return String(message);
+}
 var LightningSocketError = class extends Error {
   constructor(event, message) {
-    super(`[${event}] ${message}`);
+    super(`[${event}] ${serializeMessage(message)}`);
     this.name = "LightningSocketError";
     this.event = "";
     this.rejectMessage = "";
@@ -469,7 +504,7 @@ var sendEvent = (context, event, payload) => {
       if (error.rejectMessage) {
         extras.rejection_reason = error.rejectMessage;
       }
-      Sentry.captureException(error, (scope) => {
+      Sentry2.captureException(error, (scope) => {
         scope.setContext("run", context2);
         scope.setExtras(extras);
         return scope;
@@ -802,8 +837,8 @@ function execute(channel, engine, logger, plan, input, options = {}, onFinish =
     options,
     onFinish
   };
-  Sentry2.withIsolationScope(async () => {
-    Sentry2.addBreadcrumb({
+  Sentry3.withIsolationScope(async () => {
+    Sentry3.addBreadcrumb({
       category: "run",
       message: "Executing run: loading metadata",
       level: "info",
@@ -815,7 +850,7 @@ function execute(channel, engine, logger, plan, input, options = {}, onFinish =
     const addEvent = (eventName, handler) => {
       const wrappedFn = async (event) => {
         if (eventName !== "workflow-log") {
-          Sentry2.addBreadcrumb({
+          Sentry3.addBreadcrumb({
             category: "event",
             message: eventName,
             level: "info"
@@ -827,7 +862,7 @@ function execute(channel, engine, logger, plan, input, options = {}, onFinish =
           logger.info(`${plan.id} :: ${lightningEvent} :: OK`);
         } catch (e) {
           if (!e.reportedToSentry) {
-            Sentry2.captureException(e);
+            Sentry3.captureException(e);
             logger.error(e);
           }
         }
@@ -871,7 +906,7 @@ function execute(channel, engine, logger, plan, input, options = {}, onFinish =
         }
       }
       try {
-        Sentry2.addBreadcrumb({
+        Sentry3.addBreadcrumb({
           category: "run",
           message: "run metadata loaded: starting run",
           level: "info",
@@ -881,7 +916,7 @@ function execute(channel, engine, logger, plan, input, options = {}, onFinish =
         });
         engine.execute(plan, loadedInput, { resolvers, ...options });
       } catch (e) {
-        Sentry2.addBreadcrumb({
+        Sentry3.addBreadcrumb({
           category: "run",
           message: "exception in run",
           level: "info",
@@ -948,7 +983,7 @@ var healthcheck_default = (ctx) => {
 };
 
 // src/channels/run.ts
-import * as Sentry3 from "@sentry/node";
+import * as Sentry4 from "@sentry/node";
 var joinRunChannel = (socket, token, runId, logger, timeout = 30) => {
   return new Promise((resolve, reject) => {
     let didReceiveOk = false;
@@ -967,12 +1002,12 @@ var joinRunChannel = (socket, token, runId, logger, timeout = 30) => {
         resolve({ channel, run });
       }
     }).receive("error", (err) => {
-      Sentry3.captureException(err);
+      Sentry4.captureException(err);
       logger.error(`error connecting to ${channelName}`, err);
       channel?.leave();
       reject(err);
     }).receive("timeout", (err) => {
-      Sentry3.captureException(err);
+      Sentry4.captureException(err);
       logger.error(`Timeout for ${channelName}`, err);
       channel?.leave();
       reject(err);
@@ -990,7 +1025,7 @@ var run_default = joinRunChannel;
 
 // src/channels/worker-queue.ts
 import EventEmitter from "node:events";
-import * as Sentry4 from "@sentry/node";
+import * as Sentry5 from "@sentry/node";
 import { Socket as PhxSocket } from "phoenix";
 import { WebSocket } from "ws";
 import { API_VERSION } from "@openfn/lexicon/lightning";
@@ -1019,13 +1054,13 @@ var worker_token_default = generateWorkerToken;
 // src/channels/worker-queue.ts
 var connectToWorkerQueue = (endpoint, serverId, secret, timeout = 10, logger, SocketConstructor = PhxSocket) => {
   const events = new EventEmitter();
-  Sentry4.addBreadcrumb({
+  Sentry5.addBreadcrumb({
     category: "lifecycle",
     message: "Connecting to worker queue",
     level: "info"
   });
   worker_token_default(secret, serverId, logger).then(async (token) => {
-    Sentry4.addBreadcrumb({
+    Sentry5.addBreadcrumb({
       category: "lifecycle",
       message: "Worker token generated",
       level: "info"
@@ -1044,7 +1079,7 @@ var connectToWorkerQueue = (endpoint, serverId, secret, timeout = 10, logger, So
     let didOpen = false;
     let shouldReportConnectionError = true;
     socket.onOpen(() => {
-      Sentry4.addBreadcrumb({
+      Sentry5.addBreadcrumb({
         category: "lifecycle",
         message: "Web socket connected",
         level: "info"
@@ -1070,7 +1105,7 @@ var connectToWorkerQueue = (endpoint, serverId, secret, timeout = 10, logger, So
       events.emit("disconnect");
     });
     socket.onError((e) => {
-      Sentry4.addBreadcrumb({
+      Sentry5.addBreadcrumb({
         category: "lifecycle",
         message: "Error in web socket connection",
         level: "info"
@@ -1078,7 +1113,7 @@ var connectToWorkerQueue = (endpoint, serverId, secret, timeout = 10, logger, So
       if (shouldReportConnectionError) {
         logger.debug("Reporting connection error to sentry");
         shouldReportConnectionError = false;
-        Sentry4.captureException(e);
+        Sentry5.captureException(e);
       }
       if (!didOpen) {
         events.emit("error", e.message);
@@ -1185,11 +1220,11 @@ function createServer(engine, options = {}) {
   app.events = new EventEmitter2();
   app.engine = engine;
   if (options.sentryDsn) {
-    Sentry5.init({
+    Sentry6.init({
       environment: options.sentryEnv,
       dsn: options.sentryDsn
     });
-    Sentry5.setupKoaErrorHandler(app);
+    Sentry6.setupKoaErrorHandler(app);
   }
   app.use(bodyParser());
   app.use(
@@ -1206,7 +1241,7 @@ function createServer(engine, options = {}) {
   router.get("/", healthcheck_default);
   app.options = options;
   app.resumeWorkloop = () => {
-    if (options.noLoop) {
+    if (options.noLoop || app.destroyed) {
       return;
     }
     if (!app.workloop || app.workloop?.isStopped()) {
@@ -1306,7 +1341,7 @@ function createServer(engine, options = {}) {
       shutdown = true;
       logger.always(`${signal} RECEIVED: CLOSING SERVER`);
       await app.destroy();
-      process.exit();
+      process.exit(0);
     }
   };
   process.on("SIGINT", () => exit("SIGINT"));

package/dist/start.js

@@ -142,7 +142,7 @@ var runtime_engine_default = createMock;
 import { EventEmitter as EventEmitter3 } from "node:events";
 import { promisify } from "node:util";
 import { exec as _exec } from "node:child_process";
-import * as Sentry5 from "@sentry/node";
+import * as Sentry6 from "@sentry/node";
 import Koa from "koa";
 import bodyParser from "koa-bodyparser";
 import koaLogger from "koa-logger";
@@ -222,6 +222,10 @@ var tryWithBackoff = (fn, opts = {}) => {
       await fn();
       resolve5();
     } catch (e) {
+      if (e?.abort) {
+        cancelled = true;
+        return reject();
+      }
       if (opts.isCancelled()) {
         return resolve5();
       }
@@ -251,6 +255,7 @@ var tryWithBackoff = (fn, opts = {}) => {
 var try_with_backoff_default = tryWithBackoff;
 
 // src/api/claim.ts
+import * as Sentry from "@sentry/node";
 import crypto2 from "node:crypto";
 import * as jose from "jose";
 import { createMockLogger } from "@openfn/logger";
@@ -268,6 +273,13 @@ var verifyToken = async (token, publicKey) => {
 };
 var { DEPLOYED_POD_NAME, WORKER_NAME } = process.env;
 var NAME = WORKER_NAME || DEPLOYED_POD_NAME;
+var ClaimError = class extends Error {
+  constructor(e) {
+    super(e);
+    // This breaks the parenting backoff loop
+    this.abort = true;
+  }
+};
 var claim = (app, logger2 = mockLogger, options = {}) => {
   return new Promise((resolve5, reject) => {
     const { maxWorkers = 5 } = options;
@@ -275,11 +287,17 @@ var claim = (app, logger2 = mockLogger, options = {}) => {
     const activeWorkers = Object.keys(app.workflows).length;
     if (activeWorkers >= maxWorkers) {
       app.workloop?.stop(`server at capacity (${activeWorkers}/${maxWorkers})`);
-      return reject(new Error("Server at capacity"));
+      return reject(new ClaimError("Server at capacity"));
     }
     if (!app.queueChannel) {
-      logger2.debug("skipping claim attempt: websocket unavailable");
-      return reject(new Error("No websocket available"));
+      logger2.warn("skipping claim attempt: websocket unavailable");
+      return reject(new ClaimError("No websocket available"));
+    }
+    if (app.queueChannel.state === "closed") {
+      const e = new ClaimError("queue closed");
+      Sentry.captureException(e);
+      logger2.warn("skipping claim attempt: channel closed");
+      return reject(e);
     }
     logger2.debug(`requesting run (capacity ${activeWorkers}/${maxWorkers})`);
     const start = Date.now();
@@ -342,6 +360,7 @@ var startWorkloop = (app, logger2, minBackoff2, maxBackoff2, maxWorkers) => {
         if (!cancelled) {
           setTimeout(workLoop, minBackoff2);
         }
+      }).catch(() => {
       });
     }
   };
@@ -360,7 +379,7 @@ var startWorkloop = (app, logger2, minBackoff2, maxBackoff2, maxWorkers) => {
 var workloop_default = startWorkloop;
 
 // src/api/execute.ts
-import * as Sentry2 from "@sentry/node";
+import * as Sentry3 from "@sentry/node";
 
 // src/util/convert-lightning-plan.ts
 import crypto3 from "node:crypto";
@@ -575,12 +594,28 @@ var create_run_state_default = (plan, input) => {
 };
 
 // src/util/send-event.ts
-import * as Sentry from "@sentry/node";
+import * as Sentry2 from "@sentry/node";
 
 // src/errors.ts
+function serializeMessage(message) {
+  if (typeof message === "string") {
+    return message;
+  }
+  if (message instanceof Error) {
+    return message.toString();
+  }
+  if (message && typeof message === "object") {
+    try {
+      return JSON.stringify(message);
+    } catch {
+      return String(message);
+    }
+  }
+  return String(message);
+}
 var LightningSocketError = class extends Error {
   constructor(event, message) {
-    super(`[${event}] ${message}`);
+    super(`[${event}] ${serializeMessage(message)}`);
     this.name = "LightningSocketError";
     this.event = "";
     this.rejectMessage = "";
@@ -609,7 +644,7 @@ var sendEvent = (context, event, payload) => {
       if (error.rejectMessage) {
         extras.rejection_reason = error.rejectMessage;
       }
-      Sentry.captureException(error, (scope) => {
+      Sentry2.captureException(error, (scope) => {
         scope.setContext("run", context2);
         scope.setExtras(extras);
         return scope;
@@ -942,8 +977,8 @@ function execute(channel, engine, logger2, plan, input, options = {}, onFinish =
     options,
     onFinish
   };
-  Sentry2.withIsolationScope(async () => {
-    Sentry2.addBreadcrumb({
+  Sentry3.withIsolationScope(async () => {
+    Sentry3.addBreadcrumb({
       category: "run",
       message: "Executing run: loading metadata",
       level: "info",
@@ -955,7 +990,7 @@ function execute(channel, engine, logger2, plan, input, options = {}, onFinish =
     const addEvent = (eventName, handler) => {
       const wrappedFn = async (event) => {
         if (eventName !== "workflow-log") {
-          Sentry2.addBreadcrumb({
+          Sentry3.addBreadcrumb({
             category: "event",
             message: eventName,
             level: "info"
@@ -967,7 +1002,7 @@ function execute(channel, engine, logger2, plan, input, options = {}, onFinish =
           logger2.info(`${plan.id} :: ${lightningEvent} :: OK`);
         } catch (e) {
           if (!e.reportedToSentry) {
-            Sentry2.captureException(e);
+            Sentry3.captureException(e);
             logger2.error(e);
           }
         }
@@ -1011,7 +1046,7 @@ function execute(channel, engine, logger2, plan, input, options = {}, onFinish =
         }
       }
       try {
-        Sentry2.addBreadcrumb({
+        Sentry3.addBreadcrumb({
           category: "run",
           message: "run metadata loaded: starting run",
           level: "info",
@@ -1021,7 +1056,7 @@ function execute(channel, engine, logger2, plan, input, options = {}, onFinish =
         });
         engine.execute(plan, loadedInput, { resolvers, ...options });
       } catch (e) {
-        Sentry2.addBreadcrumb({
+        Sentry3.addBreadcrumb({
           category: "run",
           message: "exception in run",
           level: "info",
@@ -1088,7 +1123,7 @@ var healthcheck_default = (ctx) => {
 };
 
 // src/channels/run.ts
-import * as Sentry3 from "@sentry/node";
+import * as Sentry4 from "@sentry/node";
 var joinRunChannel = (socket, token, runId, logger2, timeout = 30) => {
   return new Promise((resolve5, reject) => {
     let didReceiveOk = false;
@@ -1107,12 +1142,12 @@ var joinRunChannel = (socket, token, runId, logger2, timeout = 30) => {
         resolve5({ channel, run: run2 });
       }
     }).receive("error", (err) => {
-      Sentry3.captureException(err);
+      Sentry4.captureException(err);
       logger2.error(`error connecting to ${channelName}`, err);
       channel?.leave();
       reject(err);
     }).receive("timeout", (err) => {
-      Sentry3.captureException(err);
+      Sentry4.captureException(err);
       logger2.error(`Timeout for ${channelName}`, err);
       channel?.leave();
       reject(err);
@@ -1130,7 +1165,7 @@ var run_default = joinRunChannel;
 
 // src/channels/worker-queue.ts
 import EventEmitter2 from "node:events";
-import * as Sentry4 from "@sentry/node";
+import * as Sentry5 from "@sentry/node";
 import { Socket as PhxSocket } from "phoenix";
 import { WebSocket } from "ws";
 import { API_VERSION } from "@openfn/lexicon/lightning";
@@ -1159,13 +1194,13 @@ var worker_token_default = generateWorkerToken;
 // src/channels/worker-queue.ts
 var connectToWorkerQueue = (endpoint, serverId, secret, timeout = 10, logger2, SocketConstructor = PhxSocket) => {
   const events = new EventEmitter2();
-  Sentry4.addBreadcrumb({
+  Sentry5.addBreadcrumb({
     category: "lifecycle",
     message: "Connecting to worker queue",
     level: "info"
   });
   worker_token_default(secret, serverId, logger2).then(async (token) => {
-    Sentry4.addBreadcrumb({
+    Sentry5.addBreadcrumb({
       category: "lifecycle",
       message: "Worker token generated",
       level: "info"
@@ -1184,7 +1219,7 @@ var connectToWorkerQueue = (endpoint, serverId, secret, timeout = 10, logger2, S
     let didOpen = false;
     let shouldReportConnectionError = true;
     socket.onOpen(() => {
-      Sentry4.addBreadcrumb({
+      Sentry5.addBreadcrumb({
         category: "lifecycle",
         message: "Web socket connected",
         level: "info"
@@ -1210,7 +1245,7 @@ var connectToWorkerQueue = (endpoint, serverId, secret, timeout = 10, logger2, S
       events.emit("disconnect");
     });
     socket.onError((e) => {
-      Sentry4.addBreadcrumb({
+      Sentry5.addBreadcrumb({
         category: "lifecycle",
         message: "Error in web socket connection",
         level: "info"
@@ -1218,7 +1253,7 @@ var connectToWorkerQueue = (endpoint, serverId, secret, timeout = 10, logger2, S
       if (shouldReportConnectionError) {
         logger2.debug("Reporting connection error to sentry");
         shouldReportConnectionError = false;
-        Sentry4.captureException(e);
+        Sentry5.captureException(e);
       }
       if (!didOpen) {
         events.emit("error", e.message);
@@ -1325,11 +1360,11 @@ function createServer(engine, options = {}) {
   app.events = new EventEmitter3();
   app.engine = engine;
   if (options.sentryDsn) {
-    Sentry5.init({
+    Sentry6.init({
       environment: options.sentryEnv,
       dsn: options.sentryDsn
     });
-    Sentry5.setupKoaErrorHandler(app);
+    Sentry6.setupKoaErrorHandler(app);
   }
   app.use(bodyParser());
   app.use(
@@ -1346,7 +1381,7 @@ function createServer(engine, options = {}) {
   router.get("/", healthcheck_default);
   app.options = options;
   app.resumeWorkloop = () => {
-    if (options.noLoop) {
+    if (options.noLoop || app.destroyed) {
       return;
     }
     if (!app.workloop || app.workloop?.isStopped()) {
@@ -1446,7 +1481,7 @@ function createServer(engine, options = {}) {
       shutdown = true;
       logger2.always(`${signal} RECEIVED: CLOSING SERVER`);
       await app.destroy();
-      process.exit();
+      process.exit(0);
     }
   };
   process.on("SIGINT", () => exit("SIGINT"));
@@ -6350,7 +6385,10 @@ function parseArgs(argv) {
     WORKER_SOCKET_TIMEOUT_SECONDS,
     WORKER_STATE_PROPS_TO_REMOVE
   } = process.env;
-  const parser2 = yargs_default(hideBin(argv)).command("server", "Start a ws-worker server").option("port", {
+  const parser2 = yargs_default(hideBin(argv)).command("server", "Start a ws-worker server").option("debug", {
+    hidden: true,
+    type: "boolean"
+  }).option("port", {
     alias: "p",
     description: `Port to run the server on. Default ${DEFAULT_PORT2}. Env: WORKER_PORT`,
     type: "number"
@@ -6472,11 +6510,11 @@ if (args.lightning === "mock") {
   if (!args.secret) {
     args.secret = "abdefg";
   }
-} else if (!args.secret) {
+} else if (!args.debug && !args.secret) {
   logger.error("WORKER_SECRET is not set");
   process.exit(1);
 }
-var [minBackoff, maxBackoff] = args.backoff.split("/").map((n) => parseInt(n, 10) * 1e3);
+var [minBackoff, maxBackoff] = args.backoff.split("/").map((n) => parseFloat(n) * 1e3);
 function engineReady(engine) {
   logger.debug("Creating worker instance");
   const workerOptions = {
@@ -6487,7 +6525,6 @@ function engineReady(engine) {
     sentryDsn: args.sentryDsn,
     sentryEnv: args.sentryEnv,
     noLoop: !args.loop,
-    // TODO need to feed this through properly
     backoff: {
       min: minBackoff,
       max: maxBackoff

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openfn/ws-worker",
-  "version": "1.14.0",
+  "version": "1.14.2",
   "description": "A Websocket Worker to connect Lightning to a Runtime Engine",
   "main": "dist/index.js",
   "type": "module",
@@ -23,10 +23,10 @@
     "koa-logger": "^3.2.1",
     "phoenix": "1.7.10",
     "ws": "^8.18.0",
-    "@openfn/engine-multi": "1.6.7",
+    "@openfn/engine-multi": "1.6.8",
     "@openfn/lexicon": "^1.2.2",
-    "@openfn/runtime": "1.7.0",
-    "@openfn/logger": "1.0.5"
+    "@openfn/logger": "1.0.5",
+    "@openfn/runtime": "1.7.1"
   },
   "devDependencies": {
     "@types/koa": "^2.13.5",
@@ -43,7 +43,7 @@
     "tsup": "^6.2.3",
     "typescript": "^4.6.4",
     "yargs": "^17.6.2",
-    "@openfn/lightning-mock": "2.2.1"
+    "@openfn/lightning-mock": "2.2.2"
   },
   "files": [
     "dist",