@openfn/ws-worker 1.13.0 → 1.13.1

package/CHANGELOG.md

@@ -1,5 +1,12 @@
 # ws-worker
 
+## 1.13.1
+
+### Patch Changes
+
+- b83d13c: Add DEFAULT_MESSAGE_TIMEOUT_SECONDS env var and tweaked some error handling around lightning messaging
+- 0bd4adf: Include pod name in logs when claiming
+
 ## 1.13.0
 
 ### Minor Changes

package/dist/index.d.ts

@@ -77,6 +77,7 @@ type ServerOptions = {
     sentryDsn?: string;
     sentryEnv?: string;
     socketTimeoutSeconds?: number;
+    messageTimeoutSeconds?: number;
     payloadLimitMb?: number;
     collectionsVersion?: string;
     collectionsUrl?: string;

package/dist/index.js

@@ -124,9 +124,11 @@ var verifyToken = async (token, publicKey) => {
     return true;
   }
 };
+var { DEPLOYED_POD_NAME } = process.env;
 var claim = (app, logger = mockLogger, options = {}) => {
   return new Promise((resolve, reject) => {
     const { maxWorkers = 5 } = options;
+    const podName = DEPLOYED_POD_NAME ? `[${DEPLOYED_POD_NAME}] ` : "";
     const activeWorkers = Object.keys(app.workflows).length;
     if (activeWorkers >= maxWorkers) {
       app.workloop?.stop(`server at capacity (${activeWorkers}/${maxWorkers})`);
@@ -138,10 +140,10 @@ var claim = (app, logger = mockLogger, options = {}) => {
     }
     logger.debug(`requesting run (capacity ${activeWorkers}/${maxWorkers})`);
     const start = Date.now();
-    app.queueChannel.push(CLAIM, { demand: 1 }).receive("ok", ({ runs }) => {
+    app.queueChannel.push(CLAIM, { demand: 1, pod_name: DEPLOYED_POD_NAME }).receive("ok", ({ runs }) => {
       const duration = Date.now() - start;
       logger.debug(
-        `claimed ${runs.length} runs in ${duration}ms (${runs.length ? runs.map((r) => r.id).join(",") : "-"})`
+        `${podName}claimed ${runs.length} runs in ${duration}ms (${runs.length ? runs.map((r) => r.id).join(",") : "-"})`
       );
       if (!runs?.length) {
         return reject(new Error("No runs returned"));
@@ -161,7 +163,7 @@ var claim = (app, logger = mockLogger, options = {}) => {
         } else {
           logger.debug("skipping run token validation for", run.id);
         }
-        logger.debug("starting run", run.id);
+        logger.debug(`${podName} starting run ${run.id}`);
         app.execute(run);
         resolve();
       });
@@ -939,13 +941,14 @@ var healthcheck_default = (ctx) => {
 
 // src/channels/run.ts
 import * as Sentry3 from "@sentry/node";
-var joinRunChannel = (socket, token, runId, logger) => {
+var joinRunChannel = (socket, token, runId, logger, timeout = 30) => {
   return new Promise((resolve, reject) => {
     let didReceiveOk = false;
     const channelName = `run:${runId}`;
-    logger.debug("connecting to ", channelName);
+    logger.info(`JOINING ${channelName}`);
+    logger.debug(`connecting to ${channelName} with timeout ${timeout}s`);
     const channel = socket.channel(channelName, { token });
-    channel.join().receive("ok", async (e) => {
+    channel.join(timeout * 1e3).receive("ok", async (e) => {
       if (!didReceiveOk) {
         didReceiveOk = true;
         logger.success(`connected to ${channelName}`, e);
@@ -958,10 +961,12 @@ var joinRunChannel = (socket, token, runId, logger) => {
     }).receive("error", (err) => {
       Sentry3.captureException(err);
       logger.error(`error connecting to ${channelName}`, err);
+      channel?.leave();
       reject(err);
     }).receive("timeout", (err) => {
       Sentry3.captureException(err);
       logger.error(`Timeout for ${channelName}`, err);
+      channel?.leave();
       reject(err);
     });
     channel.onClose(() => {
@@ -1216,7 +1221,8 @@ function createServer(engine, options = {}) {
           app.socket,
           token,
           id,
-          logger
+          logger,
+          app.options.messageTimeoutSeconds
         );
         const { plan, options: options2, input } = convert_lightning_plan_default(run, {
           collectionsVersion: app.options.collectionsVersion,
@@ -1253,6 +1259,8 @@ function createServer(engine, options = {}) {
         );
         app.workflows[id] = context;
       } catch (e) {
+        delete app.workflows[id];
+        app.resumeWorkloop();
         logger.error(`Unexpected error executing ${id}`);
         logger.error(e);
       }

package/dist/start.js

@@ -264,9 +264,11 @@ var verifyToken = async (token, publicKey) => {
     return true;
   }
 };
+var { DEPLOYED_POD_NAME } = process.env;
 var claim = (app, logger2 = mockLogger, options = {}) => {
   return new Promise((resolve5, reject) => {
     const { maxWorkers = 5 } = options;
+    const podName = DEPLOYED_POD_NAME ? `[${DEPLOYED_POD_NAME}] ` : "";
     const activeWorkers = Object.keys(app.workflows).length;
     if (activeWorkers >= maxWorkers) {
       app.workloop?.stop(`server at capacity (${activeWorkers}/${maxWorkers})`);
@@ -278,10 +280,10 @@ var claim = (app, logger2 = mockLogger, options = {}) => {
     }
     logger2.debug(`requesting run (capacity ${activeWorkers}/${maxWorkers})`);
     const start = Date.now();
-    app.queueChannel.push(CLAIM, { demand: 1 }).receive("ok", ({ runs }) => {
+    app.queueChannel.push(CLAIM, { demand: 1, pod_name: DEPLOYED_POD_NAME }).receive("ok", ({ runs }) => {
       const duration = Date.now() - start;
       logger2.debug(
-        `claimed ${runs.length} runs in ${duration}ms (${runs.length ? runs.map((r) => r.id).join(",") : "-"})`
+        `${podName}claimed ${runs.length} runs in ${duration}ms (${runs.length ? runs.map((r) => r.id).join(",") : "-"})`
       );
       if (!runs?.length) {
         return reject(new Error("No runs returned"));
@@ -301,7 +303,7 @@ var claim = (app, logger2 = mockLogger, options = {}) => {
         } else {
           logger2.debug("skipping run token validation for", run2.id);
         }
-        logger2.debug("starting run", run2.id);
+        logger2.debug(`${podName} starting run ${run2.id}`);
         app.execute(run2);
         resolve5();
       });
@@ -1079,13 +1081,14 @@ var healthcheck_default = (ctx) => {
 
 // src/channels/run.ts
 import * as Sentry3 from "@sentry/node";
-var joinRunChannel = (socket, token, runId, logger2) => {
+var joinRunChannel = (socket, token, runId, logger2, timeout = 30) => {
   return new Promise((resolve5, reject) => {
     let didReceiveOk = false;
     const channelName = `run:${runId}`;
-    logger2.debug("connecting to ", channelName);
+    logger2.info(`JOINING ${channelName}`);
+    logger2.debug(`connecting to ${channelName} with timeout ${timeout}s`);
     const channel = socket.channel(channelName, { token });
-    channel.join().receive("ok", async (e) => {
+    channel.join(timeout * 1e3).receive("ok", async (e) => {
       if (!didReceiveOk) {
         didReceiveOk = true;
         logger2.success(`connected to ${channelName}`, e);
@@ -1098,10 +1101,12 @@ var joinRunChannel = (socket, token, runId, logger2) => {
     }).receive("error", (err) => {
       Sentry3.captureException(err);
       logger2.error(`error connecting to ${channelName}`, err);
+      channel?.leave();
       reject(err);
     }).receive("timeout", (err) => {
       Sentry3.captureException(err);
       logger2.error(`Timeout for ${channelName}`, err);
+      channel?.leave();
       reject(err);
     });
     channel.onClose(() => {
@@ -1356,7 +1361,8 @@ function createServer(engine, options = {}) {
           app.socket,
           token,
           id,
-          logger2
+          logger2,
+          app.options.messageTimeoutSeconds
         );
         const { plan, options: options2, input } = convert_lightning_plan_default(run2, {
           collectionsVersion: app.options.collectionsVersion,
@@ -1393,6 +1399,8 @@ function createServer(engine, options = {}) {
         );
         app.workflows[id] = context;
       } catch (e) {
+        delete app.workflows[id];
+        app.resumeWorkloop();
         logger2.error(`Unexpected error executing ${id}`);
         logger2.error(e);
       }
@@ -6302,6 +6310,7 @@ var yargs_default = Yargs;
 var DEFAULT_PORT2 = 2222;
 var DEFAULT_WORKER_CAPACITY = 5;
 var DEFAULT_SOCKET_TIMEOUT_SECONDS = 10;
+var DEFAULT_MESSAGE_TIMEOUT_SECONDS = 30;
 function setArg(argValue, envValue, defaultValue) {
   if (Array.isArray(defaultValue) && !argValue && typeof envValue === "string") {
     return envValue.split(",");
@@ -6313,24 +6322,25 @@ function setArg(argValue, envValue, defaultValue) {
 }
 function parseArgs(argv) {
   const {
+    OPENFN_ADAPTORS_REPO,
     WORKER_BACKOFF,
     WORKER_CAPACITY,
-    WORKER_COLLECTIONS_VERSION,
     WORKER_COLLECTIONS_URL,
+    WORKER_COLLECTIONS_VERSION,
     WORKER_LIGHTNING_PUBLIC_KEY,
     WORKER_LIGHTNING_SERVICE_URL,
     WORKER_LOG_LEVEL,
     WORKER_MAX_PAYLOAD_MB,
     WORKER_MAX_RUN_DURATION_SECONDS,
     WORKER_MAX_RUN_MEMORY_MB,
+    WORKER_MESSAGE_TIMEOUT_SECONDS,
     WORKER_PORT,
     WORKER_REPO_DIR,
     WORKER_SECRET,
     WORKER_SENTRY_DSN,
     WORKER_SENTRY_ENV,
-    WORKER_STATE_PROPS_TO_REMOVE,
     WORKER_SOCKET_TIMEOUT_SECONDS,
-    OPENFN_ADAPTORS_REPO
+    WORKER_STATE_PROPS_TO_REMOVE
   } = process.env;
   const parser2 = yargs_default(hideBin(argv)).command("server", "Start a ws-worker server").option("port", {
     alias: "p",
@@ -6344,7 +6354,7 @@ function parseArgs(argv) {
     description: "Path to the runtime repo (where modules will be installed). Env: WORKER_REPO_DIR"
   }).option("monorepo-dir", {
     alias: "m",
-    description: "Path to the adaptors mono repo, from where @local adaptors will be loaded. Env: OPENFN_ADAPTORS_REPO"
+    description: "Path to the adaptors monorepo, from where @local adaptors will be loaded. Env: OPENFN_ADAPTORS_REPO"
   }).option("secret", {
     alias: "s",
     description: "Worker secret. (comes from WORKER_SECRET by default). Env: WORKER_SECRET"
@@ -6354,7 +6364,9 @@ function parseArgs(argv) {
   }).option("sentry-env", {
     description: "Sentry environment. Defaults to 'dev'. Env: WORKER_SENTRY_ENV"
   }).option("socket-timeout", {
-    description: `Timeout for websockets to Lighting, in seconds. Defaults to 10.`
+    description: `Timeout for websockets to Lightning, in seconds. Defaults to 10.Env: WORKER_SOCKET_TIMEOUT_SECONDS`
+  }).option("message-timeout", {
+    description: `Timeout for messages in the run channel in seconds. Defaults to 1. Env: WORKER_MESSAGE_TIMEOUT_SECONDS`
   }).option("lightning-public-key", {
     description: "Base64-encoded public key. Used to verify run tokens. Env: WORKER_LIGHTNING_PUBLIC_KEY"
   }).option("log", {
@@ -6430,6 +6442,11 @@ function parseArgs(argv) {
       WORKER_SOCKET_TIMEOUT_SECONDS,
       DEFAULT_SOCKET_TIMEOUT_SECONDS
     ),
+    messageTimeoutSeconds: setArg(
+      args2.messageTimeoutSeconds,
+      WORKER_MESSAGE_TIMEOUT_SECONDS,
+      DEFAULT_MESSAGE_TIMEOUT_SECONDS
+    ),
     collectionsVersion: setArg(
       args2.collectionsVersion,
       WORKER_COLLECTIONS_VERSION
@@ -6471,7 +6488,9 @@ function engineReady(engine) {
     payloadLimitMb: args.payloadMemory,
     collectionsVersion: args.collectionsVersion,
     collectionsUrl: args.collectionsUrl,
-    monorepoDir: args.monorepoDir
+    monorepoDir: args.monorepoDir,
+    messageTimeoutSeconds: args.messageTimeoutSeconds,
+    socketTimeoutSeconds: args.socketTimeoutSeconds
   };
   if (args.lightningPublicKey) {
     logger.info(

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openfn/ws-worker",
-  "version": "1.13.0",
+  "version": "1.13.1",
   "description": "A Websocket Worker to connect Lightning to a Runtime Engine",
   "main": "dist/index.js",
   "type": "module",
@@ -23,10 +23,10 @@
     "koa-logger": "^3.2.1",
     "phoenix": "1.7.10",
     "ws": "^8.18.0",
-    "@openfn/engine-multi": "1.6.2",
+    "@openfn/lexicon": "^1.2.0",
     "@openfn/logger": "1.0.5",
     "@openfn/runtime": "1.6.4",
-    "@openfn/lexicon": "^1.2.0"
+    "@openfn/engine-multi": "1.6.2"
   },
   "devDependencies": {
     "@types/koa": "^2.13.5",