@openfn/ws-worker 1.20.0 → 1.20.2

package/CHANGELOG.md

@@ -1,5 +1,22 @@
 # ws-worker
 
+## 1.20.2
+
+### Patch Changes
+
+- Add internal fallback timeout to events. Also adjust error handling to try and catch an uncaught error
+- q
+
+## 1.20.1
+
+### Patch Changes
+
+- Updated dependencies [a1bfdc1]
+- Updated dependencies [c70369b]
+  - @openfn/runtime@1.7.7
+  - @openfn/logger@1.1.1
+  - @openfn/engine-multi@1.9.1
+
 ## 1.20.0
 
 ### Minor Changes

package/dist/index.d.ts

@@ -56,6 +56,7 @@ type WorkerRunOptions = ExecuteOptions & {
     batchLogs?: boolean;
     batchInterval?: number;
     batchLimit?: number;
+    eventTimeoutSeconds?: number;
 };
 
 type Context = {

package/dist/index.js

@@ -936,21 +936,37 @@ function eventProcessor(engine, context, callbacks, options = {}) {
   const { id: planId, logger } = context;
   const {
     batchLimit: limit = DEFAULT_BATCH_LIMIT,
-    batchInterval: interval = DEFAULT_BATCH_INTERVAL
+    batchInterval: interval = DEFAULT_BATCH_INTERVAL,
+    timeout_ms
   } = options;
   const queue = [];
+  let activeBatch = null;
+  let batch = [];
+  let batchTimeout;
+  let didFinish = false;
+  let timeoutHandle;
   const next = async () => {
     const evt = queue[0];
     if (evt) {
+      didFinish = false;
+      const finish = () => {
+        clearTimeout(timeoutHandle);
+        if (!didFinish) {
+          didFinish = true;
+          queue.shift();
+          setImmediate(next);
+        }
+      };
+      if (timeout_ms) {
+        timeoutHandle = setTimeout(() => {
+          logger.error(`${planId} :: ${evt.name} :: timeout (fallback)`);
+          finish();
+        }, timeout_ms);
+      }
       await process2(evt.name, evt.event);
-      queue.shift();
-      next();
+      finish();
     }
   };
-  let activeBatch = null;
-  let batch = [];
-  let start = -1;
-  let batchTimeout;
   const sendBatch = async (name) => {
     clearTimeout(batchTimeout);
     activeBatch = null;
@@ -958,16 +974,24 @@ function eventProcessor(engine, context, callbacks, options = {}) {
     batch = [];
   };
   const send = async (name, payload, batchSize) => {
-    const lightningEvent = eventMap[name] ?? name;
-    await callbacks[name](context, payload);
-    if (batchSize) {
-      logger.info(
-        `${planId} :: sent ${lightningEvent} (${batchSize}):: OK :: ${Date.now() - start}ms`
-      );
-    } else {
-      logger.info(
-        `${planId} :: sent ${lightningEvent} :: OK :: ${Date.now() - start}ms`
-      );
+    try {
+      const start = Date.now();
+      const lightningEvent = eventMap[name] ?? name;
+      await callbacks[name](context, payload);
+      if (batchSize) {
+        logger.info(
+          `${planId} :: sent ${lightningEvent} (${batchSize}):: OK :: ${Date.now() - start}ms`
+        );
+      } else {
+        logger.info(
+          `${planId} :: sent ${lightningEvent} :: OK :: ${Date.now() - start}ms`
+        );
+      }
+    } catch (e2) {
+      if (!e2.reportedToSentry) {
+        Sentry3.captureException(e2);
+        logger.error(e2);
+      }
     }
   };
   const process2 = async (name, event) => {
@@ -988,32 +1012,24 @@ function eventProcessor(engine, context, callbacks, options = {}) {
       await sendBatch(activeBatch);
     }
     if (name in callbacks) {
-      try {
-        start = Date.now();
-        if (options?.batch?.[name]) {
-          activeBatch = name;
-          batch.push(event);
-          while (queue.length > 1 && queue[1].name === name) {
-            const [nextBatchItem] = queue.splice(1, 1);
-            batch.push(nextBatchItem.event);
-            if (batch.length >= limit) {
-              return sendBatch(name);
-            }
+      if (options?.batch?.[name]) {
+        activeBatch = name;
+        batch.push(event);
+        while (queue.length > 1 && queue[1].name === name) {
+          const [nextBatchItem] = queue.splice(1, 1);
+          batch.push(nextBatchItem.event);
+          if (batch.length >= limit) {
+            return sendBatch(name);
           }
-          if (!batchTimeout) {
-            const batchName = activeBatch;
-            batchTimeout = setTimeout(async () => {
-              sendBatch(batchName);
-            }, interval);
-          }
-          return;
         }
-        await send(name, event);
-      } catch (e2) {
-        if (!e2.reportedToSentry) {
-          Sentry3.captureException(e2);
-          logger.error(e2);
+        if (!batchTimeout) {
+          const batchName = activeBatch;
+          batchTimeout = setTimeout(async () => {
+            sendBatch(batchName);
+          }, interval);
         }
+      } else {
+        await send(name, event);
       }
     } else {
       logger.warn("no event bound for", name);
@@ -1056,6 +1072,10 @@ function execute(channel, engine, logger, plan, input, options = {}, onFinish =
         runId: plan.id
       }
     });
+    const batch = {};
+    if (options.batchLogs) {
+      batch[WORKFLOW_LOG2] = true;
+    }
     eventProcessor(
       engine,
       context,
@@ -1069,11 +1089,10 @@ function execute(channel, engine, logger, plan, input, options = {}, onFinish =
         [WORKFLOW_ERROR2]: onRunError
       },
       {
-        batch: options.batchLogs ? {
-          [WORKFLOW_LOG2]: true
-        } : {},
+        batch,
         batchInterval: options.batchInterval,
-        batchLimit: options.batchLimit
+        batchLimit: options.batchLimit,
+        timeout_ms: (options.eventTimeoutSeconds ?? 0) * 1e3 * 1.1
       }
     );
     const resolvers = {
@@ -1468,6 +1487,7 @@ function createServer(engine, options = {}) {
         }
         options2.timeoutRetryCount = app.options.timeoutRetryCount;
         options2.timeoutRetryDelay = app.options.timeoutRetryDelayMs ?? app.options.socketTimeoutSeconds;
+        options2.eventTimeoutSeconds = app.options.messageTimeoutSeconds;
         options2.batchLogs = app.options.batchLogs;
         options2.batchInterval = app.options.batchInterval;
         options2.batchLimit = app.options.batchLimit;

package/dist/start.js

@@ -1085,21 +1085,37 @@ function eventProcessor(engine, context, callbacks, options = {}) {
   const { id: planId, logger: logger2 } = context;
   const {
     batchLimit: limit = DEFAULT_BATCH_LIMIT,
-    batchInterval: interval = DEFAULT_BATCH_INTERVAL
+    batchInterval: interval = DEFAULT_BATCH_INTERVAL,
+    timeout_ms
   } = options;
   const queue = [];
+  let activeBatch = null;
+  let batch = [];
+  let batchTimeout;
+  let didFinish = false;
+  let timeoutHandle;
   const next = async () => {
     const evt = queue[0];
     if (evt) {
+      didFinish = false;
+      const finish = () => {
+        clearTimeout(timeoutHandle);
+        if (!didFinish) {
+          didFinish = true;
+          queue.shift();
+          setImmediate(next);
+        }
+      };
+      if (timeout_ms) {
+        timeoutHandle = setTimeout(() => {
+          logger2.error(`${planId} :: ${evt.name} :: timeout (fallback)`);
+          finish();
+        }, timeout_ms);
+      }
       await process2(evt.name, evt.event);
-      queue.shift();
-      next();
+      finish();
     }
   };
-  let activeBatch = null;
-  let batch = [];
-  let start = -1;
-  let batchTimeout;
   const sendBatch = async (name) => {
     clearTimeout(batchTimeout);
     activeBatch = null;
@@ -1107,16 +1123,24 @@ function eventProcessor(engine, context, callbacks, options = {}) {
     batch = [];
   };
   const send = async (name, payload, batchSize) => {
-    const lightningEvent = eventMap[name] ?? name;
-    await callbacks[name](context, payload);
-    if (batchSize) {
-      logger2.info(
-        `${planId} :: sent ${lightningEvent} (${batchSize}):: OK :: ${Date.now() - start}ms`
-      );
-    } else {
-      logger2.info(
-        `${planId} :: sent ${lightningEvent} :: OK :: ${Date.now() - start}ms`
-      );
+    try {
+      const start = Date.now();
+      const lightningEvent = eventMap[name] ?? name;
+      await callbacks[name](context, payload);
+      if (batchSize) {
+        logger2.info(
+          `${planId} :: sent ${lightningEvent} (${batchSize}):: OK :: ${Date.now() - start}ms`
+        );
+      } else {
+        logger2.info(
+          `${planId} :: sent ${lightningEvent} :: OK :: ${Date.now() - start}ms`
+        );
+      }
+    } catch (e2) {
+      if (!e2.reportedToSentry) {
+        Sentry3.captureException(e2);
+        logger2.error(e2);
+      }
     }
   };
   const process2 = async (name, event) => {
@@ -1137,32 +1161,24 @@ function eventProcessor(engine, context, callbacks, options = {}) {
       await sendBatch(activeBatch);
     }
     if (name in callbacks) {
-      try {
-        start = Date.now();
-        if (options?.batch?.[name]) {
-          activeBatch = name;
-          batch.push(event);
-          while (queue.length > 1 && queue[1].name === name) {
-            const [nextBatchItem] = queue.splice(1, 1);
-            batch.push(nextBatchItem.event);
-            if (batch.length >= limit) {
-              return sendBatch(name);
-            }
-          }
-          if (!batchTimeout) {
-            const batchName = activeBatch;
-            batchTimeout = setTimeout(async () => {
-              sendBatch(batchName);
-            }, interval);
+      if (options?.batch?.[name]) {
+        activeBatch = name;
+        batch.push(event);
+        while (queue.length > 1 && queue[1].name === name) {
+          const [nextBatchItem] = queue.splice(1, 1);
+          batch.push(nextBatchItem.event);
+          if (batch.length >= limit) {
+            return sendBatch(name);
           }
-          return;
         }
-        await send(name, event);
-      } catch (e2) {
-        if (!e2.reportedToSentry) {
-          Sentry3.captureException(e2);
-          logger2.error(e2);
+        if (!batchTimeout) {
+          const batchName = activeBatch;
+          batchTimeout = setTimeout(async () => {
+            sendBatch(batchName);
+          }, interval);
         }
+      } else {
+        await send(name, event);
       }
     } else {
       logger2.warn("no event bound for", name);
@@ -1205,6 +1221,10 @@ function execute(channel, engine, logger2, plan, input, options = {}, onFinish =
         runId: plan.id
       }
     });
+    const batch = {};
+    if (options.batchLogs) {
+      batch[WORKFLOW_LOG2] = true;
+    }
     eventProcessor(
       engine,
       context,
@@ -1218,11 +1238,10 @@ function execute(channel, engine, logger2, plan, input, options = {}, onFinish =
         [WORKFLOW_ERROR2]: onRunError
       },
       {
-        batch: options.batchLogs ? {
-          [WORKFLOW_LOG2]: true
-        } : {},
+        batch,
         batchInterval: options.batchInterval,
-        batchLimit: options.batchLimit
+        batchLimit: options.batchLimit,
+        timeout_ms: (options.eventTimeoutSeconds ?? 0) * 1e3 * 1.1
       }
     );
     const resolvers = {
@@ -1617,6 +1636,7 @@ function createServer(engine, options = {}) {
         }
         options2.timeoutRetryCount = app.options.timeoutRetryCount;
         options2.timeoutRetryDelay = app.options.timeoutRetryDelayMs ?? app.options.socketTimeoutSeconds;
+        options2.eventTimeoutSeconds = app.options.messageTimeoutSeconds;
         options2.batchLogs = app.options.batchLogs;
         options2.batchInterval = app.options.batchInterval;
         options2.batchLimit = app.options.batchLimit;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openfn/ws-worker",
-  "version": "1.20.0",
+  "version": "1.20.2",
   "description": "A Websocket Worker to connect Lightning to a Runtime Engine",
   "main": "dist/index.js",
   "type": "module",
@@ -23,9 +23,9 @@
     "koa-logger": "^3.2.1",
     "phoenix": "1.7.10",
     "ws": "^8.18.3",
-    "@openfn/engine-multi": "1.9.0",
-    "@openfn/logger": "1.1.0",
-    "@openfn/runtime": "1.7.6",
+    "@openfn/engine-multi": "1.9.1",
+    "@openfn/logger": "1.1.1",
+    "@openfn/runtime": "1.7.7",
     "@openfn/lexicon": "^1.3.0"
   },
   "devDependencies": {
@@ -43,7 +43,7 @@
     "tsup": "^6.7.0",
     "typescript": "^4.9.5",
     "yargs": "^17.7.2",
-    "@openfn/lightning-mock": "2.3.9"
+    "@openfn/lightning-mock": "2.3.10"
   },
   "files": [
     "dist",