@openfn/ws-worker 1.20.1 → 1.21.0

package/CHANGELOG.md

@@ -1,5 +1,28 @@
 # ws-worker
 
+## 1.21.0
+
+### Minor Changes
+
+- 064933d: Measure the size of state objects at the end of each step, and throw if they exceed a limit
+
+  In the Worker, this limit is set to 25% of the available runtime memory.
+
+### Patch Changes
+
+- f089f8d: Remove special edge condition mapping (this is now handled by the runtime)
+- Updated dependencies [f089f8d]
+- Updated dependencies [064933d]
+  - @openfn/runtime@1.8.0
+  - @openfn/engine-multi@1.10.0
+
+## 1.20.2
+
+### Patch Changes
+
+- Add internal fallback timeout to events. Also adjust error handling to try and catch an uncaught error
+- q
+
 ## 1.20.1
 
 ### Patch Changes

package/dist/index.d.ts

@@ -56,6 +56,7 @@ type WorkerRunOptions = ExecuteOptions & {
     batchLogs?: boolean;
     batchInterval?: number;
     batchLimit?: number;
+    eventTimeoutSeconds?: number;
 };
 
 type Context = {

package/dist/index.js

@@ -286,19 +286,6 @@ import {
 import crypto2 from "node:crypto";
 import path from "node:path";
 import { getNameAndVersion } from "@openfn/runtime";
-var conditions = {
-  on_job_success: (upstreamId) => `Boolean(!state?.errors?.["${upstreamId}"] ?? true)`,
-  on_job_failure: (upstreamId) => `Boolean(state?.errors && state.errors["${upstreamId}"])`,
-  always: (_upstreamId) => null
-};
-var mapEdgeCondition = (edge) => {
-  const { condition } = edge;
-  if (condition && condition in conditions) {
-    const upstream = edge.source_job_id || edge.source_trigger_id;
-    return conditions[condition](upstream);
-  }
-  return condition;
-};
 var mapTriggerEdgeCondition = (edge) => {
   const { condition } = edge;
   if (condition === void 0 || condition === "always")
@@ -414,9 +401,8 @@ var convert_lightning_plan_default = (run, options = {}) => {
       }
       const next = edges.filter((e) => e.source_job_id === id).reduce((obj, edge) => {
         const newEdge = {};
-        const condition = mapEdgeCondition(edge);
-        if (condition) {
-          newEdge.condition = condition;
+        if (edge.condition) {
+          newEdge.condition = edge.condition;
         }
         if (edge.enabled === false) {
           newEdge.disabled = true;
@@ -936,21 +922,37 @@ function eventProcessor(engine, context, callbacks, options = {}) {
   const { id: planId, logger } = context;
   const {
     batchLimit: limit = DEFAULT_BATCH_LIMIT,
-    batchInterval: interval = DEFAULT_BATCH_INTERVAL
+    batchInterval: interval = DEFAULT_BATCH_INTERVAL,
+    timeout_ms
   } = options;
   const queue = [];
+  let activeBatch = null;
+  let batch = [];
+  let batchTimeout;
+  let didFinish = false;
+  let timeoutHandle;
   const next = async () => {
     const evt = queue[0];
     if (evt) {
+      didFinish = false;
+      const finish = () => {
+        clearTimeout(timeoutHandle);
+        if (!didFinish) {
+          didFinish = true;
+          queue.shift();
+          setImmediate(next);
+        }
+      };
+      if (timeout_ms) {
+        timeoutHandle = setTimeout(() => {
+          logger.error(`${planId} :: ${evt.name} :: timeout (fallback)`);
+          finish();
+        }, timeout_ms);
+      }
       await process2(evt.name, evt.event);
-      queue.shift();
-      next();
+      finish();
     }
   };
-  let activeBatch = null;
-  let batch = [];
-  let start = -1;
-  let batchTimeout;
   const sendBatch = async (name) => {
     clearTimeout(batchTimeout);
     activeBatch = null;
@@ -958,16 +960,24 @@ function eventProcessor(engine, context, callbacks, options = {}) {
     batch = [];
   };
   const send = async (name, payload, batchSize) => {
-    const lightningEvent = eventMap[name] ?? name;
-    await callbacks[name](context, payload);
-    if (batchSize) {
-      logger.info(
-        `${planId} :: sent ${lightningEvent} (${batchSize}):: OK :: ${Date.now() - start}ms`
-      );
-    } else {
-      logger.info(
-        `${planId} :: sent ${lightningEvent} :: OK :: ${Date.now() - start}ms`
-      );
+    try {
+      const start = Date.now();
+      const lightningEvent = eventMap[name] ?? name;
+      await callbacks[name](context, payload);
+      if (batchSize) {
+        logger.info(
+          `${planId} :: sent ${lightningEvent} (${batchSize}):: OK :: ${Date.now() - start}ms`
+        );
+      } else {
+        logger.info(
+          `${planId} :: sent ${lightningEvent} :: OK :: ${Date.now() - start}ms`
+        );
+      }
+    } catch (e2) {
+      if (!e2.reportedToSentry) {
+        Sentry3.captureException(e2);
+        logger.error(e2);
+      }
     }
   };
   const process2 = async (name, event) => {
@@ -988,32 +998,24 @@ function eventProcessor(engine, context, callbacks, options = {}) {
       await sendBatch(activeBatch);
     }
     if (name in callbacks) {
-      try {
-        start = Date.now();
-        if (options?.batch?.[name]) {
-          activeBatch = name;
-          batch.push(event);
-          while (queue.length > 1 && queue[1].name === name) {
-            const [nextBatchItem] = queue.splice(1, 1);
-            batch.push(nextBatchItem.event);
-            if (batch.length >= limit) {
-              return sendBatch(name);
-            }
-          }
-          if (!batchTimeout) {
-            const batchName = activeBatch;
-            batchTimeout = setTimeout(async () => {
-              sendBatch(batchName);
-            }, interval);
+      if (options?.batch?.[name]) {
+        activeBatch = name;
+        batch.push(event);
+        while (queue.length > 1 && queue[1].name === name) {
+          const [nextBatchItem] = queue.splice(1, 1);
+          batch.push(nextBatchItem.event);
+          if (batch.length >= limit) {
+            return sendBatch(name);
           }
-          return;
         }
-        await send(name, event);
-      } catch (e2) {
-        if (!e2.reportedToSentry) {
-          Sentry3.captureException(e2);
-          logger.error(e2);
+        if (!batchTimeout) {
+          const batchName = activeBatch;
+          batchTimeout = setTimeout(async () => {
+            sendBatch(batchName);
+          }, interval);
         }
+      } else {
+        await send(name, event);
       }
     } else {
       logger.warn("no event bound for", name);
@@ -1056,6 +1058,10 @@ function execute(channel, engine, logger, plan, input, options = {}, onFinish =
         runId: plan.id
       }
     });
+    const batch = {};
+    if (options.batchLogs) {
+      batch[WORKFLOW_LOG2] = true;
+    }
     eventProcessor(
       engine,
       context,
@@ -1069,11 +1075,10 @@ function execute(channel, engine, logger, plan, input, options = {}, onFinish =
         [WORKFLOW_ERROR2]: onRunError
       },
       {
-        batch: options.batchLogs ? {
-          [WORKFLOW_LOG2]: true
-        } : {},
+        batch,
         batchInterval: options.batchInterval,
-        batchLimit: options.batchLimit
+        batchLimit: options.batchLimit,
+        timeout_ms: (options.eventTimeoutSeconds ?? 0) * 1e3 * 1.1
       }
     );
     const resolvers = {
@@ -1468,6 +1473,7 @@ function createServer(engine, options = {}) {
         }
         options2.timeoutRetryCount = app.options.timeoutRetryCount;
         options2.timeoutRetryDelay = app.options.timeoutRetryDelayMs ?? app.options.socketTimeoutSeconds;
+        options2.eventTimeoutSeconds = app.options.messageTimeoutSeconds;
         options2.batchLogs = app.options.batchLogs;
         options2.batchInterval = app.options.batchInterval;
         options2.batchLimit = app.options.batchLimit;

package/dist/start.js

@@ -435,19 +435,6 @@ import {
 import crypto3 from "node:crypto";
 import path from "node:path";
 import { getNameAndVersion } from "@openfn/runtime";
-var conditions = {
-  on_job_success: (upstreamId) => `Boolean(!state?.errors?.["${upstreamId}"] ?? true)`,
-  on_job_failure: (upstreamId) => `Boolean(state?.errors && state.errors["${upstreamId}"])`,
-  always: (_upstreamId) => null
-};
-var mapEdgeCondition = (edge) => {
-  const { condition } = edge;
-  if (condition && condition in conditions) {
-    const upstream = edge.source_job_id || edge.source_trigger_id;
-    return conditions[condition](upstream);
-  }
-  return condition;
-};
 var mapTriggerEdgeCondition = (edge) => {
   const { condition } = edge;
   if (condition === void 0 || condition === "always")
@@ -563,9 +550,8 @@ var convert_lightning_plan_default = (run2, options = {}) => {
       }
       const next = edges.filter((e) => e.source_job_id === id).reduce((obj, edge) => {
         const newEdge = {};
-        const condition = mapEdgeCondition(edge);
-        if (condition) {
-          newEdge.condition = condition;
+        if (edge.condition) {
+          newEdge.condition = edge.condition;
         }
         if (edge.enabled === false) {
           newEdge.disabled = true;
@@ -1085,21 +1071,37 @@ function eventProcessor(engine, context, callbacks, options = {}) {
   const { id: planId, logger: logger2 } = context;
   const {
     batchLimit: limit = DEFAULT_BATCH_LIMIT,
-    batchInterval: interval = DEFAULT_BATCH_INTERVAL
+    batchInterval: interval = DEFAULT_BATCH_INTERVAL,
+    timeout_ms
   } = options;
   const queue = [];
+  let activeBatch = null;
+  let batch = [];
+  let batchTimeout;
+  let didFinish = false;
+  let timeoutHandle;
   const next = async () => {
     const evt = queue[0];
     if (evt) {
+      didFinish = false;
+      const finish = () => {
+        clearTimeout(timeoutHandle);
+        if (!didFinish) {
+          didFinish = true;
+          queue.shift();
+          setImmediate(next);
+        }
+      };
+      if (timeout_ms) {
+        timeoutHandle = setTimeout(() => {
+          logger2.error(`${planId} :: ${evt.name} :: timeout (fallback)`);
+          finish();
+        }, timeout_ms);
+      }
       await process2(evt.name, evt.event);
-      queue.shift();
-      next();
+      finish();
     }
   };
-  let activeBatch = null;
-  let batch = [];
-  let start = -1;
-  let batchTimeout;
   const sendBatch = async (name) => {
     clearTimeout(batchTimeout);
     activeBatch = null;
@@ -1107,16 +1109,24 @@ function eventProcessor(engine, context, callbacks, options = {}) {
     batch = [];
   };
   const send = async (name, payload, batchSize) => {
-    const lightningEvent = eventMap[name] ?? name;
-    await callbacks[name](context, payload);
-    if (batchSize) {
-      logger2.info(
-        `${planId} :: sent ${lightningEvent} (${batchSize}):: OK :: ${Date.now() - start}ms`
-      );
-    } else {
-      logger2.info(
-        `${planId} :: sent ${lightningEvent} :: OK :: ${Date.now() - start}ms`
-      );
+    try {
+      const start = Date.now();
+      const lightningEvent = eventMap[name] ?? name;
+      await callbacks[name](context, payload);
+      if (batchSize) {
+        logger2.info(
+          `${planId} :: sent ${lightningEvent} (${batchSize}):: OK :: ${Date.now() - start}ms`
+        );
+      } else {
+        logger2.info(
+          `${planId} :: sent ${lightningEvent} :: OK :: ${Date.now() - start}ms`
+        );
+      }
+    } catch (e2) {
+      if (!e2.reportedToSentry) {
+        Sentry3.captureException(e2);
+        logger2.error(e2);
+      }
     }
   };
   const process2 = async (name, event) => {
@@ -1137,32 +1147,24 @@ function eventProcessor(engine, context, callbacks, options = {}) {
       await sendBatch(activeBatch);
     }
     if (name in callbacks) {
-      try {
-        start = Date.now();
-        if (options?.batch?.[name]) {
-          activeBatch = name;
-          batch.push(event);
-          while (queue.length > 1 && queue[1].name === name) {
-            const [nextBatchItem] = queue.splice(1, 1);
-            batch.push(nextBatchItem.event);
-            if (batch.length >= limit) {
-              return sendBatch(name);
-            }
+      if (options?.batch?.[name]) {
+        activeBatch = name;
+        batch.push(event);
+        while (queue.length > 1 && queue[1].name === name) {
+          const [nextBatchItem] = queue.splice(1, 1);
+          batch.push(nextBatchItem.event);
+          if (batch.length >= limit) {
+            return sendBatch(name);
           }
-          if (!batchTimeout) {
-            const batchName = activeBatch;
-            batchTimeout = setTimeout(async () => {
-              sendBatch(batchName);
-            }, interval);
-          }
-          return;
         }
-        await send(name, event);
-      } catch (e2) {
-        if (!e2.reportedToSentry) {
-          Sentry3.captureException(e2);
-          logger2.error(e2);
+        if (!batchTimeout) {
+          const batchName = activeBatch;
+          batchTimeout = setTimeout(async () => {
+            sendBatch(batchName);
+          }, interval);
         }
+      } else {
+        await send(name, event);
       }
     } else {
       logger2.warn("no event bound for", name);
@@ -1205,6 +1207,10 @@ function execute(channel, engine, logger2, plan, input, options = {}, onFinish =
         runId: plan.id
       }
     });
+    const batch = {};
+    if (options.batchLogs) {
+      batch[WORKFLOW_LOG2] = true;
+    }
     eventProcessor(
       engine,
       context,
@@ -1218,11 +1224,10 @@ function execute(channel, engine, logger2, plan, input, options = {}, onFinish =
         [WORKFLOW_ERROR2]: onRunError
       },
       {
-        batch: options.batchLogs ? {
-          [WORKFLOW_LOG2]: true
-        } : {},
+        batch,
         batchInterval: options.batchInterval,
-        batchLimit: options.batchLimit
+        batchLimit: options.batchLimit,
+        timeout_ms: (options.eventTimeoutSeconds ?? 0) * 1e3 * 1.1
       }
     );
     const resolvers = {
@@ -1617,6 +1622,7 @@ function createServer(engine, options = {}) {
         }
         options2.timeoutRetryCount = app.options.timeoutRetryCount;
         options2.timeoutRetryDelay = app.options.timeoutRetryDelayMs ?? app.options.socketTimeoutSeconds;
+        options2.eventTimeoutSeconds = app.options.messageTimeoutSeconds;
         options2.batchLogs = app.options.batchLogs;
         options2.batchInterval = app.options.batchInterval;
         options2.batchLimit = app.options.batchLimit;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openfn/ws-worker",
-  "version": "1.20.1",
+  "version": "1.21.0",
   "description": "A Websocket Worker to connect Lightning to a Runtime Engine",
   "main": "dist/index.js",
   "type": "module",
@@ -23,10 +23,10 @@
     "koa-logger": "^3.2.1",
     "phoenix": "1.7.10",
     "ws": "^8.18.3",
-    "@openfn/engine-multi": "1.9.1",
     "@openfn/lexicon": "^1.3.0",
     "@openfn/logger": "1.1.1",
-    "@openfn/runtime": "1.7.7"
+    "@openfn/engine-multi": "1.10.0",
+    "@openfn/runtime": "1.8.0"
   },
   "devDependencies": {
     "@types/koa": "^2.15.0",
@@ -43,7 +43,7 @@
     "tsup": "^6.7.0",
     "typescript": "^4.9.5",
     "yargs": "^17.7.2",
-    "@openfn/lightning-mock": "2.3.10"
+    "@openfn/lightning-mock": "2.4.1"
   },
   "files": [
     "dist",