@openfn/ws-worker 1.19.2 → 1.19.5

package/CHANGELOG.md

@@ -1,5 +1,26 @@
 # ws-worker
 
+## 1.19.5
+
+### Patch Changes
+
+- b42b0b0: Force disable message retries
+
+## 1.19.4
+
+### Patch Changes
+
+- 3e0aac7: Retry events if they are timed out. Will retry WORKER_TIMEOUT_RETRY_COUNT times and wait for WORKER_TIMEOUT_RETRY_DELAY_MS or WORKER_MESSAGE_TIMEOUT_SECONDS between retries
+
+## 1.19.3
+
+### Patch Changes
+
+- 62f24b0: Fix an edge case where the server can shut down in-between claim and start (lost runs edge case)
+- Updated dependencies [fe06f44]
+  - @openfn/runtime@1.7.5
+  - @openfn/engine-multi@1.8.3
+
 ## 1.19.2
 
 ### Patch Changes

package/dist/index.d.ts

@@ -3,8 +3,8 @@ import Koa from 'koa';
 import { LogLevel, Logger } from '@openfn/logger';
 import * as l from '@openfn/lexicon/lightning';
 import { ClaimRun } from '@openfn/lexicon/lightning';
+import { ExecutionPlan, Lazy, State, UUID } from '@openfn/lexicon';
 import { ExecuteOptions, RuntimeEngine } from '@openfn/engine-multi';
-import { ExecutionPlan, Lazy, State } from '@openfn/lexicon';
 import { Channel as Channel$1 } from 'phoenix';
 import { Server } from 'http';
 
@@ -50,10 +50,12 @@ type WorkerRunOptions = ExecuteOptions & {
     outputDataclips?: boolean;
     payloadLimitMb?: number;
     jobLogLevel?: LogLevel;
+    timeoutRetryCount?: number;
+    timeoutRetryDelay?: number;
 };
 
 type Context = {
-    id: string;
+    id: UUID;
     channel: Channel;
     state: RunState;
     logger: Logger;
@@ -83,6 +85,8 @@ type ServerOptions = {
     collectionsVersion?: string;
     collectionsUrl?: string;
     monorepoDir?: string;
+    timeoutRetryCount?: number;
+    timeoutRetryDelayMs?: number;
 };
 interface ServerApp extends Koa {
     id: string;

package/dist/index.js

@@ -190,7 +190,7 @@ var claim = (app, logger = mockLogger, options = {}) => {
     app.queueChannel.push(CLAIM, {
       demand,
       worker_name: NAME || null
-    }).receive("ok", ({ runs }) => {
+    }).receive("ok", async ({ runs }) => {
       delete app.openClaims[claimId];
       const duration = Date.now() - start;
       logger.debug(
@@ -200,7 +200,7 @@ var claim = (app, logger = mockLogger, options = {}) => {
         app.events.emit(INTERNAL_CLAIM_COMPLETE, { runs });
         return reject(new Error("No runs returned"));
       }
-      runs.forEach(async (run) => {
+      for (const run of runs) {
         if (app.options?.runPublicKey) {
           try {
             await verifyToken(run.token, app.options.runPublicKey);
@@ -217,7 +217,7 @@ var claim = (app, logger = mockLogger, options = {}) => {
         }
         logger.debug(`${podName} starting run ${run.id}`);
         app.execute(run);
-      });
+      }
       resolve();
       app.events.emit(INTERNAL_CLAIM_COMPLETE, { runs });
     }).receive("error", (e) => {
@@ -525,7 +525,10 @@ var LightningTimeoutError = class extends Error {
 };
 
 // src/util/send-event.ts
-var sendEvent = (context, event, payload) => {
+var allowRetryOntimeout = false;
+var sendEvent = (context, event, payload, attempts) => {
+  const { timeoutRetryCount = 1, timeoutRetryDelay = 1 } = context.options ?? {};
+  const thisAttempt = attempts ?? 1;
   const { channel, logger, id: runId = "<unknown run>" } = context;
   return new Promise((resolve, reject) => {
     const report = (error) => {
@@ -549,7 +552,16 @@ var sendEvent = (context, event, payload) => {
     channel.push(event, payload).receive("error", (message) => {
       report(new LightningSocketError(event, message));
     }).receive("timeout", () => {
-      report(new LightningTimeoutError(event));
+      if (!allowRetryOntimeout || thisAttempt >= timeoutRetryCount) {
+        report(new LightningTimeoutError(event));
+      } else {
+        logger.warn(
+          `${runId} event ${event} timed out, will retry in ${timeoutRetryDelay}ms (attempt ${thisAttempt + 1} of ${timeoutRetryCount})`
+        );
+        setTimeout(() => {
+          sendEvent(context, event, payload, thisAttempt + 1).then(resolve).catch(reject);
+        }, timeoutRetryDelay);
+      }
     }).receive("ok", resolve);
   });
 };
@@ -957,7 +969,6 @@ function execute(channel, engine, logger, plan, input, options = {}, onFinish =
         });
         engine.execute(plan, loadedInput, { resolvers, ...options });
       } catch (e) {
-        process.exit(66);
         Sentry3.addBreadcrumb({
           category: "run",
           message: "exception in run",
@@ -996,7 +1007,7 @@ function onJobLog(context, event) {
     message = JSON.parse(event.message);
   }
   const log = {
-    run_id: state.plan.id,
+    run_id: `${state.plan.id}`,
     message,
     source: event.name,
     level: event.level,
@@ -1038,7 +1049,7 @@ var joinRunChannel = (socket, token, runId, logger, timeout = 30) => {
         didReceiveOk = true;
         logger.success(`connected to ${channelName}`, e);
         const run = await send_event_default(
-          { channel, logger, id: runId },
+          { channel, logger, id: runId, options: {} },
           GET_PLAN
         );
         resolve({ channel, run });
@@ -1336,6 +1347,8 @@ function createServer(engine, options = {}) {
         if (!("payloadLimitMb" in options2)) {
           options2.payloadLimitMb = app.options.payloadLimitMb;
         }
+        options2.timeoutRetryCount = app.options.timeoutRetryCount;
+        options2.timeoutRetryDelay = app.options.timeoutRetryDelayMs ?? app.options.socketTimeoutSeconds;
         const onFinish = () => {
           const duration = (Date.now() - start) / 1e3;
           logger.debug(

package/dist/start.js

@@ -339,7 +339,7 @@ var claim = (app, logger2 = mockLogger, options = {}) => {
     app.queueChannel.push(CLAIM, {
       demand,
       worker_name: NAME || null
-    }).receive("ok", ({ runs }) => {
+    }).receive("ok", async ({ runs }) => {
       delete app.openClaims[claimId];
       const duration = Date.now() - start;
       logger2.debug(
@@ -349,7 +349,7 @@ var claim = (app, logger2 = mockLogger, options = {}) => {
         app.events.emit(INTERNAL_CLAIM_COMPLETE, { runs });
         return reject(new Error("No runs returned"));
       }
-      runs.forEach(async (run2) => {
+      for (const run2 of runs) {
         if (app.options?.runPublicKey) {
           try {
             await verifyToken(run2.token, app.options.runPublicKey);
@@ -366,7 +366,7 @@ var claim = (app, logger2 = mockLogger, options = {}) => {
         }
         logger2.debug(`${podName} starting run ${run2.id}`);
         app.execute(run2);
-      });
+      }
       resolve5();
       app.events.emit(INTERNAL_CLAIM_COMPLETE, { runs });
     }).receive("error", (e) => {
@@ -674,7 +674,10 @@ var LightningTimeoutError = class extends Error {
 };
 
 // src/util/send-event.ts
-var sendEvent = (context, event, payload) => {
+var allowRetryOntimeout = false;
+var sendEvent = (context, event, payload, attempts) => {
+  const { timeoutRetryCount = 1, timeoutRetryDelay = 1 } = context.options ?? {};
+  const thisAttempt = attempts ?? 1;
   const { channel, logger: logger2, id: runId = "<unknown run>" } = context;
   return new Promise((resolve5, reject) => {
     const report = (error) => {
@@ -698,7 +701,16 @@ var sendEvent = (context, event, payload) => {
     channel.push(event, payload).receive("error", (message) => {
       report(new LightningSocketError(event, message));
     }).receive("timeout", () => {
-      report(new LightningTimeoutError(event));
+      if (!allowRetryOntimeout || thisAttempt >= timeoutRetryCount) {
+        report(new LightningTimeoutError(event));
+      } else {
+        logger2.warn(
+          `${runId} event ${event} timed out, will retry in ${timeoutRetryDelay}ms (attempt ${thisAttempt + 1} of ${timeoutRetryCount})`
+        );
+        setTimeout(() => {
+          sendEvent(context, event, payload, thisAttempt + 1).then(resolve5).catch(reject);
+        }, timeoutRetryDelay);
+      }
     }).receive("ok", resolve5);
   });
 };
@@ -1106,7 +1118,6 @@ function execute(channel, engine, logger2, plan, input, options = {}, onFinish =
         });
         engine.execute(plan, loadedInput, { resolvers, ...options });
       } catch (e) {
-        process.exit(66);
         Sentry3.addBreadcrumb({
           category: "run",
           message: "exception in run",
@@ -1145,7 +1156,7 @@ function onJobLog(context, event) {
     message = JSON.parse(event.message);
   }
   const log = {
-    run_id: state.plan.id,
+    run_id: `${state.plan.id}`,
     message,
     source: event.name,
     level: event.level,
@@ -1187,7 +1198,7 @@ var joinRunChannel = (socket, token, runId, logger2, timeout = 30) => {
         didReceiveOk = true;
         logger2.success(`connected to ${channelName}`, e);
         const run2 = await send_event_default(
-          { channel, logger: logger2, id: runId },
+          { channel, logger: logger2, id: runId, options: {} },
           GET_PLAN
         );
         resolve5({ channel, run: run2 });
@@ -1485,6 +1496,8 @@ function createServer(engine, options = {}) {
         if (!("payloadLimitMb" in options2)) {
           options2.payloadLimitMb = app.options.payloadLimitMb;
         }
+        options2.timeoutRetryCount = app.options.timeoutRetryCount;
+        options2.timeoutRetryDelay = app.options.timeoutRetryDelayMs ?? app.options.socketTimeoutSeconds;
         const onFinish = () => {
           const duration = (Date.now() - start) / 1e3;
           logger2.debug(
@@ -6449,14 +6462,16 @@ function parseArgs(argv) {
     WORKER_MAX_RUN_MEMORY_MB,
     WORKER_MESSAGE_TIMEOUT_SECONDS,
     WORKER_PORT,
-    WORKER_PROFILE,
     WORKER_PROFILE_POLL_INTERVAL_MS,
+    WORKER_PROFILE,
     WORKER_REPO_DIR,
     WORKER_SECRET,
     WORKER_SENTRY_DSN,
     WORKER_SENTRY_ENV,
     WORKER_SOCKET_TIMEOUT_SECONDS,
     WORKER_STATE_PROPS_TO_REMOVE,
+    WORKER_TIMEOUT_RETRY_COUNT,
+    WORKER_TIMEOUT_RETRY_DELAY_MS,
     WORKER_VALIDATION_RETRIES,
     WORKER_VALIDATION_TIMEOUT_MS
   } = process.env;
@@ -6539,6 +6554,12 @@ function parseArgs(argv) {
   }).option("profile-poll-interval-ms", {
     description: "Interval for polling profile data, in milliseconds. Default 10. Env: WORKER_PROFILE_POLL_INTERVAL_MS",
     type: "number"
+  }).option("timeout-retry-count", {
+    description: "When a websocket event receives a timeout, this option sets how many times the worker should retry it. Default 10. Env: WORKER_TIMEOUT_RETRY_COUNT",
+    type: "number"
+  }).option("timeout-retry-delay", {
+    description: "When a websocket event receives a timeout, this option sets how log to wait before retrying Default 30000. Env: WORKER_TIMEOUT_RETRY_DELAY_MS",
+    type: "number"
   });
   const args2 = parser2.parse();
   return {
@@ -6607,6 +6628,16 @@ function parseArgs(argv) {
       args2.profilePollIntervalMs,
       WORKER_PROFILE_POLL_INTERVAL_MS,
       10
+    ),
+    timeoutRetryCount: setArg(
+      args2.timeoutRetryCount,
+      WORKER_TIMEOUT_RETRY_COUNT,
+      10
+    ),
+    timeoutRetryDelayMs: setArg(
+      args2.timeoutRetryDelayMs,
+      WORKER_TIMEOUT_RETRY_DELAY_MS,
+      30 * 1e3
     )
   };
 }
@@ -6647,9 +6678,11 @@ function engineReady(engine) {
     messageTimeoutSeconds: args.messageTimeoutSeconds,
     claimTimeoutSeconds: args.claimTimeoutSeconds,
     // deprecated!
-    socketTimeoutSeconds: args.socketTimeoutSeconds
+    socketTimeoutSeconds: args.socketTimeoutSeconds,
+    timeoutRetryCount: args.timeoutRetryCount,
+    timeoutRetryDelayMs: args.timeoutRetryDelayMs
   };
-  if ("socketTimeoutSeconds" in args) {
+  if (args.socketTimeoutSeconds) {
     logger.warn(
       "WARNING: deprecated socketTimeoutSeconds value passed.\n\nThis will be respected as the default socket timeout value, but will be removed from future versions of the worker."
     );

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openfn/ws-worker",
-  "version": "1.19.2",
+  "version": "1.19.5",
   "description": "A Websocket Worker to connect Lightning to a Runtime Engine",
   "main": "dist/index.js",
   "type": "module",
@@ -23,10 +23,10 @@
     "koa-logger": "^3.2.1",
     "phoenix": "1.7.10",
     "ws": "^8.18.3",
-    "@openfn/lexicon": "^1.2.5",
-    "@openfn/engine-multi": "1.8.2",
-    "@openfn/logger": "1.0.6",
-    "@openfn/runtime": "1.7.4"
+    "@openfn/runtime": "1.7.5",
+    "@openfn/lexicon": "^1.2.6",
+    "@openfn/engine-multi": "1.8.3",
+    "@openfn/logger": "1.0.6"
   },
   "devDependencies": {
     "@types/koa": "^2.15.0",
@@ -43,7 +43,7 @@
     "tsup": "^6.7.0",
     "typescript": "^4.9.5",
     "yargs": "^17.7.2",
-    "@openfn/lightning-mock": "2.3.5"
+    "@openfn/lightning-mock": "2.3.6"
   },
   "files": [
     "dist",