@openfn/ws-worker 1.6.6 → 1.7.0

package/CHANGELOG.md

@@ -1,5 +1,22 @@
 # ws-worker
 
+## 1.7.0
+
+### Minor Changes
+
+- ae55a6a: Include timestamp on step complete even if the step failed
+
+### Patch Changes
+
+- Updated dependencies [ae55a6a]
+  - @openfn/engine-multi@1.3.0
+
+## 1.6.7
+
+### Patch Changes
+
+- 42883f8: Better handliung of claim backoffs when at capacity
+
 ## 1.6.6
 
 ### Patch Changes

package/dist/index.d.ts

@@ -8,6 +8,11 @@ import { ExecutionPlan, Lazy, State } from '@openfn/lexicon';
 import { Channel as Channel$1 } from 'phoenix';
 import { Server } from 'http';
 
+declare type Workloop = {
+    stop: (reason?: string) => void;
+    isStopped: () => boolean;
+};
+
 // Internal server state for each run
 type RunState = {
   activeStep?: string;
@@ -80,9 +85,10 @@ interface ServerApp extends Koa {
     server: Server;
     engine: RuntimeEngine;
     options: ServerOptions;
+    workloop?: Workloop;
     execute: ({ id, token }: ClaimRun) => Promise<void>;
     destroy: () => void;
-    killWorkloop?: () => void;
+    resumeWorkloop: () => void;
 }
 declare function createServer(engine: RuntimeEngine, options?: ServerOptions): ServerApp;
 

package/dist/index.js

@@ -29,7 +29,7 @@ var name, version, description, main, type, scripts, bin, author, license, depen
 var init_package = __esm({
   "package.json"() {
     name = "@openfn/ws-worker";
-    version = "1.6.6";
+    version = "1.7.0";
     description = "A Websocket Worker to connect Lightning to a Runtime Engine";
     main = "dist/index.js";
     type = "module";
@@ -132,7 +132,7 @@ var destroy = async (app, logger) => {
   await Promise.all([
     new Promise((resolve) => {
       app.destroyed = true;
-      app.killWorkloop?.();
+      app.workloop?.stop("server closed");
       app.queueChannel?.leave();
       app.server.close(async () => {
         resolve();
@@ -232,9 +232,7 @@ var claim = (app, logger = mockLogger, options = {}) => {
     const { maxWorkers = 5 } = options;
     const activeWorkers = Object.keys(app.workflows).length;
     if (activeWorkers >= maxWorkers) {
-      logger.debug(
-        `skipping claim attempt: server at capacity (${activeWorkers}/${maxWorkers})`
-      );
+      app.workloop?.stop(`server at capacity (${activeWorkers}/${maxWorkers})`);
       return reject(new Error("Server at capacity"));
     }
     if (!app.queueChannel) {
@@ -303,11 +301,15 @@ var startWorkloop = (app, logger, minBackoff, maxBackoff, maxWorkers) => {
     }
   };
   workLoop();
-  return () => {
-    logger.debug("cancelling workloop");
-    cancelled = true;
-    promise.cancel();
-    app.queueChannel?.leave();
+  return {
+    stop: (reason = "reason unknown") => {
+      if (!cancelled) {
+        logger.info(`cancelling workloop: ${reason}`);
+        cancelled = true;
+        promise.cancel();
+      }
+    },
+    isStopped: () => cancelled
   };
 };
 var workloop_default = startWorkloop;
@@ -1018,16 +1020,7 @@ function connect(app, logger, options = {}) {
     logger.success("Connected to Lightning at", options.lightning);
     app.socket = socket;
     app.queueChannel = channel;
-    if (!options.noLoop) {
-      logger.info("Starting workloop");
-      app.killWorkloop = workloop_default(
-        app,
-        logger,
-        options.backoff?.min || MIN_BACKOFF,
-        options.backoff?.max || MAX_BACKOFF,
-        options.maxWorkflows
-      );
-    } else {
+    if (options.noLoop) {
       const port = app.server?.address().port;
       logger.break();
       logger.warn("Noloop active: workloop has not started");
@@ -1036,17 +1029,17 @@ function connect(app, logger, options = {}) {
       logger.info(`  curl -X POST http://localhost:${port}/claim`);
       logger.break();
     }
+    app.resumeWorkloop();
   };
   const onDisconnect = () => {
-    if (app.killWorkloop) {
-      app.killWorkloop();
-      delete app.killWorkloop;
-      if (!app.destroyed) {
-        logger.info("Connection to lightning lost");
-        logger.info(
-          "Worker will automatically reconnect when lightning is back online"
-        );
-      }
+    if (!app.workloop?.isStopped()) {
+      app.workloop?.stop("Socket disconnected unexpectedly");
+    }
+    if (!app.destroyed) {
+      logger.info("Connection to lightning lost");
+      logger.info(
+        "Worker will automatically reconnect when lightning is back online"
+      );
     }
   };
   const onError = (e) => {
@@ -1089,6 +1082,21 @@ function createServer(engine, options = {}) {
   router.get("/livez", healthcheck_default);
   router.get("/", healthcheck_default);
   app.options = options;
+  app.resumeWorkloop = () => {
+    if (options.noLoop) {
+      return;
+    }
+    if (!app.workloop || app.workloop?.isStopped()) {
+      logger.info("Starting workloop");
+      app.workloop = workloop_default(
+        app,
+        logger,
+        options.backoff?.min || MIN_BACKOFF,
+        options.backoff?.max || MAX_BACKOFF,
+        options.maxWorkflows
+      );
+    }
+  };
   app.execute = async ({ id, token }) => {
     if (app.socket) {
       try {
@@ -1111,6 +1119,7 @@ function createServer(engine, options = {}) {
           delete app.workflows[id];
           runChannel.leave();
           app.events.emit(INTERNAL_RUN_COMPLETE);
+          app.resumeWorkloop();
         };
         const context = execute(
           runChannel,

package/dist/start.js

@@ -37,7 +37,7 @@ var name, version, description, main, type, scripts, bin, author, license, depen
 var init_package = __esm({
   "package.json"() {
     name = "@openfn/ws-worker";
-    version = "1.6.6";
+    version = "1.7.0";
     description = "A Websocket Worker to connect Lightning to a Runtime Engine";
     main = "dist/index.js";
     type = "module";
@@ -271,7 +271,7 @@ var destroy = async (app, logger2) => {
   await Promise.all([
     new Promise((resolve5) => {
       app.destroyed = true;
-      app.killWorkloop?.();
+      app.workloop?.stop("server closed");
       app.queueChannel?.leave();
       app.server.close(async () => {
         resolve5();
@@ -371,9 +371,7 @@ var claim = (app, logger2 = mockLogger, options = {}) => {
     const { maxWorkers = 5 } = options;
     const activeWorkers = Object.keys(app.workflows).length;
     if (activeWorkers >= maxWorkers) {
-      logger2.debug(
-        `skipping claim attempt: server at capacity (${activeWorkers}/${maxWorkers})`
-      );
+      app.workloop?.stop(`server at capacity (${activeWorkers}/${maxWorkers})`);
       return reject(new Error("Server at capacity"));
     }
     if (!app.queueChannel) {
@@ -442,11 +440,15 @@ var startWorkloop = (app, logger2, minBackoff2, maxBackoff2, maxWorkers) => {
     }
   };
   workLoop();
-  return () => {
-    logger2.debug("cancelling workloop");
-    cancelled = true;
-    promise.cancel();
-    app.queueChannel?.leave();
+  return {
+    stop: (reason = "reason unknown") => {
+      if (!cancelled) {
+        logger2.info(`cancelling workloop: ${reason}`);
+        cancelled = true;
+        promise.cancel();
+      }
+    },
+    isStopped: () => cancelled
   };
 };
 var workloop_default = startWorkloop;
@@ -1157,16 +1159,7 @@ function connect(app, logger2, options = {}) {
     logger2.success("Connected to Lightning at", options.lightning);
     app.socket = socket;
     app.queueChannel = channel;
-    if (!options.noLoop) {
-      logger2.info("Starting workloop");
-      app.killWorkloop = workloop_default(
-        app,
-        logger2,
-        options.backoff?.min || MIN_BACKOFF,
-        options.backoff?.max || MAX_BACKOFF,
-        options.maxWorkflows
-      );
-    } else {
+    if (options.noLoop) {
       const port = app.server?.address().port;
       logger2.break();
       logger2.warn("Noloop active: workloop has not started");
@@ -1175,17 +1168,17 @@ function connect(app, logger2, options = {}) {
       logger2.info(`  curl -X POST http://localhost:${port}/claim`);
       logger2.break();
     }
+    app.resumeWorkloop();
   };
   const onDisconnect = () => {
-    if (app.killWorkloop) {
-      app.killWorkloop();
-      delete app.killWorkloop;
-      if (!app.destroyed) {
-        logger2.info("Connection to lightning lost");
-        logger2.info(
-          "Worker will automatically reconnect when lightning is back online"
-        );
-      }
+    if (!app.workloop?.isStopped()) {
+      app.workloop?.stop("Socket disconnected unexpectedly");
+    }
+    if (!app.destroyed) {
+      logger2.info("Connection to lightning lost");
+      logger2.info(
+        "Worker will automatically reconnect when lightning is back online"
+      );
     }
   };
   const onError = (e) => {
@@ -1228,6 +1221,21 @@ function createServer(engine, options = {}) {
   router.get("/livez", healthcheck_default);
   router.get("/", healthcheck_default);
   app.options = options;
+  app.resumeWorkloop = () => {
+    if (options.noLoop) {
+      return;
+    }
+    if (!app.workloop || app.workloop?.isStopped()) {
+      logger2.info("Starting workloop");
+      app.workloop = workloop_default(
+        app,
+        logger2,
+        options.backoff?.min || MIN_BACKOFF,
+        options.backoff?.max || MAX_BACKOFF,
+        options.maxWorkflows
+      );
+    }
+  };
   app.execute = async ({ id, token }) => {
     if (app.socket) {
       try {
@@ -1250,6 +1258,7 @@ function createServer(engine, options = {}) {
           delete app.workflows[id];
           runChannel.leave();
           app.events.emit(INTERNAL_RUN_COMPLETE);
+          app.resumeWorkloop();
         };
         const context = execute(
           runChannel,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openfn/ws-worker",
-  "version": "1.6.6",
+  "version": "1.7.0",
   "description": "A Websocket Worker to connect Lightning to a Runtime Engine",
   "main": "dist/index.js",
   "type": "module",
@@ -22,8 +22,8 @@
     "koa-logger": "^3.2.1",
     "phoenix": "1.7.10",
     "ws": "^8.18.0",
-    "@openfn/engine-multi": "1.2.5",
     "@openfn/lexicon": "^1.1.0",
+    "@openfn/engine-multi": "1.3.0",
     "@openfn/runtime": "1.4.2",
     "@openfn/logger": "1.0.2"
   },
@@ -42,7 +42,7 @@
     "tsup": "^6.2.3",
     "typescript": "^4.6.4",
     "yargs": "^17.6.2",
-    "@openfn/lightning-mock": "2.0.19"
+    "@openfn/lightning-mock": "2.0.20"
   },
   "files": [
     "dist",