@openfn/ws-worker 1.6.4 → 1.6.7

package/CHANGELOG.md

@@ -1,5 +1,26 @@
 # ws-worker
 
+## 1.6.7
+
+### Patch Changes
+
+- 42883f8: Better handliung of claim backoffs when at capacity
+
+## 1.6.6
+
+### Patch Changes
+
+- Log claim event duration
+
+## 1.6.5
+
+### Patch Changes
+
+- 5db5862: Dont log compiled job code
+- f581c6b: log duration of runs and server capacity
+- 3e6eba2: Trap errors coming out of the websocket
+  - @openfn/engine-multi@1.2.5
+
 ## 1.6.4
 
 ### Patch Changes

package/dist/index.d.ts

@@ -8,6 +8,11 @@ import { ExecutionPlan, Lazy, State } from '@openfn/lexicon';
 import { Channel as Channel$1 } from 'phoenix';
 import { Server } from 'http';
 
+declare type Workloop = {
+    stop: (reason?: string) => void;
+    isStopped: () => boolean;
+};
+
 // Internal server state for each run
 type RunState = {
   activeStep?: string;
@@ -80,9 +85,10 @@ interface ServerApp extends Koa {
     server: Server;
     engine: RuntimeEngine;
     options: ServerOptions;
+    workloop?: Workloop;
     execute: ({ id, token }: ClaimRun) => Promise<void>;
     destroy: () => void;
-    killWorkloop?: () => void;
+    resumeWorkloop: () => void;
 }
 declare function createServer(engine: RuntimeEngine, options?: ServerOptions): ServerApp;
 

package/dist/index.js

@@ -29,7 +29,7 @@ var name, version, description, main, type, scripts, bin, author, license, depen
 var init_package = __esm({
   "package.json"() {
     name = "@openfn/ws-worker";
-    version = "1.6.4";
+    version = "1.6.7";
     description = "A Websocket Worker to connect Lightning to a Runtime Engine";
     main = "dist/index.js";
     type = "module";
@@ -132,7 +132,7 @@ var destroy = async (app, logger) => {
   await Promise.all([
     new Promise((resolve) => {
       app.destroyed = true;
-      app.killWorkloop?.();
+      app.workloop?.stop("server closed");
       app.queueChannel?.leave();
       app.server.close(async () => {
         resolve();
@@ -232,18 +232,19 @@ var claim = (app, logger = mockLogger, options = {}) => {
     const { maxWorkers = 5 } = options;
     const activeWorkers = Object.keys(app.workflows).length;
     if (activeWorkers >= maxWorkers) {
-      logger.debug("skipping claim attempt: server at capacity");
+      app.workloop?.stop(`server at capacity (${activeWorkers}/${maxWorkers})`);
       return reject(new Error("Server at capacity"));
     }
     if (!app.queueChannel) {
       logger.debug("skipping claim attempt: websocket unavailable");
       return reject(new Error("No websocket available"));
     }
-    logger.debug("requesting run...");
+    logger.debug(`requesting run (capacity ${activeWorkers}/${maxWorkers})`);
+    const start = Date.now();
     app.queueChannel.push(CLAIM, { demand: 1 }).receive("ok", ({ runs }) => {
+      const duration = Date.now() - start;
       logger.debug(
-        `claimed ${runs.length} runs: `,
-        runs.map((r) => r.id).join(",")
+        `claimed ${runs.length} runs in ${duration}ms (${runs.length ? runs.map((r) => r.id).join(",") : "-"})`
       );
       if (!runs?.length) {
         return reject(new Error("No runs returned"));
@@ -300,11 +301,15 @@ var startWorkloop = (app, logger, minBackoff, maxBackoff, maxWorkers) => {
     }
   };
   workLoop();
-  return () => {
-    logger.debug("cancelling workloop");
-    cancelled = true;
-    promise.cancel();
-    app.queueChannel?.leave();
+  return {
+    stop: (reason = "reason unknown") => {
+      if (!cancelled) {
+        logger.info(`cancelling workloop: ${reason}`);
+        cancelled = true;
+        promise.cancel();
+      }
+    },
+    isStopped: () => cancelled
   };
 };
 var workloop_default = startWorkloop;
@@ -1015,16 +1020,7 @@ function connect(app, logger, options = {}) {
     logger.success("Connected to Lightning at", options.lightning);
     app.socket = socket;
     app.queueChannel = channel;
-    if (!options.noLoop) {
-      logger.info("Starting workloop");
-      app.killWorkloop = workloop_default(
-        app,
-        logger,
-        options.backoff?.min || MIN_BACKOFF,
-        options.backoff?.max || MAX_BACKOFF,
-        options.maxWorkflows
-      );
-    } else {
+    if (options.noLoop) {
       const port = app.server?.address().port;
       logger.break();
       logger.warn("Noloop active: workloop has not started");
@@ -1033,17 +1029,17 @@ function connect(app, logger, options = {}) {
       logger.info(`  curl -X POST http://localhost:${port}/claim`);
       logger.break();
     }
+    app.resumeWorkloop();
   };
   const onDisconnect = () => {
-    if (app.killWorkloop) {
-      app.killWorkloop();
-      delete app.killWorkloop;
-      if (!app.destroyed) {
-        logger.info("Connection to lightning lost");
-        logger.info(
-          "Worker will automatically reconnect when lightning is back online"
-        );
-      }
+    if (!app.workloop?.isStopped()) {
+      app.workloop?.stop("Socket disconnected unexpectedly");
+    }
+    if (!app.destroyed) {
+      logger.info("Connection to lightning lost");
+      logger.info(
+        "Worker will automatically reconnect when lightning is back online"
+      );
     }
   };
   const onError = (e) => {
@@ -1086,34 +1082,59 @@ function createServer(engine, options = {}) {
   router.get("/livez", healthcheck_default);
   router.get("/", healthcheck_default);
   app.options = options;
+  app.resumeWorkloop = () => {
+    if (options.noLoop) {
+      return;
+    }
+    if (!app.workloop || app.workloop?.isStopped()) {
+      logger.info("Starting workloop");
+      app.workloop = workloop_default(
+        app,
+        logger,
+        options.backoff?.min || MIN_BACKOFF,
+        options.backoff?.max || MAX_BACKOFF,
+        options.maxWorkflows
+      );
+    }
+  };
   app.execute = async ({ id, token }) => {
     if (app.socket) {
-      app.workflows[id] = true;
-      const {
-        channel: runChannel,
-        plan,
-        options: options2 = {},
-        input
-      } = await run_default(app.socket, token, id, logger);
-      if (!("payloadLimitMb" in options2)) {
-        options2.payloadLimitMb = app.options.payloadLimitMb;
+      try {
+        const start = Date.now();
+        app.workflows[id] = true;
+        const {
+          channel: runChannel,
+          plan,
+          options: options2 = {},
+          input
+        } = await run_default(app.socket, token, id, logger);
+        if (!("payloadLimitMb" in options2)) {
+          options2.payloadLimitMb = app.options.payloadLimitMb;
+        }
+        const onFinish = () => {
+          const duration = (Date.now() - start) / 1e3;
+          logger.debug(
+            `workflow ${id} complete in ${duration}s: releasing worker`
+          );
+          delete app.workflows[id];
+          runChannel.leave();
+          app.events.emit(INTERNAL_RUN_COMPLETE);
+          app.resumeWorkloop();
+        };
+        const context = execute(
+          runChannel,
+          engine,
+          logger,
+          plan,
+          input,
+          options2,
+          onFinish
+        );
+        app.workflows[id] = context;
+      } catch (e) {
+        logger.error(`Unexpected error executing ${id}`);
+        logger.error(e);
       }
-      const onFinish = () => {
-        logger.debug(`workflow ${id} complete: releasing worker`);
-        delete app.workflows[id];
-        runChannel.leave();
-        app.events.emit(INTERNAL_RUN_COMPLETE);
-      };
-      const context = execute(
-        runChannel,
-        engine,
-        logger,
-        plan,
-        input,
-        options2,
-        onFinish
-      );
-      app.workflows[id] = context;
     } else {
       logger.error("No lightning socket established");
     }

package/dist/start.js

@@ -37,7 +37,7 @@ var name, version, description, main, type, scripts, bin, author, license, depen
 var init_package = __esm({
   "package.json"() {
     name = "@openfn/ws-worker";
-    version = "1.6.4";
+    version = "1.6.7";
     description = "A Websocket Worker to connect Lightning to a Runtime Engine";
     main = "dist/index.js";
     type = "module";
@@ -271,7 +271,7 @@ var destroy = async (app, logger2) => {
   await Promise.all([
     new Promise((resolve5) => {
       app.destroyed = true;
-      app.killWorkloop?.();
+      app.workloop?.stop("server closed");
       app.queueChannel?.leave();
       app.server.close(async () => {
         resolve5();
@@ -371,18 +371,19 @@ var claim = (app, logger2 = mockLogger, options = {}) => {
     const { maxWorkers = 5 } = options;
     const activeWorkers = Object.keys(app.workflows).length;
     if (activeWorkers >= maxWorkers) {
-      logger2.debug("skipping claim attempt: server at capacity");
+      app.workloop?.stop(`server at capacity (${activeWorkers}/${maxWorkers})`);
       return reject(new Error("Server at capacity"));
     }
     if (!app.queueChannel) {
       logger2.debug("skipping claim attempt: websocket unavailable");
       return reject(new Error("No websocket available"));
     }
-    logger2.debug("requesting run...");
+    logger2.debug(`requesting run (capacity ${activeWorkers}/${maxWorkers})`);
+    const start = Date.now();
     app.queueChannel.push(CLAIM, { demand: 1 }).receive("ok", ({ runs }) => {
+      const duration = Date.now() - start;
       logger2.debug(
-        `claimed ${runs.length} runs: `,
-        runs.map((r) => r.id).join(",")
+        `claimed ${runs.length} runs in ${duration}ms (${runs.length ? runs.map((r) => r.id).join(",") : "-"})`
       );
       if (!runs?.length) {
         return reject(new Error("No runs returned"));
@@ -439,11 +440,15 @@ var startWorkloop = (app, logger2, minBackoff2, maxBackoff2, maxWorkers) => {
     }
   };
   workLoop();
-  return () => {
-    logger2.debug("cancelling workloop");
-    cancelled = true;
-    promise.cancel();
-    app.queueChannel?.leave();
+  return {
+    stop: (reason = "reason unknown") => {
+      if (!cancelled) {
+        logger2.info(`cancelling workloop: ${reason}`);
+        cancelled = true;
+        promise.cancel();
+      }
+    },
+    isStopped: () => cancelled
   };
 };
 var workloop_default = startWorkloop;
@@ -1154,16 +1159,7 @@ function connect(app, logger2, options = {}) {
     logger2.success("Connected to Lightning at", options.lightning);
     app.socket = socket;
     app.queueChannel = channel;
-    if (!options.noLoop) {
-      logger2.info("Starting workloop");
-      app.killWorkloop = workloop_default(
-        app,
-        logger2,
-        options.backoff?.min || MIN_BACKOFF,
-        options.backoff?.max || MAX_BACKOFF,
-        options.maxWorkflows
-      );
-    } else {
+    if (options.noLoop) {
       const port = app.server?.address().port;
       logger2.break();
       logger2.warn("Noloop active: workloop has not started");
@@ -1172,17 +1168,17 @@ function connect(app, logger2, options = {}) {
       logger2.info(`  curl -X POST http://localhost:${port}/claim`);
       logger2.break();
     }
+    app.resumeWorkloop();
   };
   const onDisconnect = () => {
-    if (app.killWorkloop) {
-      app.killWorkloop();
-      delete app.killWorkloop;
-      if (!app.destroyed) {
-        logger2.info("Connection to lightning lost");
-        logger2.info(
-          "Worker will automatically reconnect when lightning is back online"
-        );
-      }
+    if (!app.workloop?.isStopped()) {
+      app.workloop?.stop("Socket disconnected unexpectedly");
+    }
+    if (!app.destroyed) {
+      logger2.info("Connection to lightning lost");
+      logger2.info(
+        "Worker will automatically reconnect when lightning is back online"
+      );
     }
   };
   const onError = (e) => {
@@ -1225,34 +1221,59 @@ function createServer(engine, options = {}) {
   router.get("/livez", healthcheck_default);
   router.get("/", healthcheck_default);
   app.options = options;
-  app.execute = async ({ id, token }) => {
-    if (app.socket) {
-      app.workflows[id] = true;
-      const {
-        channel: runChannel,
-        plan,
-        options: options2 = {},
-        input
-      } = await run_default(app.socket, token, id, logger2);
-      if (!("payloadLimitMb" in options2)) {
-        options2.payloadLimitMb = app.options.payloadLimitMb;
-      }
-      const onFinish = () => {
-        logger2.debug(`workflow ${id} complete: releasing worker`);
-        delete app.workflows[id];
-        runChannel.leave();
-        app.events.emit(INTERNAL_RUN_COMPLETE);
-      };
-      const context = execute(
-        runChannel,
-        engine,
+  app.resumeWorkloop = () => {
+    if (options.noLoop) {
+      return;
+    }
+    if (!app.workloop || app.workloop?.isStopped()) {
+      logger2.info("Starting workloop");
+      app.workloop = workloop_default(
+        app,
         logger2,
-        plan,
-        input,
-        options2,
-        onFinish
+        options.backoff?.min || MIN_BACKOFF,
+        options.backoff?.max || MAX_BACKOFF,
+        options.maxWorkflows
       );
-      app.workflows[id] = context;
+    }
+  };
+  app.execute = async ({ id, token }) => {
+    if (app.socket) {
+      try {
+        const start = Date.now();
+        app.workflows[id] = true;
+        const {
+          channel: runChannel,
+          plan,
+          options: options2 = {},
+          input
+        } = await run_default(app.socket, token, id, logger2);
+        if (!("payloadLimitMb" in options2)) {
+          options2.payloadLimitMb = app.options.payloadLimitMb;
+        }
+        const onFinish = () => {
+          const duration = (Date.now() - start) / 1e3;
+          logger2.debug(
+            `workflow ${id} complete in ${duration}s: releasing worker`
+          );
+          delete app.workflows[id];
+          runChannel.leave();
+          app.events.emit(INTERNAL_RUN_COMPLETE);
+          app.resumeWorkloop();
+        };
+        const context = execute(
+          runChannel,
+          engine,
+          logger2,
+          plan,
+          input,
+          options2,
+          onFinish
+        );
+        app.workflows[id] = context;
+      } catch (e) {
+        logger2.error(`Unexpected error executing ${id}`);
+        logger2.error(e);
+      }
     } else {
       logger2.error("No lightning socket established");
     }
@@ -6264,6 +6285,7 @@ function parseArgs(argv) {
 // src/start.ts
 var args = parseArgs(process.argv);
 var logger = createLogger("SRV", { level: args.log });
+logger.info("Starting worker server...");
 if (args.lightning === "mock") {
   args.lightning = "ws://localhost:8888/worker";
   if (!args.secret) {
@@ -6275,7 +6297,7 @@ if (args.lightning === "mock") {
 }
 var [minBackoff, maxBackoff] = args.backoff.split("/").map((n) => parseInt(n, 10) * 1e3);
 function engineReady(engine) {
-  logger.debug("Creating worker server...");
+  logger.debug("Creating worker instance");
   const workerOptions = {
     port: args.port,
     lightning: args.lightning,
@@ -6306,6 +6328,7 @@ function engineReady(engine) {
   } = workerOptions;
   logger.debug("Worker options:", humanOptions);
   server_default(engine, workerOptions);
+  logger.success("Worker started OK");
 }
 if (args.mock) {
   runtime_engine_default().then((engine) => {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openfn/ws-worker",
-  "version": "1.6.4",
+  "version": "1.6.7",
   "description": "A Websocket Worker to connect Lightning to a Runtime Engine",
   "main": "dist/index.js",
   "type": "module",
@@ -22,10 +22,10 @@
     "koa-logger": "^3.2.1",
     "phoenix": "1.7.10",
     "ws": "^8.18.0",
-    "@openfn/engine-multi": "1.2.4",
+    "@openfn/engine-multi": "1.2.5",
     "@openfn/lexicon": "^1.1.0",
-    "@openfn/runtime": "1.4.2",
-    "@openfn/logger": "1.0.2"
+    "@openfn/logger": "1.0.2",
+    "@openfn/runtime": "1.4.2"
   },
   "devDependencies": {
     "@types/koa": "^2.13.5",
@@ -42,7 +42,7 @@
     "tsup": "^6.2.3",
     "typescript": "^4.6.4",
     "yargs": "^17.6.2",
-    "@openfn/lightning-mock": "2.0.18"
+    "@openfn/lightning-mock": "2.0.19"
   },
   "files": [
     "dist",