@openfn/ws-worker 0.2.9 → 0.2.10

package/CHANGELOG.md

@@ -1,5 +1,16 @@
 # ws-worker
 
+## 0.2.10
+
+### Patch Changes
+
+- 30da946: Better conversion of edge conditions to only take the upstream job into account
+- c1aa9b3: Leave attempt queue channel on disconnect
+  Allow outstanding work to finish before closing down on SIGTERM
+- 60b6fba: Add a healthcheck at /livez and respond with 200 at root
+- Updated dependencies [a6dd44b]
+  - @openfn/engine-multi@0.2.1
+
 ## 0.2.9
 
 ### Patch Changes

package/dist/index.d.ts

@@ -1,6 +1,8 @@
+import { EventEmitter } from 'node:events';
 import Koa from 'koa';
 import { SanitizePolicies, Logger } from '@openfn/logger';
 import { Channel as Channel$1 } from 'phoenix';
+import { Server } from 'http';
 import { RuntimeEngine } from '@openfn/engine-multi';
 
 type ExitReasonStrings =
@@ -87,13 +89,6 @@ interface Channel extends Channel$1 {
   // join: () => ReceiveHook;
 }
 
-declare type Context = {
-    channel: Channel;
-    state: AttemptState;
-    logger: Logger;
-    onFinish: (result: any) => void;
-};
-
 declare const CLAIM = "claim";
 declare type ClaimPayload = {
     demand?: number;
@@ -154,6 +149,14 @@ declare type RunCompletePayload = ExitReason & {
     output_dataclip_id?: string;
 };
 declare type RunCompleteReply = void;
+declare const INTERNAL_ATTEMPT_COMPLETE = "server:attempt-complete";
+
+declare type Context = {
+    channel: Channel;
+    state: AttemptState;
+    logger: Logger;
+    onFinish: (result: any) => void;
+};
 
 declare type ServerOptions = {
     maxWorkflows?: number;
@@ -169,13 +172,17 @@ declare type ServerOptions = {
 };
 interface ServerApp extends Koa {
     id: string;
-    socket: any;
-    channel: Channel;
+    socket?: any;
+    queueChannel?: Channel;
     workflows: Record<string, true | Context>;
+    destroyed: boolean;
+    events: EventEmitter;
+    server: Server;
+    engine: RuntimeEngine;
     execute: ({ id, token }: ClaimAttempt) => Promise<void>;
     destroy: () => void;
     killWorkloop?: () => void;
 }
 declare function createServer(engine: RuntimeEngine, options?: ServerOptions): ServerApp;
 
-export { ATTEMPT_COMPLETE, ATTEMPT_LOG, ATTEMPT_START, AttemptCompletePayload, AttemptCompleteReply, AttemptLogPayload, AttemptLogReply, AttemptStartPayload, AttemptStartReply, CLAIM, ClaimAttempt, ClaimPayload, ClaimReply, GET_ATTEMPT, GET_CREDENTIAL, GET_DATACLIP, GetAttemptPayload, GetAttemptReply, GetCredentialPayload, GetCredentialReply, GetDataClipReply, GetDataclipPayload, RUN_COMPLETE, RUN_START, RunCompletePayload, RunCompleteReply, RunStartPayload, RunStartReply, createServer as default };
+export { ATTEMPT_COMPLETE, ATTEMPT_LOG, ATTEMPT_START, AttemptCompletePayload, AttemptCompleteReply, AttemptLogPayload, AttemptLogReply, AttemptStartPayload, AttemptStartReply, CLAIM, ClaimAttempt, ClaimPayload, ClaimReply, GET_ATTEMPT, GET_CREDENTIAL, GET_DATACLIP, GetAttemptPayload, GetAttemptReply, GetCredentialPayload, GetCredentialReply, GetDataClipReply, GetDataclipPayload, INTERNAL_ATTEMPT_COMPLETE, RUN_COMPLETE, RUN_START, RunCompletePayload, RunCompleteReply, RunStartPayload, RunStartReply, createServer as default };

package/dist/index.js

@@ -1,4 +1,5 @@
 // src/server.ts
+import { EventEmitter as EventEmitter2 } from "node:events";
 import Koa from "koa";
 import bodyParser from "koa-bodyparser";
 import koaLogger from "koa-logger";
@@ -6,6 +7,63 @@ import Router from "@koa/router";
 import { humanId } from "human-id";
 import { createMockLogger as createMockLogger2 } from "@openfn/logger";
 
+// src/events.ts
+var CLAIM = "claim";
+var GET_ATTEMPT = "fetch:attempt";
+var GET_CREDENTIAL = "fetch:credential";
+var GET_DATACLIP = "fetch:dataclip";
+var ATTEMPT_START = "attempt:start";
+var ATTEMPT_COMPLETE = "attempt:complete";
+var ATTEMPT_LOG = "attempt:log";
+var RUN_START = "run:start";
+var RUN_COMPLETE = "run:complete";
+var INTERNAL_ATTEMPT_COMPLETE = "server:attempt-complete";
+
+// src/api/destroy.ts
+var destroy = async (app, logger) => {
+  logger.info("Closing server...");
+  await Promise.all([
+    new Promise((resolve) => {
+      app.destroyed = true;
+      app.killWorkloop?.();
+      app.queueChannel?.leave();
+      app.server.close(async () => {
+        resolve();
+      });
+    }),
+    new Promise(async (resolve) => {
+      await waitForAttempts(app, logger);
+      await app.engine.destroy();
+      app.socket?.disconnect();
+      resolve();
+    })
+  ]);
+  logger.success("Server closed");
+};
+var waitForAttempts = (app, logger) => new Promise((resolve) => {
+  const log = () => {
+    logger.debug(
+      `Waiting for ${Object.keys(app.workflows).length} attempts to complete...`
+    );
+  };
+  const onAttemptComplete = () => {
+    if (Object.keys(app.workflows).length === 0) {
+      logger.debug("All attempts completed!");
+      app.events.off(INTERNAL_ATTEMPT_COMPLETE, onAttemptComplete);
+      resolve();
+    } else {
+      log();
+    }
+  };
+  if (Object.keys(app.workflows).length) {
+    log();
+    app.events.on(INTERNAL_ATTEMPT_COMPLETE, onAttemptComplete);
+  } else {
+    resolve();
+  }
+});
+var destroy_default = destroy;
+
 // src/util/try-with-backoff.ts
 var BACKOFF_MULTIPLIER = 1.15;
 var tryWithBackoff = (fn, opts = {}) => {
@@ -49,19 +107,6 @@ var try_with_backoff_default = tryWithBackoff;
 
 // src/api/claim.ts
 import { createMockLogger } from "@openfn/logger";
-
-// src/events.ts
-var CLAIM = "claim";
-var GET_ATTEMPT = "fetch:attempt";
-var GET_CREDENTIAL = "fetch:credential";
-var GET_DATACLIP = "fetch:dataclip";
-var ATTEMPT_START = "attempt:start";
-var ATTEMPT_COMPLETE = "attempt:complete";
-var ATTEMPT_LOG = "attempt:log";
-var RUN_START = "run:start";
-var RUN_COMPLETE = "run:complete";
-
-// src/api/claim.ts
 var mockLogger = createMockLogger();
 var claim = (app, logger = mockLogger, maxWorkers = 5) => {
   return new Promise((resolve, reject) => {
@@ -69,8 +114,11 @@ var claim = (app, logger = mockLogger, maxWorkers = 5) => {
     if (activeWorkers >= maxWorkers) {
       return reject(new Error("Server at capacity"));
     }
+    if (!app.queueChannel) {
+      return reject(new Error("No websocket available"));
+    }
     logger.debug("requesting attempt...");
-    app.channel.push(CLAIM, { demand: 1 }).receive("ok", ({ attempts }) => {
+    app.queueChannel.push(CLAIM, { demand: 1 }).receive("ok", ({ attempts }) => {
       logger.debug(`pulled ${attempts.length} attempts`);
       if (!attempts?.length) {
         return reject(new Error("No attempts returned"));
@@ -112,6 +160,7 @@ var startWorkloop = (app, logger, minBackoff, maxBackoff, maxWorkers) => {
     logger.debug("cancelling workloop");
     cancelled = true;
     promise.cancel();
+    app.queueChannel?.leave();
   };
 };
 var workloop_default = startWorkloop;
@@ -122,13 +171,15 @@ import crypto2 from "node:crypto";
 // src/util/convert-attempt.ts
 import crypto from "node:crypto";
 var conditions = {
-  on_job_success: "!state.errors",
-  on_job_failure: "state.errors",
-  always: null
+  on_job_success: (upstreamId) => `Boolean(!state.errors?.["${upstreamId}"] ?? true)`,
+  on_job_failure: (upstreamId) => `Boolean(state.errors && state.errors["${upstreamId}"])`,
+  always: (_upstreamId) => null
 };
-var mapEdgeCondition = (condition) => {
+var mapEdgeCondition = (edge) => {
+  const { condition } = edge;
   if (condition && condition in conditions) {
-    return conditions[condition];
+    const upstream = edge.source_job_id || edge.source_trigger_id;
+    return conditions[condition](upstream);
   }
   return condition;
 };
@@ -177,7 +228,7 @@ var convert_attempt_default = (attempt) => {
       }
       const next = edges.filter((e) => e.source_job_id === id).reduce((obj, edge) => {
         const newEdge = {};
-        const condition = mapEdgeCondition(edge.condition);
+        const condition = mapEdgeCondition(edge);
         if (condition) {
           newEdge.condition = condition;
         }
@@ -446,6 +497,11 @@ async function loadCredential(channel, credentialId) {
   return get_with_reply_default(channel, GET_CREDENTIAL, { id: credentialId });
 }
 
+// src/middleware/healthcheck.ts
+var healthcheck_default = (ctx) => {
+  ctx.status = 200;
+};
+
 // src/channels/attempt.ts
 var joinAttemptChannel = (socket, token, attemptId, logger) => {
   return new Promise((resolve, reject) => {
@@ -481,14 +537,14 @@ import { WebSocket } from "ws";
 // src/util/worker-token.ts
 import * as jose from "jose";
 var alg = "HS256";
-var generateWorkerToken = async (secret, workerId) => {
+var generateWorkerToken = async (secret, workerId, logger) => {
   if (!secret) {
-    console.warn();
-    console.warn("WARNING: Worker Secret not provided!");
-    console.warn(
+    logger.warn();
+    logger.warn("WARNING: Worker Secret not provided!");
+    logger.warn(
       "This worker will attempt to connect to Lightning with default secret"
     );
-    console.warn();
+    logger.warn();
   }
   const encodedSecret = new TextEncoder().encode(secret || "<secret>");
   const claims = {
@@ -502,7 +558,7 @@ var worker_token_default = generateWorkerToken;
 // src/channels/worker-queue.ts
 var connectToWorkerQueue = (endpoint, serverId, secret, logger, SocketConstructor = PhxSocket) => {
   const events = new EventEmitter();
-  worker_token_default(secret, serverId).then((token) => {
+  worker_token_default(secret, serverId, logger).then((token) => {
     const socket = new SocketConstructor(endpoint, {
       params: { token },
       transport: WebSocket
@@ -545,7 +601,7 @@ function connect(app, logger, options = {}) {
   const onConnect = ({ socket, channel }) => {
     logger.success("Connected to Lightning at", options.lightning);
     app.socket = socket;
-    app.channel = channel;
+    app.queueChannel = channel;
     if (!options.noLoop) {
       logger.info("Starting workloop");
       app.killWorkloop = workloop_default(
@@ -592,6 +648,8 @@ function createServer(engine, options = {}) {
   const app = new Koa();
   app.id = humanId({ separator: "-", capitalize: false });
   const router = new Router();
+  app.events = new EventEmitter2();
+  app.engine = engine;
   app.use(bodyParser());
   app.use(
     koaLogger((str, _args) => {
@@ -599,8 +657,12 @@ function createServer(engine, options = {}) {
     })
   );
   app.workflows = {};
-  const server = app.listen(port);
+  app.destroyed = false;
+  app.server = app.listen(port);
   logger.success(`ws-worker ${app.id} listening on ${port}`);
+  process.send?.("READY");
+  router.get("/livez", healthcheck_default);
+  router.get("/", healthcheck_default);
   app.execute = async ({ id, token }) => {
     if (app.socket) {
       app.workflows[id] = true;
@@ -612,6 +674,7 @@ function createServer(engine, options = {}) {
       const onFinish = () => {
         delete app.workflows[id];
         attemptChannel.leave();
+        app.events.emit(INTERNAL_ATTEMPT_COMPLETE);
       };
       const context = execute(
         attemptChannel,
@@ -638,19 +701,24 @@ function createServer(engine, options = {}) {
       ctx.status = 204;
     });
   });
-  app.destroy = async () => {
-    logger.info("Closing server...");
-    server.close();
-    await engine.destroy();
-    app.killWorkloop?.();
-    logger.success("Server closed");
-  };
+  app.destroy = () => destroy_default(app, logger);
   app.use(router.routes());
   if (options.lightning) {
     connect(app, logger, options);
   } else {
     logger.warn("No lightning URL provided");
   }
+  let shutdown = false;
+  const exit = async (signal) => {
+    if (!shutdown) {
+      shutdown = true;
+      logger.always(`${signal} RECEIVED: CLOSING SERVER`);
+      await app.destroy();
+      process.exit();
+    }
+  };
+  process.on("SIGINT", () => exit("SIGINT"));
+  process.on("SIGTERM", () => exit("SIGTERM"));
   app.on = (...args) => {
     return engine.on(...args);
   };
@@ -668,6 +736,7 @@ export {
   GET_ATTEMPT,
   GET_CREDENTIAL,
   GET_DATACLIP,
+  INTERNAL_ATTEMPT_COMPLETE,
   RUN_COMPLETE,
   RUN_START,
   src_default as default

package/dist/start.js

@@ -4966,17 +4966,20 @@ async function createMock() {
       active: Object.keys(activeWorkflows).length
     };
   };
+  const destroy2 = async () => true;
   return {
     on,
     once,
     execute: execute2,
     getStatus,
-    listen
+    listen,
+    destroy: destroy2
   };
 }
 var runtime_engine_default = createMock;
 
 // src/server.ts
+import { EventEmitter as EventEmitter3 } from "node:events";
 import Koa from "koa";
 import bodyParser from "koa-bodyparser";
 import koaLogger from "koa-logger";
@@ -4984,6 +4987,63 @@ import Router from "@koa/router";
 import { humanId } from "human-id";
 import { createMockLogger as createMockLogger2 } from "@openfn/logger";
 
+// src/events.ts
+var CLAIM = "claim";
+var GET_ATTEMPT = "fetch:attempt";
+var GET_CREDENTIAL = "fetch:credential";
+var GET_DATACLIP = "fetch:dataclip";
+var ATTEMPT_START = "attempt:start";
+var ATTEMPT_COMPLETE = "attempt:complete";
+var ATTEMPT_LOG = "attempt:log";
+var RUN_START = "run:start";
+var RUN_COMPLETE = "run:complete";
+var INTERNAL_ATTEMPT_COMPLETE = "server:attempt-complete";
+
+// src/api/destroy.ts
+var destroy = async (app, logger2) => {
+  logger2.info("Closing server...");
+  await Promise.all([
+    new Promise((resolve5) => {
+      app.destroyed = true;
+      app.killWorkloop?.();
+      app.queueChannel?.leave();
+      app.server.close(async () => {
+        resolve5();
+      });
+    }),
+    new Promise(async (resolve5) => {
+      await waitForAttempts(app, logger2);
+      await app.engine.destroy();
+      app.socket?.disconnect();
+      resolve5();
+    })
+  ]);
+  logger2.success("Server closed");
+};
+var waitForAttempts = (app, logger2) => new Promise((resolve5) => {
+  const log = () => {
+    logger2.debug(
+      `Waiting for ${Object.keys(app.workflows).length} attempts to complete...`
+    );
+  };
+  const onAttemptComplete = () => {
+    if (Object.keys(app.workflows).length === 0) {
+      logger2.debug("All attempts completed!");
+      app.events.off(INTERNAL_ATTEMPT_COMPLETE, onAttemptComplete);
+      resolve5();
+    } else {
+      log();
+    }
+  };
+  if (Object.keys(app.workflows).length) {
+    log();
+    app.events.on(INTERNAL_ATTEMPT_COMPLETE, onAttemptComplete);
+  } else {
+    resolve5();
+  }
+});
+var destroy_default = destroy;
+
 // src/util/try-with-backoff.ts
 var BACKOFF_MULTIPLIER = 1.15;
 var tryWithBackoff = (fn, opts = {}) => {
@@ -5027,19 +5087,6 @@ var try_with_backoff_default = tryWithBackoff;
 
 // src/api/claim.ts
 import { createMockLogger } from "@openfn/logger";
-
-// src/events.ts
-var CLAIM = "claim";
-var GET_ATTEMPT = "fetch:attempt";
-var GET_CREDENTIAL = "fetch:credential";
-var GET_DATACLIP = "fetch:dataclip";
-var ATTEMPT_START = "attempt:start";
-var ATTEMPT_COMPLETE = "attempt:complete";
-var ATTEMPT_LOG = "attempt:log";
-var RUN_START = "run:start";
-var RUN_COMPLETE = "run:complete";
-
-// src/api/claim.ts
 var mockLogger = createMockLogger();
 var claim = (app, logger2 = mockLogger, maxWorkers = 5) => {
   return new Promise((resolve5, reject) => {
@@ -5047,8 +5094,11 @@ var claim = (app, logger2 = mockLogger, maxWorkers = 5) => {
     if (activeWorkers >= maxWorkers) {
       return reject(new Error("Server at capacity"));
     }
+    if (!app.queueChannel) {
+      return reject(new Error("No websocket available"));
+    }
     logger2.debug("requesting attempt...");
-    app.channel.push(CLAIM, { demand: 1 }).receive("ok", ({ attempts }) => {
+    app.queueChannel.push(CLAIM, { demand: 1 }).receive("ok", ({ attempts }) => {
       logger2.debug(`pulled ${attempts.length} attempts`);
       if (!attempts?.length) {
         return reject(new Error("No attempts returned"));
@@ -5090,6 +5140,7 @@ var startWorkloop = (app, logger2, minBackoff2, maxBackoff2, maxWorkers) => {
     logger2.debug("cancelling workloop");
     cancelled = true;
     promise.cancel();
+    app.queueChannel?.leave();
   };
 };
 var workloop_default = startWorkloop;
@@ -5100,13 +5151,15 @@ import crypto2 from "node:crypto";
 // src/util/convert-attempt.ts
 import crypto from "node:crypto";
 var conditions = {
-  on_job_success: "!state.errors",
-  on_job_failure: "state.errors",
-  always: null
+  on_job_success: (upstreamId) => `Boolean(!state.errors?.["${upstreamId}"] ?? true)`,
+  on_job_failure: (upstreamId) => `Boolean(state.errors && state.errors["${upstreamId}"])`,
+  always: (_upstreamId) => null
 };
-var mapEdgeCondition = (condition) => {
+var mapEdgeCondition = (edge) => {
+  const { condition } = edge;
   if (condition && condition in conditions) {
-    return conditions[condition];
+    const upstream = edge.source_job_id || edge.source_trigger_id;
+    return conditions[condition](upstream);
   }
   return condition;
 };
@@ -5155,7 +5208,7 @@ var convert_attempt_default = (attempt) => {
       }
       const next = edges.filter((e) => e.source_job_id === id).reduce((obj, edge) => {
         const newEdge = {};
-        const condition = mapEdgeCondition(edge.condition);
+        const condition = mapEdgeCondition(edge);
         if (condition) {
           newEdge.condition = condition;
         }
@@ -5424,6 +5477,11 @@ async function loadCredential(channel, credentialId) {
   return get_with_reply_default(channel, GET_CREDENTIAL, { id: credentialId });
 }
 
+// src/middleware/healthcheck.ts
+var healthcheck_default = (ctx) => {
+  ctx.status = 200;
+};
+
 // src/channels/attempt.ts
 var joinAttemptChannel = (socket, token, attemptId, logger2) => {
   return new Promise((resolve5, reject) => {
@@ -5459,14 +5517,14 @@ import { WebSocket } from "ws";
 // src/util/worker-token.ts
 import * as jose from "jose";
 var alg = "HS256";
-var generateWorkerToken = async (secret, workerId) => {
+var generateWorkerToken = async (secret, workerId, logger2) => {
   if (!secret) {
-    console.warn();
-    console.warn("WARNING: Worker Secret not provided!");
-    console.warn(
+    logger2.warn();
+    logger2.warn("WARNING: Worker Secret not provided!");
+    logger2.warn(
       "This worker will attempt to connect to Lightning with default secret"
     );
-    console.warn();
+    logger2.warn();
   }
   const encodedSecret = new TextEncoder().encode(secret || "<secret>");
   const claims = {
@@ -5480,7 +5538,7 @@ var worker_token_default = generateWorkerToken;
 // src/channels/worker-queue.ts
 var connectToWorkerQueue = (endpoint, serverId, secret, logger2, SocketConstructor = PhxSocket) => {
   const events = new EventEmitter2();
-  worker_token_default(secret, serverId).then((token) => {
+  worker_token_default(secret, serverId, logger2).then((token) => {
     const socket = new SocketConstructor(endpoint, {
       params: { token },
       transport: WebSocket
@@ -5523,7 +5581,7 @@ function connect(app, logger2, options = {}) {
   const onConnect = ({ socket, channel }) => {
     logger2.success("Connected to Lightning at", options.lightning);
     app.socket = socket;
-    app.channel = channel;
+    app.queueChannel = channel;
     if (!options.noLoop) {
       logger2.info("Starting workloop");
       app.killWorkloop = workloop_default(
@@ -5570,6 +5628,8 @@ function createServer(engine, options = {}) {
   const app = new Koa();
   app.id = humanId({ separator: "-", capitalize: false });
   const router = new Router();
+  app.events = new EventEmitter3();
+  app.engine = engine;
   app.use(bodyParser());
   app.use(
     koaLogger((str, _args) => {
@@ -5577,8 +5637,12 @@ function createServer(engine, options = {}) {
     })
   );
   app.workflows = {};
-  const server = app.listen(port);
+  app.destroyed = false;
+  app.server = app.listen(port);
   logger2.success(`ws-worker ${app.id} listening on ${port}`);
+  process.send?.("READY");
+  router.get("/livez", healthcheck_default);
+  router.get("/", healthcheck_default);
   app.execute = async ({ id, token }) => {
     if (app.socket) {
       app.workflows[id] = true;
@@ -5590,6 +5654,7 @@ function createServer(engine, options = {}) {
       const onFinish = () => {
         delete app.workflows[id];
         attemptChannel.leave();
+        app.events.emit(INTERNAL_ATTEMPT_COMPLETE);
       };
       const context = execute(
         attemptChannel,
@@ -5616,19 +5681,24 @@ function createServer(engine, options = {}) {
       ctx.status = 204;
     });
   });
-  app.destroy = async () => {
-    logger2.info("Closing server...");
-    server.close();
-    await engine.destroy();
-    app.killWorkloop?.();
-    logger2.success("Server closed");
-  };
+  app.destroy = () => destroy_default(app, logger2);
   app.use(router.routes());
   if (options.lightning) {
     connect(app, logger2, options);
   } else {
     logger2.warn("No lightning URL provided");
   }
+  let shutdown = false;
+  const exit = async (signal) => {
+    if (!shutdown) {
+      shutdown = true;
+      logger2.always(`${signal} RECEIVED: CLOSING SERVER`);
+      await app.destroy();
+      process.exit();
+    }
+  };
+  process.on("SIGINT", () => exit("SIGINT"));
+  process.on("SIGTERM", () => exit("SIGTERM"));
   app.on = (...args2) => {
     return engine.on(...args2);
   };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openfn/ws-worker",
-  "version": "0.2.9",
+  "version": "0.2.10",
   "description": "A Websocket Worker to connect Lightning to a Runtime Engine",
   "main": "dist/index.js",
   "type": "module",
@@ -21,7 +21,7 @@
     "koa-logger": "^3.2.1",
     "phoenix": "^1.7.7",
     "ws": "^8.14.1",
-    "@openfn/engine-multi": "0.2.0",
+    "@openfn/engine-multi": "0.2.1",
     "@openfn/logger": "0.0.19",
     "@openfn/runtime": "0.2.0"
   },
@@ -40,7 +40,7 @@
     "tsup": "^6.2.3",
     "typescript": "^4.6.4",
     "yargs": "^17.6.2",
-    "@openfn/lightning-mock": "1.1.2"
+    "@openfn/lightning-mock": "1.1.3"
   },
   "files": [
     "dist",