@openfn/ws-worker 0.2.9 → 0.2.11

package/CHANGELOG.md

@@ -1,5 +1,27 @@
 # ws-worker
 
+## 0.2.11
+
+### Patch Changes
+
+- 22339c6: Add MAX_RUN_MEMORY env var and option to limit the memory available to each run
+- 04ac3cc: Include duration and threadid in run-complete
+- 340b96e: Send memory usage to lightning on run:complete
+- Updated dependencies
+  - @openfn/engine-multi@0.2.2
+  - @openfn/runtime@0.2.1
+
+## 0.2.10
+
+### Patch Changes
+
+- 30da946: Better conversion of edge conditions to only take the upstream job into account
+- c1aa9b3: Leave attempt queue channel on disconnect
+  Allow outstanding work to finish before closing down on SIGTERM
+- 60b6fba: Add a healthcheck at /livez and respond with 200 at root
+- Updated dependencies [a6dd44b]
+  - @openfn/engine-multi@0.2.1
+
 ## 0.2.9
 
 ### Patch Changes

package/dist/index.d.ts

@@ -1,6 +1,8 @@
+import { EventEmitter } from 'node:events';
 import Koa from 'koa';
 import { SanitizePolicies, Logger } from '@openfn/logger';
 import { Channel as Channel$1 } from 'phoenix';
+import { Server } from 'http';
 import { RuntimeEngine } from '@openfn/engine-multi';
 
 type ExitReasonStrings =
@@ -87,13 +89,6 @@ interface Channel extends Channel$1 {
   // join: () => ReceiveHook;
 }
 
-declare type Context = {
-    channel: Channel;
-    state: AttemptState;
-    logger: Logger;
-    onFinish: (result: any) => void;
-};
-
 declare const CLAIM = "claim";
 declare type ClaimPayload = {
     demand?: number;
@@ -154,6 +149,14 @@ declare type RunCompletePayload = ExitReason & {
     output_dataclip_id?: string;
 };
 declare type RunCompleteReply = void;
+declare const INTERNAL_ATTEMPT_COMPLETE = "server:attempt-complete";
+
+declare type Context = {
+    channel: Channel;
+    state: AttemptState;
+    logger: Logger;
+    onFinish: (result: any) => void;
+};
 
 declare type ServerOptions = {
     maxWorkflows?: number;
@@ -169,13 +172,17 @@ declare type ServerOptions = {
 };
 interface ServerApp extends Koa {
     id: string;
-    socket: any;
-    channel: Channel;
+    socket?: any;
+    queueChannel?: Channel;
     workflows: Record<string, true | Context>;
+    destroyed: boolean;
+    events: EventEmitter;
+    server: Server;
+    engine: RuntimeEngine;
     execute: ({ id, token }: ClaimAttempt) => Promise<void>;
     destroy: () => void;
     killWorkloop?: () => void;
 }
 declare function createServer(engine: RuntimeEngine, options?: ServerOptions): ServerApp;
 
-export { ATTEMPT_COMPLETE, ATTEMPT_LOG, ATTEMPT_START, AttemptCompletePayload, AttemptCompleteReply, AttemptLogPayload, AttemptLogReply, AttemptStartPayload, AttemptStartReply, CLAIM, ClaimAttempt, ClaimPayload, ClaimReply, GET_ATTEMPT, GET_CREDENTIAL, GET_DATACLIP, GetAttemptPayload, GetAttemptReply, GetCredentialPayload, GetCredentialReply, GetDataClipReply, GetDataclipPayload, RUN_COMPLETE, RUN_START, RunCompletePayload, RunCompleteReply, RunStartPayload, RunStartReply, createServer as default };
+export { ATTEMPT_COMPLETE, ATTEMPT_LOG, ATTEMPT_START, AttemptCompletePayload, AttemptCompleteReply, AttemptLogPayload, AttemptLogReply, AttemptStartPayload, AttemptStartReply, CLAIM, ClaimAttempt, ClaimPayload, ClaimReply, GET_ATTEMPT, GET_CREDENTIAL, GET_DATACLIP, GetAttemptPayload, GetAttemptReply, GetCredentialPayload, GetCredentialReply, GetDataClipReply, GetDataclipPayload, INTERNAL_ATTEMPT_COMPLETE, RUN_COMPLETE, RUN_START, RunCompletePayload, RunCompleteReply, RunStartPayload, RunStartReply, createServer as default };

package/dist/index.js

@@ -1,4 +1,5 @@
 // src/server.ts
+import { EventEmitter as EventEmitter2 } from "node:events";
 import Koa from "koa";
 import bodyParser from "koa-bodyparser";
 import koaLogger from "koa-logger";
@@ -6,6 +7,63 @@ import Router from "@koa/router";
 import { humanId } from "human-id";
 import { createMockLogger as createMockLogger2 } from "@openfn/logger";
 
+// src/events.ts
+var CLAIM = "claim";
+var GET_ATTEMPT = "fetch:attempt";
+var GET_CREDENTIAL = "fetch:credential";
+var GET_DATACLIP = "fetch:dataclip";
+var ATTEMPT_START = "attempt:start";
+var ATTEMPT_COMPLETE = "attempt:complete";
+var ATTEMPT_LOG = "attempt:log";
+var RUN_START = "run:start";
+var RUN_COMPLETE = "run:complete";
+var INTERNAL_ATTEMPT_COMPLETE = "server:attempt-complete";
+
+// src/api/destroy.ts
+var destroy = async (app, logger) => {
+  logger.info("Closing server...");
+  await Promise.all([
+    new Promise((resolve) => {
+      app.destroyed = true;
+      app.killWorkloop?.();
+      app.queueChannel?.leave();
+      app.server.close(async () => {
+        resolve();
+      });
+    }),
+    new Promise(async (resolve) => {
+      await waitForAttempts(app, logger);
+      await app.engine.destroy();
+      app.socket?.disconnect();
+      resolve();
+    })
+  ]);
+  logger.success("Server closed");
+};
+var waitForAttempts = (app, logger) => new Promise((resolve) => {
+  const log = () => {
+    logger.debug(
+      `Waiting for ${Object.keys(app.workflows).length} attempts to complete...`
+    );
+  };
+  const onAttemptComplete = () => {
+    if (Object.keys(app.workflows).length === 0) {
+      logger.debug("All attempts completed!");
+      app.events.off(INTERNAL_ATTEMPT_COMPLETE, onAttemptComplete);
+      resolve();
+    } else {
+      log();
+    }
+  };
+  if (Object.keys(app.workflows).length) {
+    log();
+    app.events.on(INTERNAL_ATTEMPT_COMPLETE, onAttemptComplete);
+  } else {
+    resolve();
+  }
+});
+var destroy_default = destroy;
+
 // src/util/try-with-backoff.ts
 var BACKOFF_MULTIPLIER = 1.15;
 var tryWithBackoff = (fn, opts = {}) => {
@@ -49,19 +107,6 @@ var try_with_backoff_default = tryWithBackoff;
 
 // src/api/claim.ts
 import { createMockLogger } from "@openfn/logger";
-
-// src/events.ts
-var CLAIM = "claim";
-var GET_ATTEMPT = "fetch:attempt";
-var GET_CREDENTIAL = "fetch:credential";
-var GET_DATACLIP = "fetch:dataclip";
-var ATTEMPT_START = "attempt:start";
-var ATTEMPT_COMPLETE = "attempt:complete";
-var ATTEMPT_LOG = "attempt:log";
-var RUN_START = "run:start";
-var RUN_COMPLETE = "run:complete";
-
-// src/api/claim.ts
 var mockLogger = createMockLogger();
 var claim = (app, logger = mockLogger, maxWorkers = 5) => {
   return new Promise((resolve, reject) => {
@@ -69,8 +114,11 @@ var claim = (app, logger = mockLogger, maxWorkers = 5) => {
     if (activeWorkers >= maxWorkers) {
       return reject(new Error("Server at capacity"));
     }
+    if (!app.queueChannel) {
+      return reject(new Error("No websocket available"));
+    }
     logger.debug("requesting attempt...");
-    app.channel.push(CLAIM, { demand: 1 }).receive("ok", ({ attempts }) => {
+    app.queueChannel.push(CLAIM, { demand: 1 }).receive("ok", ({ attempts }) => {
       logger.debug(`pulled ${attempts.length} attempts`);
       if (!attempts?.length) {
         return reject(new Error("No attempts returned"));
@@ -112,6 +160,7 @@ var startWorkloop = (app, logger, minBackoff, maxBackoff, maxWorkers) => {
     logger.debug("cancelling workloop");
     cancelled = true;
     promise.cancel();
+    app.queueChannel?.leave();
   };
 };
 var workloop_default = startWorkloop;
@@ -122,13 +171,15 @@ import crypto2 from "node:crypto";
 // src/util/convert-attempt.ts
 import crypto from "node:crypto";
 var conditions = {
-  on_job_success: "!state.errors",
-  on_job_failure: "state.errors",
-  always: null
+  on_job_success: (upstreamId) => `Boolean(!state.errors?.["${upstreamId}"] ?? true)`,
+  on_job_failure: (upstreamId) => `Boolean(state.errors && state.errors["${upstreamId}"])`,
+  always: (_upstreamId) => null
 };
-var mapEdgeCondition = (condition) => {
+var mapEdgeCondition = (edge) => {
+  const { condition } = edge;
   if (condition && condition in conditions) {
-    return conditions[condition];
+    const upstream = edge.source_job_id || edge.source_trigger_id;
+    return conditions[condition](upstream);
   }
   return condition;
 };
@@ -177,7 +228,7 @@ var convert_attempt_default = (attempt) => {
       }
       const next = edges.filter((e) => e.source_job_id === id).reduce((obj, edge) => {
         const newEdge = {};
-        const condition = mapEdgeCondition(edge.condition);
+        const condition = mapEdgeCondition(edge);
         if (condition) {
           newEdge.condition = condition;
         }
@@ -373,12 +424,12 @@ function onJobComplete({ channel, state }, event, error) {
     state.dataclips = {};
   }
   state.dataclips[dataclipId] = event.state;
+  delete state.activeRun;
+  delete state.activeJob;
   state.lastDataclipId = dataclipId;
   event.next?.forEach((nextJobId) => {
     state.inputDataclips[nextJobId] = dataclipId;
   });
-  delete state.activeRun;
-  delete state.activeJob;
   const { reason, error_message, error_type } = calculateJobExitReason(
     job_id,
     event.state,
@@ -392,7 +443,10 @@ function onJobComplete({ channel, state }, event, error) {
     output_dataclip: stringify_default(event.state),
     reason,
     error_message,
-    error_type
+    error_type,
+    mem: event.mem,
+    duration: event.duration,
+    thread_id: event.threadId
   };
   return sendEvent(channel, RUN_COMPLETE, evt);
 }
@@ -446,6 +500,11 @@ async function loadCredential(channel, credentialId) {
   return get_with_reply_default(channel, GET_CREDENTIAL, { id: credentialId });
 }
 
+// src/middleware/healthcheck.ts
+var healthcheck_default = (ctx) => {
+  ctx.status = 200;
+};
+
 // src/channels/attempt.ts
 var joinAttemptChannel = (socket, token, attemptId, logger) => {
   return new Promise((resolve, reject) => {
@@ -481,14 +540,14 @@ import { WebSocket } from "ws";
 // src/util/worker-token.ts
 import * as jose from "jose";
 var alg = "HS256";
-var generateWorkerToken = async (secret, workerId) => {
+var generateWorkerToken = async (secret, workerId, logger) => {
   if (!secret) {
-    console.warn();
-    console.warn("WARNING: Worker Secret not provided!");
-    console.warn(
+    logger.warn();
+    logger.warn("WARNING: Worker Secret not provided!");
+    logger.warn(
       "This worker will attempt to connect to Lightning with default secret"
     );
-    console.warn();
+    logger.warn();
   }
   const encodedSecret = new TextEncoder().encode(secret || "<secret>");
   const claims = {
@@ -502,7 +561,7 @@ var worker_token_default = generateWorkerToken;
 // src/channels/worker-queue.ts
 var connectToWorkerQueue = (endpoint, serverId, secret, logger, SocketConstructor = PhxSocket) => {
   const events = new EventEmitter();
-  worker_token_default(secret, serverId).then((token) => {
+  worker_token_default(secret, serverId, logger).then((token) => {
     const socket = new SocketConstructor(endpoint, {
       params: { token },
       transport: WebSocket
@@ -545,7 +604,7 @@ function connect(app, logger, options = {}) {
   const onConnect = ({ socket, channel }) => {
     logger.success("Connected to Lightning at", options.lightning);
     app.socket = socket;
-    app.channel = channel;
+    app.queueChannel = channel;
     if (!options.noLoop) {
       logger.info("Starting workloop");
       app.killWorkloop = workloop_default(
@@ -592,6 +651,8 @@ function createServer(engine, options = {}) {
   const app = new Koa();
   app.id = humanId({ separator: "-", capitalize: false });
   const router = new Router();
+  app.events = new EventEmitter2();
+  app.engine = engine;
   app.use(bodyParser());
   app.use(
     koaLogger((str, _args) => {
@@ -599,8 +660,12 @@ function createServer(engine, options = {}) {
     })
   );
   app.workflows = {};
-  const server = app.listen(port);
+  app.destroyed = false;
+  app.server = app.listen(port);
   logger.success(`ws-worker ${app.id} listening on ${port}`);
+  process.send?.("READY");
+  router.get("/livez", healthcheck_default);
+  router.get("/", healthcheck_default);
   app.execute = async ({ id, token }) => {
     if (app.socket) {
       app.workflows[id] = true;
@@ -612,6 +677,7 @@ function createServer(engine, options = {}) {
       const onFinish = () => {
         delete app.workflows[id];
         attemptChannel.leave();
+        app.events.emit(INTERNAL_ATTEMPT_COMPLETE);
       };
       const context = execute(
         attemptChannel,
@@ -638,19 +704,24 @@ function createServer(engine, options = {}) {
       ctx.status = 204;
     });
   });
-  app.destroy = async () => {
-    logger.info("Closing server...");
-    server.close();
-    await engine.destroy();
-    app.killWorkloop?.();
-    logger.success("Server closed");
-  };
+  app.destroy = () => destroy_default(app, logger);
   app.use(router.routes());
   if (options.lightning) {
     connect(app, logger, options);
   } else {
     logger.warn("No lightning URL provided");
   }
+  let shutdown = false;
+  const exit = async (signal) => {
+    if (!shutdown) {
+      shutdown = true;
+      logger.always(`${signal} RECEIVED: CLOSING SERVER`);
+      await app.destroy();
+      process.exit();
+    }
+  };
+  process.on("SIGINT", () => exit("SIGINT"));
+  process.on("SIGTERM", () => exit("SIGTERM"));
   app.on = (...args) => {
     return engine.on(...args);
   };
@@ -668,6 +739,7 @@ export {
   GET_ATTEMPT,
   GET_CREDENTIAL,
   GET_DATACLIP,
+  INTERNAL_ATTEMPT_COMPLETE,
   RUN_COMPLETE,
   RUN_START,
   src_default as default

package/dist/start.js

@@ -4865,6 +4865,7 @@ import createRTE from "@openfn/engine-multi";
 
 // src/mock/runtime-engine.ts
 import { EventEmitter } from "node:events";
+import crypto from "node:crypto";
 import run from "@openfn/runtime";
 
 // src/mock/resolvers.ts
@@ -4911,6 +4912,7 @@ async function createMock() {
   }) => {
     const { id, jobs } = xplan;
     activeWorkflows[id] = true;
+    const threadId = crypto.randomUUID();
     for (const job of jobs) {
       if (typeof job.configuration === "string") {
         job.configuration = await options.resolvers?.credential?.(
@@ -4925,6 +4927,7 @@ async function createMock() {
       log: (...args2) => {
         dispatch("workflow-log", {
           workflowId: id,
+          threadId,
           level: "info",
           json: true,
           message: args2,
@@ -4941,24 +4944,26 @@ async function createMock() {
         notify: (name, payload) => {
           dispatch(name, {
             workflowId: id,
+            threadId,
             ...payload
           });
         }
       }
     };
     setTimeout(async () => {
-      dispatch("workflow-start", { workflowId: id });
+      dispatch("workflow-start", { workflowId: id, threadId });
       try {
         await run(xplan, void 0, opts);
       } catch (e) {
         dispatch("workflow-error", {
+          threadId,
           workflowId: id,
           type: e.name,
           message: e.message
         });
       }
       delete activeWorkflows[id];
-      dispatch("workflow-complete", { workflowId: id });
+      dispatch("workflow-complete", { workflowId: id, threadId });
     }, 1);
   };
   const getStatus = () => {
@@ -4966,17 +4971,20 @@ async function createMock() {
       active: Object.keys(activeWorkflows).length
     };
   };
+  const destroy2 = async () => true;
   return {
     on,
     once,
     execute: execute2,
     getStatus,
-    listen
+    listen,
+    destroy: destroy2
   };
 }
 var runtime_engine_default = createMock;
 
 // src/server.ts
+import { EventEmitter as EventEmitter3 } from "node:events";
 import Koa from "koa";
 import bodyParser from "koa-bodyparser";
 import koaLogger from "koa-logger";
@@ -4984,6 +4992,63 @@ import Router from "@koa/router";
 import { humanId } from "human-id";
 import { createMockLogger as createMockLogger2 } from "@openfn/logger";
 
+// src/events.ts
+var CLAIM = "claim";
+var GET_ATTEMPT = "fetch:attempt";
+var GET_CREDENTIAL = "fetch:credential";
+var GET_DATACLIP = "fetch:dataclip";
+var ATTEMPT_START = "attempt:start";
+var ATTEMPT_COMPLETE = "attempt:complete";
+var ATTEMPT_LOG = "attempt:log";
+var RUN_START = "run:start";
+var RUN_COMPLETE = "run:complete";
+var INTERNAL_ATTEMPT_COMPLETE = "server:attempt-complete";
+
+// src/api/destroy.ts
+var destroy = async (app, logger2) => {
+  logger2.info("Closing server...");
+  await Promise.all([
+    new Promise((resolve5) => {
+      app.destroyed = true;
+      app.killWorkloop?.();
+      app.queueChannel?.leave();
+      app.server.close(async () => {
+        resolve5();
+      });
+    }),
+    new Promise(async (resolve5) => {
+      await waitForAttempts(app, logger2);
+      await app.engine.destroy();
+      app.socket?.disconnect();
+      resolve5();
+    })
+  ]);
+  logger2.success("Server closed");
+};
+var waitForAttempts = (app, logger2) => new Promise((resolve5) => {
+  const log = () => {
+    logger2.debug(
+      `Waiting for ${Object.keys(app.workflows).length} attempts to complete...`
+    );
+  };
+  const onAttemptComplete = () => {
+    if (Object.keys(app.workflows).length === 0) {
+      logger2.debug("All attempts completed!");
+      app.events.off(INTERNAL_ATTEMPT_COMPLETE, onAttemptComplete);
+      resolve5();
+    } else {
+      log();
+    }
+  };
+  if (Object.keys(app.workflows).length) {
+    log();
+    app.events.on(INTERNAL_ATTEMPT_COMPLETE, onAttemptComplete);
+  } else {
+    resolve5();
+  }
+});
+var destroy_default = destroy;
+
 // src/util/try-with-backoff.ts
 var BACKOFF_MULTIPLIER = 1.15;
 var tryWithBackoff = (fn, opts = {}) => {
@@ -5027,19 +5092,6 @@ var try_with_backoff_default = tryWithBackoff;
 
 // src/api/claim.ts
 import { createMockLogger } from "@openfn/logger";
-
-// src/events.ts
-var CLAIM = "claim";
-var GET_ATTEMPT = "fetch:attempt";
-var GET_CREDENTIAL = "fetch:credential";
-var GET_DATACLIP = "fetch:dataclip";
-var ATTEMPT_START = "attempt:start";
-var ATTEMPT_COMPLETE = "attempt:complete";
-var ATTEMPT_LOG = "attempt:log";
-var RUN_START = "run:start";
-var RUN_COMPLETE = "run:complete";
-
-// src/api/claim.ts
 var mockLogger = createMockLogger();
 var claim = (app, logger2 = mockLogger, maxWorkers = 5) => {
   return new Promise((resolve5, reject) => {
@@ -5047,8 +5099,11 @@ var claim = (app, logger2 = mockLogger, maxWorkers = 5) => {
     if (activeWorkers >= maxWorkers) {
       return reject(new Error("Server at capacity"));
     }
+    if (!app.queueChannel) {
+      return reject(new Error("No websocket available"));
+    }
     logger2.debug("requesting attempt...");
-    app.channel.push(CLAIM, { demand: 1 }).receive("ok", ({ attempts }) => {
+    app.queueChannel.push(CLAIM, { demand: 1 }).receive("ok", ({ attempts }) => {
       logger2.debug(`pulled ${attempts.length} attempts`);
       if (!attempts?.length) {
         return reject(new Error("No attempts returned"));
@@ -5090,23 +5145,26 @@ var startWorkloop = (app, logger2, minBackoff2, maxBackoff2, maxWorkers) => {
     logger2.debug("cancelling workloop");
     cancelled = true;
     promise.cancel();
+    app.queueChannel?.leave();
   };
 };
 var workloop_default = startWorkloop;
 
 // src/api/execute.ts
-import crypto2 from "node:crypto";
+import crypto3 from "node:crypto";
 
 // src/util/convert-attempt.ts
-import crypto from "node:crypto";
+import crypto2 from "node:crypto";
 var conditions = {
-  on_job_success: "!state.errors",
-  on_job_failure: "state.errors",
-  always: null
+  on_job_success: (upstreamId) => `Boolean(!state.errors?.["${upstreamId}"] ?? true)`,
+  on_job_failure: (upstreamId) => `Boolean(state.errors && state.errors["${upstreamId}"])`,
+  always: (_upstreamId) => null
 };
-var mapEdgeCondition = (condition) => {
+var mapEdgeCondition = (edge) => {
+  const { condition } = edge;
   if (condition && condition in conditions) {
-    return conditions[condition];
+    const upstream = edge.source_job_id || edge.source_trigger_id;
+    return conditions[condition](upstream);
   }
   return condition;
 };
@@ -5143,7 +5201,7 @@ var convert_attempt_default = (attempt) => {
   }
   if (attempt.jobs?.length) {
     attempt.jobs.forEach((job) => {
-      const id = job.id || crypto.randomUUID();
+      const id = job.id || crypto2.randomUUID();
       nodes[id] = {
         id,
         configuration: job.credential_id,
@@ -5155,7 +5213,7 @@ var convert_attempt_default = (attempt) => {
       }
       const next = edges.filter((e) => e.source_job_id === id).reduce((obj, edge) => {
         const newEdge = {};
-        const condition = mapEdgeCondition(edge.condition);
+        const condition = mapEdgeCondition(edge);
         if (condition) {
           newEdge.condition = condition;
         }
@@ -5326,7 +5384,7 @@ var sendEvent = (channel, event, payload) => new Promise((resolve5, reject) => {
   channel.push(event, payload).receive("error", reject).receive("timeout", () => reject(new Error("timeout"))).receive("ok", resolve5);
 });
 function onJobStart({ channel, state }, event) {
-  state.activeRun = crypto2.randomUUID();
+  state.activeRun = crypto3.randomUUID();
   state.activeJob = event.jobId;
   const input_dataclip_id = state.inputDataclips[event.jobId];
   return sendEvent(channel, RUN_START, {
@@ -5344,19 +5402,19 @@ function onJobError(context, event) {
   }
 }
 function onJobComplete({ channel, state }, event, error) {
-  const dataclipId = crypto2.randomUUID();
+  const dataclipId = crypto3.randomUUID();
   const run_id = state.activeRun;
   const job_id = state.activeJob;
   if (!state.dataclips) {
     state.dataclips = {};
   }
   state.dataclips[dataclipId] = event.state;
+  delete state.activeRun;
+  delete state.activeJob;
   state.lastDataclipId = dataclipId;
   event.next?.forEach((nextJobId) => {
     state.inputDataclips[nextJobId] = dataclipId;
   });
-  delete state.activeRun;
-  delete state.activeJob;
   const { reason, error_message, error_type } = calculateJobExitReason(
     job_id,
     event.state,
@@ -5370,7 +5428,10 @@ function onJobComplete({ channel, state }, event, error) {
     output_dataclip: stringify_default(event.state),
     reason,
     error_message,
-    error_type
+    error_type,
+    mem: event.mem,
+    duration: event.duration,
+    thread_id: event.threadId
   };
   return sendEvent(channel, RUN_COMPLETE, evt);
 }
@@ -5424,6 +5485,11 @@ async function loadCredential(channel, credentialId) {
   return get_with_reply_default(channel, GET_CREDENTIAL, { id: credentialId });
 }
 
+// src/middleware/healthcheck.ts
+var healthcheck_default = (ctx) => {
+  ctx.status = 200;
+};
+
 // src/channels/attempt.ts
 var joinAttemptChannel = (socket, token, attemptId, logger2) => {
   return new Promise((resolve5, reject) => {
@@ -5459,14 +5525,14 @@ import { WebSocket } from "ws";
 // src/util/worker-token.ts
 import * as jose from "jose";
 var alg = "HS256";
-var generateWorkerToken = async (secret, workerId) => {
+var generateWorkerToken = async (secret, workerId, logger2) => {
   if (!secret) {
-    console.warn();
-    console.warn("WARNING: Worker Secret not provided!");
-    console.warn(
+    logger2.warn();
+    logger2.warn("WARNING: Worker Secret not provided!");
+    logger2.warn(
       "This worker will attempt to connect to Lightning with default secret"
     );
-    console.warn();
+    logger2.warn();
   }
   const encodedSecret = new TextEncoder().encode(secret || "<secret>");
   const claims = {
@@ -5480,7 +5546,7 @@ var worker_token_default = generateWorkerToken;
 // src/channels/worker-queue.ts
 var connectToWorkerQueue = (endpoint, serverId, secret, logger2, SocketConstructor = PhxSocket) => {
   const events = new EventEmitter2();
-  worker_token_default(secret, serverId).then((token) => {
+  worker_token_default(secret, serverId, logger2).then((token) => {
     const socket = new SocketConstructor(endpoint, {
       params: { token },
       transport: WebSocket
@@ -5523,7 +5589,7 @@ function connect(app, logger2, options = {}) {
   const onConnect = ({ socket, channel }) => {
     logger2.success("Connected to Lightning at", options.lightning);
     app.socket = socket;
-    app.channel = channel;
+    app.queueChannel = channel;
     if (!options.noLoop) {
       logger2.info("Starting workloop");
       app.killWorkloop = workloop_default(
@@ -5570,6 +5636,8 @@ function createServer(engine, options = {}) {
   const app = new Koa();
   app.id = humanId({ separator: "-", capitalize: false });
   const router = new Router();
+  app.events = new EventEmitter3();
+  app.engine = engine;
   app.use(bodyParser());
   app.use(
     koaLogger((str, _args) => {
@@ -5577,8 +5645,12 @@ function createServer(engine, options = {}) {
     })
   );
   app.workflows = {};
-  const server = app.listen(port);
+  app.destroyed = false;
+  app.server = app.listen(port);
   logger2.success(`ws-worker ${app.id} listening on ${port}`);
+  process.send?.("READY");
+  router.get("/livez", healthcheck_default);
+  router.get("/", healthcheck_default);
   app.execute = async ({ id, token }) => {
     if (app.socket) {
       app.workflows[id] = true;
@@ -5590,6 +5662,7 @@ function createServer(engine, options = {}) {
       const onFinish = () => {
         delete app.workflows[id];
         attemptChannel.leave();
+        app.events.emit(INTERNAL_ATTEMPT_COMPLETE);
       };
       const context = execute(
         attemptChannel,
@@ -5616,19 +5689,24 @@ function createServer(engine, options = {}) {
       ctx.status = 204;
     });
   });
-  app.destroy = async () => {
-    logger2.info("Closing server...");
-    server.close();
-    await engine.destroy();
-    app.killWorkloop?.();
-    logger2.success("Server closed");
-  };
+  app.destroy = () => destroy_default(app, logger2);
   app.use(router.routes());
   if (options.lightning) {
     connect(app, logger2, options);
   } else {
     logger2.warn("No lightning URL provided");
   }
+  let shutdown = false;
+  const exit = async (signal) => {
+    if (!shutdown) {
+      shutdown = true;
+      logger2.always(`${signal} RECEIVED: CLOSING SERVER`);
+      await app.destroy();
+      process.exit();
+    }
+  };
+  process.on("SIGINT", () => exit("SIGINT"));
+  process.on("SIGTERM", () => exit("SIGTERM"));
   app.on = (...args2) => {
     return engine.on(...args2);
   };
@@ -5637,7 +5715,7 @@ function createServer(engine, options = {}) {
 var server_default = createServer;
 
 // src/start.ts
-var { WORKER_REPO_DIR, WORKER_SECRET } = process.env;
+var { WORKER_REPO_DIR, WORKER_SECRET, MAX_RUN_MEMORY } = process.env;
 var args = yargs_default(hideBin(process.argv)).command("server", "Start a ws-worker server").option("port", {
   alias: "p",
   description: "Port to run the server on",
@@ -5673,6 +5751,10 @@ var args = yargs_default(hideBin(process.argv)).command("server", "Start a ws-wo
   description: "max concurrent workers",
   default: 5,
   type: "number"
+}).option("run-memory", {
+  description: "Maximum memory allocated to a single run, in mb",
+  type: "number",
+  default: MAX_RUN_MEMORY ? parseInt(MAX_RUN_MEMORY) : 500
 }).parse();
 var logger = createLogger("SRV", { level: args.log });
 if (args.lightning === "mock") {
@@ -5708,10 +5790,12 @@ if (args.mock) {
     engineReady(engine);
   });
 } else {
-  createRTE({ repoDir: args.repoDir }).then((engine) => {
-    logger.debug("engine created");
-    engineReady(engine);
-  });
+  createRTE({ repoDir: args.repoDir, memoryLimitMb: args.runMemory }).then(
+    (engine) => {
+      logger.debug("engine created");
+      engineReady(engine);
+    }
+  );
 }
 /**
  * @fileoverview Main entrypoint for libraries using yargs-parser in Node.js

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openfn/ws-worker",
-  "version": "0.2.9",
+  "version": "0.2.11",
   "description": "A Websocket Worker to connect Lightning to a Runtime Engine",
   "main": "dist/index.js",
   "type": "module",
@@ -21,9 +21,9 @@
     "koa-logger": "^3.2.1",
     "phoenix": "^1.7.7",
     "ws": "^8.14.1",
-    "@openfn/engine-multi": "0.2.0",
+    "@openfn/engine-multi": "0.2.2",
     "@openfn/logger": "0.0.19",
-    "@openfn/runtime": "0.2.0"
+    "@openfn/runtime": "0.2.1"
   },
   "devDependencies": {
     "@types/koa": "^2.13.5",
@@ -40,7 +40,7 @@
     "tsup": "^6.2.3",
     "typescript": "^4.6.4",
     "yargs": "^17.6.2",
-    "@openfn/lightning-mock": "1.1.2"
+    "@openfn/lightning-mock": "1.1.4"
   },
   "files": [
     "dist",