@openfn/ws-worker 1.4.1 → 1.5.1

package/CHANGELOG.md

@@ -1,5 +1,18 @@
 # ws-worker
 
+## 1.5.1
+
+### Patch Changes
+
+- a08fb47: Update CLI docs
+  Add WORKER_MAX_SOCKET_TIMEOUT_SECONDS
+
+## 1.5.0
+
+### Minor Changes
+
+- f363254: Allow a payload limit to be set for large dataclips and logs (payload_limit_mb)
+
 ## 1.4.1
 
 ### Patch Changes

package/dist/index.d.ts

@@ -41,6 +41,7 @@ interface Channel extends Channel$1 {
 
 declare type WorkerRunOptions = ExecuteOptions & {
     outputDataclips?: boolean;
+    payloadLimitMb?: number;
 };
 
 declare type Context = {
@@ -64,6 +65,8 @@ declare type ServerOptions = {
         min?: number;
         max?: number;
     };
+    socketTimeoutSeconds?: number;
+    payloadLimitMb?: number;
 };
 interface ServerApp extends Koa {
     id: string;

package/dist/index.js

@@ -29,7 +29,7 @@ var name, version, description, main, type, scripts, bin, author, license, depen
 var init_package = __esm({
   "package.json"() {
     name = "@openfn/ws-worker";
-    version = "1.4.1";
+    version = "1.5.1";
     description = "A Websocket Worker to connect Lightning to a Runtime Engine";
     main = "dist/index.js";
     type = "module";
@@ -167,7 +167,7 @@ var waitForRuns = (app, logger) => new Promise((resolve) => {
     log();
     app.events.on(INTERNAL_RUN_COMPLETE, onRunComplete);
   } else {
-    logger.debug("No active rns detected");
+    logger.debug("No active runs detected, closing immediately");
     resolve();
   }
 });
@@ -242,7 +242,10 @@ var claim = (app, logger = mockLogger, options = {}) => {
     }
     logger.debug("requesting run...");
     app.queueChannel.push(CLAIM, { demand: 1 }).receive("ok", ({ runs }) => {
-      logger.debug(`pulled ${runs.length} runs`);
+      logger.debug(
+        `claimed ${runs.length} runs: `,
+        runs.map((r) => r.id).join(",")
+      );
       if (!runs?.length) {
         return reject(new Error("No runs returned"));
       }
@@ -265,10 +268,10 @@ var claim = (app, logger = mockLogger, options = {}) => {
         app.execute(run);
         resolve();
       });
-    }).receive("error", () => {
-      logger.debug("pull err");
+    }).receive("error", (e) => {
+      logger.error("Error on claim", e);
     }).receive("timeout", () => {
-      logger.debug("pull timeout");
+      logger.error("TIMEOUT on claim. Runs may be lost.");
       reject(new Error("timeout"));
     });
   });
@@ -332,13 +335,19 @@ var convert_lightning_plan_default = (run) => {
   const runtimeOpts = {};
   const engineOpts = {};
   if (run.options) {
-    if (run.options.run_timeout_ms) {
+    if ("run_timeout_ms" in run.options) {
       engineOpts.runTimeoutMs = run.options.run_timeout_ms;
     }
-    if (run.options.sanitize) {
+    if ("payload_limit_mb" in run.options) {
+      engineOpts.payloadLimitMb = run.options.payload_limit_mb;
+    }
+    if ("run_memory_limit_mb" in run.options) {
+      engineOpts.memoryLimitMb = run.options.run_memory_limit_mb;
+    }
+    if ("sanitize" in run.options) {
       engineOpts.sanitize = run.options.sanitize;
     }
-    if (run.options.hasOwnProperty("output_dataclips")) {
+    if ("output_dataclips" in run.options) {
       engineOpts.outputDataclips = run.options.output_dataclips;
     }
   }
@@ -527,7 +536,7 @@ ${prefix("worker")}${versions.worker || "unknown"}`;
 // src/events/run-start.ts
 init_package();
 async function onRunStart(context, event) {
-  const { channel, state } = context;
+  const { channel, state, options = {} } = context;
   const time = (timestamp() - BigInt(1e7)).toString();
   const versionLogContext = {
     ...context,
@@ -541,6 +550,14 @@ async function onRunStart(context, event) {
     ...event.versions
   };
   await sendEvent(channel, RUN_START, { versions });
+  if ("payloadLimitMb" in options) {
+    await onJobLog(versionLogContext, {
+      time,
+      message: [`Payload limit: ${options.payloadLimitMb}mb`],
+      level: "info",
+      name: "RTE"
+    });
+  }
   const versionMessage = versions_default(versions);
   await onJobLog(versionLogContext, {
     time,
@@ -552,6 +569,7 @@ async function onRunStart(context, event) {
 
 // src/events/step-complete.ts
 import crypto3 from "node:crypto";
+import { timestamp as timestamp2 } from "@openfn/logger";
 
 // src/api/reasons.ts
 var calculateJobExitReason = (jobId, state = { data: {} }, error) => {
@@ -586,8 +604,25 @@ var calculateRunExitReason = (state) => {
   return { reason: "success", error_type: null, error_message: null };
 };
 
+// src/util/ensure-payload-size.ts
+var ensure_payload_size_default = (payload, limit_mb) => {
+  if (!isNaN(limit_mb)) {
+    const limit = limit_mb;
+    const size_bytes = Buffer.byteLength(payload, "utf8");
+    const size_mb = size_bytes / 1024 / 1024;
+    if (size_mb > limit) {
+      const e = new Error();
+      e.severity = "kill";
+      e.name = "PAYLOAD_TOO_LARGE";
+      e.message = `The payload exceeded the size limit of ${limit}mb`;
+      throw e;
+    }
+  }
+};
+
 // src/events/step-complete.ts
-function onStepComplete({ channel, state, options }, event, error) {
+async function onStepComplete(context, event, error) {
+  const { channel, state, options } = context;
   const dataclipId = crypto3.randomUUID();
   const step_id = state.activeStep;
   const job_id = state.activeJob;
@@ -602,26 +637,35 @@ function onStepComplete({ channel, state, options }, event, error) {
   event.next?.forEach((nextJobId) => {
     state.inputDataclips[nextJobId] = dataclipId;
   });
-  const { reason, error_message, error_type } = calculateJobExitReason(
-    job_id,
-    event.state,
-    error
-  );
-  state.reasons[job_id] = { reason, error_message, error_type };
   const evt = {
     step_id,
     job_id,
-    output_dataclip_id: dataclipId,
-    reason,
-    error_message,
-    error_type,
     mem: event.mem,
     duration: event.duration,
     thread_id: event.threadId
   };
-  if (!options || options.outputDataclips !== false) {
-    evt.output_dataclip = stringify_default(outputState);
+  try {
+    if (!options || options.outputDataclips !== false) {
+      const payload = stringify_default(outputState);
+      ensure_payload_size_default(payload, options?.payloadLimitMb);
+      evt.output_dataclip = payload;
+    }
+    evt.output_dataclip_id = dataclipId;
+  } catch (e) {
+    evt.output_dataclip_error = "DATACLIP_TOO_LARGE";
+    const time = (timestamp2() - BigInt(1e7)).toString();
+    await onJobLog(context, {
+      time,
+      message: [
+        "Dataclip too large. This dataclip will not be sent back to lighting."
+      ],
+      level: "info",
+      name: "R/T"
+    });
   }
+  const reason = calculateJobExitReason(job_id, event.state, error);
+  state.reasons[job_id] = reason;
+  Object.assign(evt, reason);
   return sendEvent(channel, STEP_COMPLETE, evt);
 }
 
@@ -640,9 +684,9 @@ async function onStepStart(context, event) {
 }
 
 // src/util/log-final-reason.ts
-import { timestamp as timestamp2 } from "@openfn/logger";
+import { timestamp as timestamp3 } from "@openfn/logger";
 var log_final_reason_default = async (context, reason) => {
-  const time = (timestamp2() - BigInt(1e7)).toString();
+  const time = (timestamp3() - BigInt(1e7)).toString();
   let message = `Run complete with status: ${reason.reason}`;
   if (reason.reason !== "success") {
     message += `
@@ -792,11 +836,25 @@ function onJobError(context, event) {
     return onStepComplete(context, event, event.error);
   }
 }
-function onJobLog({ channel, state }, event) {
+function onJobLog({ channel, state, options }, event) {
   const timeInMicroseconds = BigInt(event.time) / BigInt(1e3);
+  let message = event.message;
+  try {
+    if (typeof event.message === "string") {
+      ensure_payload_size_default(event.message, options?.payloadLimitMb);
+      message = JSON.parse(message);
+    } else if (event.message) {
+      const payload = stringify_default(event.message);
+      ensure_payload_size_default(payload, options?.payloadLimitMb);
+    }
+  } catch (e) {
+    message = [
+      `(Log message redacted: exceeds ${options.payloadLimitMb}mb memory limit)`
+    ];
+  }
   const log = {
     run_id: state.plan.id,
-    message: typeof event.message === "string" ? JSON.parse(event.message) : event.message,
+    message,
     source: event.name,
     level: event.level,
     timestamp: timeInMicroseconds.toString()
@@ -887,7 +945,7 @@ var generateWorkerToken = async (secret, workerId, logger) => {
 var worker_token_default = generateWorkerToken;
 
 // src/channels/worker-queue.ts
-var connectToWorkerQueue = (endpoint, serverId, secret, logger, SocketConstructor = PhxSocket) => {
+var connectToWorkerQueue = (endpoint, serverId, secret, timeout = 10, logger, SocketConstructor = PhxSocket) => {
   const events = new EventEmitter();
   worker_token_default(secret, serverId, logger).then(async (token) => {
     const pkg = await Promise.resolve().then(() => (init_package(), package_exports));
@@ -898,7 +956,8 @@ var connectToWorkerQueue = (endpoint, serverId, secret, logger, SocketConstructo
     };
     const socket = new SocketConstructor(endpoint, {
       params,
-      transport: WebSocket
+      transport: WebSocket,
+      timeout: timeout * 1e3
     });
     let didOpen = false;
     socket.onOpen(() => {
@@ -930,7 +989,7 @@ var connectToWorkerQueue = (endpoint, serverId, secret, logger, SocketConstructo
 var worker_queue_default = connectToWorkerQueue;
 
 // src/server.ts
-var DEFAULT_PORT = 1234;
+var DEFAULT_PORT = 2222;
 var MIN_BACKOFF = 1e3;
 var MAX_BACKOFF = 1e3 * 30;
 function connect(app, logger, options = {}) {
@@ -952,13 +1011,12 @@ function connect(app, logger, options = {}) {
         options.maxWorkflows
       );
     } else {
+      const port = app.server?.address().port;
       logger.break();
-      logger.warn("Workloop not starting");
+      logger.warn("Noloop active: workloop has not started");
       logger.info("This server will not auto-pull work from lightning.");
       logger.info("You can manually claim by posting to /claim, eg:");
-      logger.info(
-        `  curl -X POST http://locahost:${options.port || DEFAULT_PORT}/claim`
-      );
+      logger.info(`  curl -X POST http://localhost:${port}/claim`);
       logger.break();
     }
   };
@@ -982,7 +1040,13 @@ function connect(app, logger, options = {}) {
     );
     logger.debug(e);
   };
-  worker_queue_default(options.lightning, app.id, options.secret, logger).on("connect", onConnect).on("disconnect", onDisconnect).on("error", onError);
+  worker_queue_default(
+    options.lightning,
+    app.id,
+    options.secret,
+    options.socketTimeoutSeconds,
+    logger
+  ).on("connect", onConnect).on("disconnect", onDisconnect).on("error", onError);
 }
 function createServer(engine, options = {}) {
   const logger = options.logger || createMockLogger2();
@@ -1005,16 +1069,19 @@ function createServer(engine, options = {}) {
   process.send?.("READY");
   router.get("/livez", healthcheck_default);
   router.get("/", healthcheck_default);
-  app.options = options || {};
+  app.options = options;
   app.execute = async ({ id, token }) => {
     if (app.socket) {
       app.workflows[id] = true;
       const {
         channel: runChannel,
         plan,
-        options: options2,
+        options: options2 = {},
         input
       } = await run_default(app.socket, token, id, logger);
+      if (!("payloadLimitMb" in options2)) {
+        options2.payloadLimitMb = app.options.payloadLimitMb;
+      }
       const onFinish = () => {
         logger.debug(`workflow ${id} complete: releasing worker`);
         delete app.workflows[id];

package/dist/start.js

@@ -37,7 +37,7 @@ var name, version, description, main, type, scripts, bin, author, license, depen
 var init_package = __esm({
   "package.json"() {
     name = "@openfn/ws-worker";
-    version = "1.4.1";
+    version = "1.5.1";
     description = "A Websocket Worker to connect Lightning to a Runtime Engine";
     main = "dist/index.js";
     type = "module";
@@ -306,7 +306,7 @@ var waitForRuns = (app, logger2) => new Promise((resolve5) => {
     log();
     app.events.on(INTERNAL_RUN_COMPLETE, onRunComplete);
   } else {
-    logger2.debug("No active rns detected");
+    logger2.debug("No active runs detected, closing immediately");
     resolve5();
   }
 });
@@ -381,7 +381,10 @@ var claim = (app, logger2 = mockLogger, options = {}) => {
     }
     logger2.debug("requesting run...");
     app.queueChannel.push(CLAIM, { demand: 1 }).receive("ok", ({ runs }) => {
-      logger2.debug(`pulled ${runs.length} runs`);
+      logger2.debug(
+        `claimed ${runs.length} runs: `,
+        runs.map((r) => r.id).join(",")
+      );
       if (!runs?.length) {
         return reject(new Error("No runs returned"));
       }
@@ -404,10 +407,10 @@ var claim = (app, logger2 = mockLogger, options = {}) => {
         app.execute(run2);
         resolve5();
       });
-    }).receive("error", () => {
-      logger2.debug("pull err");
+    }).receive("error", (e) => {
+      logger2.error("Error on claim", e);
     }).receive("timeout", () => {
-      logger2.debug("pull timeout");
+      logger2.error("TIMEOUT on claim. Runs may be lost.");
       reject(new Error("timeout"));
     });
   });
@@ -471,13 +474,19 @@ var convert_lightning_plan_default = (run2) => {
   const runtimeOpts = {};
   const engineOpts = {};
   if (run2.options) {
-    if (run2.options.run_timeout_ms) {
+    if ("run_timeout_ms" in run2.options) {
       engineOpts.runTimeoutMs = run2.options.run_timeout_ms;
     }
-    if (run2.options.sanitize) {
+    if ("payload_limit_mb" in run2.options) {
+      engineOpts.payloadLimitMb = run2.options.payload_limit_mb;
+    }
+    if ("run_memory_limit_mb" in run2.options) {
+      engineOpts.memoryLimitMb = run2.options.run_memory_limit_mb;
+    }
+    if ("sanitize" in run2.options) {
       engineOpts.sanitize = run2.options.sanitize;
     }
-    if (run2.options.hasOwnProperty("output_dataclips")) {
+    if ("output_dataclips" in run2.options) {
       engineOpts.outputDataclips = run2.options.output_dataclips;
     }
   }
@@ -666,7 +675,7 @@ ${prefix("worker")}${versions.worker || "unknown"}`;
 // src/events/run-start.ts
 init_package();
 async function onRunStart(context, event) {
-  const { channel, state } = context;
+  const { channel, state, options = {} } = context;
   const time = (timestamp() - BigInt(1e7)).toString();
   const versionLogContext = {
     ...context,
@@ -680,6 +689,14 @@ async function onRunStart(context, event) {
     ...event.versions
   };
   await sendEvent(channel, RUN_START, { versions });
+  if ("payloadLimitMb" in options) {
+    await onJobLog(versionLogContext, {
+      time,
+      message: [`Payload limit: ${options.payloadLimitMb}mb`],
+      level: "info",
+      name: "RTE"
+    });
+  }
   const versionMessage = versions_default(versions);
   await onJobLog(versionLogContext, {
     time,
@@ -691,6 +708,7 @@ async function onRunStart(context, event) {
 
 // src/events/step-complete.ts
 import crypto4 from "node:crypto";
+import { timestamp as timestamp2 } from "@openfn/logger";
 
 // src/api/reasons.ts
 var calculateJobExitReason = (jobId, state = { data: {} }, error) => {
@@ -725,8 +743,25 @@ var calculateRunExitReason = (state) => {
   return { reason: "success", error_type: null, error_message: null };
 };
 
+// src/util/ensure-payload-size.ts
+var ensure_payload_size_default = (payload, limit_mb) => {
+  if (!isNaN(limit_mb)) {
+    const limit = limit_mb;
+    const size_bytes = Buffer.byteLength(payload, "utf8");
+    const size_mb = size_bytes / 1024 / 1024;
+    if (size_mb > limit) {
+      const e = new Error();
+      e.severity = "kill";
+      e.name = "PAYLOAD_TOO_LARGE";
+      e.message = `The payload exceeded the size limit of ${limit}mb`;
+      throw e;
+    }
+  }
+};
+
 // src/events/step-complete.ts
-function onStepComplete({ channel, state, options }, event, error) {
+async function onStepComplete(context, event, error) {
+  const { channel, state, options } = context;
   const dataclipId = crypto4.randomUUID();
   const step_id = state.activeStep;
   const job_id = state.activeJob;
@@ -741,26 +776,35 @@ function onStepComplete({ channel, state, options }, event, error) {
   event.next?.forEach((nextJobId) => {
     state.inputDataclips[nextJobId] = dataclipId;
   });
-  const { reason, error_message, error_type } = calculateJobExitReason(
-    job_id,
-    event.state,
-    error
-  );
-  state.reasons[job_id] = { reason, error_message, error_type };
   const evt = {
     step_id,
     job_id,
-    output_dataclip_id: dataclipId,
-    reason,
-    error_message,
-    error_type,
     mem: event.mem,
     duration: event.duration,
     thread_id: event.threadId
   };
-  if (!options || options.outputDataclips !== false) {
-    evt.output_dataclip = stringify_default(outputState);
+  try {
+    if (!options || options.outputDataclips !== false) {
+      const payload = stringify_default(outputState);
+      ensure_payload_size_default(payload, options?.payloadLimitMb);
+      evt.output_dataclip = payload;
+    }
+    evt.output_dataclip_id = dataclipId;
+  } catch (e) {
+    evt.output_dataclip_error = "DATACLIP_TOO_LARGE";
+    const time = (timestamp2() - BigInt(1e7)).toString();
+    await onJobLog(context, {
+      time,
+      message: [
+        "Dataclip too large. This dataclip will not be sent back to lighting."
+      ],
+      level: "info",
+      name: "R/T"
+    });
   }
+  const reason = calculateJobExitReason(job_id, event.state, error);
+  state.reasons[job_id] = reason;
+  Object.assign(evt, reason);
   return sendEvent(channel, STEP_COMPLETE, evt);
 }
 
@@ -779,9 +823,9 @@ async function onStepStart(context, event) {
 }
 
 // src/util/log-final-reason.ts
-import { timestamp as timestamp2 } from "@openfn/logger";
+import { timestamp as timestamp3 } from "@openfn/logger";
 var log_final_reason_default = async (context, reason) => {
-  const time = (timestamp2() - BigInt(1e7)).toString();
+  const time = (timestamp3() - BigInt(1e7)).toString();
   let message = `Run complete with status: ${reason.reason}`;
   if (reason.reason !== "success") {
     message += `
@@ -931,11 +975,25 @@ function onJobError(context, event) {
     return onStepComplete(context, event, event.error);
   }
 }
-function onJobLog({ channel, state }, event) {
+function onJobLog({ channel, state, options }, event) {
   const timeInMicroseconds = BigInt(event.time) / BigInt(1e3);
+  let message = event.message;
+  try {
+    if (typeof event.message === "string") {
+      ensure_payload_size_default(event.message, options?.payloadLimitMb);
+      message = JSON.parse(message);
+    } else if (event.message) {
+      const payload = stringify_default(event.message);
+      ensure_payload_size_default(payload, options?.payloadLimitMb);
+    }
+  } catch (e) {
+    message = [
+      `(Log message redacted: exceeds ${options.payloadLimitMb}mb memory limit)`
+    ];
+  }
   const log = {
     run_id: state.plan.id,
-    message: typeof event.message === "string" ? JSON.parse(event.message) : event.message,
+    message,
     source: event.name,
     level: event.level,
     timestamp: timeInMicroseconds.toString()
@@ -1026,7 +1084,7 @@ var generateWorkerToken = async (secret, workerId, logger2) => {
 var worker_token_default = generateWorkerToken;
 
 // src/channels/worker-queue.ts
-var connectToWorkerQueue = (endpoint, serverId, secret, logger2, SocketConstructor = PhxSocket) => {
+var connectToWorkerQueue = (endpoint, serverId, secret, timeout = 10, logger2, SocketConstructor = PhxSocket) => {
   const events = new EventEmitter2();
   worker_token_default(secret, serverId, logger2).then(async (token) => {
     const pkg = await Promise.resolve().then(() => (init_package(), package_exports));
@@ -1037,7 +1095,8 @@ var connectToWorkerQueue = (endpoint, serverId, secret, logger2, SocketConstruct
     };
     const socket = new SocketConstructor(endpoint, {
       params,
-      transport: WebSocket
+      transport: WebSocket,
+      timeout: timeout * 1e3
     });
     let didOpen = false;
     socket.onOpen(() => {
@@ -1069,7 +1128,7 @@ var connectToWorkerQueue = (endpoint, serverId, secret, logger2, SocketConstruct
 var worker_queue_default = connectToWorkerQueue;
 
 // src/server.ts
-var DEFAULT_PORT = 1234;
+var DEFAULT_PORT = 2222;
 var MIN_BACKOFF = 1e3;
 var MAX_BACKOFF = 1e3 * 30;
 function connect(app, logger2, options = {}) {
@@ -1091,13 +1150,12 @@ function connect(app, logger2, options = {}) {
         options.maxWorkflows
       );
     } else {
+      const port = app.server?.address().port;
       logger2.break();
-      logger2.warn("Workloop not starting");
+      logger2.warn("Noloop active: workloop has not started");
       logger2.info("This server will not auto-pull work from lightning.");
       logger2.info("You can manually claim by posting to /claim, eg:");
-      logger2.info(
-        `  curl -X POST http://locahost:${options.port || DEFAULT_PORT}/claim`
-      );
+      logger2.info(`  curl -X POST http://localhost:${port}/claim`);
       logger2.break();
     }
   };
@@ -1121,7 +1179,13 @@ function connect(app, logger2, options = {}) {
     );
     logger2.debug(e);
   };
-  worker_queue_default(options.lightning, app.id, options.secret, logger2).on("connect", onConnect).on("disconnect", onDisconnect).on("error", onError);
+  worker_queue_default(
+    options.lightning,
+    app.id,
+    options.secret,
+    options.socketTimeoutSeconds,
+    logger2
+  ).on("connect", onConnect).on("disconnect", onDisconnect).on("error", onError);
 }
 function createServer(engine, options = {}) {
   const logger2 = options.logger || createMockLogger2();
@@ -1144,16 +1208,19 @@ function createServer(engine, options = {}) {
   process.send?.("READY");
   router.get("/livez", healthcheck_default);
   router.get("/", healthcheck_default);
-  app.options = options || {};
+  app.options = options;
   app.execute = async ({ id, token }) => {
     if (app.socket) {
       app.workflows[id] = true;
       const {
         channel: runChannel,
         plan,
-        options: options2,
+        options: options2 = {},
         input
       } = await run_default(app.socket, token, id, logger2);
+      if (!("payloadLimitMb" in options2)) {
+        options2.payloadLimitMb = app.options.payloadLimitMb;
+      }
       const onFinish = () => {
         logger2.debug(`workflow ${id} complete: releasing worker`);
         delete app.workflows[id];
@@ -6066,6 +6133,9 @@ var Yargs = YargsFactory(esm_default);
 var yargs_default = Yargs;
 
 // src/util/cli.ts
+var DEFAULT_PORT2 = 2222;
+var DEFAULT_WORKER_CAPACITY = 5;
+var DEFAULT_SOCKET_TIMEOUT_SECONDS = 10;
 function setArg(argValue, envValue, defaultValue) {
   if (Array.isArray(defaultValue) && !argValue && typeof envValue === "string") {
     return envValue.split(",");
@@ -6082,16 +6152,18 @@ function parseArgs(argv) {
     WORKER_LIGHTNING_PUBLIC_KEY,
     WORKER_LIGHTNING_SERVICE_URL,
     WORKER_LOG_LEVEL,
+    WORKER_MAX_PAYLOAD_MB,
     WORKER_MAX_RUN_DURATION_SECONDS,
     WORKER_MAX_RUN_MEMORY_MB,
     WORKER_PORT,
     WORKER_REPO_DIR,
     WORKER_SECRET,
-    WORKER_STATE_PROPS_TO_REMOVE
+    WORKER_STATE_PROPS_TO_REMOVE,
+    WORKER_SOCKET_TIMEOUT_SECONDS
   } = process.env;
   const parser2 = yargs_default(hideBin(argv)).command("server", "Start a ws-worker server").option("port", {
     alias: "p",
-    description: "Port to run the server on. Env: WORKER_PORT",
+    description: `Port to run the server on. Default ${DEFAULT_PORT2}. Env: WORKER_PORT`,
     type: "number"
   }).option("lightning", {
     alias: ["l", "lightning-service-url"],
@@ -6102,6 +6174,8 @@ function parseArgs(argv) {
   }).option("secret", {
     alias: "s",
     description: "Worker secret. (comes from WORKER_SECRET by default). Env: WORKER_SECRET"
+  }).option("socket-timeout", {
+    description: `Timeout for websockets to Lighting, in seconds. Defaults to 10.`
   }).option("lightning-public-key", {
     description: "Base64-encoded public key. Used to verify run tokens. Env: WORKER_LIGHTNING_PUBLIC_KEY"
   }).option("log", {
@@ -6117,7 +6191,7 @@ function parseArgs(argv) {
   }).option("backoff", {
     description: "Claim backoff rules: min/max (in seconds). Env: WORKER_BACKOFF"
   }).option("capacity", {
-    description: "max concurrent workers. Env: WORKER_CAPACITY",
+    description: `max concurrent workers. Default ${DEFAULT_WORKER_CAPACITY}. Env: WORKER_CAPACITY`,
     type: "number"
   }).option("state-props-to-remove", {
     description: "A list of properties to remove from the final state returned by a job. Env: WORKER_STATE_PROPS_TO_REMOVE",
@@ -6125,6 +6199,9 @@ function parseArgs(argv) {
   }).option("run-memory", {
     description: "Maximum memory allocated to a single run, in mb. Env: WORKER_MAX_RUN_MEMORY_MB",
     type: "number"
+  }).option("payload-memory", {
+    description: "Maximum memory allocated to a single run, in mb. Env: WORKER_MAX_PAYLOAD_MB",
+    type: "number"
   }).option("max-run-duration-seconds", {
     alias: "t",
     description: "Default run timeout for the server, in seconds. Env: WORKER_MAX_RUN_DURATION_SECONDS",
@@ -6133,7 +6210,7 @@ function parseArgs(argv) {
   const args2 = parser2.parse();
   return {
     ...args2,
-    port: setArg(args2.port, WORKER_PORT, 2222),
+    port: setArg(args2.port, WORKER_PORT, DEFAULT_PORT2),
     lightning: setArg(
       args2.lightning,
       WORKER_LIGHTNING_SERVICE_URL,
@@ -6147,17 +6224,23 @@ function parseArgs(argv) {
     ),
     log: setArg(args2.log, WORKER_LOG_LEVEL, "debug"),
     backoff: setArg(args2.backoff, WORKER_BACKOFF, "1/10"),
-    capacity: setArg(args2.capacity, WORKER_CAPACITY, 5),
+    capacity: setArg(args2.capacity, WORKER_CAPACITY, DEFAULT_WORKER_CAPACITY),
     statePropsToRemove: setArg(
       args2.statePropsToRemove,
       WORKER_STATE_PROPS_TO_REMOVE,
       ["configuration", "response"]
     ),
     runMemory: setArg(args2.runMemory, WORKER_MAX_RUN_MEMORY_MB, 500),
+    payloadMemory: setArg(args2.payloadMemory, WORKER_MAX_PAYLOAD_MB, 10),
     maxRunDurationSeconds: setArg(
       args2.maxRunDurationSeconds,
       WORKER_MAX_RUN_DURATION_SECONDS,
       300
+    ),
+    socketTimeoutSeconds: setArg(
+      args2.socketTimeoutSeconds,
+      WORKER_SOCKET_TIMEOUT_SECONDS,
+      DEFAULT_SOCKET_TIMEOUT_SECONDS
     )
   };
 }
@@ -6187,7 +6270,8 @@ function engineReady(engine) {
       min: minBackoff,
       max: maxBackoff
     },
-    maxWorkflows: args.capacity
+    maxWorkflows: args.capacity,
+    payloadLimitMb: args.payloadMemory
   };
   if (args.lightningPublicKey) {
     logger.info(

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openfn/ws-worker",
-  "version": "1.4.1",
+  "version": "1.5.1",
   "description": "A Websocket Worker to connect Lightning to a Runtime Engine",
   "main": "dist/index.js",
   "type": "module",
@@ -24,8 +24,8 @@
     "ws": "^8.14.1",
     "@openfn/engine-multi": "1.2.1",
     "@openfn/lexicon": "^1.0.2",
-    "@openfn/runtime": "1.4.1",
-    "@openfn/logger": "1.0.1"
+    "@openfn/logger": "1.0.1",
+    "@openfn/runtime": "1.4.1"
   },
   "devDependencies": {
     "@types/koa": "^2.13.5",