@openfn/ws-worker 1.4.1 → 1.5.0

package/CHANGELOG.md

@@ -1,5 +1,11 @@
 # ws-worker
 
+## 1.5.0
+
+### Minor Changes
+
+- f363254: Allow a payload limit to be set for large dataclips and logs (payload_limit_mb)
+
 ## 1.4.1
 
 ### Patch Changes

package/dist/index.d.ts

@@ -41,6 +41,7 @@ interface Channel extends Channel$1 {
 
 declare type WorkerRunOptions = ExecuteOptions & {
     outputDataclips?: boolean;
+    payloadLimitMb?: number;
 };
 
 declare type Context = {
@@ -64,6 +65,7 @@ declare type ServerOptions = {
         min?: number;
         max?: number;
     };
+    payloadLimitMb?: number;
 };
 interface ServerApp extends Koa {
     id: string;

package/dist/index.js

@@ -29,7 +29,7 @@ var name, version, description, main, type, scripts, bin, author, license, depen
 var init_package = __esm({
   "package.json"() {
     name = "@openfn/ws-worker";
-    version = "1.4.1";
+    version = "1.5.0";
     description = "A Websocket Worker to connect Lightning to a Runtime Engine";
     main = "dist/index.js";
     type = "module";
@@ -332,13 +332,19 @@ var convert_lightning_plan_default = (run) => {
   const runtimeOpts = {};
   const engineOpts = {};
   if (run.options) {
-    if (run.options.run_timeout_ms) {
+    if ("run_timeout_ms" in run.options) {
       engineOpts.runTimeoutMs = run.options.run_timeout_ms;
     }
-    if (run.options.sanitize) {
+    if ("payload_limit_mb" in run.options) {
+      engineOpts.payloadLimitMb = run.options.payload_limit_mb;
+    }
+    if ("run_memory_limit_mb" in run.options) {
+      engineOpts.memoryLimitMb = run.options.run_memory_limit_mb;
+    }
+    if ("sanitize" in run.options) {
       engineOpts.sanitize = run.options.sanitize;
     }
-    if (run.options.hasOwnProperty("output_dataclips")) {
+    if ("output_dataclips" in run.options) {
       engineOpts.outputDataclips = run.options.output_dataclips;
     }
   }
@@ -527,7 +533,7 @@ ${prefix("worker")}${versions.worker || "unknown"}`;
 // src/events/run-start.ts
 init_package();
 async function onRunStart(context, event) {
-  const { channel, state } = context;
+  const { channel, state, options = {} } = context;
   const time = (timestamp() - BigInt(1e7)).toString();
   const versionLogContext = {
     ...context,
@@ -541,6 +547,14 @@ async function onRunStart(context, event) {
     ...event.versions
   };
   await sendEvent(channel, RUN_START, { versions });
+  if ("payloadLimitMb" in options) {
+    await onJobLog(versionLogContext, {
+      time,
+      message: [`Payload limit: ${options.payloadLimitMb}mb`],
+      level: "info",
+      name: "RTE"
+    });
+  }
   const versionMessage = versions_default(versions);
   await onJobLog(versionLogContext, {
     time,
@@ -552,6 +566,7 @@ async function onRunStart(context, event) {
 
 // src/events/step-complete.ts
 import crypto3 from "node:crypto";
+import { timestamp as timestamp2 } from "@openfn/logger";
 
 // src/api/reasons.ts
 var calculateJobExitReason = (jobId, state = { data: {} }, error) => {
@@ -586,8 +601,25 @@ var calculateRunExitReason = (state) => {
   return { reason: "success", error_type: null, error_message: null };
 };
 
+// src/util/ensure-payload-size.ts
+var ensure_payload_size_default = (payload, limit_mb) => {
+  if (!isNaN(limit_mb)) {
+    const limit = limit_mb;
+    const size_bytes = Buffer.byteLength(payload, "utf8");
+    const size_mb = size_bytes / 1024 / 1024;
+    if (size_mb > limit) {
+      const e = new Error();
+      e.severity = "kill";
+      e.name = "PAYLOAD_TOO_LARGE";
+      e.message = `The payload exceeded the size limit of ${limit}mb`;
+      throw e;
+    }
+  }
+};
+
 // src/events/step-complete.ts
-function onStepComplete({ channel, state, options }, event, error) {
+async function onStepComplete(context, event, error) {
+  const { channel, state, options } = context;
   const dataclipId = crypto3.randomUUID();
   const step_id = state.activeStep;
   const job_id = state.activeJob;
@@ -602,26 +634,35 @@ function onStepComplete({ channel, state, options }, event, error) {
   event.next?.forEach((nextJobId) => {
     state.inputDataclips[nextJobId] = dataclipId;
   });
-  const { reason, error_message, error_type } = calculateJobExitReason(
-    job_id,
-    event.state,
-    error
-  );
-  state.reasons[job_id] = { reason, error_message, error_type };
   const evt = {
     step_id,
     job_id,
-    output_dataclip_id: dataclipId,
-    reason,
-    error_message,
-    error_type,
     mem: event.mem,
     duration: event.duration,
     thread_id: event.threadId
   };
-  if (!options || options.outputDataclips !== false) {
-    evt.output_dataclip = stringify_default(outputState);
+  try {
+    if (!options || options.outputDataclips !== false) {
+      const payload = stringify_default(outputState);
+      ensure_payload_size_default(payload, options?.payloadLimitMb);
+      evt.output_dataclip = payload;
+    }
+    evt.output_dataclip_id = dataclipId;
+  } catch (e) {
+    evt.output_dataclip_error = "DATACLIP_TOO_LARGE";
+    const time = (timestamp2() - BigInt(1e7)).toString();
+    await onJobLog(context, {
+      time,
+      message: [
+        "Dataclip too large. This dataclip will not be sent back to lighting."
+      ],
+      level: "info",
+      name: "R/T"
+    });
   }
+  const reason = calculateJobExitReason(job_id, event.state, error);
+  state.reasons[job_id] = reason;
+  Object.assign(evt, reason);
   return sendEvent(channel, STEP_COMPLETE, evt);
 }
 
@@ -640,9 +681,9 @@ async function onStepStart(context, event) {
 }
 
 // src/util/log-final-reason.ts
-import { timestamp as timestamp2 } from "@openfn/logger";
+import { timestamp as timestamp3 } from "@openfn/logger";
 var log_final_reason_default = async (context, reason) => {
-  const time = (timestamp2() - BigInt(1e7)).toString();
+  const time = (timestamp3() - BigInt(1e7)).toString();
   let message = `Run complete with status: ${reason.reason}`;
   if (reason.reason !== "success") {
     message += `
@@ -792,11 +833,25 @@ function onJobError(context, event) {
     return onStepComplete(context, event, event.error);
   }
 }
-function onJobLog({ channel, state }, event) {
+function onJobLog({ channel, state, options }, event) {
   const timeInMicroseconds = BigInt(event.time) / BigInt(1e3);
+  let message = event.message;
+  try {
+    if (typeof event.message === "string") {
+      ensure_payload_size_default(event.message, options?.payloadLimitMb);
+      message = JSON.parse(message);
+    } else if (event.message) {
+      const payload = stringify_default(event.message);
+      ensure_payload_size_default(payload, options?.payloadLimitMb);
+    }
+  } catch (e) {
+    message = [
+      `(Log message redacted: exceeds ${options.payloadLimitMb}mb memory limit)`
+    ];
+  }
   const log = {
     run_id: state.plan.id,
-    message: typeof event.message === "string" ? JSON.parse(event.message) : event.message,
+    message,
     source: event.name,
     level: event.level,
     timestamp: timeInMicroseconds.toString()
@@ -1005,16 +1060,19 @@ function createServer(engine, options = {}) {
   process.send?.("READY");
   router.get("/livez", healthcheck_default);
   router.get("/", healthcheck_default);
-  app.options = options || {};
+  app.options = options;
   app.execute = async ({ id, token }) => {
     if (app.socket) {
       app.workflows[id] = true;
       const {
         channel: runChannel,
         plan,
-        options: options2,
+        options: options2 = {},
         input
       } = await run_default(app.socket, token, id, logger);
+      if (!("payloadLimitMb" in options2)) {
+        options2.payloadLimitMb = app.options.payloadLimitMb;
+      }
       const onFinish = () => {
         logger.debug(`workflow ${id} complete: releasing worker`);
         delete app.workflows[id];

package/dist/start.js

@@ -37,7 +37,7 @@ var name, version, description, main, type, scripts, bin, author, license, depen
 var init_package = __esm({
   "package.json"() {
     name = "@openfn/ws-worker";
-    version = "1.4.1";
+    version = "1.5.0";
     description = "A Websocket Worker to connect Lightning to a Runtime Engine";
     main = "dist/index.js";
     type = "module";
@@ -471,13 +471,19 @@ var convert_lightning_plan_default = (run2) => {
   const runtimeOpts = {};
   const engineOpts = {};
   if (run2.options) {
-    if (run2.options.run_timeout_ms) {
+    if ("run_timeout_ms" in run2.options) {
       engineOpts.runTimeoutMs = run2.options.run_timeout_ms;
     }
-    if (run2.options.sanitize) {
+    if ("payload_limit_mb" in run2.options) {
+      engineOpts.payloadLimitMb = run2.options.payload_limit_mb;
+    }
+    if ("run_memory_limit_mb" in run2.options) {
+      engineOpts.memoryLimitMb = run2.options.run_memory_limit_mb;
+    }
+    if ("sanitize" in run2.options) {
       engineOpts.sanitize = run2.options.sanitize;
     }
-    if (run2.options.hasOwnProperty("output_dataclips")) {
+    if ("output_dataclips" in run2.options) {
       engineOpts.outputDataclips = run2.options.output_dataclips;
     }
   }
@@ -666,7 +672,7 @@ ${prefix("worker")}${versions.worker || "unknown"}`;
 // src/events/run-start.ts
 init_package();
 async function onRunStart(context, event) {
-  const { channel, state } = context;
+  const { channel, state, options = {} } = context;
   const time = (timestamp() - BigInt(1e7)).toString();
   const versionLogContext = {
     ...context,
@@ -680,6 +686,14 @@ async function onRunStart(context, event) {
     ...event.versions
   };
   await sendEvent(channel, RUN_START, { versions });
+  if ("payloadLimitMb" in options) {
+    await onJobLog(versionLogContext, {
+      time,
+      message: [`Payload limit: ${options.payloadLimitMb}mb`],
+      level: "info",
+      name: "RTE"
+    });
+  }
   const versionMessage = versions_default(versions);
   await onJobLog(versionLogContext, {
     time,
@@ -691,6 +705,7 @@ async function onRunStart(context, event) {
 
 // src/events/step-complete.ts
 import crypto4 from "node:crypto";
+import { timestamp as timestamp2 } from "@openfn/logger";
 
 // src/api/reasons.ts
 var calculateJobExitReason = (jobId, state = { data: {} }, error) => {
@@ -725,8 +740,25 @@ var calculateRunExitReason = (state) => {
   return { reason: "success", error_type: null, error_message: null };
 };
 
+// src/util/ensure-payload-size.ts
+var ensure_payload_size_default = (payload, limit_mb) => {
+  if (!isNaN(limit_mb)) {
+    const limit = limit_mb;
+    const size_bytes = Buffer.byteLength(payload, "utf8");
+    const size_mb = size_bytes / 1024 / 1024;
+    if (size_mb > limit) {
+      const e = new Error();
+      e.severity = "kill";
+      e.name = "PAYLOAD_TOO_LARGE";
+      e.message = `The payload exceeded the size limit of ${limit}mb`;
+      throw e;
+    }
+  }
+};
+
 // src/events/step-complete.ts
-function onStepComplete({ channel, state, options }, event, error) {
+async function onStepComplete(context, event, error) {
+  const { channel, state, options } = context;
   const dataclipId = crypto4.randomUUID();
   const step_id = state.activeStep;
   const job_id = state.activeJob;
@@ -741,26 +773,35 @@ function onStepComplete({ channel, state, options }, event, error) {
   event.next?.forEach((nextJobId) => {
     state.inputDataclips[nextJobId] = dataclipId;
   });
-  const { reason, error_message, error_type } = calculateJobExitReason(
-    job_id,
-    event.state,
-    error
-  );
-  state.reasons[job_id] = { reason, error_message, error_type };
   const evt = {
     step_id,
     job_id,
-    output_dataclip_id: dataclipId,
-    reason,
-    error_message,
-    error_type,
     mem: event.mem,
     duration: event.duration,
     thread_id: event.threadId
   };
-  if (!options || options.outputDataclips !== false) {
-    evt.output_dataclip = stringify_default(outputState);
+  try {
+    if (!options || options.outputDataclips !== false) {
+      const payload = stringify_default(outputState);
+      ensure_payload_size_default(payload, options?.payloadLimitMb);
+      evt.output_dataclip = payload;
+    }
+    evt.output_dataclip_id = dataclipId;
+  } catch (e) {
+    evt.output_dataclip_error = "DATACLIP_TOO_LARGE";
+    const time = (timestamp2() - BigInt(1e7)).toString();
+    await onJobLog(context, {
+      time,
+      message: [
+        "Dataclip too large. This dataclip will not be sent back to lighting."
+      ],
+      level: "info",
+      name: "R/T"
+    });
   }
+  const reason = calculateJobExitReason(job_id, event.state, error);
+  state.reasons[job_id] = reason;
+  Object.assign(evt, reason);
   return sendEvent(channel, STEP_COMPLETE, evt);
 }
 
@@ -779,9 +820,9 @@ async function onStepStart(context, event) {
 }
 
 // src/util/log-final-reason.ts
-import { timestamp as timestamp2 } from "@openfn/logger";
+import { timestamp as timestamp3 } from "@openfn/logger";
 var log_final_reason_default = async (context, reason) => {
-  const time = (timestamp2() - BigInt(1e7)).toString();
+  const time = (timestamp3() - BigInt(1e7)).toString();
   let message = `Run complete with status: ${reason.reason}`;
   if (reason.reason !== "success") {
     message += `
@@ -931,11 +972,25 @@ function onJobError(context, event) {
     return onStepComplete(context, event, event.error);
   }
 }
-function onJobLog({ channel, state }, event) {
+function onJobLog({ channel, state, options }, event) {
   const timeInMicroseconds = BigInt(event.time) / BigInt(1e3);
+  let message = event.message;
+  try {
+    if (typeof event.message === "string") {
+      ensure_payload_size_default(event.message, options?.payloadLimitMb);
+      message = JSON.parse(message);
+    } else if (event.message) {
+      const payload = stringify_default(event.message);
+      ensure_payload_size_default(payload, options?.payloadLimitMb);
+    }
+  } catch (e) {
+    message = [
+      `(Log message redacted: exceeds ${options.payloadLimitMb}mb memory limit)`
+    ];
+  }
   const log = {
     run_id: state.plan.id,
-    message: typeof event.message === "string" ? JSON.parse(event.message) : event.message,
+    message,
     source: event.name,
     level: event.level,
     timestamp: timeInMicroseconds.toString()
@@ -1144,16 +1199,19 @@ function createServer(engine, options = {}) {
   process.send?.("READY");
   router.get("/livez", healthcheck_default);
   router.get("/", healthcheck_default);
-  app.options = options || {};
+  app.options = options;
   app.execute = async ({ id, token }) => {
     if (app.socket) {
       app.workflows[id] = true;
       const {
         channel: runChannel,
         plan,
-        options: options2,
+        options: options2 = {},
         input
       } = await run_default(app.socket, token, id, logger2);
+      if (!("payloadLimitMb" in options2)) {
+        options2.payloadLimitMb = app.options.payloadLimitMb;
+      }
       const onFinish = () => {
         logger2.debug(`workflow ${id} complete: releasing worker`);
         delete app.workflows[id];
@@ -6082,6 +6140,7 @@ function parseArgs(argv) {
     WORKER_LIGHTNING_PUBLIC_KEY,
     WORKER_LIGHTNING_SERVICE_URL,
     WORKER_LOG_LEVEL,
+    WORKER_MAX_PAYLOAD_MB,
     WORKER_MAX_RUN_DURATION_SECONDS,
     WORKER_MAX_RUN_MEMORY_MB,
     WORKER_PORT,
@@ -6125,6 +6184,9 @@ function parseArgs(argv) {
   }).option("run-memory", {
     description: "Maximum memory allocated to a single run, in mb. Env: WORKER_MAX_RUN_MEMORY_MB",
     type: "number"
+  }).option("payload-memory", {
+    description: "Maximum memory allocated to a single run, in mb. Env: WORKER_MAX_PAYLOAD_MB",
+    type: "number"
   }).option("max-run-duration-seconds", {
     alias: "t",
     description: "Default run timeout for the server, in seconds. Env: WORKER_MAX_RUN_DURATION_SECONDS",
@@ -6154,6 +6216,7 @@ function parseArgs(argv) {
       ["configuration", "response"]
     ),
     runMemory: setArg(args2.runMemory, WORKER_MAX_RUN_MEMORY_MB, 500),
+    payloadMemory: setArg(args2.payloadMemory, WORKER_MAX_PAYLOAD_MB, 10),
     maxRunDurationSeconds: setArg(
       args2.maxRunDurationSeconds,
       WORKER_MAX_RUN_DURATION_SECONDS,
@@ -6187,7 +6250,8 @@ function engineReady(engine) {
       min: minBackoff,
       max: maxBackoff
     },
-    maxWorkflows: args.capacity
+    maxWorkflows: args.capacity,
+    payloadLimitMb: args.payloadMemory
   };
   if (args.lightningPublicKey) {
     logger.info(

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openfn/ws-worker",
-  "version": "1.4.1",
+  "version": "1.5.0",
   "description": "A Websocket Worker to connect Lightning to a Runtime Engine",
   "main": "dist/index.js",
   "type": "module",
@@ -23,9 +23,9 @@
     "phoenix": "1.7.10",
     "ws": "^8.14.1",
     "@openfn/engine-multi": "1.2.1",
+    "@openfn/logger": "1.0.1",
     "@openfn/lexicon": "^1.0.2",
-    "@openfn/runtime": "1.4.1",
-    "@openfn/logger": "1.0.1"
+    "@openfn/runtime": "1.4.1"
   },
   "devDependencies": {
     "@types/koa": "^2.13.5",