@openfn/ws-worker 1.11.1 → 1.12.1

package/CHANGELOG.md

@@ -1,5 +1,25 @@
 # ws-worker
 
+## 1.12.1
+
+### Patch Changes
+
+- e2f1197: Better logging on credential errors
+- Updated dependencies [e2f1197]
+  - @openfn/engine-multi@1.6.1
+
+## 1.12.0
+
+### Minor Changes
+
+- d50c05d: Fix an issue where large payloads can cause the worker to OOM crash
+
+### Patch Changes
+
+- Updated dependencies [deb7293]
+- Updated dependencies [d50c05d]
+  - @openfn/engine-multi@1.6.0
+
 ## 1.11.1
 
 ### Patch Changes

package/dist/index.js

@@ -598,22 +598,6 @@ var calculateRunExitReason = (state) => {
   return { reason: "success", error_type: null, error_message: null };
 };
 
-// src/util/ensure-payload-size.ts
-var ensure_payload_size_default = (payload, limit_mb) => {
-  if (!isNaN(limit_mb)) {
-    const limit = limit_mb;
-    const size_bytes = Buffer.byteLength(payload, "utf8");
-    const size_mb = size_bytes / 1024 / 1024;
-    if (size_mb > limit) {
-      const e = new Error();
-      e.severity = "kill";
-      e.name = "PAYLOAD_TOO_LARGE";
-      e.message = `The payload exceeded the size limit of ${limit}mb`;
-      throw e;
-    }
-  }
-};
-
 // src/events/step-complete.ts
 async function onStepComplete(context, event, error) {
   const { channel, state, options } = context;
@@ -639,25 +623,24 @@ async function onStepComplete(context, event, error) {
     thread_id: event.threadId,
     timestamp: timeInMicroseconds(event.time)
   };
-  try {
-    if (!options || options.outputDataclips !== false) {
-      const payload = stringify_default(outputState);
-      ensure_payload_size_default(payload, options?.payloadLimitMb);
-      evt.output_dataclip = payload;
-    }
-    evt.output_dataclip_id = dataclipId;
-  } catch (e) {
+  if (event.redacted) {
     state.withheldDataclips[dataclipId] = true;
     evt.output_dataclip_error = "DATACLIP_TOO_LARGE";
     const time = (timestamp2() - BigInt(1e7)).toString();
     await onJobLog(context, {
       time,
       message: [
-        "Dataclip too large. This dataclip will not be sent back to lighting."
+        "Dataclip exceeds payload limit: output will not be sent back to the app."
       ],
       level: "info",
       name: "R/T"
     });
+  } else {
+    evt.output_dataclip_id = dataclipId;
+    if (!options || options.outputDataclips !== false) {
+      const payload = stringify_default(outputState);
+      evt.output_dataclip = payload;
+    }
   }
   const reason = calculateJobExitReason(job_id, event.state, error);
   state.reasons[job_id] = reason;
@@ -843,24 +826,19 @@ function onJobError(context, event) {
 }
 function onJobLog({ channel, state, options }, event) {
   let message = event.message;
-  try {
-    if (typeof event.message === "string") {
-      ensure_payload_size_default(event.message, options?.payloadLimitMb);
-      message = JSON.parse(message);
-    } else if (event.message) {
-      const payload = stringify_default(event.message);
-      ensure_payload_size_default(payload, options?.payloadLimitMb);
-    }
-  } catch (e) {
+  if (event.redacted) {
     message = [
       `(Log message redacted: exceeds ${options.payloadLimitMb}mb memory limit)`
     ];
+  } else if (typeof event.message === "string") {
+    message = JSON.parse(event.message);
   }
   const log = {
     run_id: state.plan.id,
     message,
     source: event.name,
     level: event.level,
+    // @ts-ignore
     timestamp: timeInMicroseconds(event.time)
   };
   if (state.activeStep) {

package/dist/start.js

@@ -738,22 +738,6 @@ var calculateRunExitReason = (state) => {
   return { reason: "success", error_type: null, error_message: null };
 };
 
-// src/util/ensure-payload-size.ts
-var ensure_payload_size_default = (payload, limit_mb) => {
-  if (!isNaN(limit_mb)) {
-    const limit = limit_mb;
-    const size_bytes = Buffer.byteLength(payload, "utf8");
-    const size_mb = size_bytes / 1024 / 1024;
-    if (size_mb > limit) {
-      const e = new Error();
-      e.severity = "kill";
-      e.name = "PAYLOAD_TOO_LARGE";
-      e.message = `The payload exceeded the size limit of ${limit}mb`;
-      throw e;
-    }
-  }
-};
-
 // src/events/step-complete.ts
 async function onStepComplete(context, event, error) {
   const { channel, state, options } = context;
@@ -779,25 +763,24 @@ async function onStepComplete(context, event, error) {
     thread_id: event.threadId,
     timestamp: timeInMicroseconds(event.time)
   };
-  try {
-    if (!options || options.outputDataclips !== false) {
-      const payload = stringify_default(outputState);
-      ensure_payload_size_default(payload, options?.payloadLimitMb);
-      evt.output_dataclip = payload;
-    }
-    evt.output_dataclip_id = dataclipId;
-  } catch (e) {
+  if (event.redacted) {
     state.withheldDataclips[dataclipId] = true;
     evt.output_dataclip_error = "DATACLIP_TOO_LARGE";
     const time = (timestamp2() - BigInt(1e7)).toString();
     await onJobLog(context, {
       time,
       message: [
-        "Dataclip too large. This dataclip will not be sent back to lighting."
+        "Dataclip exceeds payload limit: output will not be sent back to the app."
       ],
       level: "info",
       name: "R/T"
     });
+  } else {
+    evt.output_dataclip_id = dataclipId;
+    if (!options || options.outputDataclips !== false) {
+      const payload = stringify_default(outputState);
+      evt.output_dataclip = payload;
+    }
   }
   const reason = calculateJobExitReason(job_id, event.state, error);
   state.reasons[job_id] = reason;
@@ -983,24 +966,19 @@ function onJobError(context, event) {
 }
 function onJobLog({ channel, state, options }, event) {
   let message = event.message;
-  try {
-    if (typeof event.message === "string") {
-      ensure_payload_size_default(event.message, options?.payloadLimitMb);
-      message = JSON.parse(message);
-    } else if (event.message) {
-      const payload = stringify_default(event.message);
-      ensure_payload_size_default(payload, options?.payloadLimitMb);
-    }
-  } catch (e) {
+  if (event.redacted) {
     message = [
       `(Log message redacted: exceeds ${options.payloadLimitMb}mb memory limit)`
     ];
+  } else if (typeof event.message === "string") {
+    message = JSON.parse(event.message);
   }
   const log = {
     run_id: state.plan.id,
     message,
     source: event.name,
     level: event.level,
+    // @ts-ignore
     timestamp: timeInMicroseconds(event.time)
   };
   if (state.activeStep) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openfn/ws-worker",
-  "version": "1.11.1",
+  "version": "1.12.1",
   "description": "A Websocket Worker to connect Lightning to a Runtime Engine",
   "main": "dist/index.js",
   "type": "module",
@@ -22,10 +22,10 @@
     "koa-logger": "^3.2.1",
     "phoenix": "1.7.10",
     "ws": "^8.18.0",
-    "@openfn/engine-multi": "1.5.1",
-    "@openfn/lexicon": "^1.2.0",
+    "@openfn/engine-multi": "1.6.1",
+    "@openfn/logger": "1.0.4",
     "@openfn/runtime": "1.6.3",
-    "@openfn/logger": "1.0.4"
+    "@openfn/lexicon": "^1.2.0"
   },
   "devDependencies": {
     "@types/koa": "^2.13.5",
@@ -41,7 +41,7 @@
     "tsup": "^6.2.3",
     "typescript": "^4.6.4",
     "yargs": "^17.6.2",
-    "@openfn/lightning-mock": "2.1.1"
+    "@openfn/lightning-mock": "2.1.3"
   },
   "files": [
     "dist",