@openfn/ws-worker 0.2.10 → 0.2.12

package/CHANGELOG.md

@@ -1,5 +1,25 @@
 # ws-worker
 
+## 0.2.12
+
+### Patch Changes
+
+- 6c3e9e42: Ensure capacity is also set on the engine
+- Updated dependencies [05ccc10b]
+- Updated dependencies [7235bf5e]
+  - @openfn/engine-multi@0.2.3
+
+## 0.2.11
+
+### Patch Changes
+
+- 22339c6: Add MAX_RUN_MEMORY env var and option to limit the memory available to each run
+- 04ac3cc: Include duration and threadid in run-complete
+- 340b96e: Send memory usage to lightning on run:complete
+- Updated dependencies
+  - @openfn/engine-multi@0.2.2
+  - @openfn/runtime@0.2.1
+
 ## 0.2.10
 
 ### Patch Changes

package/dist/index.js

@@ -409,7 +409,7 @@ function onJobStart({ channel, state }, event) {
   });
 }
 function onJobError(context, event) {
-  const { state, error, jobId } = event;
+  const { state = {}, error, jobId } = event;
   if (state.errors?.[jobId]?.message === error.message) {
     onJobComplete(context, event);
   } else {
@@ -424,12 +424,12 @@ function onJobComplete({ channel, state }, event, error) {
     state.dataclips = {};
   }
   state.dataclips[dataclipId] = event.state;
+  delete state.activeRun;
+  delete state.activeJob;
   state.lastDataclipId = dataclipId;
   event.next?.forEach((nextJobId) => {
     state.inputDataclips[nextJobId] = dataclipId;
   });
-  delete state.activeRun;
-  delete state.activeJob;
   const { reason, error_message, error_type } = calculateJobExitReason(
     job_id,
     event.state,
@@ -443,7 +443,10 @@ function onJobComplete({ channel, state }, event, error) {
     output_dataclip: stringify_default(event.state),
     reason,
     error_message,
-    error_type
+    error_type,
+    mem: event.mem,
+    duration: event.duration,
+    thread_id: event.threadId
   };
   return sendEvent(channel, RUN_COMPLETE, evt);
 }
@@ -459,9 +462,13 @@ async function onWorkflowComplete({ state, channel, onFinish }, _event) {
   });
   onFinish({ reason, state: result });
 }
-async function onWorkflowError({ state, channel, logger, onFinish }, event) {
+async function onWorkflowError(context, event) {
+  const { state, channel, logger, onFinish } = context;
   try {
     const reason = calculateJobExitReason("", { data: {} }, event);
+    if (state.activeJob) {
+      await onJobError(context, { error: event });
+    }
     await sendEvent(channel, ATTEMPT_COMPLETE, {
       final_dataclip_id: state.lastDataclipId,
       ...reason
@@ -470,6 +477,7 @@ async function onWorkflowError({ state, channel, logger, onFinish }, event) {
   } catch (e) {
     logger.error("ERROR in workflow-error handler:", e.message);
     logger.error(e);
+    onFinish({});
   }
 }
 function onJobLog({ channel, state }, event) {

package/dist/start.js

@@ -4865,6 +4865,7 @@ import createRTE from "@openfn/engine-multi";
 
 // src/mock/runtime-engine.ts
 import { EventEmitter } from "node:events";
+import crypto from "node:crypto";
 import run from "@openfn/runtime";
 
 // src/mock/resolvers.ts
@@ -4911,6 +4912,7 @@ async function createMock() {
   }) => {
     const { id, jobs } = xplan;
     activeWorkflows[id] = true;
+    const threadId = crypto.randomUUID();
     for (const job of jobs) {
       if (typeof job.configuration === "string") {
         job.configuration = await options.resolvers?.credential?.(
@@ -4925,6 +4927,7 @@ async function createMock() {
       log: (...args2) => {
         dispatch("workflow-log", {
           workflowId: id,
+          threadId,
           level: "info",
           json: true,
           message: args2,
@@ -4941,24 +4944,27 @@ async function createMock() {
         notify: (name, payload) => {
           dispatch(name, {
             workflowId: id,
+            threadId,
             ...payload
           });
         }
       }
     };
     setTimeout(async () => {
-      dispatch("workflow-start", { workflowId: id });
+      dispatch("workflow-start", { workflowId: id, threadId });
       try {
         await run(xplan, void 0, opts);
+        dispatch("workflow-complete", { workflowId: id, threadId });
       } catch (e) {
         dispatch("workflow-error", {
+          threadId,
           workflowId: id,
           type: e.name,
           message: e.message
         });
+      } finally {
+        delete activeWorkflows[id];
       }
-      delete activeWorkflows[id];
-      dispatch("workflow-complete", { workflowId: id });
     }, 1);
   };
   const getStatus = () => {
@@ -5146,10 +5152,10 @@ var startWorkloop = (app, logger2, minBackoff2, maxBackoff2, maxWorkers) => {
 var workloop_default = startWorkloop;
 
 // src/api/execute.ts
-import crypto2 from "node:crypto";
+import crypto3 from "node:crypto";
 
 // src/util/convert-attempt.ts
-import crypto from "node:crypto";
+import crypto2 from "node:crypto";
 var conditions = {
   on_job_success: (upstreamId) => `Boolean(!state.errors?.["${upstreamId}"] ?? true)`,
   on_job_failure: (upstreamId) => `Boolean(state.errors && state.errors["${upstreamId}"])`,
@@ -5196,7 +5202,7 @@ var convert_attempt_default = (attempt) => {
   }
   if (attempt.jobs?.length) {
     attempt.jobs.forEach((job) => {
-      const id = job.id || crypto.randomUUID();
+      const id = job.id || crypto2.randomUUID();
       nodes[id] = {
         id,
         configuration: job.credential_id,
@@ -5379,7 +5385,7 @@ var sendEvent = (channel, event, payload) => new Promise((resolve5, reject) => {
   channel.push(event, payload).receive("error", reject).receive("timeout", () => reject(new Error("timeout"))).receive("ok", resolve5);
 });
 function onJobStart({ channel, state }, event) {
-  state.activeRun = crypto2.randomUUID();
+  state.activeRun = crypto3.randomUUID();
   state.activeJob = event.jobId;
   const input_dataclip_id = state.inputDataclips[event.jobId];
   return sendEvent(channel, RUN_START, {
@@ -5389,7 +5395,7 @@ function onJobStart({ channel, state }, event) {
   });
 }
 function onJobError(context, event) {
-  const { state, error, jobId } = event;
+  const { state = {}, error, jobId } = event;
   if (state.errors?.[jobId]?.message === error.message) {
     onJobComplete(context, event);
   } else {
@@ -5397,19 +5403,19 @@ function onJobError(context, event) {
   }
 }
 function onJobComplete({ channel, state }, event, error) {
-  const dataclipId = crypto2.randomUUID();
+  const dataclipId = crypto3.randomUUID();
   const run_id = state.activeRun;
   const job_id = state.activeJob;
   if (!state.dataclips) {
     state.dataclips = {};
   }
   state.dataclips[dataclipId] = event.state;
+  delete state.activeRun;
+  delete state.activeJob;
   state.lastDataclipId = dataclipId;
   event.next?.forEach((nextJobId) => {
     state.inputDataclips[nextJobId] = dataclipId;
   });
-  delete state.activeRun;
-  delete state.activeJob;
   const { reason, error_message, error_type } = calculateJobExitReason(
     job_id,
     event.state,
@@ -5423,7 +5429,10 @@ function onJobComplete({ channel, state }, event, error) {
     output_dataclip: stringify_default(event.state),
     reason,
     error_message,
-    error_type
+    error_type,
+    mem: event.mem,
+    duration: event.duration,
+    thread_id: event.threadId
   };
   return sendEvent(channel, RUN_COMPLETE, evt);
 }
@@ -5439,9 +5448,13 @@ async function onWorkflowComplete({ state, channel, onFinish }, _event) {
   });
   onFinish({ reason, state: result });
 }
-async function onWorkflowError({ state, channel, logger: logger2, onFinish }, event) {
+async function onWorkflowError(context, event) {
+  const { state, channel, logger: logger2, onFinish } = context;
   try {
     const reason = calculateJobExitReason("", { data: {} }, event);
+    if (state.activeJob) {
+      await onJobError(context, { error: event });
+    }
     await sendEvent(channel, ATTEMPT_COMPLETE, {
       final_dataclip_id: state.lastDataclipId,
       ...reason
@@ -5450,6 +5463,7 @@ async function onWorkflowError({ state, channel, logger: logger2, onFinish }, ev
   } catch (e) {
     logger2.error("ERROR in workflow-error handler:", e.message);
     logger2.error(e);
+    onFinish({});
   }
 }
 function onJobLog({ channel, state }, event) {
@@ -5707,7 +5721,7 @@ function createServer(engine, options = {}) {
 var server_default = createServer;
 
 // src/start.ts
-var { WORKER_REPO_DIR, WORKER_SECRET } = process.env;
+var { WORKER_REPO_DIR, WORKER_SECRET, MAX_RUN_MEMORY } = process.env;
 var args = yargs_default(hideBin(process.argv)).command("server", "Start a ws-worker server").option("port", {
   alias: "p",
   description: "Port to run the server on",
@@ -5743,6 +5757,10 @@ var args = yargs_default(hideBin(process.argv)).command("server", "Start a ws-wo
   description: "max concurrent workers",
   default: 5,
   type: "number"
+}).option("run-memory", {
+  description: "Maximum memory allocated to a single run, in mb",
+  type: "number",
+  default: MAX_RUN_MEMORY ? parseInt(MAX_RUN_MEMORY) : 500
 }).parse();
 var logger = createLogger("SRV", { level: args.log });
 if (args.lightning === "mock") {
@@ -5778,7 +5796,11 @@ if (args.mock) {
     engineReady(engine);
   });
 } else {
-  createRTE({ repoDir: args.repoDir }).then((engine) => {
+  createRTE({
+    repoDir: args.repoDir,
+    memoryLimitMb: args.runMemory,
+    maxWorkers: args.capacity
+  }).then((engine) => {
     logger.debug("engine created");
     engineReady(engine);
   });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openfn/ws-worker",
-  "version": "0.2.10",
+  "version": "0.2.12",
   "description": "A Websocket Worker to connect Lightning to a Runtime Engine",
   "main": "dist/index.js",
   "type": "module",
@@ -21,9 +21,9 @@
     "koa-logger": "^3.2.1",
     "phoenix": "^1.7.7",
     "ws": "^8.14.1",
-    "@openfn/engine-multi": "0.2.1",
+    "@openfn/engine-multi": "0.2.3",
     "@openfn/logger": "0.0.19",
-    "@openfn/runtime": "0.2.0"
+    "@openfn/runtime": "0.2.1"
   },
   "devDependencies": {
     "@types/koa": "^2.13.5",
@@ -40,7 +40,7 @@
     "tsup": "^6.2.3",
     "typescript": "^4.6.4",
     "yargs": "^17.6.2",
-    "@openfn/lightning-mock": "1.1.3"
+    "@openfn/lightning-mock": "1.1.5"
   },
   "files": [
     "dist",