@openfn/ws-worker 0.4.0 → 0.6.0

package/CHANGELOG.md

@@ -1,5 +1,39 @@
 # ws-worker
 
+## 0.6.0
+
+### Patch Changes
+
+- eb10b1f: Updated start env vars and arguments
+- 281391b: Support attemptTimeoutMs in attempt options
+  Better server logging at startup
+  Support start arguments from the environment (but prefer CLI)
+- 2857fe6: Send the exit reason to the attempt logs
+- Updated dependencies [281391b]
+  - @openfn/engine-multi@0.3.0
+
+## 0.6.0
+
+### Minor Changes
+
+- 9b9ca0c: New worker pool engine
+
+### Patch Changes
+
+- Updated dependencies [0f22694]
+- Updated dependencies [9b9ca0c]
+  - @openfn/runtime@0.2.5
+  - @openfn/engine-multi@0.3.0
+
+## 0.5.0
+
+### Minor Changes
+
+- Add state-props-to-remove option
+- Updated dependencies [56b6e44]
+  - @openfn/engine-multi@0.2.6
+  - @openfn/runtime@0.2.4
+
 ## 0.4.0
 
 ### Minor Changes

package/README.md

@@ -24,8 +24,6 @@ The worker will use the WORKER_SECRET env var (which you should have set for Lig
 mix lightning.gen_worker_keys
 ```
 
-### WS Server
-
 To start a `ws-worker` server, run:
 
 ```
@@ -40,6 +38,10 @@ Pass a custom lightining url with `-l ws://localhost:1234/worker`. (Note that yo
 
 Use `-l mock` to connect to a lightning mock server (on the default port).
 
+## Options
+
+For a list of supported worker and engine options, see src/start.ts
+
 ## Watched Server
 
 You can start a dev server (which rebuilds on save) by running:

package/dist/index.d.ts

@@ -56,7 +56,18 @@ type Attempt = {
 };
 
 type AttemptOptions = {
+  // This is what Lightning will ssend us
+  // Note that this is the NEW terminology, so it's the timeout  for the whole "attempt"
+  runTimeout?: number;
+
+  // this is the internal old terminology, which will be deprecated soon
+  attemptTimeoutMs?: number;
+
+  attemptTimeout?: number; // deprecated
+
+  // deprecated alias for timeout. Maps to "attemptTimeout" internally
   timeout?: number;
+
   sanitize?: SanitizePolicies;
 };
 

package/dist/index.js

@@ -186,6 +186,14 @@ var mapTriggerEdgeCondition = (edge) => {
     return true;
   return condition;
 };
+var mapOptions = (options) => {
+  const { attemptTimeout, timeout, runTimeout, ...opts } = options;
+  const to = runTimeout || attemptTimeout || timeout;
+  if (to) {
+    opts.attemptTimeoutMs = to;
+  }
+  return opts;
+};
 var convert_attempt_default = (attempt) => {
   const options = attempt.options || {};
   const plan = {
@@ -249,7 +257,7 @@ var convert_attempt_default = (attempt) => {
   plan.jobs = Object.values(nodes);
   return {
     plan,
-    options
+    options: mapOptions(options)
   };
 };
 
@@ -298,6 +306,9 @@ var create_attempt_state_default = (plan, options = {}) => {
   return state;
 };
 
+// src/events/run-complete.ts
+import crypto2 from "node:crypto";
+
 // src/api/reasons.ts
 var calculateJobExitReason = (jobId, state = { data: {} }, error) => {
   let reason = "success";
@@ -332,7 +343,6 @@ var calculateAttemptExitReason = (state) => {
 };
 
 // src/events/run-complete.ts
-import crypto2 from "node:crypto";
 function onRunComplete({ channel, state }, event, error) {
   const dataclipId = crypto2.randomUUID();
   const run_id = state.activeRun;
@@ -376,7 +386,7 @@ import { timestamp } from "@openfn/logger";
 // package.json
 var package_default = {
   name: "@openfn/ws-worker",
-  version: "0.4.0",
+  version: "0.6.0",
   description: "A Websocket Worker to connect Lightning to a Runtime Engine",
   main: "dist/index.js",
   type: "module",
@@ -499,6 +509,57 @@ async function onRunStart(context, event) {
   return;
 }
 
+// src/util/log-final-reason.ts
+import { timestamp as timestamp2 } from "@openfn/logger";
+var log_final_reason_default = async (context, reason) => {
+  const time = (timestamp2() - BigInt(1e7)).toString();
+  let message = `Run complete with status: ${reason.reason}`;
+  if (reason.reason !== "success") {
+    message += `
+${reason.error_type}: ${reason.error_message || "unknown"}`;
+  }
+  await onJobLog(context, {
+    time,
+    message: [message],
+    level: "info",
+    name: "R/T"
+  });
+};
+
+// src/events/attempt-complete.ts
+async function onWorkflowComplete(context, _event) {
+  const { state, channel, onFinish } = context;
+  const result = state.dataclips[state.lastDataclipId];
+  const reason = calculateAttemptExitReason(state);
+  await log_final_reason_default(context, reason);
+  await sendEvent(channel, ATTEMPT_COMPLETE, {
+    final_dataclip_id: state.lastDataclipId,
+    ...reason
+  });
+  onFinish({ reason, state: result });
+}
+
+// src/events/attempt-error.ts
+async function onAttemptError(context, event) {
+  const { state, channel, logger, onFinish } = context;
+  try {
+    const reason = calculateJobExitReason("", { data: {} }, event);
+    if (state.activeJob) {
+      await onJobError(context, { error: event });
+    }
+    await log_final_reason_default(context, reason);
+    await sendEvent(channel, ATTEMPT_COMPLETE, {
+      final_dataclip_id: state.lastDataclipId,
+      ...reason
+    });
+    onFinish({ reason });
+  } catch (e) {
+    logger.error("ERROR in workflow-error handler:", e.message);
+    logger.error(e);
+    onFinish({});
+  }
+}
+
 // src/util/throttle.ts
 var createThrottler = () => {
   const q = [];
@@ -566,7 +627,7 @@ function execute(channel, engine, logger, plan, options = {}, onFinish = (_resul
     addEvent("job-error", throttle(onJobError)),
     addEvent("workflow-log", throttle(onJobLog)),
     addEvent("workflow-complete", throttle(onWorkflowComplete)),
-    addEvent("workflow-error", throttle(onWorkflowError))
+    addEvent("workflow-error", throttle(onAttemptError))
   );
   engine.listen(plan.id, listeners);
   const resolvers = {
@@ -584,7 +645,7 @@ function execute(channel, engine, logger, plan, options = {}, onFinish = (_resul
     try {
       engine.execute(plan, { resolvers, ...options });
     } catch (e) {
-      onWorkflowError(context, {
+      onAttemptError(context, {
         workflowId: plan.id,
         message: e.message,
         type: e.type,
@@ -610,33 +671,6 @@ function onJobError(context, event) {
 function onWorkflowStart({ channel }, _event) {
   return sendEvent(channel, ATTEMPT_START);
 }
-async function onWorkflowComplete({ state, channel, onFinish }, _event) {
-  const result = state.dataclips[state.lastDataclipId];
-  const reason = calculateAttemptExitReason(state);
-  await sendEvent(channel, ATTEMPT_COMPLETE, {
-    final_dataclip_id: state.lastDataclipId,
-    ...reason
-  });
-  onFinish({ reason, state: result });
-}
-async function onWorkflowError(context, event) {
-  const { state, channel, logger, onFinish } = context;
-  try {
-    const reason = calculateJobExitReason("", { data: {} }, event);
-    if (state.activeJob) {
-      await onJobError(context, { error: event });
-    }
-    await sendEvent(channel, ATTEMPT_COMPLETE, {
-      final_dataclip_id: state.lastDataclipId,
-      ...reason
-    });
-    onFinish({ reason });
-  } catch (e) {
-    logger.error("ERROR in workflow-error handler:", e.message);
-    logger.error(e);
-    onFinish({});
-  }
-}
 function onJobLog({ channel, state }, event) {
   const timeInMicroseconds = BigInt(event.time) / BigInt(1e3);
   const log = {
@@ -809,7 +843,6 @@ function connect(app, logger, options = {}) {
 function createServer(engine, options = {}) {
   const logger = options.logger || createMockLogger2();
   const port = options.port || DEFAULT_PORT;
-  logger.debug("Starting server");
   const app = new Koa();
   app.id = humanId({ separator: "-", capitalize: false });
   const router = new Router();
@@ -824,7 +857,7 @@ function createServer(engine, options = {}) {
   app.workflows = {};
   app.destroyed = false;
   app.server = app.listen(port);
-  logger.success(`ws-worker ${app.id} listening on ${port}`);
+  logger.success(`Worker ${app.id} listening on ${port}`);
   process.send?.("READY");
   router.get("/livez", healthcheck_default);
   router.get("/", healthcheck_default);

package/dist/start.js

@@ -4890,7 +4890,10 @@ var resolvers_default = {
 // src/mock/runtime-engine.ts
 var helpers = {
   fn: (f) => (s) => f(s),
-  wait: (duration) => (s) => new Promise((resolve5) => setTimeout(() => resolve5(s), duration))
+  wait: (duration) => (s) => new Promise((resolve5) => setTimeout(() => resolve5(s), duration)),
+  err: () => {
+    throw new Error("test_err");
+  }
 };
 async function createMock() {
   const activeWorkflows = {};
@@ -5172,6 +5175,14 @@ var mapTriggerEdgeCondition = (edge) => {
     return true;
   return condition;
 };
+var mapOptions = (options) => {
+  const { attemptTimeout, timeout, runTimeout, ...opts } = options;
+  const to = runTimeout || attemptTimeout || timeout;
+  if (to) {
+    opts.attemptTimeoutMs = to;
+  }
+  return opts;
+};
 var convert_attempt_default = (attempt) => {
   const options = attempt.options || {};
   const plan = {
@@ -5235,7 +5246,7 @@ var convert_attempt_default = (attempt) => {
   plan.jobs = Object.values(nodes);
   return {
     plan,
-    options
+    options: mapOptions(options)
   };
 };
 
@@ -5284,6 +5295,9 @@ var create_attempt_state_default = (plan, options = {}) => {
   return state;
 };
 
+// src/events/run-complete.ts
+import crypto3 from "node:crypto";
+
 // src/api/reasons.ts
 var calculateJobExitReason = (jobId, state = { data: {} }, error) => {
   let reason = "success";
@@ -5318,7 +5332,6 @@ var calculateAttemptExitReason = (state) => {
 };
 
 // src/events/run-complete.ts
-import crypto3 from "node:crypto";
 function onRunComplete({ channel, state }, event, error) {
   const dataclipId = crypto3.randomUUID();
   const run_id = state.activeRun;
@@ -5362,7 +5375,7 @@ import { timestamp } from "@openfn/logger";
 // package.json
 var package_default = {
   name: "@openfn/ws-worker",
-  version: "0.4.0",
+  version: "0.6.0",
   description: "A Websocket Worker to connect Lightning to a Runtime Engine",
   main: "dist/index.js",
   type: "module",
@@ -5485,6 +5498,57 @@ async function onRunStart(context, event) {
   return;
 }
 
+// src/util/log-final-reason.ts
+import { timestamp as timestamp2 } from "@openfn/logger";
+var log_final_reason_default = async (context, reason) => {
+  const time = (timestamp2() - BigInt(1e7)).toString();
+  let message = `Run complete with status: ${reason.reason}`;
+  if (reason.reason !== "success") {
+    message += `
+${reason.error_type}: ${reason.error_message || "unknown"}`;
+  }
+  await onJobLog(context, {
+    time,
+    message: [message],
+    level: "info",
+    name: "R/T"
+  });
+};
+
+// src/events/attempt-complete.ts
+async function onWorkflowComplete(context, _event) {
+  const { state, channel, onFinish } = context;
+  const result = state.dataclips[state.lastDataclipId];
+  const reason = calculateAttemptExitReason(state);
+  await log_final_reason_default(context, reason);
+  await sendEvent(channel, ATTEMPT_COMPLETE, {
+    final_dataclip_id: state.lastDataclipId,
+    ...reason
+  });
+  onFinish({ reason, state: result });
+}
+
+// src/events/attempt-error.ts
+async function onAttemptError(context, event) {
+  const { state, channel, logger: logger2, onFinish } = context;
+  try {
+    const reason = calculateJobExitReason("", { data: {} }, event);
+    if (state.activeJob) {
+      await onJobError(context, { error: event });
+    }
+    await log_final_reason_default(context, reason);
+    await sendEvent(channel, ATTEMPT_COMPLETE, {
+      final_dataclip_id: state.lastDataclipId,
+      ...reason
+    });
+    onFinish({ reason });
+  } catch (e) {
+    logger2.error("ERROR in workflow-error handler:", e.message);
+    logger2.error(e);
+    onFinish({});
+  }
+}
+
 // src/util/throttle.ts
 var createThrottler = () => {
   const q = [];
@@ -5552,7 +5616,7 @@ function execute(channel, engine, logger2, plan, options = {}, onFinish = (_resu
     addEvent("job-error", throttle(onJobError)),
     addEvent("workflow-log", throttle(onJobLog)),
     addEvent("workflow-complete", throttle(onWorkflowComplete)),
-    addEvent("workflow-error", throttle(onWorkflowError))
+    addEvent("workflow-error", throttle(onAttemptError))
   );
   engine.listen(plan.id, listeners);
   const resolvers = {
@@ -5570,7 +5634,7 @@ function execute(channel, engine, logger2, plan, options = {}, onFinish = (_resu
     try {
       engine.execute(plan, { resolvers, ...options });
     } catch (e) {
-      onWorkflowError(context, {
+      onAttemptError(context, {
         workflowId: plan.id,
         message: e.message,
         type: e.type,
@@ -5596,33 +5660,6 @@ function onJobError(context, event) {
 function onWorkflowStart({ channel }, _event) {
   return sendEvent(channel, ATTEMPT_START);
 }
-async function onWorkflowComplete({ state, channel, onFinish }, _event) {
-  const result = state.dataclips[state.lastDataclipId];
-  const reason = calculateAttemptExitReason(state);
-  await sendEvent(channel, ATTEMPT_COMPLETE, {
-    final_dataclip_id: state.lastDataclipId,
-    ...reason
-  });
-  onFinish({ reason, state: result });
-}
-async function onWorkflowError(context, event) {
-  const { state, channel, logger: logger2, onFinish } = context;
-  try {
-    const reason = calculateJobExitReason("", { data: {} }, event);
-    if (state.activeJob) {
-      await onJobError(context, { error: event });
-    }
-    await sendEvent(channel, ATTEMPT_COMPLETE, {
-      final_dataclip_id: state.lastDataclipId,
-      ...reason
-    });
-    onFinish({ reason });
-  } catch (e) {
-    logger2.error("ERROR in workflow-error handler:", e.message);
-    logger2.error(e);
-    onFinish({});
-  }
-}
 function onJobLog({ channel, state }, event) {
   const timeInMicroseconds = BigInt(event.time) / BigInt(1e3);
   const log = {
@@ -5795,7 +5832,6 @@ function connect(app, logger2, options = {}) {
 function createServer(engine, options = {}) {
   const logger2 = options.logger || createMockLogger2();
   const port = options.port || DEFAULT_PORT;
-  logger2.debug("Starting server");
   const app = new Koa();
   app.id = humanId({ separator: "-", capitalize: false });
   const router = new Router();
@@ -5810,7 +5846,7 @@ function createServer(engine, options = {}) {
   app.workflows = {};
   app.destroyed = false;
   app.server = app.listen(port);
-  logger2.success(`ws-worker ${app.id} listening on ${port}`);
+  logger2.success(`Worker ${app.id} listening on ${port}`);
   process.send?.("READY");
   router.get("/livez", healthcheck_default);
   router.get("/", healthcheck_default);
@@ -5878,26 +5914,38 @@ function createServer(engine, options = {}) {
 var server_default = createServer;
 
 // src/start.ts
-var { WORKER_REPO_DIR, WORKER_SECRET, MAX_RUN_MEMORY } = process.env;
+var {
+  WORKER_BACKOFF,
+  WORKER_CAPACITY,
+  WORKER_LIGHTNING_SERVICE_URL,
+  WORKER_LOG_LEVEL,
+  WORKER_MAX_RUN_DURATION_SECONDS,
+  WORKER_MAX_RUN_MEMORY_MB,
+  WORKER_PORT,
+  WORKER_REPO_DIR,
+  WORKER_SECRET,
+  WORKER_STATE_PROPS_TO_REMOVE
+} = process.env;
 var args = yargs_default(hideBin(process.argv)).command("server", "Start a ws-worker server").option("port", {
   alias: "p",
-  description: "Port to run the server on",
+  description: "Port to run the server on. Env: WORKER_PORT",
   type: "number",
-  default: 2222
+  default: WORKER_PORT || 2222
 }).option("lightning", {
-  alias: "l",
-  description: 'Base url to Lightning websocket endpoint, eg, ws://localhost:4000/worker. Set to "mock" to use the default mock server',
-  default: "ws://localhost:4000/worker"
+  alias: ["l", "lightning-service-url"],
+  description: 'Base url to Lightning websocket endpoint, eg, ws://localhost:4000/worker. Set to "mock" to use the default mock server. Env: WORKER_LIGHTNING_SERVICE_URL',
+  default: WORKER_LIGHTNING_SERVICE_URL || "ws://localhost:4000/worker"
 }).option("repo-dir", {
   alias: "d",
-  description: "Path to the runtime repo (where modules will be installed)",
+  description: "Path to the runtime repo (where modules will be installed). Env: WORKER_REPO_DIR",
   default: WORKER_REPO_DIR
 }).option("secret", {
   alias: "s",
-  description: "Worker secret (comes from WORKER_SECRET by default)"
+  description: "Worker secret. (comes from WORKER_SECRET by default). Env: WORKER_SECRET",
+  default: WORKER_SECRET
 }).option("log", {
-  description: "Worker secret (comes from WORKER_SECRET by default)",
-  default: "info",
+  description: "Set the log level for stdout (default to info, set to debug for verbose output). Env: WORKER_LOG_LEVEL",
+  default: WORKER_LOG_LEVEL || "debug",
   type: "string"
 }).option("loop", {
   description: "Disable the claims loop",
@@ -5908,16 +5956,25 @@ var args = yargs_default(hideBin(process.argv)).command("server", "Start a ws-wo
   default: false,
   type: "boolean"
 }).option("backoff", {
-  description: "Claim backoff rules: min/max (s)",
-  default: "1/10"
+  description: "Claim backoff rules: min/max (in seconds). Env: WORKER_BACKOFF",
+  default: WORKER_BACKOFF || "1/10"
 }).option("capacity", {
-  description: "max concurrent workers",
-  default: 5,
+  description: "max concurrent workers. Env: WORKER_CAPACITY",
+  default: WORKER_CAPACITY ? parseInt(WORKER_CAPACITY) : 5,
   type: "number"
+}).option("state-props-to-remove", {
+  description: "A list of properties to remove from the final state returned by a job. Env: WORKER_STATE_PROPS_TO_REMOVE",
+  default: WORKER_STATE_PROPS_TO_REMOVE ?? ["configuration", "response"],
+  type: "array"
 }).option("run-memory", {
-  description: "Maximum memory allocated to a single run, in mb",
+  description: "Maximum memory allocated to a single run, in mb. Env: WORKER_MAX_RUN_MEMORY_MB",
+  type: "number",
+  default: WORKER_MAX_RUN_MEMORY_MB ? parseInt(WORKER_MAX_RUN_MEMORY_MB) : 500
+}).option("max-run-duration-seconds", {
+  alias: "t",
+  description: "Default attempt timeout for the server, in seconds. Env: WORKER_MAX_RUN_DURATION_SECONDS",
   type: "number",
-  default: MAX_RUN_MEMORY ? parseInt(MAX_RUN_MEMORY) : 500
+  default: WORKER_MAX_RUN_DURATION_SECONDS || 60 * 5
 }).parse();
 var logger = createLogger("SRV", { level: args.log });
 if (args.lightning === "mock") {
@@ -5926,15 +5983,13 @@ if (args.lightning === "mock") {
     args.secret = "abdefg";
   }
 } else if (!args.secret) {
-  if (!WORKER_SECRET) {
-    logger.error("WORKER_SECRET is not set");
-    process.exit(1);
-  }
-  args.secret = WORKER_SECRET;
+  logger.error("WORKER_SECRET is not set");
+  process.exit(1);
 }
 var [minBackoff, maxBackoff] = args.backoff.split("/").map((n) => parseInt(n, 10) * 1e3);
 function engineReady(engine) {
-  server_default(engine, {
+  logger.debug("Creating worker server...");
+  const workerOptions = {
     port: args.port,
     lightning: args.lightning,
     logger,
@@ -5945,7 +6000,10 @@ function engineReady(engine) {
       max: maxBackoff
     },
     maxWorkflows: args.capacity
-  });
+  };
+  const { logger: _l, secret: _s, ...humanOptions } = workerOptions;
+  logger.debug("Worker options:", humanOptions);
+  server_default(engine, workerOptions);
 }
 if (args.mock) {
   runtime_engine_default().then((engine) => {
@@ -5953,12 +6011,17 @@ if (args.mock) {
     engineReady(engine);
   });
 } else {
-  createRTE({
+  const engineOptions = {
     repoDir: args.repoDir,
     memoryLimitMb: args.runMemory,
-    maxWorkers: args.capacity
-  }).then((engine) => {
-    logger.debug("engine created");
+    maxWorkers: args.capacity,
+    statePropsToRemove: args.statePropsToRemove,
+    attemptTimeoutMs: args.maxRunDurationSeconds * 1e3
+  };
+  logger.debug("Creating runtime engine...");
+  logger.debug("Engine options:", engineOptions);
+  createRTE(engineOptions).then((engine) => {
+    logger.debug("Engine created!");
     engineReady(engine);
   });
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openfn/ws-worker",
-  "version": "0.4.0",
+  "version": "0.6.0",
   "description": "A Websocket Worker to connect Lightning to a Runtime Engine",
   "main": "dist/index.js",
   "type": "module",
@@ -22,9 +22,9 @@
     "koa-logger": "^3.2.1",
     "phoenix": "^1.7.7",
     "ws": "^8.14.1",
-    "@openfn/engine-multi": "0.2.5",
+    "@openfn/engine-multi": "0.3.0",
     "@openfn/logger": "0.0.19",
-    "@openfn/runtime": "0.2.3"
+    "@openfn/runtime": "0.2.5"
   },
   "devDependencies": {
     "@types/koa": "^2.13.5",
@@ -41,7 +41,7 @@
     "tsup": "^6.2.3",
     "typescript": "^4.6.4",
     "yargs": "^17.6.2",
-    "@openfn/lightning-mock": "1.1.7"
+    "@openfn/lightning-mock": "1.1.10"
   },
   "files": [
     "dist",