@openfn/ws-worker 0.5.0 → 0.7.0

package/CHANGELOG.md

@@ -1,5 +1,36 @@
 # ws-worker
 
+## 0.7.0
+
+### Minor Changes
+
+- 39af8e1: Ensure that we refer to the child of a 'run' (aka attempt) as a 'step'
+
+## 0.6.0
+
+### Patch Changes
+
+- eb10b1f: Updated start env vars and arguments
+- 281391b: Support attemptTimeoutMs in attempt options
+  Better server logging at startup
+  Support start arguments from the environment (but prefer CLI)
+- 2857fe6: Send the exit reason to the attempt logs
+- Updated dependencies [281391b]
+  - @openfn/engine-multi@0.3.0
+
+## 0.6.0
+
+### Minor Changes
+
+- 9b9ca0c: New worker pool engine
+
+### Patch Changes
+
+- Updated dependencies [0f22694]
+- Updated dependencies [9b9ca0c]
+  - @openfn/runtime@0.2.5
+  - @openfn/engine-multi@0.3.0
+
 ## 0.5.0
 
 ### Minor Changes

package/dist/index.d.ts

@@ -56,13 +56,24 @@ type Attempt = {
 };
 
 type AttemptOptions = {
+  // This is what Lightning will ssend us
+  // Note that this is the NEW terminology, so it's the timeout  for the whole "attempt"
+  runTimeout?: number;
+
+  // this is the internal old terminology, which will be deprecated soon
+  attemptTimeoutMs?: number;
+
+  attemptTimeout?: number; // deprecated
+
+  // deprecated alias for timeout. Maps to "attemptTimeout" internally
   timeout?: number;
+
   sanitize?: SanitizePolicies;
 };
 
 // Internal server state for each attempt
 type AttemptState = {
-  activeRun?: string;
+  activeStep?: string;
   activeJob?: string;
   plan: ExecutionPlan;
   options: AttemptOptions;
@@ -131,27 +142,27 @@ declare type AttemptLogPayload = {
     level?: string;
     source?: string;
     job_id?: string;
-    run_id?: string;
+    step_id?: string;
 };
 declare type AttemptLogReply = void;
-declare const RUN_START = "run:start";
-declare type RunStartPayload = {
+declare const STEP_START = "step:start";
+declare type StepStartPayload = {
     job_id: string;
-    run_id: string;
+    step_id: string;
     attempt_id?: string;
     input_dataclip_id?: string;
     versions: Record<string, string>;
 };
-declare type RunStartReply = void;
-declare const RUN_COMPLETE = "run:complete";
-declare type RunCompletePayload = ExitReason & {
+declare type StepStartReply = void;
+declare const STEP_COMPLETE = "step:complete";
+declare type StepCompletePayload = ExitReason & {
     attempt_id?: string;
     job_id: string;
-    run_id: string;
+    step_id: string;
     output_dataclip?: string;
     output_dataclip_id?: string;
 };
-declare type RunCompleteReply = void;
+declare type StepCompleteReply = void;
 declare const INTERNAL_ATTEMPT_COMPLETE = "server:attempt-complete";
 
 declare type Context = {
@@ -189,4 +200,4 @@ interface ServerApp extends Koa {
 }
 declare function createServer(engine: RuntimeEngine, options?: ServerOptions): ServerApp;
 
-export { ATTEMPT_COMPLETE, ATTEMPT_LOG, ATTEMPT_START, AttemptCompletePayload, AttemptCompleteReply, AttemptLogPayload, AttemptLogReply, AttemptStartPayload, AttemptStartReply, CLAIM, ClaimAttempt, ClaimPayload, ClaimReply, GET_ATTEMPT, GET_CREDENTIAL, GET_DATACLIP, GetAttemptPayload, GetAttemptReply, GetCredentialPayload, GetCredentialReply, GetDataClipReply, GetDataclipPayload, INTERNAL_ATTEMPT_COMPLETE, RUN_COMPLETE, RUN_START, RunCompletePayload, RunCompleteReply, RunStartPayload, RunStartReply, createServer as default };
+export { ATTEMPT_COMPLETE, ATTEMPT_LOG, ATTEMPT_START, AttemptCompletePayload, AttemptCompleteReply, AttemptLogPayload, AttemptLogReply, AttemptStartPayload, AttemptStartReply, CLAIM, ClaimAttempt, ClaimPayload, ClaimReply, GET_ATTEMPT, GET_CREDENTIAL, GET_DATACLIP, GetAttemptPayload, GetAttemptReply, GetCredentialPayload, GetCredentialReply, GetDataClipReply, GetDataclipPayload, INTERNAL_ATTEMPT_COMPLETE, STEP_COMPLETE, STEP_START, StepCompletePayload, StepCompleteReply, StepStartPayload, StepStartReply, createServer as default };

package/dist/index.js

@@ -15,8 +15,8 @@ var GET_DATACLIP = "fetch:dataclip";
 var ATTEMPT_START = "attempt:start";
 var ATTEMPT_COMPLETE = "attempt:complete";
 var ATTEMPT_LOG = "attempt:log";
-var RUN_START = "run:start";
-var RUN_COMPLETE = "run:complete";
+var STEP_START = "step:start";
+var STEP_COMPLETE = "step:complete";
 var INTERNAL_ATTEMPT_COMPLETE = "server:attempt-complete";
 
 // src/api/destroy.ts
@@ -186,6 +186,14 @@ var mapTriggerEdgeCondition = (edge) => {
     return true;
   return condition;
 };
+var mapOptions = (options) => {
+  const { attemptTimeout, timeout, runTimeout, ...opts } = options;
+  const to = runTimeout || attemptTimeout || timeout;
+  if (to) {
+    opts.attemptTimeoutMs = to;
+  }
+  return opts;
+};
 var convert_attempt_default = (attempt) => {
   const options = attempt.options || {};
   const plan = {
@@ -249,7 +257,7 @@ var convert_attempt_default = (attempt) => {
   plan.jobs = Object.values(nodes);
   return {
     plan,
-    options
+    options: mapOptions(options)
   };
 };
 
@@ -298,6 +306,9 @@ var create_attempt_state_default = (plan, options = {}) => {
   return state;
 };
 
+// src/events/step-complete.ts
+import crypto2 from "node:crypto";
+
 // src/api/reasons.ts
 var calculateJobExitReason = (jobId, state = { data: {} }, error) => {
   let reason = "success";
@@ -331,18 +342,17 @@ var calculateAttemptExitReason = (state) => {
   return { reason: "success", error_type: null, error_message: null };
 };
 
-// src/events/run-complete.ts
-import crypto2 from "node:crypto";
-function onRunComplete({ channel, state }, event, error) {
+// src/events/step-complete.ts
+function onStepComplete({ channel, state }, event, error) {
   const dataclipId = crypto2.randomUUID();
-  const run_id = state.activeRun;
+  const step_id = state.activeStep;
   const job_id = state.activeJob;
   if (!state.dataclips) {
     state.dataclips = {};
   }
   const outputState = event.state || {};
   state.dataclips[dataclipId] = event.state;
-  delete state.activeRun;
+  delete state.activeStep;
   delete state.activeJob;
   state.lastDataclipId = dataclipId;
   event.next?.forEach((nextJobId) => {
@@ -355,7 +365,7 @@ function onRunComplete({ channel, state }, event, error) {
   );
   state.reasons[job_id] = { reason, error_message, error_type };
   const evt = {
-    run_id,
+    step_id,
     job_id,
     output_dataclip_id: dataclipId,
     output_dataclip: stringify_default(outputState),
@@ -366,17 +376,17 @@ function onRunComplete({ channel, state }, event, error) {
     duration: event.duration,
     thread_id: event.threadId
   };
-  return sendEvent(channel, RUN_COMPLETE, evt);
+  return sendEvent(channel, STEP_COMPLETE, evt);
 }
 
-// src/events/run-start.ts
+// src/events/step-start.ts
 import crypto3 from "node:crypto";
 import { timestamp } from "@openfn/logger";
 
 // package.json
 var package_default = {
   name: "@openfn/ws-worker",
-  version: "0.5.0",
+  version: "0.7.0",
   description: "A Websocket Worker to connect Lightning to a Runtime Engine",
   main: "dist/index.js",
   type: "module",
@@ -439,14 +449,14 @@ var package_default = {
 // src/util/versions.ts
 import { mainSymbols } from "figures";
 var { triangleRightSmall: t } = mainSymbols;
-var versions_default = (runId, versions, adaptor) => {
+var versions_default = (stepId, versions, adaptor) => {
   let longest = "compiler".length;
   for (const v in versions) {
     longest = Math.max(v.length, longest);
   }
   const { node, compiler, engine, worker, runtime, ...adaptors } = versions;
   const prefix = (str2) => `    ${t} ${str2.padEnd(longest + 4, " ")}`;
-  let str = `Versions for run ${runId}:
+  let str = `Versions for step ${stepId}:
 ${prefix("node.js")}${versions.node || "unknown"}
 ${prefix("worker")}${versions.worker || "unknown"}
 ${prefix("engine")}${versions.engine || "unknown"}`;
@@ -460,11 +470,11 @@ ${prefix("engine")}${versions.engine || "unknown"}`;
   return str;
 };
 
-// src/events/run-start.ts
-async function onRunStart(context, event) {
+// src/events/step-start.ts
+async function onStepStart(context, event) {
   const time = (timestamp() - BigInt(1e7)).toString();
   const { channel, state } = context;
-  state.activeRun = crypto3.randomUUID();
+  state.activeStep = crypto3.randomUUID();
   state.activeJob = event.jobId;
   const job = state.plan.jobs.find(({ id }) => id === event.jobId);
   const input_dataclip_id = state.inputDataclips[event.jobId];
@@ -476,17 +486,17 @@ async function onRunStart(context, event) {
     ...context,
     state: {
       ...state,
-      activeRun: state.activeRun
+      activeStep: state.activeStep
     }
   };
-  await sendEvent(channel, RUN_START, {
-    run_id: state.activeRun,
+  await sendEvent(channel, STEP_START, {
+    step_id: state.activeStep,
     job_id: state.activeJob,
     input_dataclip_id,
     versions
   });
   const versionMessage = versions_default(
-    versionLogContext.state.activeRun,
+    versionLogContext.state.activeStep,
     versions,
     job?.adaptor
   );
@@ -499,6 +509,57 @@ async function onRunStart(context, event) {
   return;
 }
 
+// src/util/log-final-reason.ts
+import { timestamp as timestamp2 } from "@openfn/logger";
+var log_final_reason_default = async (context, reason) => {
+  const time = (timestamp2() - BigInt(1e7)).toString();
+  let message = `Run complete with status: ${reason.reason}`;
+  if (reason.reason !== "success") {
+    message += `
+${reason.error_type}: ${reason.error_message || "unknown"}`;
+  }
+  await onJobLog(context, {
+    time,
+    message: [message],
+    level: "info",
+    name: "R/T"
+  });
+};
+
+// src/events/attempt-complete.ts
+async function onWorkflowComplete(context, _event) {
+  const { state, channel, onFinish } = context;
+  const result = state.dataclips[state.lastDataclipId];
+  const reason = calculateAttemptExitReason(state);
+  await log_final_reason_default(context, reason);
+  await sendEvent(channel, ATTEMPT_COMPLETE, {
+    final_dataclip_id: state.lastDataclipId,
+    ...reason
+  });
+  onFinish({ reason, state: result });
+}
+
+// src/events/attempt-error.ts
+async function onAttemptError(context, event) {
+  const { state, channel, logger, onFinish } = context;
+  try {
+    const reason = calculateJobExitReason("", { data: {} }, event);
+    if (state.activeJob) {
+      await onJobError(context, { error: event });
+    }
+    await log_final_reason_default(context, reason);
+    await sendEvent(channel, ATTEMPT_COMPLETE, {
+      final_dataclip_id: state.lastDataclipId,
+      ...reason
+    });
+    onFinish({ reason });
+  } catch (e) {
+    logger.error("ERROR in workflow-error handler:", e.message);
+    logger.error(e);
+    onFinish({});
+  }
+}
+
 // src/util/throttle.ts
 var createThrottler = () => {
   const q = [];
@@ -530,8 +591,8 @@ var throttle_default = createThrottler;
 var enc = new TextDecoder("utf-8");
 var eventMap = {
   "workflow-start": ATTEMPT_START,
-  "job-start": RUN_START,
-  "job-complete": RUN_COMPLETE,
+  "job-start": STEP_START,
+  "job-complete": STEP_COMPLETE,
   "workflow-log": ATTEMPT_LOG,
   "workflow-complete": ATTEMPT_COMPLETE
 };
@@ -561,12 +622,12 @@ function execute(channel, engine, logger, plan, options = {}, onFinish = (_resul
   const listeners = Object.assign(
     {},
     addEvent("workflow-start", throttle(onWorkflowStart)),
-    addEvent("job-start", throttle(onRunStart)),
-    addEvent("job-complete", throttle(onRunComplete)),
+    addEvent("job-start", throttle(onStepStart)),
+    addEvent("job-complete", throttle(onStepComplete)),
     addEvent("job-error", throttle(onJobError)),
     addEvent("workflow-log", throttle(onJobLog)),
     addEvent("workflow-complete", throttle(onWorkflowComplete)),
-    addEvent("workflow-error", throttle(onWorkflowError))
+    addEvent("workflow-error", throttle(onAttemptError))
   );
   engine.listen(plan.id, listeners);
   const resolvers = {
@@ -584,7 +645,7 @@ function execute(channel, engine, logger, plan, options = {}, onFinish = (_resul
     try {
       engine.execute(plan, { resolvers, ...options });
     } catch (e) {
-      onWorkflowError(context, {
+      onAttemptError(context, {
         workflowId: plan.id,
         message: e.message,
         type: e.type,
@@ -602,41 +663,14 @@ var sendEvent = (channel, event, payload) => new Promise((resolve, reject) => {
 function onJobError(context, event) {
   const { state, error, jobId } = event;
   if (state?.errors?.[jobId]?.message === error.message) {
-    return onRunComplete(context, event);
+    return onStepComplete(context, event);
   } else {
-    return onRunComplete(context, event, event.error);
+    return onStepComplete(context, event, event.error);
   }
 }
 function onWorkflowStart({ channel }, _event) {
   return sendEvent(channel, ATTEMPT_START);
 }
-async function onWorkflowComplete({ state, channel, onFinish }, _event) {
-  const result = state.dataclips[state.lastDataclipId];
-  const reason = calculateAttemptExitReason(state);
-  await sendEvent(channel, ATTEMPT_COMPLETE, {
-    final_dataclip_id: state.lastDataclipId,
-    ...reason
-  });
-  onFinish({ reason, state: result });
-}
-async function onWorkflowError(context, event) {
-  const { state, channel, logger, onFinish } = context;
-  try {
-    const reason = calculateJobExitReason("", { data: {} }, event);
-    if (state.activeJob) {
-      await onJobError(context, { error: event });
-    }
-    await sendEvent(channel, ATTEMPT_COMPLETE, {
-      final_dataclip_id: state.lastDataclipId,
-      ...reason
-    });
-    onFinish({ reason });
-  } catch (e) {
-    logger.error("ERROR in workflow-error handler:", e.message);
-    logger.error(e);
-    onFinish({});
-  }
-}
 function onJobLog({ channel, state }, event) {
   const timeInMicroseconds = BigInt(event.time) / BigInt(1e3);
   const log = {
@@ -646,8 +680,8 @@ function onJobLog({ channel, state }, event) {
     level: event.level,
     timestamp: timeInMicroseconds.toString()
   };
-  if (state.activeRun) {
-    log.run_id = state.activeRun;
+  if (state.activeStep) {
+    log.step_id = state.activeStep;
   }
   return sendEvent(channel, ATTEMPT_LOG, log);
 }
@@ -809,7 +843,6 @@ function connect(app, logger, options = {}) {
 function createServer(engine, options = {}) {
   const logger = options.logger || createMockLogger2();
   const port = options.port || DEFAULT_PORT;
-  logger.debug("Starting server");
   const app = new Koa();
   app.id = humanId({ separator: "-", capitalize: false });
   const router = new Router();
@@ -824,7 +857,7 @@ function createServer(engine, options = {}) {
   app.workflows = {};
   app.destroyed = false;
   app.server = app.listen(port);
-  logger.success(`ws-worker ${app.id} listening on ${port}`);
+  logger.success(`Worker ${app.id} listening on ${port}`);
   process.send?.("READY");
   router.get("/livez", healthcheck_default);
   router.get("/", healthcheck_default);
@@ -902,7 +935,7 @@ export {
   GET_CREDENTIAL,
   GET_DATACLIP,
   INTERNAL_ATTEMPT_COMPLETE,
-  RUN_COMPLETE,
-  RUN_START,
+  STEP_COMPLETE,
+  STEP_START,
   src_default as default
 };

package/dist/start.js

@@ -4890,7 +4890,10 @@ var resolvers_default = {
 // src/mock/runtime-engine.ts
 var helpers = {
   fn: (f) => (s) => f(s),
-  wait: (duration) => (s) => new Promise((resolve5) => setTimeout(() => resolve5(s), duration))
+  wait: (duration) => (s) => new Promise((resolve5) => setTimeout(() => resolve5(s), duration)),
+  err: () => {
+    throw new Error("test_err");
+  }
 };
 async function createMock() {
   const activeWorkflows = {};
@@ -5001,8 +5004,8 @@ var GET_DATACLIP = "fetch:dataclip";
 var ATTEMPT_START = "attempt:start";
 var ATTEMPT_COMPLETE = "attempt:complete";
 var ATTEMPT_LOG = "attempt:log";
-var RUN_START = "run:start";
-var RUN_COMPLETE = "run:complete";
+var STEP_START = "step:start";
+var STEP_COMPLETE = "step:complete";
 var INTERNAL_ATTEMPT_COMPLETE = "server:attempt-complete";
 
 // src/api/destroy.ts
@@ -5172,6 +5175,14 @@ var mapTriggerEdgeCondition = (edge) => {
     return true;
   return condition;
 };
+var mapOptions = (options) => {
+  const { attemptTimeout, timeout, runTimeout, ...opts } = options;
+  const to = runTimeout || attemptTimeout || timeout;
+  if (to) {
+    opts.attemptTimeoutMs = to;
+  }
+  return opts;
+};
 var convert_attempt_default = (attempt) => {
   const options = attempt.options || {};
   const plan = {
@@ -5235,7 +5246,7 @@ var convert_attempt_default = (attempt) => {
   plan.jobs = Object.values(nodes);
   return {
     plan,
-    options
+    options: mapOptions(options)
   };
 };
 
@@ -5284,6 +5295,9 @@ var create_attempt_state_default = (plan, options = {}) => {
   return state;
 };
 
+// src/events/step-complete.ts
+import crypto3 from "node:crypto";
+
 // src/api/reasons.ts
 var calculateJobExitReason = (jobId, state = { data: {} }, error) => {
   let reason = "success";
@@ -5317,18 +5331,17 @@ var calculateAttemptExitReason = (state) => {
   return { reason: "success", error_type: null, error_message: null };
 };
 
-// src/events/run-complete.ts
-import crypto3 from "node:crypto";
-function onRunComplete({ channel, state }, event, error) {
+// src/events/step-complete.ts
+function onStepComplete({ channel, state }, event, error) {
   const dataclipId = crypto3.randomUUID();
-  const run_id = state.activeRun;
+  const step_id = state.activeStep;
   const job_id = state.activeJob;
   if (!state.dataclips) {
     state.dataclips = {};
   }
   const outputState = event.state || {};
   state.dataclips[dataclipId] = event.state;
-  delete state.activeRun;
+  delete state.activeStep;
   delete state.activeJob;
   state.lastDataclipId = dataclipId;
   event.next?.forEach((nextJobId) => {
@@ -5341,7 +5354,7 @@ function onRunComplete({ channel, state }, event, error) {
   );
   state.reasons[job_id] = { reason, error_message, error_type };
   const evt = {
-    run_id,
+    step_id,
     job_id,
     output_dataclip_id: dataclipId,
     output_dataclip: stringify_default(outputState),
@@ -5352,17 +5365,17 @@ function onRunComplete({ channel, state }, event, error) {
     duration: event.duration,
     thread_id: event.threadId
   };
-  return sendEvent(channel, RUN_COMPLETE, evt);
+  return sendEvent(channel, STEP_COMPLETE, evt);
 }
 
-// src/events/run-start.ts
+// src/events/step-start.ts
 import crypto4 from "node:crypto";
 import { timestamp } from "@openfn/logger";
 
 // package.json
 var package_default = {
   name: "@openfn/ws-worker",
-  version: "0.5.0",
+  version: "0.7.0",
   description: "A Websocket Worker to connect Lightning to a Runtime Engine",
   main: "dist/index.js",
   type: "module",
@@ -5425,14 +5438,14 @@ var package_default = {
 // src/util/versions.ts
 import { mainSymbols } from "figures";
 var { triangleRightSmall: t } = mainSymbols;
-var versions_default = (runId, versions, adaptor) => {
+var versions_default = (stepId, versions, adaptor) => {
   let longest = "compiler".length;
   for (const v in versions) {
     longest = Math.max(v.length, longest);
   }
   const { node, compiler, engine, worker, runtime, ...adaptors } = versions;
   const prefix = (str2) => `    ${t} ${str2.padEnd(longest + 4, " ")}`;
-  let str = `Versions for run ${runId}:
+  let str = `Versions for step ${stepId}:
 ${prefix("node.js")}${versions.node || "unknown"}
 ${prefix("worker")}${versions.worker || "unknown"}
 ${prefix("engine")}${versions.engine || "unknown"}`;
@@ -5446,11 +5459,11 @@ ${prefix("engine")}${versions.engine || "unknown"}`;
   return str;
 };
 
-// src/events/run-start.ts
-async function onRunStart(context, event) {
+// src/events/step-start.ts
+async function onStepStart(context, event) {
   const time = (timestamp() - BigInt(1e7)).toString();
   const { channel, state } = context;
-  state.activeRun = crypto4.randomUUID();
+  state.activeStep = crypto4.randomUUID();
   state.activeJob = event.jobId;
   const job = state.plan.jobs.find(({ id }) => id === event.jobId);
   const input_dataclip_id = state.inputDataclips[event.jobId];
@@ -5462,17 +5475,17 @@ async function onRunStart(context, event) {
     ...context,
     state: {
       ...state,
-      activeRun: state.activeRun
+      activeStep: state.activeStep
     }
   };
-  await sendEvent(channel, RUN_START, {
-    run_id: state.activeRun,
+  await sendEvent(channel, STEP_START, {
+    step_id: state.activeStep,
     job_id: state.activeJob,
     input_dataclip_id,
     versions
   });
   const versionMessage = versions_default(
-    versionLogContext.state.activeRun,
+    versionLogContext.state.activeStep,
     versions,
     job?.adaptor
   );
@@ -5485,6 +5498,57 @@ async function onRunStart(context, event) {
   return;
 }
 
+// src/util/log-final-reason.ts
+import { timestamp as timestamp2 } from "@openfn/logger";
+var log_final_reason_default = async (context, reason) => {
+  const time = (timestamp2() - BigInt(1e7)).toString();
+  let message = `Run complete with status: ${reason.reason}`;
+  if (reason.reason !== "success") {
+    message += `
+${reason.error_type}: ${reason.error_message || "unknown"}`;
+  }
+  await onJobLog(context, {
+    time,
+    message: [message],
+    level: "info",
+    name: "R/T"
+  });
+};
+
+// src/events/attempt-complete.ts
+async function onWorkflowComplete(context, _event) {
+  const { state, channel, onFinish } = context;
+  const result = state.dataclips[state.lastDataclipId];
+  const reason = calculateAttemptExitReason(state);
+  await log_final_reason_default(context, reason);
+  await sendEvent(channel, ATTEMPT_COMPLETE, {
+    final_dataclip_id: state.lastDataclipId,
+    ...reason
+  });
+  onFinish({ reason, state: result });
+}
+
+// src/events/attempt-error.ts
+async function onAttemptError(context, event) {
+  const { state, channel, logger: logger2, onFinish } = context;
+  try {
+    const reason = calculateJobExitReason("", { data: {} }, event);
+    if (state.activeJob) {
+      await onJobError(context, { error: event });
+    }
+    await log_final_reason_default(context, reason);
+    await sendEvent(channel, ATTEMPT_COMPLETE, {
+      final_dataclip_id: state.lastDataclipId,
+      ...reason
+    });
+    onFinish({ reason });
+  } catch (e) {
+    logger2.error("ERROR in workflow-error handler:", e.message);
+    logger2.error(e);
+    onFinish({});
+  }
+}
+
 // src/util/throttle.ts
 var createThrottler = () => {
   const q = [];
@@ -5516,8 +5580,8 @@ var throttle_default = createThrottler;
 var enc = new TextDecoder("utf-8");
 var eventMap = {
   "workflow-start": ATTEMPT_START,
-  "job-start": RUN_START,
-  "job-complete": RUN_COMPLETE,
+  "job-start": STEP_START,
+  "job-complete": STEP_COMPLETE,
   "workflow-log": ATTEMPT_LOG,
   "workflow-complete": ATTEMPT_COMPLETE
 };
@@ -5547,12 +5611,12 @@ function execute(channel, engine, logger2, plan, options = {}, onFinish = (_resu
   const listeners = Object.assign(
     {},
     addEvent("workflow-start", throttle(onWorkflowStart)),
-    addEvent("job-start", throttle(onRunStart)),
-    addEvent("job-complete", throttle(onRunComplete)),
+    addEvent("job-start", throttle(onStepStart)),
+    addEvent("job-complete", throttle(onStepComplete)),
     addEvent("job-error", throttle(onJobError)),
     addEvent("workflow-log", throttle(onJobLog)),
     addEvent("workflow-complete", throttle(onWorkflowComplete)),
-    addEvent("workflow-error", throttle(onWorkflowError))
+    addEvent("workflow-error", throttle(onAttemptError))
   );
   engine.listen(plan.id, listeners);
   const resolvers = {
@@ -5570,7 +5634,7 @@ function execute(channel, engine, logger2, plan, options = {}, onFinish = (_resu
     try {
       engine.execute(plan, { resolvers, ...options });
     } catch (e) {
-      onWorkflowError(context, {
+      onAttemptError(context, {
         workflowId: plan.id,
         message: e.message,
         type: e.type,
@@ -5588,41 +5652,14 @@ var sendEvent = (channel, event, payload) => new Promise((resolve5, reject) => {
 function onJobError(context, event) {
   const { state, error, jobId } = event;
   if (state?.errors?.[jobId]?.message === error.message) {
-    return onRunComplete(context, event);
+    return onStepComplete(context, event);
   } else {
-    return onRunComplete(context, event, event.error);
+    return onStepComplete(context, event, event.error);
   }
 }
 function onWorkflowStart({ channel }, _event) {
   return sendEvent(channel, ATTEMPT_START);
 }
-async function onWorkflowComplete({ state, channel, onFinish }, _event) {
-  const result = state.dataclips[state.lastDataclipId];
-  const reason = calculateAttemptExitReason(state);
-  await sendEvent(channel, ATTEMPT_COMPLETE, {
-    final_dataclip_id: state.lastDataclipId,
-    ...reason
-  });
-  onFinish({ reason, state: result });
-}
-async function onWorkflowError(context, event) {
-  const { state, channel, logger: logger2, onFinish } = context;
-  try {
-    const reason = calculateJobExitReason("", { data: {} }, event);
-    if (state.activeJob) {
-      await onJobError(context, { error: event });
-    }
-    await sendEvent(channel, ATTEMPT_COMPLETE, {
-      final_dataclip_id: state.lastDataclipId,
-      ...reason
-    });
-    onFinish({ reason });
-  } catch (e) {
-    logger2.error("ERROR in workflow-error handler:", e.message);
-    logger2.error(e);
-    onFinish({});
-  }
-}
 function onJobLog({ channel, state }, event) {
   const timeInMicroseconds = BigInt(event.time) / BigInt(1e3);
   const log = {
@@ -5632,8 +5669,8 @@ function onJobLog({ channel, state }, event) {
     level: event.level,
     timestamp: timeInMicroseconds.toString()
   };
-  if (state.activeRun) {
-    log.run_id = state.activeRun;
+  if (state.activeStep) {
+    log.step_id = state.activeStep;
   }
   return sendEvent(channel, ATTEMPT_LOG, log);
 }
@@ -5795,7 +5832,6 @@ function connect(app, logger2, options = {}) {
 function createServer(engine, options = {}) {
   const logger2 = options.logger || createMockLogger2();
   const port = options.port || DEFAULT_PORT;
-  logger2.debug("Starting server");
   const app = new Koa();
   app.id = humanId({ separator: "-", capitalize: false });
   const router = new Router();
@@ -5810,7 +5846,7 @@ function createServer(engine, options = {}) {
   app.workflows = {};
   app.destroyed = false;
   app.server = app.listen(port);
-  logger2.success(`ws-worker ${app.id} listening on ${port}`);
+  logger2.success(`Worker ${app.id} listening on ${port}`);
   process.send?.("READY");
   router.get("/livez", healthcheck_default);
   router.get("/", healthcheck_default);
@@ -5879,30 +5915,37 @@ var server_default = createServer;
 
 // src/start.ts
 var {
+  WORKER_BACKOFF,
+  WORKER_CAPACITY,
+  WORKER_LIGHTNING_SERVICE_URL,
+  WORKER_LOG_LEVEL,
+  WORKER_MAX_RUN_DURATION_SECONDS,
+  WORKER_MAX_RUN_MEMORY_MB,
+  WORKER_PORT,
   WORKER_REPO_DIR,
   WORKER_SECRET,
-  MAX_RUN_MEMORY,
-  STATE_PROPS_TO_REMOVE
+  WORKER_STATE_PROPS_TO_REMOVE
 } = process.env;
 var args = yargs_default(hideBin(process.argv)).command("server", "Start a ws-worker server").option("port", {
   alias: "p",
-  description: "Port to run the server on",
+  description: "Port to run the server on. Env: WORKER_PORT",
   type: "number",
-  default: 2222
+  default: WORKER_PORT || 2222
 }).option("lightning", {
-  alias: "l",
-  description: 'Base url to Lightning websocket endpoint, eg, ws://localhost:4000/worker. Set to "mock" to use the default mock server',
-  default: "ws://localhost:4000/worker"
+  alias: ["l", "lightning-service-url"],
+  description: 'Base url to Lightning websocket endpoint, eg, ws://localhost:4000/worker. Set to "mock" to use the default mock server. Env: WORKER_LIGHTNING_SERVICE_URL',
+  default: WORKER_LIGHTNING_SERVICE_URL || "ws://localhost:4000/worker"
 }).option("repo-dir", {
   alias: "d",
-  description: "Path to the runtime repo (where modules will be installed)",
+  description: "Path to the runtime repo (where modules will be installed). Env: WORKER_REPO_DIR",
   default: WORKER_REPO_DIR
 }).option("secret", {
   alias: "s",
-  description: "Worker secret (comes from WORKER_SECRET by default)"
+  description: "Worker secret. (comes from WORKER_SECRET by default). Env: WORKER_SECRET",
+  default: WORKER_SECRET
 }).option("log", {
-  description: "Worker secret (comes from WORKER_SECRET by default)",
-  default: "info",
+  description: "Set the log level for stdout (default to info, set to debug for verbose output). Env: WORKER_LOG_LEVEL",
+  default: WORKER_LOG_LEVEL || "debug",
   type: "string"
 }).option("loop", {
   description: "Disable the claims loop",
@@ -5913,20 +5956,25 @@ var args = yargs_default(hideBin(process.argv)).command("server", "Start a ws-wo
   default: false,
   type: "boolean"
 }).option("backoff", {
-  description: "Claim backoff rules: min/max (s)",
-  default: "1/10"
+  description: "Claim backoff rules: min/max (in seconds). Env: WORKER_BACKOFF",
+  default: WORKER_BACKOFF || "1/10"
 }).option("capacity", {
-  description: "max concurrent workers",
-  default: 5,
+  description: "max concurrent workers. Env: WORKER_CAPACITY",
+  default: WORKER_CAPACITY ? parseInt(WORKER_CAPACITY) : 5,
   type: "number"
 }).option("state-props-to-remove", {
-  description: "A list of properties to remove from the final state returned by a job",
-  default: STATE_PROPS_TO_REMOVE ?? ["configuration", "response"],
+  description: "A list of properties to remove from the final state returned by a job. Env: WORKER_STATE_PROPS_TO_REMOVE",
+  default: WORKER_STATE_PROPS_TO_REMOVE ?? ["configuration", "response"],
   type: "array"
 }).option("run-memory", {
-  description: "Maximum memory allocated to a single run, in mb",
+  description: "Maximum memory allocated to a single run, in mb. Env: WORKER_MAX_RUN_MEMORY_MB",
+  type: "number",
+  default: WORKER_MAX_RUN_MEMORY_MB ? parseInt(WORKER_MAX_RUN_MEMORY_MB) : 500
+}).option("max-run-duration-seconds", {
+  alias: "t",
+  description: "Default attempt timeout for the server, in seconds. Env: WORKER_MAX_RUN_DURATION_SECONDS",
   type: "number",
-  default: MAX_RUN_MEMORY ? parseInt(MAX_RUN_MEMORY) : 500
+  default: WORKER_MAX_RUN_DURATION_SECONDS || 60 * 5
 }).parse();
 var logger = createLogger("SRV", { level: args.log });
 if (args.lightning === "mock") {
@@ -5935,15 +5983,13 @@ if (args.lightning === "mock") {
     args.secret = "abdefg";
   }
 } else if (!args.secret) {
-  if (!WORKER_SECRET) {
-    logger.error("WORKER_SECRET is not set");
-    process.exit(1);
-  }
-  args.secret = WORKER_SECRET;
+  logger.error("WORKER_SECRET is not set");
+  process.exit(1);
 }
 var [minBackoff, maxBackoff] = args.backoff.split("/").map((n) => parseInt(n, 10) * 1e3);
 function engineReady(engine) {
-  server_default(engine, {
+  logger.debug("Creating worker server...");
+  const workerOptions = {
     port: args.port,
     lightning: args.lightning,
     logger,
@@ -5954,7 +6000,10 @@ function engineReady(engine) {
       max: maxBackoff
     },
     maxWorkflows: args.capacity
-  });
+  };
+  const { logger: _l, secret: _s, ...humanOptions } = workerOptions;
+  logger.debug("Worker options:", humanOptions);
+  server_default(engine, workerOptions);
 }
 if (args.mock) {
   runtime_engine_default().then((engine) => {
@@ -5962,13 +6011,17 @@ if (args.mock) {
     engineReady(engine);
   });
 } else {
-  createRTE({
+  const engineOptions = {
     repoDir: args.repoDir,
     memoryLimitMb: args.runMemory,
     maxWorkers: args.capacity,
-    statePropsToRemove: args.statePropsToRemove
-  }).then((engine) => {
-    logger.debug("engine created");
+    statePropsToRemove: args.statePropsToRemove,
+    attemptTimeoutMs: args.maxRunDurationSeconds * 1e3
+  };
+  logger.debug("Creating runtime engine...");
+  logger.debug("Engine options:", engineOptions);
+  createRTE(engineOptions).then((engine) => {
+    logger.debug("Engine created!");
     engineReady(engine);
   });
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openfn/ws-worker",
-  "version": "0.5.0",
+  "version": "0.7.0",
   "description": "A Websocket Worker to connect Lightning to a Runtime Engine",
   "main": "dist/index.js",
   "type": "module",
@@ -22,9 +22,9 @@
     "koa-logger": "^3.2.1",
     "phoenix": "^1.7.7",
     "ws": "^8.14.1",
-    "@openfn/engine-multi": "0.2.6",
-    "@openfn/runtime": "0.2.4",
-    "@openfn/logger": "0.0.19"
+    "@openfn/engine-multi": "0.3.0",
+    "@openfn/logger": "0.0.19",
+    "@openfn/runtime": "0.2.5"
   },
   "devDependencies": {
     "@types/koa": "^2.13.5",
@@ -41,7 +41,7 @@
     "tsup": "^6.2.3",
     "typescript": "^4.6.4",
     "yargs": "^17.6.2",
-    "@openfn/lightning-mock": "1.1.8"
+    "@openfn/lightning-mock": "1.1.11"
   },
   "files": [
     "dist",