@openfn/ws-worker 0.1.8 → 0.2.1

package/CHANGELOG.md

@@ -1,5 +1,27 @@
 # ws-worker
 
+## 0.2.1
+
+### Patch Changes
+
+- ad8f6e9: Improve robustness of server connectivity
+- Updated dependencies [704e7b6]
+  - @openfn/engine-multi@0.1.7
+
+## 0.2.0
+
+### Minor Changes
+
+- 0e8e20c: BREAKING: Updated exit reasons to `{ reason: "success", error_type, error_message }`
+  Add exit reasons to job and attempt complete
+
+### Patch Changes
+
+- Updated dependencies [a540888]
+- Updated dependencies [0e8e20c]
+  - @openfn/runtime@0.1.0
+  - @openfn/engine-multi@0.1.6
+
 ## 0.1.8
 
 ### Patch Changes

package/dist/index.d.ts

@@ -1,8 +1,22 @@
 import Koa from 'koa';
 import { SanitizePolicies, Logger } from '@openfn/logger';
-import { RuntimeEngine } from '@openfn/engine-multi';
 import { Channel as Channel$1 } from 'phoenix';
 import { ExecutionPlan } from '@openfn/runtime';
+import { RuntimeEngine } from '@openfn/engine-multi';
+
+type ExitReasonStrings =
+  | 'success'
+  | 'fail'
+  | 'crash'
+  | 'kill'
+  | 'cancel'
+  | 'exception';
+
+type ExitReason = {
+  reason: ExitReasonStrings;
+  error_message: string | null;
+  error_type: string | null;
+};
 
 type AttemptOptions = {
   timeout?: number;
@@ -37,6 +51,7 @@ declare type AttemptState = {
     plan: ExecutionPlan;
     options: AttemptOptions;
     dataclips: Record<string, any>;
+    reasons: Record<string, ExitReason>;
     lastDataclipId?: string;
 };
 declare type Context = {
@@ -70,7 +85,7 @@ interface ServerApp extends Koa {
     workflows: Record<string, true | Context>;
     execute: ({ id, token }: CLAIM_ATTEMPT) => Promise<void>;
     destroy: () => void;
-    killWorkloop: () => void;
+    killWorkloop?: () => void;
 }
 declare function createServer(engine: RuntimeEngine, options?: ServerOptions): ServerApp;
 

package/dist/index.js

@@ -80,6 +80,11 @@ var claim = (app, logger = mockLogger, maxWorkers = 5) => {
         app.execute(attempt);
         resolve();
       });
+    }).receive("error", () => {
+      logger.debug("pull err");
+    }).receive("timeout", () => {
+      logger.debug("pull timeout");
+      reject(new Error("timeout"));
     });
   });
 };
@@ -210,6 +215,30 @@ var stringify_default = (obj) => stringify(obj, (_key, value) => {
   return value;
 });
 
+// src/api/reasons.ts
+var calculateJobExitReason = (jobId, state, error) => {
+  let reason = "success";
+  let error_type = null;
+  let error_message = null;
+  if (error) {
+    reason = error.severity ?? "crash";
+    error_message = error.message;
+    error_type = error.subtype || error.type || error.name;
+  } else if (state.errors?.[jobId]) {
+    reason = "fail";
+    ({ message: error_message, type: error_type } = state.errors[jobId]);
+  }
+  return { reason, error_type, error_message };
+};
+var calculateAttemptExitReason = (state) => {
+  if (state.reasons) {
+    const fail = Object.values(state.reasons).find(
+      ({ reason }) => reason === "fail"
+    );
+    return fail || { reason: "success", error_type: null, error_message: null };
+  }
+};
+
 // src/api/execute.ts
 var enc = new TextDecoder("utf-8");
 var eventMap = {
@@ -219,15 +248,17 @@ var eventMap = {
   "workflow-log": ATTEMPT_LOG,
   "workflow-complete": ATTEMPT_COMPLETE
 };
+var createAttemptState = (plan, options = {}) => ({
+  plan,
+  lastDataclipId: plan.initialState,
+  dataclips: {},
+  reasons: {},
+  options
+});
 function execute(channel, engine, logger, plan, options = {}, onComplete = (_result) => {
 }) {
   logger.info("execute...");
-  const state = {
-    plan,
-    lastDataclipId: plan.initialState,
-    dataclips: {},
-    options
-  };
+  const state = createAttemptState(plan, options);
   const context = { channel, state, logger, onComplete };
   const addEvent = (eventName, handler) => {
     const wrappedFn = async (event) => {
@@ -251,6 +282,7 @@ function execute(channel, engine, logger, plan, options = {}, onComplete = (_res
     addEvent("workflow-start", onWorkflowStart),
     addEvent("job-start", onJobStart),
     addEvent("job-complete", onJobComplete),
+    addEvent("job-error", onJobError),
     addEvent("workflow-log", onJobLog),
     addEvent("workflow-complete", onWorkflowComplete),
     addEvent("workflow-error", onWorkflowError)
@@ -288,7 +320,15 @@ function onJobStart({ channel, state }, event) {
     input_dataclip_id: state.lastDataclipId
   });
 }
-function onJobComplete({ channel, state }, event) {
+function onJobError(context, event) {
+  const { state, error, jobId } = event;
+  if (state.errors?.[jobId]?.message === error.message) {
+    onJobComplete(context, event);
+  } else {
+    onJobComplete(context, event, event.error);
+  }
+}
+function onJobComplete({ channel, state }, event, error) {
   const dataclipId = crypto2.randomUUID();
   const run_id = state.activeRun;
   const job_id = state.activeJob;
@@ -299,12 +339,20 @@ function onJobComplete({ channel, state }, event) {
   state.lastDataclipId = dataclipId;
   delete state.activeRun;
   delete state.activeJob;
+  const { reason, error_message, error_type } = calculateJobExitReason(
+    job_id,
+    event.state,
+    error
+  );
+  state.reasons[job_id] = { reason, error_message, error_type };
   return sendEvent(channel, RUN_COMPLETE, {
     run_id,
     job_id,
     output_dataclip_id: dataclipId,
     output_dataclip: stringify_default(event.state),
-    reason: "success"
+    reason,
+    error_message,
+    error_type
   });
 }
 function onWorkflowStart({ channel }, _event) {
@@ -312,20 +360,20 @@ function onWorkflowStart({ channel }, _event) {
 }
 async function onWorkflowComplete({ state, channel, onComplete }, _event) {
   const result = state.dataclips[state.lastDataclipId];
+  const reason = calculateAttemptExitReason(state);
   await sendEvent(channel, ATTEMPT_COMPLETE, {
     final_dataclip_id: state.lastDataclipId,
-    status: "success",
-    reason: "ok"
+    ...reason
   });
-  onComplete(result);
+  onComplete({ reason, state: result });
 }
 async function onWorkflowError({ state, channel, onComplete }, event) {
+  const reason = calculateJobExitReason("", { data: {} }, event);
   await sendEvent(channel, ATTEMPT_COMPLETE, {
-    reason: "fail",
     final_dataclip_id: state.lastDataclipId,
-    message: event.message
+    ...reason
   });
-  onComplete({});
+  onComplete({ reason });
 }
 function onJobLog({ channel, state }, event) {
   const timeInMicroseconds = BigInt(event.time) / BigInt(1e3);
@@ -380,6 +428,7 @@ async function loadAttempt(channel) {
 }
 
 // src/channels/worker-queue.ts
+import EventEmitter from "node:events";
 import { Socket as PhxSocket } from "phoenix";
 import { WebSocket } from "ws";
 
@@ -405,9 +454,9 @@ var generateWorkerToken = async (secret, workerId) => {
 var worker_token_default = generateWorkerToken;
 
 // src/channels/worker-queue.ts
-var connectToWorkerQueue = (endpoint, serverId, secret, SocketConstructor = PhxSocket) => {
-  return new Promise(async (done, reject) => {
-    const token = await worker_token_default(secret, serverId);
+var connectToWorkerQueue = (endpoint, serverId, secret, logger, SocketConstructor = PhxSocket) => {
+  const events = new EventEmitter();
+  worker_token_default(secret, serverId).then((token) => {
     const socket = new SocketConstructor(endpoint, {
       params: { token },
       transport: WebSocket
@@ -417,20 +466,27 @@ var connectToWorkerQueue = (endpoint, serverId, secret, SocketConstructor = PhxS
       didOpen = true;
       const channel = socket.channel("worker:queue");
       channel.join().receive("ok", () => {
-        done({ socket, channel });
+        logger.debug("Connected to worker queue socket");
+        events.emit("connect", { socket, channel });
       }).receive("error", (e) => {
-        console.log("ERROR", e);
+        logger.error("ERROR", e);
       }).receive("timeout", (e) => {
-        console.log("TIMEOUT", e);
+        logger.error("TIMEOUT", e);
       });
     });
+    socket.onClose((_e) => {
+      logger.debug("queue socket closed");
+      events.emit("disconnect");
+    });
     socket.onError((e) => {
       if (!didOpen) {
-        reject(e);
+        events.emit("error", e.message);
+        didOpen = false;
       }
     });
     socket.connect();
   });
+  return events;
 };
 var worker_queue_default = connectToWorkerQueue;
 
@@ -438,17 +494,13 @@ var worker_queue_default = connectToWorkerQueue;
 var DEFAULT_PORT = 1234;
 var MIN_BACKOFF = 1e3;
 var MAX_BACKOFF = 1e3 * 30;
-function connect(app, engine, logger, options = {}) {
+function connect(app, logger, options = {}) {
   logger.debug("Connecting to Lightning at", options.lightning);
-  worker_queue_default(options.lightning, app.id, options.secret).then(({ socket, channel }) => {
+  const onConnect = ({ socket, channel }) => {
     logger.success("Connected to Lightning at", options.lightning);
     app.socket = socket;
     app.channel = channel;
     if (!options.noLoop) {
-      if (app.killWorkloop) {
-        logger.info("Terminating old workloop");
-        app.killWorkloop();
-      }
       logger.info("Starting workloop");
       app.killWorkloop = workloop_default(
         app,
@@ -467,17 +519,25 @@ function connect(app, engine, logger, options = {}) {
       );
       logger.break();
     }
-  }).catch((e) => {
+  };
+  const onDisconnect = () => {
+    if (app.killWorkloop) {
+      app.killWorkloop();
+      delete app.killWorkloop;
+      logger.info("Connection to lightning lost.");
+      logger.info(
+        "Worker will automatically reconnect when lightning is back online."
+      );
+    }
+  };
+  const onError = (e) => {
     logger.error(
       "CRITICAL ERROR: could not connect to lightning at",
       options.lightning
     );
     logger.debug(e);
-    app.killWorkloop?.();
-    setTimeout(() => {
-      connect(app, engine, logger, options);
-    }, 1e4);
-  });
+  };
+  worker_queue_default(options.lightning, app.id, options.secret, logger).on("connect", onConnect).on("disconnect", onDisconnect).on("error", onError);
 }
 function createServer(engine, options = {}) {
   const logger = options.logger || createMockLogger2();
@@ -540,7 +600,7 @@ function createServer(engine, options = {}) {
   };
   app.use(router.routes());
   if (options.lightning) {
-    connect(app, engine, logger, options);
+    connect(app, logger, options);
   } else {
     logger.warn("No lightning URL provided");
   }

package/dist/start.js

@@ -5062,6 +5062,11 @@ var claim = (app, logger2 = mockLogger, maxWorkers = 5) => {
         app.execute(attempt);
         resolve5();
       });
+    }).receive("error", () => {
+      logger2.debug("pull err");
+    }).receive("timeout", () => {
+      logger2.debug("pull timeout");
+      reject(new Error("timeout"));
     });
   });
 };
@@ -5192,6 +5197,30 @@ var stringify_default = (obj) => stringify(obj, (_key, value) => {
   return value;
 });
 
+// src/api/reasons.ts
+var calculateJobExitReason = (jobId, state, error) => {
+  let reason = "success";
+  let error_type = null;
+  let error_message = null;
+  if (error) {
+    reason = error.severity ?? "crash";
+    error_message = error.message;
+    error_type = error.subtype || error.type || error.name;
+  } else if (state.errors?.[jobId]) {
+    reason = "fail";
+    ({ message: error_message, type: error_type } = state.errors[jobId]);
+  }
+  return { reason, error_type, error_message };
+};
+var calculateAttemptExitReason = (state) => {
+  if (state.reasons) {
+    const fail = Object.values(state.reasons).find(
+      ({ reason }) => reason === "fail"
+    );
+    return fail || { reason: "success", error_type: null, error_message: null };
+  }
+};
+
 // src/api/execute.ts
 var enc = new TextDecoder("utf-8");
 var eventMap = {
@@ -5201,15 +5230,17 @@ var eventMap = {
   "workflow-log": ATTEMPT_LOG,
   "workflow-complete": ATTEMPT_COMPLETE
 };
+var createAttemptState = (plan, options = {}) => ({
+  plan,
+  lastDataclipId: plan.initialState,
+  dataclips: {},
+  reasons: {},
+  options
+});
 function execute(channel, engine, logger2, plan, options = {}, onComplete = (_result) => {
 }) {
   logger2.info("execute...");
-  const state = {
-    plan,
-    lastDataclipId: plan.initialState,
-    dataclips: {},
-    options
-  };
+  const state = createAttemptState(plan, options);
   const context = { channel, state, logger: logger2, onComplete };
   const addEvent = (eventName, handler) => {
     const wrappedFn = async (event) => {
@@ -5233,6 +5264,7 @@ function execute(channel, engine, logger2, plan, options = {}, onComplete = (_re
     addEvent("workflow-start", onWorkflowStart),
     addEvent("job-start", onJobStart),
     addEvent("job-complete", onJobComplete),
+    addEvent("job-error", onJobError),
     addEvent("workflow-log", onJobLog),
     addEvent("workflow-complete", onWorkflowComplete),
     addEvent("workflow-error", onWorkflowError)
@@ -5270,7 +5302,15 @@ function onJobStart({ channel, state }, event) {
     input_dataclip_id: state.lastDataclipId
   });
 }
-function onJobComplete({ channel, state }, event) {
+function onJobError(context, event) {
+  const { state, error, jobId } = event;
+  if (state.errors?.[jobId]?.message === error.message) {
+    onJobComplete(context, event);
+  } else {
+    onJobComplete(context, event, event.error);
+  }
+}
+function onJobComplete({ channel, state }, event, error) {
   const dataclipId = crypto3.randomUUID();
   const run_id = state.activeRun;
   const job_id = state.activeJob;
@@ -5281,12 +5321,20 @@ function onJobComplete({ channel, state }, event) {
   state.lastDataclipId = dataclipId;
   delete state.activeRun;
   delete state.activeJob;
+  const { reason, error_message, error_type } = calculateJobExitReason(
+    job_id,
+    event.state,
+    error
+  );
+  state.reasons[job_id] = { reason, error_message, error_type };
   return sendEvent(channel, RUN_COMPLETE, {
     run_id,
     job_id,
     output_dataclip_id: dataclipId,
     output_dataclip: stringify_default(event.state),
-    reason: "success"
+    reason,
+    error_message,
+    error_type
   });
 }
 function onWorkflowStart({ channel }, _event) {
@@ -5294,20 +5342,20 @@ function onWorkflowStart({ channel }, _event) {
 }
 async function onWorkflowComplete({ state, channel, onComplete }, _event) {
   const result = state.dataclips[state.lastDataclipId];
+  const reason = calculateAttemptExitReason(state);
   await sendEvent(channel, ATTEMPT_COMPLETE, {
     final_dataclip_id: state.lastDataclipId,
-    status: "success",
-    reason: "ok"
+    ...reason
   });
-  onComplete(result);
+  onComplete({ reason, state: result });
 }
 async function onWorkflowError({ state, channel, onComplete }, event) {
+  const reason = calculateJobExitReason("", { data: {} }, event);
   await sendEvent(channel, ATTEMPT_COMPLETE, {
-    reason: "fail",
     final_dataclip_id: state.lastDataclipId,
-    message: event.message
+    ...reason
   });
-  onComplete({});
+  onComplete({ reason });
 }
 function onJobLog({ channel, state }, event) {
   const timeInMicroseconds = BigInt(event.time) / BigInt(1e3);
@@ -5362,6 +5410,7 @@ async function loadAttempt(channel) {
 }
 
 // src/channels/worker-queue.ts
+import EventEmitter2 from "node:events";
 import { Socket as PhxSocket } from "phoenix";
 import { WebSocket } from "ws";
 
@@ -5387,9 +5436,9 @@ var generateWorkerToken = async (secret, workerId) => {
 var worker_token_default = generateWorkerToken;
 
 // src/channels/worker-queue.ts
-var connectToWorkerQueue = (endpoint, serverId, secret, SocketConstructor = PhxSocket) => {
-  return new Promise(async (done, reject) => {
-    const token = await worker_token_default(secret, serverId);
+var connectToWorkerQueue = (endpoint, serverId, secret, logger2, SocketConstructor = PhxSocket) => {
+  const events = new EventEmitter2();
+  worker_token_default(secret, serverId).then((token) => {
     const socket = new SocketConstructor(endpoint, {
       params: { token },
       transport: WebSocket
@@ -5399,20 +5448,27 @@ var connectToWorkerQueue = (endpoint, serverId, secret, SocketConstructor = PhxS
       didOpen = true;
       const channel = socket.channel("worker:queue");
       channel.join().receive("ok", () => {
-        done({ socket, channel });
+        logger2.debug("Connected to worker queue socket");
+        events.emit("connect", { socket, channel });
       }).receive("error", (e) => {
-        console.log("ERROR", e);
+        logger2.error("ERROR", e);
       }).receive("timeout", (e) => {
-        console.log("TIMEOUT", e);
+        logger2.error("TIMEOUT", e);
       });
     });
+    socket.onClose((_e) => {
+      logger2.debug("queue socket closed");
+      events.emit("disconnect");
+    });
     socket.onError((e) => {
       if (!didOpen) {
-        reject(e);
+        events.emit("error", e.message);
+        didOpen = false;
       }
     });
     socket.connect();
   });
+  return events;
 };
 var worker_queue_default = connectToWorkerQueue;
 
@@ -5420,17 +5476,13 @@ var worker_queue_default = connectToWorkerQueue;
 var DEFAULT_PORT = 1234;
 var MIN_BACKOFF = 1e3;
 var MAX_BACKOFF = 1e3 * 30;
-function connect(app, engine, logger2, options = {}) {
+function connect(app, logger2, options = {}) {
   logger2.debug("Connecting to Lightning at", options.lightning);
-  worker_queue_default(options.lightning, app.id, options.secret).then(({ socket, channel }) => {
+  const onConnect = ({ socket, channel }) => {
     logger2.success("Connected to Lightning at", options.lightning);
     app.socket = socket;
     app.channel = channel;
     if (!options.noLoop) {
-      if (app.killWorkloop) {
-        logger2.info("Terminating old workloop");
-        app.killWorkloop();
-      }
       logger2.info("Starting workloop");
       app.killWorkloop = workloop_default(
         app,
@@ -5449,17 +5501,25 @@ function connect(app, engine, logger2, options = {}) {
       );
       logger2.break();
     }
-  }).catch((e) => {
+  };
+  const onDisconnect = () => {
+    if (app.killWorkloop) {
+      app.killWorkloop();
+      delete app.killWorkloop;
+      logger2.info("Connection to lightning lost.");
+      logger2.info(
+        "Worker will automatically reconnect when lightning is back online."
+      );
+    }
+  };
+  const onError = (e) => {
     logger2.error(
       "CRITICAL ERROR: could not connect to lightning at",
       options.lightning
     );
     logger2.debug(e);
-    app.killWorkloop?.();
-    setTimeout(() => {
-      connect(app, engine, logger2, options);
-    }, 1e4);
-  });
+  };
+  worker_queue_default(options.lightning, app.id, options.secret, logger2).on("connect", onConnect).on("disconnect", onDisconnect).on("error", onError);
 }
 function createServer(engine, options = {}) {
   const logger2 = options.logger || createMockLogger2();
@@ -5522,7 +5582,7 @@ function createServer(engine, options = {}) {
   };
   app.use(router.routes());
   if (options.lightning) {
-    connect(app, engine, logger2, options);
+    connect(app, logger2, options);
   } else {
     logger2.warn("No lightning URL provided");
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openfn/ws-worker",
-  "version": "0.1.8",
+  "version": "0.2.1",
   "description": "A Websocket Worker to connect Lightning to a Runtime Engine",
   "main": "dist/index.js",
   "type": "module",
@@ -21,9 +21,9 @@
     "koa-logger": "^3.2.1",
     "phoenix": "^1.7.7",
     "ws": "^8.14.1",
-    "@openfn/engine-multi": "0.1.5",
+    "@openfn/engine-multi": "0.1.7",
     "@openfn/logger": "0.0.19",
-    "@openfn/runtime": "0.0.33"
+    "@openfn/runtime": "0.1.0"
   },
   "devDependencies": {
     "@types/koa": "^2.13.5",
@@ -40,7 +40,7 @@
     "tsup": "^6.2.3",
     "typescript": "^4.6.4",
     "yargs": "^17.6.2",
-    "@openfn/lightning-mock": "1.0.6"
+    "@openfn/lightning-mock": "1.0.8"
   },
   "files": [
     "dist",