@openfn/ws-worker 1.12.1 → 1.13.1

package/CHANGELOG.md

@@ -1,5 +1,27 @@
 # ws-worker
 
+## 1.13.1
+
+### Patch Changes
+
+- b83d13c: Add DEFAULT_MESSAGE_TIMEOUT_SECONDS env var and tweaked some error handling around lightning messaging
+- 0bd4adf: Include pod name in logs when claiming
+
+## 1.13.0
+
+### Minor Changes
+
+- ce5022a: Added sentry notifications for server and websocket errors
+
+### Patch Changes
+
+- 0a176aa: Ignore empty log lines (don't send them to lightning)
+- Updated dependencies [0a176aa]
+  - @openfn/logger@1.0.5
+  - @openfn/engine-multi@1.6.2
+  - @openfn/lexicon@1.2.0
+  - @openfn/runtime@1.6.4
+
 ## 1.12.1
 
 ### Patch Changes

package/dist/index.d.ts

@@ -53,6 +53,7 @@ type WorkerRunOptions = ExecuteOptions & {
 };
 
 type Context = {
+    id: string;
     channel: Channel;
     state: RunState;
     logger: Logger;
@@ -73,7 +74,10 @@ type ServerOptions = {
         min?: number;
         max?: number;
     };
+    sentryDsn?: string;
+    sentryEnv?: string;
     socketTimeoutSeconds?: number;
+    messageTimeoutSeconds?: number;
     payloadLimitMb?: number;
     collectionsVersion?: string;
     collectionsUrl?: string;

package/dist/index.js

@@ -2,6 +2,7 @@
 import { EventEmitter as EventEmitter2 } from "node:events";
 import { promisify } from "node:util";
 import { exec as _exec } from "node:child_process";
+import * as Sentry5 from "@sentry/node";
 import Koa from "koa";
 import bodyParser from "koa-bodyparser";
 import koaLogger from "koa-logger";
@@ -123,9 +124,11 @@ var verifyToken = async (token, publicKey) => {
     return true;
   }
 };
+var { DEPLOYED_POD_NAME } = process.env;
 var claim = (app, logger = mockLogger, options = {}) => {
   return new Promise((resolve, reject) => {
     const { maxWorkers = 5 } = options;
+    const podName = DEPLOYED_POD_NAME ? `[${DEPLOYED_POD_NAME}] ` : "";
     const activeWorkers = Object.keys(app.workflows).length;
     if (activeWorkers >= maxWorkers) {
       app.workloop?.stop(`server at capacity (${activeWorkers}/${maxWorkers})`);
@@ -137,10 +140,10 @@ var claim = (app, logger = mockLogger, options = {}) => {
     }
     logger.debug(`requesting run (capacity ${activeWorkers}/${maxWorkers})`);
     const start = Date.now();
-    app.queueChannel.push(CLAIM, { demand: 1 }).receive("ok", ({ runs }) => {
+    app.queueChannel.push(CLAIM, { demand: 1, pod_name: DEPLOYED_POD_NAME }).receive("ok", ({ runs }) => {
       const duration = Date.now() - start;
       logger.debug(
-        `claimed ${runs.length} runs in ${duration}ms (${runs.length ? runs.map((r) => r.id).join(",") : "-"})`
+        `${podName}claimed ${runs.length} runs in ${duration}ms (${runs.length ? runs.map((r) => r.id).join(",") : "-"})`
       );
       if (!runs?.length) {
         return reject(new Error("No runs returned"));
@@ -160,7 +163,7 @@ var claim = (app, logger = mockLogger, options = {}) => {
         } else {
           logger.debug("skipping run token validation for", run.id);
         }
-        logger.debug("starting run", run.id);
+        logger.debug(`${podName} starting run ${run.id}`);
         app.execute(run);
         resolve();
       });
@@ -210,6 +213,9 @@ var startWorkloop = (app, logger, minBackoff, maxBackoff, maxWorkers) => {
 };
 var workloop_default = startWorkloop;
 
+// src/api/execute.ts
+import * as Sentry2 from "@sentry/node";
+
 // src/util/convert-lightning-plan.ts
 import crypto2 from "node:crypto";
 import path from "node:path";
@@ -387,26 +393,6 @@ var convert_lightning_plan_default = (run, options = {}) => {
   };
 };
 
-// src/util/get-with-reply.ts
-var get_with_reply_default = (channel, event, payload) => new Promise((resolve, reject) => {
-  channel.push(event, payload).receive("ok", (evt) => {
-    resolve(evt);
-  }).receive("error", (e) => {
-    reject(e);
-  }).receive("timeout", (e) => {
-    reject(e);
-  });
-});
-
-// src/util/stringify.ts
-import stringify from "fast-safe-stringify";
-var stringify_default = (obj) => stringify(obj, (_key, value) => {
-  if (value instanceof Uint8Array) {
-    return Array.from(value);
-  }
-  return value;
-});
-
 // src/util/create-run-state.ts
 var create_run_state_default = (plan, input) => {
   const state = {
@@ -440,6 +426,67 @@ var create_run_state_default = (plan, input) => {
   return state;
 };
 
+// src/util/send-event.ts
+import * as Sentry from "@sentry/node";
+
+// src/errors.ts
+var LightningSocketError = class extends Error {
+  constructor(event, message) {
+    super(`[${event}] ${message}`);
+    this.name = "LightningSocketError";
+    this.event = "";
+    this.rejectMessage = "";
+    this.event = event;
+    this.rejectMessage = message;
+  }
+};
+var LightningTimeoutError = class extends Error {
+  constructor(event) {
+    super(`[${event}] timeout`);
+    this.name = "LightningTimeoutError";
+  }
+};
+
+// src/util/send-event.ts
+var sendEvent = (context, event, payload) => {
+  const { channel, logger, id: runId = "<unknown run>" } = context;
+  return new Promise((resolve, reject) => {
+    const report = (error) => {
+      logger.error(`${runId} :: ${event} :: ERR: ${error.message || error}`);
+      const context2 = {
+        run_id: runId,
+        event
+      };
+      const extras = {};
+      if (error.rejectMessage) {
+        extras.rejection_reason = error.rejectMessage;
+      }
+      Sentry.captureException(error, (scope) => {
+        scope.setContext("run", context2);
+        scope.setExtras(extras);
+        return scope;
+      });
+      error.reportedToSentry = true;
+      reject(error);
+    };
+    channel.push(event, payload).receive("error", (message) => {
+      report(new LightningSocketError(event, message));
+    }).receive("timeout", () => {
+      report(new LightningTimeoutError(event));
+    }).receive("ok", resolve);
+  });
+};
+var send_event_default = sendEvent;
+
+// src/util/stringify.ts
+import stringify from "fast-safe-stringify";
+var stringify_default = (obj) => stringify(obj, (_key, value) => {
+  if (value instanceof Uint8Array) {
+    return Array.from(value);
+  }
+  return value;
+});
+
 // src/util/throttle.ts
 var createThrottler = () => {
   const q = [];
@@ -525,7 +572,7 @@ function getVersion() {
 
 // src/events/run-start.ts
 async function onRunStart(context, event) {
-  const { channel, state, options = {} } = context;
+  const { state, options = {} } = context;
   const time = (timestamp() - BigInt(1e7)).toString();
   const versionLogContext = {
     ...context,
@@ -538,7 +585,7 @@ async function onRunStart(context, event) {
     worker: await getVersion(),
     ...event.versions
   };
-  await sendEvent(channel, RUN_START, {
+  await sendEvent(context, RUN_START, {
     versions,
     /// use the engine time in run start
     timestamp: timeInMicroseconds(event.time)
@@ -600,7 +647,7 @@ var calculateRunExitReason = (state) => {
 
 // src/events/step-complete.ts
 async function onStepComplete(context, event, error) {
-  const { channel, state, options } = context;
+  const { state, options } = context;
   const dataclipId = crypto3.randomUUID();
   const step_id = state.activeStep;
   const job_id = state.activeJob;
@@ -645,13 +692,13 @@ async function onStepComplete(context, event, error) {
   const reason = calculateJobExitReason(job_id, event.state, error);
   state.reasons[job_id] = reason;
   Object.assign(evt, reason);
-  return sendEvent(channel, STEP_COMPLETE, evt);
+  return sendEvent(context, STEP_COMPLETE, evt);
 }
 
 // src/events/step-start.ts
 import crypto4 from "node:crypto";
 async function onStepStart(context, event) {
-  const { channel, state } = context;
+  const { state } = context;
   state.activeStep = crypto4.randomUUID();
   state.activeJob = event.jobId;
   const input_dataclip_id = state.inputDataclips[event.jobId];
@@ -663,7 +710,7 @@ async function onStepStart(context, event) {
   if (!state.withheldDataclips[input_dataclip_id]) {
     evt.input_dataclip_id = input_dataclip_id;
   }
-  await sendEvent(channel, STEP_START, evt);
+  await sendEvent(context, STEP_START, evt);
 }
 
 // src/util/log-final-reason.ts
@@ -685,12 +732,12 @@ ${reason.error_type}: ${reason.error_message || "unknown"}`;
 
 // src/events/run-complete.ts
 async function onWorkflowComplete(context, event) {
-  const { state, channel, onFinish, logger } = context;
+  const { state, onFinish, logger } = context;
   const result = state.dataclips[state.lastDataclipId];
   const reason = calculateRunExitReason(state);
   await log_final_reason_default(context, reason);
   try {
-    await sendEvent(channel, RUN_COMPLETE, {
+    await sendEvent(context, RUN_COMPLETE, {
       final_dataclip_id: state.lastDataclipId,
       timestamp: timeInMicroseconds(event.time),
       ...reason
@@ -706,14 +753,14 @@ async function onWorkflowComplete(context, event) {
 
 // src/events/run-error.ts
 async function onRunError(context, event) {
-  const { state, channel, logger, onFinish } = context;
+  const { state, logger, onFinish } = context;
   try {
     const reason = calculateJobExitReason("", { data: {} }, event);
     if (state.activeJob) {
       await onJobError(context, { error: event });
     }
     await log_final_reason_default(context, reason);
-    await sendEvent(channel, RUN_COMPLETE, {
+    await sendEvent(context, RUN_COMPLETE, {
       final_dataclip_id: state.lastDataclipId,
       ...reason
     });
@@ -739,6 +786,7 @@ function execute(channel, engine, logger, plan, input, options = {}, onFinish =
   logger.info("executing ", plan.id);
   const state = create_run_state_default(plan, input);
   const context = {
+    id: plan.id,
     channel,
     state,
     logger,
@@ -746,76 +794,104 @@ function execute(channel, engine, logger, plan, input, options = {}, onFinish =
     options,
     onFinish
   };
-  const throttle = throttle_default();
-  const addEvent = (eventName, handler) => {
-    const wrappedFn = async (event) => {
-      const lightningEvent = eventMap[eventName] ?? eventName;
-      try {
-        await handler(context, event);
-        logger.info(`${plan.id} :: ${lightningEvent} :: OK`);
-      } catch (e) {
-        logger.error(
-          `${plan.id} :: ${lightningEvent} :: ERR: ${e.message || e.toString()}`
-        );
-        logger.error(e);
+  Sentry2.withIsolationScope(async () => {
+    Sentry2.addBreadcrumb({
+      category: "run",
+      message: "Executing run: loading metadata",
+      level: "info",
+      data: {
+        runId: plan.id
       }
+    });
+    const throttle = throttle_default();
+    const addEvent = (eventName, handler) => {
+      const wrappedFn = async (event) => {
+        if (eventName !== "workflow-log") {
+          Sentry2.addBreadcrumb({
+            category: "event",
+            message: eventName,
+            level: "info"
+          });
+        }
+        const lightningEvent = eventMap[eventName] ?? eventName;
+        try {
+          await handler(context, event);
+          logger.info(`${plan.id} :: ${lightningEvent} :: OK`);
+        } catch (e) {
+          if (!e.reportedToSentry) {
+            Sentry2.captureException(e);
+            logger.error(e);
+          }
+        }
+      };
+      return {
+        [eventName]: wrappedFn
+      };
     };
-    return {
-      [eventName]: wrappedFn
+    const listeners = Object.assign(
+      {},
+      addEvent("workflow-start", throttle(onRunStart)),
+      addEvent("job-start", throttle(onStepStart)),
+      addEvent("job-complete", throttle(onStepComplete)),
+      addEvent("job-error", throttle(onJobError)),
+      addEvent("workflow-log", throttle(onJobLog)),
+      // This will also resolve the promise
+      addEvent("workflow-complete", throttle(onWorkflowComplete)),
+      addEvent("workflow-error", throttle(onRunError))
+      // TODO send autoinstall logs
+    );
+    engine.listen(plan.id, listeners);
+    const resolvers = {
+      credential: (id) => loadCredential(context, id)
+      // TODO not supported right now
+      // dataclip: (id: string) => loadDataclip(channel, id),
     };
-  };
-  const listeners = Object.assign(
-    {},
-    addEvent("workflow-start", throttle(onRunStart)),
-    addEvent("job-start", throttle(onStepStart)),
-    addEvent("job-complete", throttle(onStepComplete)),
-    addEvent("job-error", throttle(onJobError)),
-    addEvent("workflow-log", throttle(onJobLog)),
-    // This will also resolve the promise
-    addEvent("workflow-complete", throttle(onWorkflowComplete)),
-    addEvent("workflow-error", throttle(onRunError))
-    // TODO send autoinstall logs
-  );
-  engine.listen(plan.id, listeners);
-  const resolvers = {
-    credential: (id) => loadCredential(channel, id)
-    // TODO not supported right now
-    // dataclip: (id: string) => loadDataclip(channel, id),
-  };
-  setTimeout(async () => {
-    let loadedInput = input;
-    if (typeof input === "string") {
-      logger.debug("loading dataclip", input);
+    setTimeout(async () => {
+      let loadedInput = input;
+      if (typeof input === "string") {
+        logger.debug("loading dataclip", input);
+        try {
+          loadedInput = await loadDataclip(context, input);
+          logger.success("dataclip loaded");
+        } catch (e) {
+          return onRunError(context, {
+            workflowId: plan.id,
+            message: `Failed to load dataclip ${input}${e.message ? `: ${e.message}` : ""}`,
+            type: "DataClipError",
+            severity: "exception"
+          });
+        }
+      }
       try {
-        loadedInput = await loadDataclip(channel, input);
-        logger.success("dataclip loaded");
+        Sentry2.addBreadcrumb({
+          category: "run",
+          message: "run metadata loaded: starting run",
+          level: "info",
+          data: {
+            runId: plan.id
+          }
+        });
+        engine.execute(plan, loadedInput, { resolvers, ...options });
       } catch (e) {
-        return onRunError(context, {
+        Sentry2.addBreadcrumb({
+          category: "run",
+          message: "exception in run",
+          level: "info",
+          data: {
+            runId: plan.id
+          }
+        });
+        onRunError(context, {
           workflowId: plan.id,
-          message: `Failed to load dataclip ${input}${e.message ? `: ${e.message}` : ""}`,
-          type: "DataClipError",
-          severity: "exception"
+          message: e.message,
+          type: e.type,
+          severity: e.severity
         });
       }
-    }
-    try {
-      engine.execute(plan, loadedInput, { resolvers, ...options });
-    } catch (e) {
-      onRunError(context, {
-        workflowId: plan.id,
-        message: e.message,
-        type: e.type,
-        severity: e.severity
-      });
-    }
+    });
   });
   return context;
 }
-var sendEvent = (channel, event, payload) => new Promise((resolve, reject) => {
-  channel.push(event, payload).receive("error", reject).receive("timeout", () => {
-    reject(new Error("timeout"));
-  }).receive("ok", resolve);
-});
 function onJobError(context, event) {
   const { state, error, jobId } = event;
   if (state?.errors?.[jobId]?.message === error.message) {
@@ -824,7 +900,8 @@ function onJobError(context, event) {
     return onStepComplete(context, event, event.error);
   }
 }
-function onJobLog({ channel, state, options }, event) {
+function onJobLog(context, event) {
+  const { state, options } = context;
   let message = event.message;
   if (event.redacted) {
     message = [
@@ -844,17 +921,17 @@ function onJobLog({ channel, state, options }, event) {
   if (state.activeStep) {
     log.step_id = state.activeStep;
   }
-  return sendEvent(channel, RUN_LOG, log);
+  return sendEvent(context, RUN_LOG, log);
 }
-async function loadDataclip(channel, stateId) {
-  const result = await get_with_reply_default(channel, GET_DATACLIP, {
+async function loadDataclip(context, stateId) {
+  const result = await sendEvent(context, GET_DATACLIP, {
     id: stateId
   });
   const str = enc.decode(new Uint8Array(result));
   return JSON.parse(str);
 }
-async function loadCredential(channel, credentialId) {
-  return get_with_reply_default(channel, GET_CREDENTIAL, { id: credentialId });
+async function loadCredential(context, credentialId) {
+  return sendEvent(context, GET_CREDENTIAL, { id: credentialId });
 }
 
 // src/middleware/healthcheck.ts
@@ -863,24 +940,33 @@ var healthcheck_default = (ctx) => {
 };
 
 // src/channels/run.ts
-var joinRunChannel = (socket, token, runId, logger) => {
+import * as Sentry3 from "@sentry/node";
+var joinRunChannel = (socket, token, runId, logger, timeout = 30) => {
   return new Promise((resolve, reject) => {
     let didReceiveOk = false;
     const channelName = `run:${runId}`;
-    logger.debug("connecting to ", channelName);
+    logger.info(`JOINING ${channelName}`);
+    logger.debug(`connecting to ${channelName} with timeout ${timeout}s`);
     const channel = socket.channel(channelName, { token });
-    channel.join().receive("ok", async (e) => {
+    channel.join(timeout * 1e3).receive("ok", async (e) => {
       if (!didReceiveOk) {
         didReceiveOk = true;
         logger.success(`connected to ${channelName}`, e);
-        const run = await get_with_reply_default(channel, GET_PLAN);
+        const run = await send_event_default(
+          { channel, logger, id: runId },
+          GET_PLAN
+        );
         resolve({ channel, run });
       }
     }).receive("error", (err) => {
+      Sentry3.captureException(err);
       logger.error(`error connecting to ${channelName}`, err);
+      channel?.leave();
       reject(err);
     }).receive("timeout", (err) => {
+      Sentry3.captureException(err);
       logger.error(`Timeout for ${channelName}`, err);
+      channel?.leave();
       reject(err);
     });
     channel.onClose(() => {
@@ -896,6 +982,7 @@ var run_default = joinRunChannel;
 
 // src/channels/worker-queue.ts
 import EventEmitter from "node:events";
+import * as Sentry4 from "@sentry/node";
 import { Socket as PhxSocket } from "phoenix";
 import { WebSocket } from "ws";
 import { API_VERSION } from "@openfn/lexicon/lightning";
@@ -924,7 +1011,17 @@ var worker_token_default = generateWorkerToken;
 // src/channels/worker-queue.ts
 var connectToWorkerQueue = (endpoint, serverId, secret, timeout = 10, logger, SocketConstructor = PhxSocket) => {
   const events = new EventEmitter();
+  Sentry4.addBreadcrumb({
+    category: "lifecycle",
+    message: "Connecting to worker queue",
+    level: "info"
+  });
   worker_token_default(secret, serverId, logger).then(async (token) => {
+    Sentry4.addBreadcrumb({
+      category: "lifecycle",
+      message: "Worker token generated",
+      level: "info"
+    });
     const params = {
       token,
       api_version: API_VERSION,
@@ -933,11 +1030,19 @@ var connectToWorkerQueue = (endpoint, serverId, secret, timeout = 10, logger, So
     const socket = new SocketConstructor(endpoint, {
       params,
       transport: WebSocket,
-      timeout: timeout * 1e3
+      timeout: timeout * 1e3,
+      reconnectAfterMs: (tries) => Math.max(tries * 1e3)
     });
     let didOpen = false;
+    let shouldReportConnectionError = true;
     socket.onOpen(() => {
+      Sentry4.addBreadcrumb({
+        category: "lifecycle",
+        message: "Web socket connected",
+        level: "info"
+      });
       didOpen = true;
+      shouldReportConnectionError = true;
       const channel = socket.channel("worker:queue");
       channel.onMessage = (ev, load) => {
         events.emit("message", ev, load);
@@ -957,6 +1062,16 @@ var connectToWorkerQueue = (endpoint, serverId, secret, timeout = 10, logger, So
       events.emit("disconnect");
     });
     socket.onError((e) => {
+      Sentry4.addBreadcrumb({
+        category: "lifecycle",
+        message: "Error in web socket connection",
+        level: "info"
+      });
+      if (shouldReportConnectionError) {
+        logger.debug("Reporting connection error to sentry");
+        shouldReportConnectionError = false;
+        Sentry4.captureException(e);
+      }
       if (!didOpen) {
         events.emit("error", e.message);
         didOpen = false;
@@ -1061,6 +1176,13 @@ function createServer(engine, options = {}) {
   const router = new Router();
   app.events = new EventEmitter2();
   app.engine = engine;
+  if (options.sentryDsn) {
+    Sentry5.init({
+      environment: options.sentryEnv,
+      dsn: options.sentryDsn
+    });
+    Sentry5.setupKoaErrorHandler(app);
+  }
   app.use(bodyParser());
   app.use(
     koaLogger((str, _args) => {
@@ -1099,7 +1221,8 @@ function createServer(engine, options = {}) {
           app.socket,
           token,
           id,
-          logger
+          logger,
+          app.options.messageTimeoutSeconds
         );
         const { plan, options: options2, input } = convert_lightning_plan_default(run, {
           collectionsVersion: app.options.collectionsVersion,
@@ -1136,6 +1259,8 @@ function createServer(engine, options = {}) {
         );
         app.workflows[id] = context;
       } catch (e) {
+        delete app.workflows[id];
+        app.resumeWorkloop();
         logger.error(`Unexpected error executing ${id}`);
         logger.error(e);
       }

package/dist/start.js

@@ -142,6 +142,7 @@ var runtime_engine_default = createMock;
 import { EventEmitter as EventEmitter3 } from "node:events";
 import { promisify } from "node:util";
 import { exec as _exec } from "node:child_process";
+import * as Sentry5 from "@sentry/node";
 import Koa from "koa";
 import bodyParser from "koa-bodyparser";
 import koaLogger from "koa-logger";
@@ -263,9 +264,11 @@ var verifyToken = async (token, publicKey) => {
     return true;
   }
 };
+var { DEPLOYED_POD_NAME } = process.env;
 var claim = (app, logger2 = mockLogger, options = {}) => {
   return new Promise((resolve5, reject) => {
     const { maxWorkers = 5 } = options;
+    const podName = DEPLOYED_POD_NAME ? `[${DEPLOYED_POD_NAME}] ` : "";
     const activeWorkers = Object.keys(app.workflows).length;
     if (activeWorkers >= maxWorkers) {
       app.workloop?.stop(`server at capacity (${activeWorkers}/${maxWorkers})`);
@@ -277,10 +280,10 @@ var claim = (app, logger2 = mockLogger, options = {}) => {
     }
     logger2.debug(`requesting run (capacity ${activeWorkers}/${maxWorkers})`);
     const start = Date.now();
-    app.queueChannel.push(CLAIM, { demand: 1 }).receive("ok", ({ runs }) => {
+    app.queueChannel.push(CLAIM, { demand: 1, pod_name: DEPLOYED_POD_NAME }).receive("ok", ({ runs }) => {
       const duration = Date.now() - start;
       logger2.debug(
-        `claimed ${runs.length} runs in ${duration}ms (${runs.length ? runs.map((r) => r.id).join(",") : "-"})`
+        `${podName}claimed ${runs.length} runs in ${duration}ms (${runs.length ? runs.map((r) => r.id).join(",") : "-"})`
       );
       if (!runs?.length) {
         return reject(new Error("No runs returned"));
@@ -300,7 +303,7 @@ var claim = (app, logger2 = mockLogger, options = {}) => {
         } else {
           logger2.debug("skipping run token validation for", run2.id);
         }
-        logger2.debug("starting run", run2.id);
+        logger2.debug(`${podName} starting run ${run2.id}`);
         app.execute(run2);
         resolve5();
       });
@@ -350,6 +353,9 @@ var startWorkloop = (app, logger2, minBackoff2, maxBackoff2, maxWorkers) => {
 };
 var workloop_default = startWorkloop;
 
+// src/api/execute.ts
+import * as Sentry2 from "@sentry/node";
+
 // src/util/convert-lightning-plan.ts
 import crypto3 from "node:crypto";
 import path from "node:path";
@@ -527,26 +533,6 @@ var convert_lightning_plan_default = (run2, options = {}) => {
   };
 };
 
-// src/util/get-with-reply.ts
-var get_with_reply_default = (channel, event, payload) => new Promise((resolve5, reject) => {
-  channel.push(event, payload).receive("ok", (evt) => {
-    resolve5(evt);
-  }).receive("error", (e) => {
-    reject(e);
-  }).receive("timeout", (e) => {
-    reject(e);
-  });
-});
-
-// src/util/stringify.ts
-import stringify from "fast-safe-stringify";
-var stringify_default = (obj) => stringify(obj, (_key, value) => {
-  if (value instanceof Uint8Array) {
-    return Array.from(value);
-  }
-  return value;
-});
-
 // src/util/create-run-state.ts
 var create_run_state_default = (plan, input) => {
   const state = {
@@ -580,6 +566,67 @@ var create_run_state_default = (plan, input) => {
   return state;
 };
 
+// src/util/send-event.ts
+import * as Sentry from "@sentry/node";
+
+// src/errors.ts
+var LightningSocketError = class extends Error {
+  constructor(event, message) {
+    super(`[${event}] ${message}`);
+    this.name = "LightningSocketError";
+    this.event = "";
+    this.rejectMessage = "";
+    this.event = event;
+    this.rejectMessage = message;
+  }
+};
+var LightningTimeoutError = class extends Error {
+  constructor(event) {
+    super(`[${event}] timeout`);
+    this.name = "LightningTimeoutError";
+  }
+};
+
+// src/util/send-event.ts
+var sendEvent = (context, event, payload) => {
+  const { channel, logger: logger2, id: runId = "<unknown run>" } = context;
+  return new Promise((resolve5, reject) => {
+    const report = (error) => {
+      logger2.error(`${runId} :: ${event} :: ERR: ${error.message || error}`);
+      const context2 = {
+        run_id: runId,
+        event
+      };
+      const extras = {};
+      if (error.rejectMessage) {
+        extras.rejection_reason = error.rejectMessage;
+      }
+      Sentry.captureException(error, (scope) => {
+        scope.setContext("run", context2);
+        scope.setExtras(extras);
+        return scope;
+      });
+      error.reportedToSentry = true;
+      reject(error);
+    };
+    channel.push(event, payload).receive("error", (message) => {
+      report(new LightningSocketError(event, message));
+    }).receive("timeout", () => {
+      report(new LightningTimeoutError(event));
+    }).receive("ok", resolve5);
+  });
+};
+var send_event_default = sendEvent;
+
+// src/util/stringify.ts
+import stringify from "fast-safe-stringify";
+var stringify_default = (obj) => stringify(obj, (_key, value) => {
+  if (value instanceof Uint8Array) {
+    return Array.from(value);
+  }
+  return value;
+});
+
 // src/util/throttle.ts
 var createThrottler = () => {
   const q = [];
@@ -665,7 +712,7 @@ function getVersion() {
 
 // src/events/run-start.ts
 async function onRunStart(context, event) {
-  const { channel, state, options = {} } = context;
+  const { state, options = {} } = context;
   const time = (timestamp() - BigInt(1e7)).toString();
   const versionLogContext = {
     ...context,
@@ -678,7 +725,7 @@ async function onRunStart(context, event) {
     worker: await getVersion(),
     ...event.versions
   };
-  await sendEvent(channel, RUN_START, {
+  await sendEvent(context, RUN_START, {
     versions,
     /// use the engine time in run start
     timestamp: timeInMicroseconds(event.time)
@@ -740,7 +787,7 @@ var calculateRunExitReason = (state) => {
 
 // src/events/step-complete.ts
 async function onStepComplete(context, event, error) {
-  const { channel, state, options } = context;
+  const { state, options } = context;
   const dataclipId = crypto4.randomUUID();
   const step_id = state.activeStep;
   const job_id = state.activeJob;
@@ -785,13 +832,13 @@ async function onStepComplete(context, event, error) {
   const reason = calculateJobExitReason(job_id, event.state, error);
   state.reasons[job_id] = reason;
   Object.assign(evt, reason);
-  return sendEvent(channel, STEP_COMPLETE, evt);
+  return sendEvent(context, STEP_COMPLETE, evt);
 }
 
 // src/events/step-start.ts
 import crypto5 from "node:crypto";
 async function onStepStart(context, event) {
-  const { channel, state } = context;
+  const { state } = context;
   state.activeStep = crypto5.randomUUID();
   state.activeJob = event.jobId;
   const input_dataclip_id = state.inputDataclips[event.jobId];
@@ -803,7 +850,7 @@ async function onStepStart(context, event) {
   if (!state.withheldDataclips[input_dataclip_id]) {
     evt.input_dataclip_id = input_dataclip_id;
   }
-  await sendEvent(channel, STEP_START, evt);
+  await sendEvent(context, STEP_START, evt);
 }
 
 // src/util/log-final-reason.ts
@@ -825,12 +872,12 @@ ${reason.error_type}: ${reason.error_message || "unknown"}`;
 
 // src/events/run-complete.ts
 async function onWorkflowComplete(context, event) {
-  const { state, channel, onFinish, logger: logger2 } = context;
+  const { state, onFinish, logger: logger2 } = context;
   const result = state.dataclips[state.lastDataclipId];
   const reason = calculateRunExitReason(state);
   await log_final_reason_default(context, reason);
   try {
-    await sendEvent(channel, RUN_COMPLETE, {
+    await sendEvent(context, RUN_COMPLETE, {
       final_dataclip_id: state.lastDataclipId,
       timestamp: timeInMicroseconds(event.time),
       ...reason
@@ -846,14 +893,14 @@ async function onWorkflowComplete(context, event) {
 
 // src/events/run-error.ts
 async function onRunError(context, event) {
-  const { state, channel, logger: logger2, onFinish } = context;
+  const { state, logger: logger2, onFinish } = context;
   try {
     const reason = calculateJobExitReason("", { data: {} }, event);
     if (state.activeJob) {
       await onJobError(context, { error: event });
     }
     await log_final_reason_default(context, reason);
-    await sendEvent(channel, RUN_COMPLETE, {
+    await sendEvent(context, RUN_COMPLETE, {
       final_dataclip_id: state.lastDataclipId,
       ...reason
     });
@@ -879,6 +926,7 @@ function execute(channel, engine, logger2, plan, input, options = {}, onFinish =
   logger2.info("executing ", plan.id);
   const state = create_run_state_default(plan, input);
   const context = {
+    id: plan.id,
     channel,
     state,
     logger: logger2,
@@ -886,76 +934,104 @@ function execute(channel, engine, logger2, plan, input, options = {}, onFinish =
     options,
     onFinish
   };
-  const throttle = throttle_default();
-  const addEvent = (eventName, handler) => {
-    const wrappedFn = async (event) => {
-      const lightningEvent = eventMap[eventName] ?? eventName;
-      try {
-        await handler(context, event);
-        logger2.info(`${plan.id} :: ${lightningEvent} :: OK`);
-      } catch (e) {
-        logger2.error(
-          `${plan.id} :: ${lightningEvent} :: ERR: ${e.message || e.toString()}`
-        );
-        logger2.error(e);
+  Sentry2.withIsolationScope(async () => {
+    Sentry2.addBreadcrumb({
+      category: "run",
+      message: "Executing run: loading metadata",
+      level: "info",
+      data: {
+        runId: plan.id
       }
+    });
+    const throttle = throttle_default();
+    const addEvent = (eventName, handler) => {
+      const wrappedFn = async (event) => {
+        if (eventName !== "workflow-log") {
+          Sentry2.addBreadcrumb({
+            category: "event",
+            message: eventName,
+            level: "info"
+          });
+        }
+        const lightningEvent = eventMap[eventName] ?? eventName;
+        try {
+          await handler(context, event);
+          logger2.info(`${plan.id} :: ${lightningEvent} :: OK`);
+        } catch (e) {
+          if (!e.reportedToSentry) {
+            Sentry2.captureException(e);
+            logger2.error(e);
+          }
+        }
+      };
+      return {
+        [eventName]: wrappedFn
+      };
     };
-    return {
-      [eventName]: wrappedFn
+    const listeners = Object.assign(
+      {},
+      addEvent("workflow-start", throttle(onRunStart)),
+      addEvent("job-start", throttle(onStepStart)),
+      addEvent("job-complete", throttle(onStepComplete)),
+      addEvent("job-error", throttle(onJobError)),
+      addEvent("workflow-log", throttle(onJobLog)),
+      // This will also resolve the promise
+      addEvent("workflow-complete", throttle(onWorkflowComplete)),
+      addEvent("workflow-error", throttle(onRunError))
+      // TODO send autoinstall logs
+    );
+    engine.listen(plan.id, listeners);
+    const resolvers = {
+      credential: (id) => loadCredential(context, id)
+      // TODO not supported right now
+      // dataclip: (id: string) => loadDataclip(channel, id),
     };
-  };
-  const listeners = Object.assign(
-    {},
-    addEvent("workflow-start", throttle(onRunStart)),
-    addEvent("job-start", throttle(onStepStart)),
-    addEvent("job-complete", throttle(onStepComplete)),
-    addEvent("job-error", throttle(onJobError)),
-    addEvent("workflow-log", throttle(onJobLog)),
-    // This will also resolve the promise
-    addEvent("workflow-complete", throttle(onWorkflowComplete)),
-    addEvent("workflow-error", throttle(onRunError))
-    // TODO send autoinstall logs
-  );
-  engine.listen(plan.id, listeners);
-  const resolvers = {
-    credential: (id) => loadCredential(channel, id)
-    // TODO not supported right now
-    // dataclip: (id: string) => loadDataclip(channel, id),
-  };
-  setTimeout(async () => {
-    let loadedInput = input;
-    if (typeof input === "string") {
-      logger2.debug("loading dataclip", input);
+    setTimeout(async () => {
+      let loadedInput = input;
+      if (typeof input === "string") {
+        logger2.debug("loading dataclip", input);
+        try {
+          loadedInput = await loadDataclip(context, input);
+          logger2.success("dataclip loaded");
+        } catch (e) {
+          return onRunError(context, {
+            workflowId: plan.id,
+            message: `Failed to load dataclip ${input}${e.message ? `: ${e.message}` : ""}`,
+            type: "DataClipError",
+            severity: "exception"
+          });
+        }
+      }
       try {
-        loadedInput = await loadDataclip(channel, input);
-        logger2.success("dataclip loaded");
+        Sentry2.addBreadcrumb({
+          category: "run",
+          message: "run metadata loaded: starting run",
+          level: "info",
+          data: {
+            runId: plan.id
+          }
+        });
+        engine.execute(plan, loadedInput, { resolvers, ...options });
       } catch (e) {
-        return onRunError(context, {
+        Sentry2.addBreadcrumb({
+          category: "run",
+          message: "exception in run",
+          level: "info",
+          data: {
+            runId: plan.id
+          }
+        });
+        onRunError(context, {
           workflowId: plan.id,
-          message: `Failed to load dataclip ${input}${e.message ? `: ${e.message}` : ""}`,
-          type: "DataClipError",
-          severity: "exception"
+          message: e.message,
+          type: e.type,
+          severity: e.severity
         });
       }
-    }
-    try {
-      engine.execute(plan, loadedInput, { resolvers, ...options });
-    } catch (e) {
-      onRunError(context, {
-        workflowId: plan.id,
-        message: e.message,
-        type: e.type,
-        severity: e.severity
-      });
-    }
+    });
   });
   return context;
 }
-var sendEvent = (channel, event, payload) => new Promise((resolve5, reject) => {
-  channel.push(event, payload).receive("error", reject).receive("timeout", () => {
-    reject(new Error("timeout"));
-  }).receive("ok", resolve5);
-});
 function onJobError(context, event) {
   const { state, error, jobId } = event;
   if (state?.errors?.[jobId]?.message === error.message) {
@@ -964,7 +1040,8 @@ function onJobError(context, event) {
     return onStepComplete(context, event, event.error);
   }
 }
-function onJobLog({ channel, state, options }, event) {
+function onJobLog(context, event) {
+  const { state, options } = context;
   let message = event.message;
   if (event.redacted) {
     message = [
@@ -984,17 +1061,17 @@ function onJobLog({ channel, state, options }, event) {
   if (state.activeStep) {
     log.step_id = state.activeStep;
   }
-  return sendEvent(channel, RUN_LOG, log);
+  return sendEvent(context, RUN_LOG, log);
 }
-async function loadDataclip(channel, stateId) {
-  const result = await get_with_reply_default(channel, GET_DATACLIP, {
+async function loadDataclip(context, stateId) {
+  const result = await sendEvent(context, GET_DATACLIP, {
     id: stateId
   });
   const str = enc.decode(new Uint8Array(result));
   return JSON.parse(str);
 }
-async function loadCredential(channel, credentialId) {
-  return get_with_reply_default(channel, GET_CREDENTIAL, { id: credentialId });
+async function loadCredential(context, credentialId) {
+  return sendEvent(context, GET_CREDENTIAL, { id: credentialId });
 }
 
 // src/middleware/healthcheck.ts
@@ -1003,24 +1080,33 @@ var healthcheck_default = (ctx) => {
 };
 
 // src/channels/run.ts
-var joinRunChannel = (socket, token, runId, logger2) => {
+import * as Sentry3 from "@sentry/node";
+var joinRunChannel = (socket, token, runId, logger2, timeout = 30) => {
   return new Promise((resolve5, reject) => {
     let didReceiveOk = false;
     const channelName = `run:${runId}`;
-    logger2.debug("connecting to ", channelName);
+    logger2.info(`JOINING ${channelName}`);
+    logger2.debug(`connecting to ${channelName} with timeout ${timeout}s`);
     const channel = socket.channel(channelName, { token });
-    channel.join().receive("ok", async (e) => {
+    channel.join(timeout * 1e3).receive("ok", async (e) => {
       if (!didReceiveOk) {
         didReceiveOk = true;
         logger2.success(`connected to ${channelName}`, e);
-        const run2 = await get_with_reply_default(channel, GET_PLAN);
+        const run2 = await send_event_default(
+          { channel, logger: logger2, id: runId },
+          GET_PLAN
+        );
         resolve5({ channel, run: run2 });
       }
     }).receive("error", (err) => {
+      Sentry3.captureException(err);
       logger2.error(`error connecting to ${channelName}`, err);
+      channel?.leave();
       reject(err);
     }).receive("timeout", (err) => {
+      Sentry3.captureException(err);
       logger2.error(`Timeout for ${channelName}`, err);
+      channel?.leave();
       reject(err);
     });
     channel.onClose(() => {
@@ -1036,6 +1122,7 @@ var run_default = joinRunChannel;
 
 // src/channels/worker-queue.ts
 import EventEmitter2 from "node:events";
+import * as Sentry4 from "@sentry/node";
 import { Socket as PhxSocket } from "phoenix";
 import { WebSocket } from "ws";
 import { API_VERSION } from "@openfn/lexicon/lightning";
@@ -1064,7 +1151,17 @@ var worker_token_default = generateWorkerToken;
 // src/channels/worker-queue.ts
 var connectToWorkerQueue = (endpoint, serverId, secret, timeout = 10, logger2, SocketConstructor = PhxSocket) => {
   const events = new EventEmitter2();
+  Sentry4.addBreadcrumb({
+    category: "lifecycle",
+    message: "Connecting to worker queue",
+    level: "info"
+  });
   worker_token_default(secret, serverId, logger2).then(async (token) => {
+    Sentry4.addBreadcrumb({
+      category: "lifecycle",
+      message: "Worker token generated",
+      level: "info"
+    });
     const params = {
       token,
       api_version: API_VERSION,
@@ -1073,11 +1170,19 @@ var connectToWorkerQueue = (endpoint, serverId, secret, timeout = 10, logger2, S
     const socket = new SocketConstructor(endpoint, {
       params,
       transport: WebSocket,
-      timeout: timeout * 1e3
+      timeout: timeout * 1e3,
+      reconnectAfterMs: (tries) => Math.max(tries * 1e3)
     });
     let didOpen = false;
+    let shouldReportConnectionError = true;
     socket.onOpen(() => {
+      Sentry4.addBreadcrumb({
+        category: "lifecycle",
+        message: "Web socket connected",
+        level: "info"
+      });
       didOpen = true;
+      shouldReportConnectionError = true;
       const channel = socket.channel("worker:queue");
       channel.onMessage = (ev, load) => {
         events.emit("message", ev, load);
@@ -1097,6 +1202,16 @@ var connectToWorkerQueue = (endpoint, serverId, secret, timeout = 10, logger2, S
       events.emit("disconnect");
     });
     socket.onError((e) => {
+      Sentry4.addBreadcrumb({
+        category: "lifecycle",
+        message: "Error in web socket connection",
+        level: "info"
+      });
+      if (shouldReportConnectionError) {
+        logger2.debug("Reporting connection error to sentry");
+        shouldReportConnectionError = false;
+        Sentry4.captureException(e);
+      }
       if (!didOpen) {
         events.emit("error", e.message);
         didOpen = false;
@@ -1201,6 +1316,13 @@ function createServer(engine, options = {}) {
   const router = new Router();
   app.events = new EventEmitter3();
   app.engine = engine;
+  if (options.sentryDsn) {
+    Sentry5.init({
+      environment: options.sentryEnv,
+      dsn: options.sentryDsn
+    });
+    Sentry5.setupKoaErrorHandler(app);
+  }
   app.use(bodyParser());
   app.use(
     koaLogger((str, _args) => {
@@ -1239,7 +1361,8 @@ function createServer(engine, options = {}) {
           app.socket,
           token,
           id,
-          logger2
+          logger2,
+          app.options.messageTimeoutSeconds
         );
         const { plan, options: options2, input } = convert_lightning_plan_default(run2, {
           collectionsVersion: app.options.collectionsVersion,
@@ -1276,6 +1399,8 @@ function createServer(engine, options = {}) {
         );
         app.workflows[id] = context;
       } catch (e) {
+        delete app.workflows[id];
+        app.resumeWorkloop();
         logger2.error(`Unexpected error executing ${id}`);
         logger2.error(e);
       }
@@ -6185,6 +6310,7 @@ var yargs_default = Yargs;
 var DEFAULT_PORT2 = 2222;
 var DEFAULT_WORKER_CAPACITY = 5;
 var DEFAULT_SOCKET_TIMEOUT_SECONDS = 10;
+var DEFAULT_MESSAGE_TIMEOUT_SECONDS = 30;
 function setArg(argValue, envValue, defaultValue) {
   if (Array.isArray(defaultValue) && !argValue && typeof envValue === "string") {
     return envValue.split(",");
@@ -6196,22 +6322,25 @@ function setArg(argValue, envValue, defaultValue) {
 }
 function parseArgs(argv) {
   const {
+    OPENFN_ADAPTORS_REPO,
     WORKER_BACKOFF,
     WORKER_CAPACITY,
-    WORKER_COLLECTIONS_VERSION,
     WORKER_COLLECTIONS_URL,
+    WORKER_COLLECTIONS_VERSION,
     WORKER_LIGHTNING_PUBLIC_KEY,
     WORKER_LIGHTNING_SERVICE_URL,
     WORKER_LOG_LEVEL,
     WORKER_MAX_PAYLOAD_MB,
     WORKER_MAX_RUN_DURATION_SECONDS,
     WORKER_MAX_RUN_MEMORY_MB,
+    WORKER_MESSAGE_TIMEOUT_SECONDS,
     WORKER_PORT,
     WORKER_REPO_DIR,
     WORKER_SECRET,
-    WORKER_STATE_PROPS_TO_REMOVE,
+    WORKER_SENTRY_DSN,
+    WORKER_SENTRY_ENV,
     WORKER_SOCKET_TIMEOUT_SECONDS,
-    OPENFN_ADAPTORS_REPO
+    WORKER_STATE_PROPS_TO_REMOVE
   } = process.env;
   const parser2 = yargs_default(hideBin(argv)).command("server", "Start a ws-worker server").option("port", {
     alias: "p",
@@ -6225,12 +6354,19 @@ function parseArgs(argv) {
     description: "Path to the runtime repo (where modules will be installed). Env: WORKER_REPO_DIR"
   }).option("monorepo-dir", {
     alias: "m",
-    description: "Path to the adaptors mono repo, from where @local adaptors will be loaded. Env: OPENFN_ADAPTORS_REPO"
+    description: "Path to the adaptors monorepo, from where @local adaptors will be loaded. Env: OPENFN_ADAPTORS_REPO"
   }).option("secret", {
     alias: "s",
     description: "Worker secret. (comes from WORKER_SECRET by default). Env: WORKER_SECRET"
+  }).option("sentry-dsn", {
+    alias: ["dsn"],
+    description: "Sentry DSN. Env: WORKER_SENTRY_DSN"
+  }).option("sentry-env", {
+    description: "Sentry environment. Defaults to 'dev'. Env: WORKER_SENTRY_ENV"
   }).option("socket-timeout", {
-    description: `Timeout for websockets to Lighting, in seconds. Defaults to 10.`
+    description: `Timeout for websockets to Lightning, in seconds. Defaults to 10.Env: WORKER_SOCKET_TIMEOUT_SECONDS`
+  }).option("message-timeout", {
+    description: `Timeout for messages in the run channel in seconds. Defaults to 1. Env: WORKER_MESSAGE_TIMEOUT_SECONDS`
   }).option("lightning-public-key", {
     description: "Base64-encoded public key. Used to verify run tokens. Env: WORKER_LIGHTNING_PUBLIC_KEY"
   }).option("log", {
@@ -6280,6 +6416,8 @@ function parseArgs(argv) {
     repoDir: setArg(args2.repoDir, WORKER_REPO_DIR),
     monorepoDir: setArg(args2.monorepoDir, OPENFN_ADAPTORS_REPO),
     secret: setArg(args2.secret, WORKER_SECRET),
+    sentryDsn: setArg(args2.sentryDsn, WORKER_SENTRY_DSN),
+    sentryEnv: setArg(args2.sentryEnv, WORKER_SENTRY_ENV, "dev"),
     lightningPublicKey: setArg(
       args2.lightningPublicKey,
       WORKER_LIGHTNING_PUBLIC_KEY
@@ -6304,6 +6442,11 @@ function parseArgs(argv) {
       WORKER_SOCKET_TIMEOUT_SECONDS,
       DEFAULT_SOCKET_TIMEOUT_SECONDS
     ),
+    messageTimeoutSeconds: setArg(
+      args2.messageTimeoutSeconds,
+      WORKER_MESSAGE_TIMEOUT_SECONDS,
+      DEFAULT_MESSAGE_TIMEOUT_SECONDS
+    ),
     collectionsVersion: setArg(
       args2.collectionsVersion,
       WORKER_COLLECTIONS_VERSION
@@ -6333,6 +6476,8 @@ function engineReady(engine) {
     lightning: args.lightning,
     logger,
     secret: args.secret,
+    sentryDsn: args.sentryDsn,
+    sentryEnv: args.sentryEnv,
     noLoop: !args.loop,
     // TODO need to feed this through properly
     backoff: {
@@ -6343,7 +6488,9 @@ function engineReady(engine) {
     payloadLimitMb: args.payloadMemory,
     collectionsVersion: args.collectionsVersion,
     collectionsUrl: args.collectionsUrl,
-    monorepoDir: args.monorepoDir
+    monorepoDir: args.monorepoDir,
+    messageTimeoutSeconds: args.messageTimeoutSeconds,
+    socketTimeoutSeconds: args.socketTimeoutSeconds
   };
   if (args.lightningPublicKey) {
     logger.info(

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openfn/ws-worker",
-  "version": "1.12.1",
+  "version": "1.13.1",
   "description": "A Websocket Worker to connect Lightning to a Runtime Engine",
   "main": "dist/index.js",
   "type": "module",
@@ -11,6 +11,7 @@
   "license": "ISC",
   "dependencies": {
     "@koa/router": "^12.0.0",
+    "@sentry/node": "^9.5.0",
     "@types/koa-logger": "^3.1.2",
     "@types/ws": "^8.5.6",
     "fast-safe-stringify": "^2.1.1",
@@ -22,10 +23,10 @@
     "koa-logger": "^3.2.1",
     "phoenix": "1.7.10",
     "ws": "^8.18.0",
-    "@openfn/engine-multi": "1.6.1",
-    "@openfn/logger": "1.0.4",
-    "@openfn/runtime": "1.6.3",
-    "@openfn/lexicon": "^1.2.0"
+    "@openfn/lexicon": "^1.2.0",
+    "@openfn/logger": "1.0.5",
+    "@openfn/runtime": "1.6.4",
+    "@openfn/engine-multi": "1.6.2"
   },
   "devDependencies": {
     "@types/koa": "^2.13.5",
@@ -37,11 +38,12 @@
     "@types/yargs": "^17.0.12",
     "ava": "5.1.0",
     "nodemon": "3.0.1",
+    "sentry-testkit": "^6.1.0",
     "tslib": "^2.4.0",
     "tsup": "^6.2.3",
     "typescript": "^4.6.4",
     "yargs": "^17.6.2",
-    "@openfn/lightning-mock": "2.1.3"
+    "@openfn/lightning-mock": "2.1.4"
   },
   "files": [
     "dist",