@openfn/ws-worker 1.12.0 → 1.13.0

package/CHANGELOG.md

@@ -1,5 +1,28 @@
 # ws-worker
 
+## 1.13.0
+
+### Minor Changes
+
+- ce5022a: Added sentry notifications for server and websocket errors
+
+### Patch Changes
+
+- 0a176aa: Ignore empty log lines (don't send them to lightning)
+- Updated dependencies [0a176aa]
+  - @openfn/logger@1.0.5
+  - @openfn/engine-multi@1.6.2
+  - @openfn/lexicon@1.2.0
+  - @openfn/runtime@1.6.4
+
+## 1.12.1
+
+### Patch Changes
+
+- e2f1197: Better logging on credential errors
+- Updated dependencies [e2f1197]
+  - @openfn/engine-multi@1.6.1
+
 ## 1.12.0
 
 ### Minor Changes

package/dist/index.d.ts

@@ -53,6 +53,7 @@ type WorkerRunOptions = ExecuteOptions & {
 };
 
 type Context = {
+    id: string;
     channel: Channel;
     state: RunState;
     logger: Logger;
@@ -73,6 +74,8 @@ type ServerOptions = {
         min?: number;
         max?: number;
     };
+    sentryDsn?: string;
+    sentryEnv?: string;
     socketTimeoutSeconds?: number;
     payloadLimitMb?: number;
     collectionsVersion?: string;

package/dist/index.js

@@ -2,6 +2,7 @@
 import { EventEmitter as EventEmitter2 } from "node:events";
 import { promisify } from "node:util";
 import { exec as _exec } from "node:child_process";
+import * as Sentry5 from "@sentry/node";
 import Koa from "koa";
 import bodyParser from "koa-bodyparser";
 import koaLogger from "koa-logger";
@@ -210,6 +211,9 @@ var startWorkloop = (app, logger, minBackoff, maxBackoff, maxWorkers) => {
 };
 var workloop_default = startWorkloop;
 
+// src/api/execute.ts
+import * as Sentry2 from "@sentry/node";
+
 // src/util/convert-lightning-plan.ts
 import crypto2 from "node:crypto";
 import path from "node:path";
@@ -387,26 +391,6 @@ var convert_lightning_plan_default = (run, options = {}) => {
   };
 };
 
-// src/util/get-with-reply.ts
-var get_with_reply_default = (channel, event, payload) => new Promise((resolve, reject) => {
-  channel.push(event, payload).receive("ok", (evt) => {
-    resolve(evt);
-  }).receive("error", (e) => {
-    reject(e);
-  }).receive("timeout", (e) => {
-    reject(e);
-  });
-});
-
-// src/util/stringify.ts
-import stringify from "fast-safe-stringify";
-var stringify_default = (obj) => stringify(obj, (_key, value) => {
-  if (value instanceof Uint8Array) {
-    return Array.from(value);
-  }
-  return value;
-});
-
 // src/util/create-run-state.ts
 var create_run_state_default = (plan, input) => {
   const state = {
@@ -440,6 +424,67 @@ var create_run_state_default = (plan, input) => {
   return state;
 };
 
+// src/util/send-event.ts
+import * as Sentry from "@sentry/node";
+
+// src/errors.ts
+var LightningSocketError = class extends Error {
+  constructor(event, message) {
+    super(`[${event}] ${message}`);
+    this.name = "LightningSocketError";
+    this.event = "";
+    this.rejectMessage = "";
+    this.event = event;
+    this.rejectMessage = message;
+  }
+};
+var LightningTimeoutError = class extends Error {
+  constructor(event) {
+    super(`[${event}] timeout`);
+    this.name = "LightningTimeoutError";
+  }
+};
+
+// src/util/send-event.ts
+var sendEvent = (context, event, payload) => {
+  const { channel, logger, id: runId = "<unknown run>" } = context;
+  return new Promise((resolve, reject) => {
+    const report = (error) => {
+      logger.error(`${runId} :: ${event} :: ERR: ${error.message || error}`);
+      const context2 = {
+        run_id: runId,
+        event
+      };
+      const extras = {};
+      if (error.rejectMessage) {
+        extras.rejection_reason = error.rejectMessage;
+      }
+      Sentry.captureException(error, (scope) => {
+        scope.setContext("run", context2);
+        scope.setExtras(extras);
+        return scope;
+      });
+      error.reportedToSentry = true;
+      reject(error);
+    };
+    channel.push(event, payload).receive("error", (message) => {
+      report(new LightningSocketError(event, message));
+    }).receive("timeout", () => {
+      report(new LightningTimeoutError(event));
+    }).receive("ok", resolve);
+  });
+};
+var send_event_default = sendEvent;
+
+// src/util/stringify.ts
+import stringify from "fast-safe-stringify";
+var stringify_default = (obj) => stringify(obj, (_key, value) => {
+  if (value instanceof Uint8Array) {
+    return Array.from(value);
+  }
+  return value;
+});
+
 // src/util/throttle.ts
 var createThrottler = () => {
   const q = [];
@@ -525,7 +570,7 @@ function getVersion() {
 
 // src/events/run-start.ts
 async function onRunStart(context, event) {
-  const { channel, state, options = {} } = context;
+  const { state, options = {} } = context;
   const time = (timestamp() - BigInt(1e7)).toString();
   const versionLogContext = {
     ...context,
@@ -538,7 +583,7 @@ async function onRunStart(context, event) {
     worker: await getVersion(),
     ...event.versions
   };
-  await sendEvent(channel, RUN_START, {
+  await sendEvent(context, RUN_START, {
     versions,
     /// use the engine time in run start
     timestamp: timeInMicroseconds(event.time)
@@ -600,7 +645,7 @@ var calculateRunExitReason = (state) => {
 
 // src/events/step-complete.ts
 async function onStepComplete(context, event, error) {
-  const { channel, state, options } = context;
+  const { state, options } = context;
   const dataclipId = crypto3.randomUUID();
   const step_id = state.activeStep;
   const job_id = state.activeJob;
@@ -645,13 +690,13 @@ async function onStepComplete(context, event, error) {
   const reason = calculateJobExitReason(job_id, event.state, error);
   state.reasons[job_id] = reason;
   Object.assign(evt, reason);
-  return sendEvent(channel, STEP_COMPLETE, evt);
+  return sendEvent(context, STEP_COMPLETE, evt);
 }
 
 // src/events/step-start.ts
 import crypto4 from "node:crypto";
 async function onStepStart(context, event) {
-  const { channel, state } = context;
+  const { state } = context;
   state.activeStep = crypto4.randomUUID();
   state.activeJob = event.jobId;
   const input_dataclip_id = state.inputDataclips[event.jobId];
@@ -663,7 +708,7 @@ async function onStepStart(context, event) {
   if (!state.withheldDataclips[input_dataclip_id]) {
     evt.input_dataclip_id = input_dataclip_id;
   }
-  await sendEvent(channel, STEP_START, evt);
+  await sendEvent(context, STEP_START, evt);
 }
 
 // src/util/log-final-reason.ts
@@ -685,12 +730,12 @@ ${reason.error_type}: ${reason.error_message || "unknown"}`;
 
 // src/events/run-complete.ts
 async function onWorkflowComplete(context, event) {
-  const { state, channel, onFinish, logger } = context;
+  const { state, onFinish, logger } = context;
   const result = state.dataclips[state.lastDataclipId];
   const reason = calculateRunExitReason(state);
   await log_final_reason_default(context, reason);
   try {
-    await sendEvent(channel, RUN_COMPLETE, {
+    await sendEvent(context, RUN_COMPLETE, {
       final_dataclip_id: state.lastDataclipId,
       timestamp: timeInMicroseconds(event.time),
       ...reason
@@ -706,14 +751,14 @@ async function onWorkflowComplete(context, event) {
 
 // src/events/run-error.ts
 async function onRunError(context, event) {
-  const { state, channel, logger, onFinish } = context;
+  const { state, logger, onFinish } = context;
   try {
     const reason = calculateJobExitReason("", { data: {} }, event);
     if (state.activeJob) {
       await onJobError(context, { error: event });
     }
     await log_final_reason_default(context, reason);
-    await sendEvent(channel, RUN_COMPLETE, {
+    await sendEvent(context, RUN_COMPLETE, {
       final_dataclip_id: state.lastDataclipId,
       ...reason
     });
@@ -739,6 +784,7 @@ function execute(channel, engine, logger, plan, input, options = {}, onFinish =
   logger.info("executing ", plan.id);
   const state = create_run_state_default(plan, input);
   const context = {
+    id: plan.id,
     channel,
     state,
     logger,
@@ -746,76 +792,104 @@ function execute(channel, engine, logger, plan, input, options = {}, onFinish =
     options,
     onFinish
   };
-  const throttle = throttle_default();
-  const addEvent = (eventName, handler) => {
-    const wrappedFn = async (event) => {
-      const lightningEvent = eventMap[eventName] ?? eventName;
-      try {
-        await handler(context, event);
-        logger.info(`${plan.id} :: ${lightningEvent} :: OK`);
-      } catch (e) {
-        logger.error(
-          `${plan.id} :: ${lightningEvent} :: ERR: ${e.message || e.toString()}`
-        );
-        logger.error(e);
+  Sentry2.withIsolationScope(async () => {
+    Sentry2.addBreadcrumb({
+      category: "run",
+      message: "Executing run: loading metadata",
+      level: "info",
+      data: {
+        runId: plan.id
       }
+    });
+    const throttle = throttle_default();
+    const addEvent = (eventName, handler) => {
+      const wrappedFn = async (event) => {
+        if (eventName !== "workflow-log") {
+          Sentry2.addBreadcrumb({
+            category: "event",
+            message: eventName,
+            level: "info"
+          });
+        }
+        const lightningEvent = eventMap[eventName] ?? eventName;
+        try {
+          await handler(context, event);
+          logger.info(`${plan.id} :: ${lightningEvent} :: OK`);
+        } catch (e) {
+          if (!e.reportedToSentry) {
+            Sentry2.captureException(e);
+            logger.error(e);
+          }
+        }
+      };
+      return {
+        [eventName]: wrappedFn
+      };
     };
-    return {
-      [eventName]: wrappedFn
+    const listeners = Object.assign(
+      {},
+      addEvent("workflow-start", throttle(onRunStart)),
+      addEvent("job-start", throttle(onStepStart)),
+      addEvent("job-complete", throttle(onStepComplete)),
+      addEvent("job-error", throttle(onJobError)),
+      addEvent("workflow-log", throttle(onJobLog)),
+      // This will also resolve the promise
+      addEvent("workflow-complete", throttle(onWorkflowComplete)),
+      addEvent("workflow-error", throttle(onRunError))
+      // TODO send autoinstall logs
+    );
+    engine.listen(plan.id, listeners);
+    const resolvers = {
+      credential: (id) => loadCredential(context, id)
+      // TODO not supported right now
+      // dataclip: (id: string) => loadDataclip(channel, id),
     };
-  };
-  const listeners = Object.assign(
-    {},
-    addEvent("workflow-start", throttle(onRunStart)),
-    addEvent("job-start", throttle(onStepStart)),
-    addEvent("job-complete", throttle(onStepComplete)),
-    addEvent("job-error", throttle(onJobError)),
-    addEvent("workflow-log", throttle(onJobLog)),
-    // This will also resolve the promise
-    addEvent("workflow-complete", throttle(onWorkflowComplete)),
-    addEvent("workflow-error", throttle(onRunError))
-    // TODO send autoinstall logs
-  );
-  engine.listen(plan.id, listeners);
-  const resolvers = {
-    credential: (id) => loadCredential(channel, id)
-    // TODO not supported right now
-    // dataclip: (id: string) => loadDataclip(channel, id),
-  };
-  setTimeout(async () => {
-    let loadedInput = input;
-    if (typeof input === "string") {
-      logger.debug("loading dataclip", input);
+    setTimeout(async () => {
+      let loadedInput = input;
+      if (typeof input === "string") {
+        logger.debug("loading dataclip", input);
+        try {
+          loadedInput = await loadDataclip(context, input);
+          logger.success("dataclip loaded");
+        } catch (e) {
+          return onRunError(context, {
+            workflowId: plan.id,
+            message: `Failed to load dataclip ${input}${e.message ? `: ${e.message}` : ""}`,
+            type: "DataClipError",
+            severity: "exception"
+          });
+        }
+      }
       try {
-        loadedInput = await loadDataclip(channel, input);
-        logger.success("dataclip loaded");
+        Sentry2.addBreadcrumb({
+          category: "run",
+          message: "run metadata loaded: starting run",
+          level: "info",
+          data: {
+            runId: plan.id
+          }
+        });
+        engine.execute(plan, loadedInput, { resolvers, ...options });
       } catch (e) {
-        return onRunError(context, {
+        Sentry2.addBreadcrumb({
+          category: "run",
+          message: "exception in run",
+          level: "info",
+          data: {
+            runId: plan.id
+          }
+        });
+        onRunError(context, {
           workflowId: plan.id,
-          message: `Failed to load dataclip ${input}${e.message ? `: ${e.message}` : ""}`,
-          type: "DataClipError",
-          severity: "exception"
+          message: e.message,
+          type: e.type,
+          severity: e.severity
         });
       }
-    }
-    try {
-      engine.execute(plan, loadedInput, { resolvers, ...options });
-    } catch (e) {
-      onRunError(context, {
-        workflowId: plan.id,
-        message: e.message,
-        type: e.type,
-        severity: e.severity
-      });
-    }
+    });
   });
   return context;
 }
-var sendEvent = (channel, event, payload) => new Promise((resolve, reject) => {
-  channel.push(event, payload).receive("error", reject).receive("timeout", () => {
-    reject(new Error("timeout"));
-  }).receive("ok", resolve);
-});
 function onJobError(context, event) {
   const { state, error, jobId } = event;
   if (state?.errors?.[jobId]?.message === error.message) {
@@ -824,7 +898,8 @@ function onJobError(context, event) {
     return onStepComplete(context, event, event.error);
   }
 }
-function onJobLog({ channel, state, options }, event) {
+function onJobLog(context, event) {
+  const { state, options } = context;
   let message = event.message;
   if (event.redacted) {
     message = [
@@ -844,17 +919,17 @@ function onJobLog({ channel, state, options }, event) {
   if (state.activeStep) {
     log.step_id = state.activeStep;
   }
-  return sendEvent(channel, RUN_LOG, log);
+  return sendEvent(context, RUN_LOG, log);
 }
-async function loadDataclip(channel, stateId) {
-  const result = await get_with_reply_default(channel, GET_DATACLIP, {
+async function loadDataclip(context, stateId) {
+  const result = await sendEvent(context, GET_DATACLIP, {
     id: stateId
   });
   const str = enc.decode(new Uint8Array(result));
   return JSON.parse(str);
 }
-async function loadCredential(channel, credentialId) {
-  return get_with_reply_default(channel, GET_CREDENTIAL, { id: credentialId });
+async function loadCredential(context, credentialId) {
+  return sendEvent(context, GET_CREDENTIAL, { id: credentialId });
 }
 
 // src/middleware/healthcheck.ts
@@ -863,6 +938,7 @@ var healthcheck_default = (ctx) => {
 };
 
 // src/channels/run.ts
+import * as Sentry3 from "@sentry/node";
 var joinRunChannel = (socket, token, runId, logger) => {
   return new Promise((resolve, reject) => {
     let didReceiveOk = false;
@@ -873,13 +949,18 @@ var joinRunChannel = (socket, token, runId, logger) => {
       if (!didReceiveOk) {
         didReceiveOk = true;
         logger.success(`connected to ${channelName}`, e);
-        const run = await get_with_reply_default(channel, GET_PLAN);
+        const run = await send_event_default(
+          { channel, logger, id: runId },
+          GET_PLAN
+        );
         resolve({ channel, run });
       }
     }).receive("error", (err) => {
+      Sentry3.captureException(err);
       logger.error(`error connecting to ${channelName}`, err);
       reject(err);
     }).receive("timeout", (err) => {
+      Sentry3.captureException(err);
       logger.error(`Timeout for ${channelName}`, err);
       reject(err);
     });
@@ -896,6 +977,7 @@ var run_default = joinRunChannel;
 
 // src/channels/worker-queue.ts
 import EventEmitter from "node:events";
+import * as Sentry4 from "@sentry/node";
 import { Socket as PhxSocket } from "phoenix";
 import { WebSocket } from "ws";
 import { API_VERSION } from "@openfn/lexicon/lightning";
@@ -924,7 +1006,17 @@ var worker_token_default = generateWorkerToken;
 // src/channels/worker-queue.ts
 var connectToWorkerQueue = (endpoint, serverId, secret, timeout = 10, logger, SocketConstructor = PhxSocket) => {
   const events = new EventEmitter();
+  Sentry4.addBreadcrumb({
+    category: "lifecycle",
+    message: "Connecting to worker queue",
+    level: "info"
+  });
   worker_token_default(secret, serverId, logger).then(async (token) => {
+    Sentry4.addBreadcrumb({
+      category: "lifecycle",
+      message: "Worker token generated",
+      level: "info"
+    });
     const params = {
       token,
       api_version: API_VERSION,
@@ -933,11 +1025,19 @@ var connectToWorkerQueue = (endpoint, serverId, secret, timeout = 10, logger, So
     const socket = new SocketConstructor(endpoint, {
       params,
       transport: WebSocket,
-      timeout: timeout * 1e3
+      timeout: timeout * 1e3,
+      reconnectAfterMs: (tries) => Math.max(tries * 1e3)
     });
     let didOpen = false;
+    let shouldReportConnectionError = true;
     socket.onOpen(() => {
+      Sentry4.addBreadcrumb({
+        category: "lifecycle",
+        message: "Web socket connected",
+        level: "info"
+      });
       didOpen = true;
+      shouldReportConnectionError = true;
       const channel = socket.channel("worker:queue");
       channel.onMessage = (ev, load) => {
         events.emit("message", ev, load);
@@ -957,6 +1057,16 @@ var connectToWorkerQueue = (endpoint, serverId, secret, timeout = 10, logger, So
       events.emit("disconnect");
     });
     socket.onError((e) => {
+      Sentry4.addBreadcrumb({
+        category: "lifecycle",
+        message: "Error in web socket connection",
+        level: "info"
+      });
+      if (shouldReportConnectionError) {
+        logger.debug("Reporting connection error to sentry");
+        shouldReportConnectionError = false;
+        Sentry4.captureException(e);
+      }
       if (!didOpen) {
         events.emit("error", e.message);
         didOpen = false;
@@ -1061,6 +1171,13 @@ function createServer(engine, options = {}) {
   const router = new Router();
   app.events = new EventEmitter2();
   app.engine = engine;
+  if (options.sentryDsn) {
+    Sentry5.init({
+      environment: options.sentryEnv,
+      dsn: options.sentryDsn
+    });
+    Sentry5.setupKoaErrorHandler(app);
+  }
   app.use(bodyParser());
   app.use(
     koaLogger((str, _args) => {

package/dist/start.js

@@ -142,6 +142,7 @@ var runtime_engine_default = createMock;
 import { EventEmitter as EventEmitter3 } from "node:events";
 import { promisify } from "node:util";
 import { exec as _exec } from "node:child_process";
+import * as Sentry5 from "@sentry/node";
 import Koa from "koa";
 import bodyParser from "koa-bodyparser";
 import koaLogger from "koa-logger";
@@ -350,6 +351,9 @@ var startWorkloop = (app, logger2, minBackoff2, maxBackoff2, maxWorkers) => {
 };
 var workloop_default = startWorkloop;
 
+// src/api/execute.ts
+import * as Sentry2 from "@sentry/node";
+
 // src/util/convert-lightning-plan.ts
 import crypto3 from "node:crypto";
 import path from "node:path";
@@ -527,26 +531,6 @@ var convert_lightning_plan_default = (run2, options = {}) => {
   };
 };
 
-// src/util/get-with-reply.ts
-var get_with_reply_default = (channel, event, payload) => new Promise((resolve5, reject) => {
-  channel.push(event, payload).receive("ok", (evt) => {
-    resolve5(evt);
-  }).receive("error", (e) => {
-    reject(e);
-  }).receive("timeout", (e) => {
-    reject(e);
-  });
-});
-
-// src/util/stringify.ts
-import stringify from "fast-safe-stringify";
-var stringify_default = (obj) => stringify(obj, (_key, value) => {
-  if (value instanceof Uint8Array) {
-    return Array.from(value);
-  }
-  return value;
-});
-
 // src/util/create-run-state.ts
 var create_run_state_default = (plan, input) => {
   const state = {
@@ -580,6 +564,67 @@ var create_run_state_default = (plan, input) => {
   return state;
 };
 
+// src/util/send-event.ts
+import * as Sentry from "@sentry/node";
+
+// src/errors.ts
+var LightningSocketError = class extends Error {
+  constructor(event, message) {
+    super(`[${event}] ${message}`);
+    this.name = "LightningSocketError";
+    this.event = "";
+    this.rejectMessage = "";
+    this.event = event;
+    this.rejectMessage = message;
+  }
+};
+var LightningTimeoutError = class extends Error {
+  constructor(event) {
+    super(`[${event}] timeout`);
+    this.name = "LightningTimeoutError";
+  }
+};
+
+// src/util/send-event.ts
+var sendEvent = (context, event, payload) => {
+  const { channel, logger: logger2, id: runId = "<unknown run>" } = context;
+  return new Promise((resolve5, reject) => {
+    const report = (error) => {
+      logger2.error(`${runId} :: ${event} :: ERR: ${error.message || error}`);
+      const context2 = {
+        run_id: runId,
+        event
+      };
+      const extras = {};
+      if (error.rejectMessage) {
+        extras.rejection_reason = error.rejectMessage;
+      }
+      Sentry.captureException(error, (scope) => {
+        scope.setContext("run", context2);
+        scope.setExtras(extras);
+        return scope;
+      });
+      error.reportedToSentry = true;
+      reject(error);
+    };
+    channel.push(event, payload).receive("error", (message) => {
+      report(new LightningSocketError(event, message));
+    }).receive("timeout", () => {
+      report(new LightningTimeoutError(event));
+    }).receive("ok", resolve5);
+  });
+};
+var send_event_default = sendEvent;
+
+// src/util/stringify.ts
+import stringify from "fast-safe-stringify";
+var stringify_default = (obj) => stringify(obj, (_key, value) => {
+  if (value instanceof Uint8Array) {
+    return Array.from(value);
+  }
+  return value;
+});
+
 // src/util/throttle.ts
 var createThrottler = () => {
   const q = [];
@@ -665,7 +710,7 @@ function getVersion() {
 
 // src/events/run-start.ts
 async function onRunStart(context, event) {
-  const { channel, state, options = {} } = context;
+  const { state, options = {} } = context;
   const time = (timestamp() - BigInt(1e7)).toString();
   const versionLogContext = {
     ...context,
@@ -678,7 +723,7 @@ async function onRunStart(context, event) {
     worker: await getVersion(),
     ...event.versions
   };
-  await sendEvent(channel, RUN_START, {
+  await sendEvent(context, RUN_START, {
     versions,
     /// use the engine time in run start
     timestamp: timeInMicroseconds(event.time)
@@ -740,7 +785,7 @@ var calculateRunExitReason = (state) => {
 
 // src/events/step-complete.ts
 async function onStepComplete(context, event, error) {
-  const { channel, state, options } = context;
+  const { state, options } = context;
   const dataclipId = crypto4.randomUUID();
   const step_id = state.activeStep;
   const job_id = state.activeJob;
@@ -785,13 +830,13 @@ async function onStepComplete(context, event, error) {
   const reason = calculateJobExitReason(job_id, event.state, error);
   state.reasons[job_id] = reason;
   Object.assign(evt, reason);
-  return sendEvent(channel, STEP_COMPLETE, evt);
+  return sendEvent(context, STEP_COMPLETE, evt);
 }
 
 // src/events/step-start.ts
 import crypto5 from "node:crypto";
 async function onStepStart(context, event) {
-  const { channel, state } = context;
+  const { state } = context;
   state.activeStep = crypto5.randomUUID();
   state.activeJob = event.jobId;
   const input_dataclip_id = state.inputDataclips[event.jobId];
@@ -803,7 +848,7 @@ async function onStepStart(context, event) {
   if (!state.withheldDataclips[input_dataclip_id]) {
     evt.input_dataclip_id = input_dataclip_id;
   }
-  await sendEvent(channel, STEP_START, evt);
+  await sendEvent(context, STEP_START, evt);
 }
 
 // src/util/log-final-reason.ts
@@ -825,12 +870,12 @@ ${reason.error_type}: ${reason.error_message || "unknown"}`;
 
 // src/events/run-complete.ts
 async function onWorkflowComplete(context, event) {
-  const { state, channel, onFinish, logger: logger2 } = context;
+  const { state, onFinish, logger: logger2 } = context;
   const result = state.dataclips[state.lastDataclipId];
   const reason = calculateRunExitReason(state);
   await log_final_reason_default(context, reason);
   try {
-    await sendEvent(channel, RUN_COMPLETE, {
+    await sendEvent(context, RUN_COMPLETE, {
       final_dataclip_id: state.lastDataclipId,
       timestamp: timeInMicroseconds(event.time),
       ...reason
@@ -846,14 +891,14 @@ async function onWorkflowComplete(context, event) {
 
 // src/events/run-error.ts
 async function onRunError(context, event) {
-  const { state, channel, logger: logger2, onFinish } = context;
+  const { state, logger: logger2, onFinish } = context;
   try {
     const reason = calculateJobExitReason("", { data: {} }, event);
     if (state.activeJob) {
       await onJobError(context, { error: event });
     }
     await log_final_reason_default(context, reason);
-    await sendEvent(channel, RUN_COMPLETE, {
+    await sendEvent(context, RUN_COMPLETE, {
       final_dataclip_id: state.lastDataclipId,
       ...reason
     });
@@ -879,6 +924,7 @@ function execute(channel, engine, logger2, plan, input, options = {}, onFinish =
   logger2.info("executing ", plan.id);
   const state = create_run_state_default(plan, input);
   const context = {
+    id: plan.id,
     channel,
     state,
     logger: logger2,
@@ -886,76 +932,104 @@ function execute(channel, engine, logger2, plan, input, options = {}, onFinish =
     options,
     onFinish
   };
-  const throttle = throttle_default();
-  const addEvent = (eventName, handler) => {
-    const wrappedFn = async (event) => {
-      const lightningEvent = eventMap[eventName] ?? eventName;
-      try {
-        await handler(context, event);
-        logger2.info(`${plan.id} :: ${lightningEvent} :: OK`);
-      } catch (e) {
-        logger2.error(
-          `${plan.id} :: ${lightningEvent} :: ERR: ${e.message || e.toString()}`
-        );
-        logger2.error(e);
+  Sentry2.withIsolationScope(async () => {
+    Sentry2.addBreadcrumb({
+      category: "run",
+      message: "Executing run: loading metadata",
+      level: "info",
+      data: {
+        runId: plan.id
       }
+    });
+    const throttle = throttle_default();
+    const addEvent = (eventName, handler) => {
+      const wrappedFn = async (event) => {
+        if (eventName !== "workflow-log") {
+          Sentry2.addBreadcrumb({
+            category: "event",
+            message: eventName,
+            level: "info"
+          });
+        }
+        const lightningEvent = eventMap[eventName] ?? eventName;
+        try {
+          await handler(context, event);
+          logger2.info(`${plan.id} :: ${lightningEvent} :: OK`);
+        } catch (e) {
+          if (!e.reportedToSentry) {
+            Sentry2.captureException(e);
+            logger2.error(e);
+          }
+        }
+      };
+      return {
+        [eventName]: wrappedFn
+      };
     };
-    return {
-      [eventName]: wrappedFn
+    const listeners = Object.assign(
+      {},
+      addEvent("workflow-start", throttle(onRunStart)),
+      addEvent("job-start", throttle(onStepStart)),
+      addEvent("job-complete", throttle(onStepComplete)),
+      addEvent("job-error", throttle(onJobError)),
+      addEvent("workflow-log", throttle(onJobLog)),
+      // This will also resolve the promise
+      addEvent("workflow-complete", throttle(onWorkflowComplete)),
+      addEvent("workflow-error", throttle(onRunError))
+      // TODO send autoinstall logs
+    );
+    engine.listen(plan.id, listeners);
+    const resolvers = {
+      credential: (id) => loadCredential(context, id)
+      // TODO not supported right now
+      // dataclip: (id: string) => loadDataclip(channel, id),
     };
-  };
-  const listeners = Object.assign(
-    {},
-    addEvent("workflow-start", throttle(onRunStart)),
-    addEvent("job-start", throttle(onStepStart)),
-    addEvent("job-complete", throttle(onStepComplete)),
-    addEvent("job-error", throttle(onJobError)),
-    addEvent("workflow-log", throttle(onJobLog)),
-    // This will also resolve the promise
-    addEvent("workflow-complete", throttle(onWorkflowComplete)),
-    addEvent("workflow-error", throttle(onRunError))
-    // TODO send autoinstall logs
-  );
-  engine.listen(plan.id, listeners);
-  const resolvers = {
-    credential: (id) => loadCredential(channel, id)
-    // TODO not supported right now
-    // dataclip: (id: string) => loadDataclip(channel, id),
-  };
-  setTimeout(async () => {
-    let loadedInput = input;
-    if (typeof input === "string") {
-      logger2.debug("loading dataclip", input);
+    setTimeout(async () => {
+      let loadedInput = input;
+      if (typeof input === "string") {
+        logger2.debug("loading dataclip", input);
+        try {
+          loadedInput = await loadDataclip(context, input);
+          logger2.success("dataclip loaded");
+        } catch (e) {
+          return onRunError(context, {
+            workflowId: plan.id,
+            message: `Failed to load dataclip ${input}${e.message ? `: ${e.message}` : ""}`,
+            type: "DataClipError",
+            severity: "exception"
+          });
+        }
+      }
       try {
-        loadedInput = await loadDataclip(channel, input);
-        logger2.success("dataclip loaded");
+        Sentry2.addBreadcrumb({
+          category: "run",
+          message: "run metadata loaded: starting run",
+          level: "info",
+          data: {
+            runId: plan.id
+          }
+        });
+        engine.execute(plan, loadedInput, { resolvers, ...options });
       } catch (e) {
-        return onRunError(context, {
+        Sentry2.addBreadcrumb({
+          category: "run",
+          message: "exception in run",
+          level: "info",
+          data: {
+            runId: plan.id
+          }
+        });
+        onRunError(context, {
           workflowId: plan.id,
-          message: `Failed to load dataclip ${input}${e.message ? `: ${e.message}` : ""}`,
-          type: "DataClipError",
-          severity: "exception"
+          message: e.message,
+          type: e.type,
+          severity: e.severity
         });
       }
-    }
-    try {
-      engine.execute(plan, loadedInput, { resolvers, ...options });
-    } catch (e) {
-      onRunError(context, {
-        workflowId: plan.id,
-        message: e.message,
-        type: e.type,
-        severity: e.severity
-      });
-    }
+    });
   });
   return context;
 }
-var sendEvent = (channel, event, payload) => new Promise((resolve5, reject) => {
-  channel.push(event, payload).receive("error", reject).receive("timeout", () => {
-    reject(new Error("timeout"));
-  }).receive("ok", resolve5);
-});
 function onJobError(context, event) {
   const { state, error, jobId } = event;
   if (state?.errors?.[jobId]?.message === error.message) {
@@ -964,7 +1038,8 @@ function onJobError(context, event) {
     return onStepComplete(context, event, event.error);
   }
 }
-function onJobLog({ channel, state, options }, event) {
+function onJobLog(context, event) {
+  const { state, options } = context;
   let message = event.message;
   if (event.redacted) {
     message = [
@@ -984,17 +1059,17 @@ function onJobLog({ channel, state, options }, event) {
   if (state.activeStep) {
     log.step_id = state.activeStep;
   }
-  return sendEvent(channel, RUN_LOG, log);
+  return sendEvent(context, RUN_LOG, log);
 }
-async function loadDataclip(channel, stateId) {
-  const result = await get_with_reply_default(channel, GET_DATACLIP, {
+async function loadDataclip(context, stateId) {
+  const result = await sendEvent(context, GET_DATACLIP, {
     id: stateId
   });
   const str = enc.decode(new Uint8Array(result));
   return JSON.parse(str);
 }
-async function loadCredential(channel, credentialId) {
-  return get_with_reply_default(channel, GET_CREDENTIAL, { id: credentialId });
+async function loadCredential(context, credentialId) {
+  return sendEvent(context, GET_CREDENTIAL, { id: credentialId });
 }
 
 // src/middleware/healthcheck.ts
@@ -1003,6 +1078,7 @@ var healthcheck_default = (ctx) => {
 };
 
 // src/channels/run.ts
+import * as Sentry3 from "@sentry/node";
 var joinRunChannel = (socket, token, runId, logger2) => {
   return new Promise((resolve5, reject) => {
     let didReceiveOk = false;
@@ -1013,13 +1089,18 @@ var joinRunChannel = (socket, token, runId, logger2) => {
       if (!didReceiveOk) {
         didReceiveOk = true;
         logger2.success(`connected to ${channelName}`, e);
-        const run2 = await get_with_reply_default(channel, GET_PLAN);
+        const run2 = await send_event_default(
+          { channel, logger: logger2, id: runId },
+          GET_PLAN
+        );
         resolve5({ channel, run: run2 });
       }
     }).receive("error", (err) => {
+      Sentry3.captureException(err);
       logger2.error(`error connecting to ${channelName}`, err);
       reject(err);
     }).receive("timeout", (err) => {
+      Sentry3.captureException(err);
       logger2.error(`Timeout for ${channelName}`, err);
       reject(err);
     });
@@ -1036,6 +1117,7 @@ var run_default = joinRunChannel;
 
 // src/channels/worker-queue.ts
 import EventEmitter2 from "node:events";
+import * as Sentry4 from "@sentry/node";
 import { Socket as PhxSocket } from "phoenix";
 import { WebSocket } from "ws";
 import { API_VERSION } from "@openfn/lexicon/lightning";
@@ -1064,7 +1146,17 @@ var worker_token_default = generateWorkerToken;
 // src/channels/worker-queue.ts
 var connectToWorkerQueue = (endpoint, serverId, secret, timeout = 10, logger2, SocketConstructor = PhxSocket) => {
   const events = new EventEmitter2();
+  Sentry4.addBreadcrumb({
+    category: "lifecycle",
+    message: "Connecting to worker queue",
+    level: "info"
+  });
   worker_token_default(secret, serverId, logger2).then(async (token) => {
+    Sentry4.addBreadcrumb({
+      category: "lifecycle",
+      message: "Worker token generated",
+      level: "info"
+    });
     const params = {
       token,
       api_version: API_VERSION,
@@ -1073,11 +1165,19 @@ var connectToWorkerQueue = (endpoint, serverId, secret, timeout = 10, logger2, S
     const socket = new SocketConstructor(endpoint, {
       params,
       transport: WebSocket,
-      timeout: timeout * 1e3
+      timeout: timeout * 1e3,
+      reconnectAfterMs: (tries) => Math.max(tries * 1e3)
     });
     let didOpen = false;
+    let shouldReportConnectionError = true;
     socket.onOpen(() => {
+      Sentry4.addBreadcrumb({
+        category: "lifecycle",
+        message: "Web socket connected",
+        level: "info"
+      });
       didOpen = true;
+      shouldReportConnectionError = true;
       const channel = socket.channel("worker:queue");
       channel.onMessage = (ev, load) => {
         events.emit("message", ev, load);
@@ -1097,6 +1197,16 @@ var connectToWorkerQueue = (endpoint, serverId, secret, timeout = 10, logger2, S
       events.emit("disconnect");
     });
     socket.onError((e) => {
+      Sentry4.addBreadcrumb({
+        category: "lifecycle",
+        message: "Error in web socket connection",
+        level: "info"
+      });
+      if (shouldReportConnectionError) {
+        logger2.debug("Reporting connection error to sentry");
+        shouldReportConnectionError = false;
+        Sentry4.captureException(e);
+      }
       if (!didOpen) {
         events.emit("error", e.message);
         didOpen = false;
@@ -1201,6 +1311,13 @@ function createServer(engine, options = {}) {
   const router = new Router();
   app.events = new EventEmitter3();
   app.engine = engine;
+  if (options.sentryDsn) {
+    Sentry5.init({
+      environment: options.sentryEnv,
+      dsn: options.sentryDsn
+    });
+    Sentry5.setupKoaErrorHandler(app);
+  }
   app.use(bodyParser());
   app.use(
     koaLogger((str, _args) => {
@@ -6209,6 +6326,8 @@ function parseArgs(argv) {
     WORKER_PORT,
     WORKER_REPO_DIR,
     WORKER_SECRET,
+    WORKER_SENTRY_DSN,
+    WORKER_SENTRY_ENV,
     WORKER_STATE_PROPS_TO_REMOVE,
     WORKER_SOCKET_TIMEOUT_SECONDS,
     OPENFN_ADAPTORS_REPO
@@ -6229,6 +6348,11 @@ function parseArgs(argv) {
   }).option("secret", {
     alias: "s",
     description: "Worker secret. (comes from WORKER_SECRET by default). Env: WORKER_SECRET"
+  }).option("sentry-dsn", {
+    alias: ["dsn"],
+    description: "Sentry DSN. Env: WORKER_SENTRY_DSN"
+  }).option("sentry-env", {
+    description: "Sentry environment. Defaults to 'dev'. Env: WORKER_SENTRY_ENV"
   }).option("socket-timeout", {
     description: `Timeout for websockets to Lighting, in seconds. Defaults to 10.`
   }).option("lightning-public-key", {
@@ -6280,6 +6404,8 @@ function parseArgs(argv) {
     repoDir: setArg(args2.repoDir, WORKER_REPO_DIR),
     monorepoDir: setArg(args2.monorepoDir, OPENFN_ADAPTORS_REPO),
     secret: setArg(args2.secret, WORKER_SECRET),
+    sentryDsn: setArg(args2.sentryDsn, WORKER_SENTRY_DSN),
+    sentryEnv: setArg(args2.sentryEnv, WORKER_SENTRY_ENV, "dev"),
     lightningPublicKey: setArg(
       args2.lightningPublicKey,
       WORKER_LIGHTNING_PUBLIC_KEY
@@ -6333,6 +6459,8 @@ function engineReady(engine) {
     lightning: args.lightning,
     logger,
     secret: args.secret,
+    sentryDsn: args.sentryDsn,
+    sentryEnv: args.sentryEnv,
     noLoop: !args.loop,
     // TODO need to feed this through properly
     backoff: {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openfn/ws-worker",
-  "version": "1.12.0",
+  "version": "1.13.0",
   "description": "A Websocket Worker to connect Lightning to a Runtime Engine",
   "main": "dist/index.js",
   "type": "module",
@@ -11,6 +11,7 @@
   "license": "ISC",
   "dependencies": {
     "@koa/router": "^12.0.0",
+    "@sentry/node": "^9.5.0",
     "@types/koa-logger": "^3.1.2",
     "@types/ws": "^8.5.6",
     "fast-safe-stringify": "^2.1.1",
@@ -22,9 +23,9 @@
     "koa-logger": "^3.2.1",
     "phoenix": "1.7.10",
     "ws": "^8.18.0",
-    "@openfn/engine-multi": "1.6.0",
-    "@openfn/logger": "1.0.4",
-    "@openfn/runtime": "1.6.3",
+    "@openfn/engine-multi": "1.6.2",
+    "@openfn/logger": "1.0.5",
+    "@openfn/runtime": "1.6.4",
     "@openfn/lexicon": "^1.2.0"
   },
   "devDependencies": {
@@ -37,11 +38,12 @@
     "@types/yargs": "^17.0.12",
     "ava": "5.1.0",
     "nodemon": "3.0.1",
+    "sentry-testkit": "^6.1.0",
     "tslib": "^2.4.0",
     "tsup": "^6.2.3",
     "typescript": "^4.6.4",
     "yargs": "^17.6.2",
-    "@openfn/lightning-mock": "2.1.2"
+    "@openfn/lightning-mock": "2.1.4"
   },
   "files": [
     "dist",