@openfn/ws-worker 1.19.7 → 1.20.1

package/CHANGELOG.md

@@ -1,5 +1,28 @@
 # ws-worker
 
+## 1.20.1
+
+### Patch Changes
+
+- Updated dependencies [a1bfdc1]
+- Updated dependencies [c70369b]
+  - @openfn/runtime@1.7.7
+  - @openfn/logger@1.1.1
+  - @openfn/engine-multi@1.9.1
+
+## 1.20.0
+
+### Minor Changes
+
+- 3a7038b: Allow logs to be sent to Lightning in batches. This behavior is disabled by default for back-compatibility: set `WORKER_BATCH_LOGS=true` to enable.
+- 5417e97: Reduce the payload size of log events sent out to Lightning. This defaults to 1mb but can be configured with `WORKER_MAX_LOG_PAYLOAD_MB`
+
+### Patch Changes
+
+- Updated dependencies [5417e97]
+  - @openfn/lexicon@1.3.0
+  - @openfn/engine-multi@1.9.0
+
 ## 1.19.7
 
 ### Patch Changes

package/dist/index.d.ts

@@ -49,9 +49,13 @@ interface Channel extends Channel$1 {
 type WorkerRunOptions = ExecuteOptions & {
     outputDataclips?: boolean;
     payloadLimitMb?: number;
+    logPayloadLimitMb?: number;
     jobLogLevel?: LogLevel;
     timeoutRetryCount?: number;
     timeoutRetryDelay?: number;
+    batchLogs?: boolean;
+    batchInterval?: number;
+    batchLimit?: number;
 };
 
 type Context = {
@@ -65,6 +69,9 @@ type Context = {
 };
 
 type ServerOptions = {
+    batchLogs?: boolean;
+    batchInterval?: number;
+    batchLimit?: number;
     maxWorkflows?: number;
     port?: number;
     lightning?: string;
@@ -82,6 +89,7 @@ type ServerOptions = {
     messageTimeoutSeconds?: number;
     claimTimeoutSeconds?: number;
     payloadLimitMb?: number;
+    logPayloadLimitMb?: number;
     collectionsVersion?: string;
     collectionsUrl?: string;
     monorepoDir?: string;

package/dist/index.js

@@ -2,7 +2,7 @@
 import { EventEmitter as EventEmitter2 } from "node:events";
 import { promisify } from "node:util";
 import { exec as _exec } from "node:child_process";
-import * as Sentry6 from "@sentry/node";
+import * as Sentry7 from "@sentry/node";
 import Koa from "koa";
 import bodyParser from "koa-bodyparser";
 import koaLogger from "koa-logger";
@@ -271,7 +271,16 @@ var startWorkloop = (app, logger, minBackoff, maxBackoff, maxWorkers) => {
 var workloop_default = startWorkloop;
 
 // src/api/execute.ts
-import * as Sentry3 from "@sentry/node";
+import * as Sentry4 from "@sentry/node";
+import {
+  JOB_COMPLETE as JOB_COMPLETE2,
+  JOB_START as JOB_START2,
+  WORKFLOW_COMPLETE as WORKFLOW_COMPLETE2,
+  WORKFLOW_LOG as WORKFLOW_LOG2,
+  WORKFLOW_START as WORKFLOW_START2,
+  JOB_ERROR as JOB_ERROR2,
+  WORKFLOW_ERROR as WORKFLOW_ERROR2
+} from "@openfn/engine-multi";
 
 // src/util/convert-lightning-plan.ts
 import crypto2 from "node:crypto";
@@ -342,6 +351,9 @@ var convert_lightning_plan_default = (run, options = {}) => {
     if ("payload_limit_mb" in run.options) {
       engineOpts.payloadLimitMb = run.options.payload_limit_mb;
     }
+    if ("log_payload_limit_mb" in run.options) {
+      engineOpts.logPayloadLimitMb = run.options.log_payload_limit_mb;
+    }
     if ("run_memory_limit_mb" in run.options) {
       engineOpts.memoryLimitMb = run.options.run_memory_limit_mb;
     }
@@ -576,43 +588,59 @@ var stringify_default = (obj) => stringify(obj, (_key, value) => {
   return value;
 });
 
-// src/util/throttle.ts
-var createThrottler = () => {
-  const q = [];
-  let activePromise;
-  const add = (fn) => {
-    return (...args) => (
-      // the wrapped function is a deferred promise
-      // that will reject or resolve as usual
-      new Promise((resolve, reject) => {
-        q.push({ fn, args, resolve, reject });
-        shift();
-      })
-    );
-  };
-  const shift = () => {
-    if (activePromise) {
-      return;
-    }
-    const next = q.shift();
-    if (next) {
-      const { fn, args, resolve, reject } = next;
-      activePromise = fn(...args).then(resolve).catch(reject).finally(() => {
-        activePromise = void 0;
-        shift();
-      });
-    }
-  };
-  return add;
-};
-var throttle_default = createThrottler;
-
 // src/util/timestamp.ts
 var timeInMicroseconds = (time) => time && (BigInt(time) / BigInt(1e3)).toString();
 
 // src/events/run-start.ts
 import { timestamp } from "@openfn/logger";
 
+// src/events/run-log.ts
+async function onRunLog(context, events) {
+  if (!Array.isArray(events)) {
+    events = [events];
+  }
+  const { state, options } = context;
+  const { batchLogs } = options ?? {};
+  const logs = events.map((evt) => {
+    let message = evt.message;
+    if (evt.redacted) {
+      message = [
+        `(Log message redacted: exceeds ${options.logPayloadLimitMb ?? options.payloadLimitMb}mb memory limit)`
+      ];
+    } else if (typeof evt.message === "string") {
+      message = JSON.parse(evt.message);
+    }
+    const logLine = {
+      message,
+      source: evt.name,
+      level: evt.level,
+      timestamp: timeInMicroseconds(evt.time)
+    };
+    if (state.activeStep) {
+      logLine.step_id = state.activeStep;
+    }
+    return logLine;
+  });
+  if (batchLogs) {
+    const payload = {
+      run_id: `${state.plan.id}`,
+      logs
+    };
+    return sendEvent(context, RUN_LOG, payload);
+  } else {
+    return new Promise(async (resolve) => {
+      for (const log of logs) {
+        const payload = {
+          run_id: `${state.plan.id}`,
+          ...log
+        };
+        await sendEvent(context, RUN_LOG, payload);
+      }
+      resolve();
+    });
+  }
+}
+
 // src/util/versions.ts
 import { mainSymbols } from "figures";
 var { triangleRightSmall: t } = mainSymbols;
@@ -680,21 +708,25 @@ async function onRunStart(context, event) {
     timestamp: timeInMicroseconds(event.time)
   });
   if ("payloadLimitMb" in options) {
-    await onJobLog(versionLogContext, {
-      // use the fake time in the log
-      time,
-      message: [`Payload limit: ${options.payloadLimitMb}mb`],
-      level: "info",
-      name: "RTE"
-    });
+    await onRunLog(versionLogContext, [
+      {
+        // use the fake time in the log
+        time,
+        message: [`Payload limit: ${options.payloadLimitMb}mb`],
+        level: "info",
+        name: "RTE"
+      }
+    ]);
   }
   const versionMessage = versions_default(versions);
-  await onJobLog(versionLogContext, {
-    time,
-    message: [versionMessage],
-    level: "info",
-    name: "VER"
-  });
+  await onRunLog(versionLogContext, [
+    {
+      time,
+      message: [versionMessage],
+      level: "info",
+      name: "VER"
+    }
+  ]);
 }
 
 // src/events/step-complete.ts
@@ -763,14 +795,16 @@ async function onStepComplete(context, event, error) {
     state.withheldDataclips[dataclipId] = true;
     evt.output_dataclip_error = "DATACLIP_TOO_LARGE";
     const time = (timestamp2() - BigInt(1e7)).toString();
-    await onJobLog(context, {
-      time,
-      message: [
-        "Dataclip exceeds payload limit: output will not be sent back to the app."
-      ],
-      level: "info",
-      name: "R/T"
-    });
+    await onRunLog(context, [
+      {
+        time,
+        message: [
+          "Dataclip exceeds payload limit: output will not be sent back to the app."
+        ],
+        level: "info",
+        name: "R/T"
+      }
+    ]);
   } else {
     evt.output_dataclip_id = dataclipId;
     if (!options || options.outputDataclips !== false) {
@@ -807,20 +841,24 @@ import { timestamp as timestamp3 } from "@openfn/logger";
 var log_final_reason_default = async (context, reason) => {
   const time = (timestamp3() - BigInt(1e7)).toString();
   const statusMessage = `Run complete with status: ${reason.reason}`;
-  await onJobLog(context, {
-    time,
-    message: [statusMessage],
-    level: "info",
-    name: "R/T"
-  });
-  if (reason.reason !== "success") {
-    const errorMessage = `${reason.error_type}: ${reason.error_message || "unknown"}`;
-    await onJobLog(context, {
+  await onRunLog(context, [
+    {
       time,
-      message: [errorMessage],
+      message: [statusMessage],
       level: "info",
       name: "R/T"
-    });
+    }
+  ]);
+  if (reason.reason !== "success") {
+    const errorMessage = `${reason.error_type}: ${reason.error_message || "unknown"}`;
+    await onRunLog(context, [
+      {
+        time,
+        message: [errorMessage],
+        level: "info",
+        name: "R/T"
+      }
+    ]);
   }
 };
 
@@ -865,15 +903,137 @@ async function onRunError(context, event) {
   }
 }
 
-// src/api/execute.ts
-var enc = new TextDecoder("utf-8");
+// src/api/process-events.ts
+import * as Sentry3 from "@sentry/node";
+import {
+  JOB_COMPLETE,
+  JOB_ERROR,
+  JOB_START,
+  WORKFLOW_COMPLETE,
+  WORKFLOW_ERROR,
+  WORKFLOW_LOG,
+  WORKFLOW_START
+} from "@openfn/engine-multi";
+var DEFAULT_BATCH_LIMIT = 10;
+var DEFAULT_BATCH_INTERVAL = 10;
 var eventMap = {
-  "workflow-start": RUN_START,
-  "job-start": STEP_START,
-  "job-complete": STEP_COMPLETE,
-  "workflow-log": RUN_LOG,
-  "workflow-complete": RUN_COMPLETE
+  [WORKFLOW_START]: RUN_START,
+  [JOB_START]: STEP_START,
+  [JOB_COMPLETE]: STEP_COMPLETE,
+  [WORKFLOW_LOG]: RUN_LOG,
+  [WORKFLOW_COMPLETE]: RUN_COMPLETE
 };
+var allEngineEvents = [
+  WORKFLOW_START,
+  WORKFLOW_COMPLETE,
+  JOB_START,
+  JOB_COMPLETE,
+  WORKFLOW_LOG,
+  JOB_ERROR,
+  WORKFLOW_ERROR
+];
+function eventProcessor(engine, context, callbacks, options = {}) {
+  const { id: planId, logger } = context;
+  const {
+    batchLimit: limit = DEFAULT_BATCH_LIMIT,
+    batchInterval: interval = DEFAULT_BATCH_INTERVAL
+  } = options;
+  const queue = [];
+  const next = async () => {
+    const evt = queue[0];
+    if (evt) {
+      await process2(evt.name, evt.event);
+      queue.shift();
+      next();
+    }
+  };
+  let activeBatch = null;
+  let batch = [];
+  let start = -1;
+  let batchTimeout;
+  const sendBatch = async (name) => {
+    clearTimeout(batchTimeout);
+    activeBatch = null;
+    await send(name, batch, batch.length);
+    batch = [];
+  };
+  const send = async (name, payload, batchSize) => {
+    const lightningEvent = eventMap[name] ?? name;
+    await callbacks[name](context, payload);
+    if (batchSize) {
+      logger.info(
+        `${planId} :: sent ${lightningEvent} (${batchSize}):: OK :: ${Date.now() - start}ms`
+      );
+    } else {
+      logger.info(
+        `${planId} :: sent ${lightningEvent} :: OK :: ${Date.now() - start}ms`
+      );
+    }
+  };
+  const process2 = async (name, event) => {
+    if (name !== "workflow-log") {
+      Sentry3.addBreadcrumb({
+        category: "event",
+        message: name,
+        level: "info"
+      });
+    }
+    if (name === activeBatch) {
+      batch.push(event);
+      if (batch.length >= limit) {
+        await sendBatch(name);
+      }
+      return;
+    } else if (activeBatch) {
+      await sendBatch(activeBatch);
+    }
+    if (name in callbacks) {
+      try {
+        start = Date.now();
+        if (options?.batch?.[name]) {
+          activeBatch = name;
+          batch.push(event);
+          while (queue.length > 1 && queue[1].name === name) {
+            const [nextBatchItem] = queue.splice(1, 1);
+            batch.push(nextBatchItem.event);
+            if (batch.length >= limit) {
+              return sendBatch(name);
+            }
+          }
+          if (!batchTimeout) {
+            const batchName = activeBatch;
+            batchTimeout = setTimeout(async () => {
+              sendBatch(batchName);
+            }, interval);
+          }
+          return;
+        }
+        await send(name, event);
+      } catch (e2) {
+        if (!e2.reportedToSentry) {
+          Sentry3.captureException(e2);
+          logger.error(e2);
+        }
+      }
+    } else {
+      logger.warn("no event bound for", name);
+    }
+  };
+  const enqueue = (name, event) => {
+    queue.push({ name, event });
+    if (queue.length == 1) {
+      next();
+    }
+  };
+  const e = allEngineEvents.reduce(
+    (obj, e2) => Object.assign(obj, { [e2]: (p) => enqueue(e2, p) }),
+    {}
+  );
+  engine.listen(planId, e);
+}
+
+// src/api/execute.ts
+var enc = new TextDecoder("utf-8");
 function execute(channel, engine, logger, plan, input, options = {}, onFinish = (_result) => {
 }) {
   logger.info("executing ", plan.id);
@@ -887,8 +1047,8 @@ function execute(channel, engine, logger, plan, input, options = {}, onFinish =
     options,
     onFinish
   };
-  Sentry3.withIsolationScope(async () => {
-    Sentry3.addBreadcrumb({
+  Sentry4.withIsolationScope(async () => {
+    Sentry4.addBreadcrumb({
       category: "run",
       message: "Executing run: loading metadata",
       level: "info",
@@ -896,47 +1056,26 @@ function execute(channel, engine, logger, plan, input, options = {}, onFinish =
         runId: plan.id
       }
     });
-    const throttle = throttle_default();
-    const addEvent = (eventName, handler) => {
-      const wrappedFn = async (event) => {
-        if (eventName !== "workflow-log") {
-          Sentry3.addBreadcrumb({
-            category: "event",
-            message: eventName,
-            level: "info"
-          });
-        }
-        const lightningEvent = eventMap[eventName] ?? eventName;
-        try {
-          let start = Date.now();
-          await handler(context, event);
-          logger.info(
-            `${plan.id} :: sent ${lightningEvent} :: OK :: ${Date.now() - start}ms`
-          );
-        } catch (e) {
-          if (!e.reportedToSentry) {
-            Sentry3.captureException(e);
-            logger.error(e);
-          }
-        }
-      };
-      return {
-        [eventName]: wrappedFn
-      };
-    };
-    const listeners = Object.assign(
-      {},
-      addEvent("workflow-start", throttle(onRunStart)),
-      addEvent("job-start", throttle(onStepStart)),
-      addEvent("job-complete", throttle(onStepComplete)),
-      addEvent("job-error", throttle(onJobError)),
-      addEvent("workflow-log", throttle(onJobLog)),
-      // This will also resolve the promise
-      addEvent("workflow-complete", throttle(onWorkflowComplete)),
-      addEvent("workflow-error", throttle(onRunError))
-      // TODO send autoinstall logs
+    eventProcessor(
+      engine,
+      context,
+      {
+        [WORKFLOW_START2]: onRunStart,
+        [JOB_START2]: onStepStart,
+        [JOB_COMPLETE2]: onStepComplete,
+        [JOB_ERROR2]: onJobError,
+        [WORKFLOW_LOG2]: onRunLog,
+        [WORKFLOW_COMPLETE2]: onWorkflowComplete,
+        [WORKFLOW_ERROR2]: onRunError
+      },
+      {
+        batch: options.batchLogs ? {
+          [WORKFLOW_LOG2]: true
+        } : {},
+        batchInterval: options.batchInterval,
+        batchLimit: options.batchLimit
+      }
     );
-    engine.listen(plan.id, listeners);
     const resolvers = {
       credential: (id) => loadCredential(context, id)
       // TODO not supported right now
@@ -959,7 +1098,7 @@ function execute(channel, engine, logger, plan, input, options = {}, onFinish =
         }
       }
       try {
-        Sentry3.addBreadcrumb({
+        Sentry4.addBreadcrumb({
           category: "run",
           message: "run metadata loaded: starting run",
           level: "info",
@@ -969,7 +1108,7 @@ function execute(channel, engine, logger, plan, input, options = {}, onFinish =
         });
         engine.execute(plan, loadedInput, { resolvers, ...options });
       } catch (e) {
-        Sentry3.addBreadcrumb({
+        Sentry4.addBreadcrumb({
           category: "run",
           message: "exception in run",
           level: "info",
@@ -996,29 +1135,6 @@ function onJobError(context, event) {
     return onStepComplete(context, event, event.error);
   }
 }
-function onJobLog(context, event) {
-  const { state, options } = context;
-  let message = event.message;
-  if (event.redacted) {
-    message = [
-      `(Log message redacted: exceeds ${options.payloadLimitMb}mb memory limit)`
-    ];
-  } else if (typeof event.message === "string") {
-    message = JSON.parse(event.message);
-  }
-  const log = {
-    run_id: `${state.plan.id}`,
-    message,
-    source: event.name,
-    level: event.level,
-    // @ts-ignore
-    timestamp: timeInMicroseconds(event.time)
-  };
-  if (state.activeStep) {
-    log.step_id = state.activeStep;
-  }
-  return sendEvent(context, RUN_LOG, log);
-}
 async function loadDataclip(context, stateId) {
   const result = await sendEvent(context, GET_DATACLIP, {
     id: stateId
@@ -1036,7 +1152,7 @@ var healthcheck_default = (ctx) => {
 };
 
 // src/channels/run.ts
-import * as Sentry4 from "@sentry/node";
+import * as Sentry5 from "@sentry/node";
 var joinRunChannel = (socket, token, runId, logger, timeout = 30) => {
   return new Promise((resolve, reject) => {
     let didReceiveOk = false;
@@ -1055,12 +1171,12 @@ var joinRunChannel = (socket, token, runId, logger, timeout = 30) => {
         resolve({ channel, run });
       }
     }).receive("error", (err) => {
-      Sentry4.captureException(err);
+      Sentry5.captureException(err);
       logger.error(`error connecting to ${channelName}`, err);
       channel?.leave();
       reject(err);
     }).receive("timeout", (err) => {
-      Sentry4.captureException(err);
+      Sentry5.captureException(err);
       logger.error(`Timeout for ${channelName}`, err);
       channel?.leave();
       reject(err);
@@ -1077,7 +1193,7 @@ var run_default = joinRunChannel;
 
 // src/channels/worker-queue.ts
 import EventEmitter from "node:events";
-import * as Sentry5 from "@sentry/node";
+import * as Sentry6 from "@sentry/node";
 import { Socket as PhxSocket } from "phoenix";
 import { WebSocket } from "ws";
 import { API_VERSION } from "@openfn/lexicon/lightning";
@@ -1116,13 +1232,13 @@ var connectToWorkerQueue = (endpoint, serverId, secret, logger, options) => {
     SocketConstructor = PhxSocket
   } = options;
   const events = new EventEmitter();
-  Sentry5.addBreadcrumb({
+  Sentry6.addBreadcrumb({
     category: "lifecycle",
     message: "Connecting to worker queue",
     level: "info"
   });
   worker_token_default(secret, serverId, logger).then(async (token) => {
-    Sentry5.addBreadcrumb({
+    Sentry6.addBreadcrumb({
       category: "lifecycle",
       message: "Worker token generated",
       level: "info"
@@ -1141,7 +1257,7 @@ var connectToWorkerQueue = (endpoint, serverId, secret, logger, options) => {
     let didOpen = false;
     let shouldReportConnectionError = true;
     socket.onOpen(() => {
-      Sentry5.addBreadcrumb({
+      Sentry6.addBreadcrumb({
         category: "lifecycle",
         message: "Web socket connected",
         level: "info"
@@ -1168,7 +1284,7 @@ var connectToWorkerQueue = (endpoint, serverId, secret, logger, options) => {
       events.emit("disconnect");
     });
     socket.onError((e) => {
-      Sentry5.addBreadcrumb({
+      Sentry6.addBreadcrumb({
         category: "lifecycle",
         message: "Error in web socket connection",
         level: "info"
@@ -1176,7 +1292,7 @@ var connectToWorkerQueue = (endpoint, serverId, secret, logger, options) => {
       if (shouldReportConnectionError) {
         logger.debug("Reporting connection error to sentry");
         shouldReportConnectionError = false;
-        Sentry5.captureException(e);
+        Sentry6.captureException(e);
       }
       if (!didOpen) {
         events.emit("error", e.message);
@@ -1286,11 +1402,11 @@ function createServer(engine, options = {}) {
   app.events = new EventEmitter2();
   app.engine = engine;
   if (options.sentryDsn) {
-    Sentry6.init({
+    Sentry7.init({
       environment: options.sentryEnv,
       dsn: options.sentryDsn
     });
-    Sentry6.setupKoaErrorHandler(app);
+    Sentry7.setupKoaErrorHandler(app);
   }
   app.use(bodyParser());
   app.use(
@@ -1347,8 +1463,14 @@ function createServer(engine, options = {}) {
         if (!("payloadLimitMb" in options2)) {
           options2.payloadLimitMb = app.options.payloadLimitMb;
         }
+        if (!("logPayloadLimitMb" in options2)) {
+          options2.logPayloadLimitMb = app.options.logPayloadLimitMb;
+        }
         options2.timeoutRetryCount = app.options.timeoutRetryCount;
         options2.timeoutRetryDelay = app.options.timeoutRetryDelayMs ?? app.options.socketTimeoutSeconds;
+        options2.batchLogs = app.options.batchLogs;
+        options2.batchInterval = app.options.batchInterval;
+        options2.batchLimit = app.options.batchLimit;
         const onFinish = () => {
           const duration = (Date.now() - start) / 1e3;
           logger.debug(

package/dist/start.js

@@ -151,7 +151,7 @@ var runtime_engine_default = createMock;
 import { EventEmitter as EventEmitter3 } from "node:events";
 import { promisify } from "node:util";
 import { exec as _exec } from "node:child_process";
-import * as Sentry6 from "@sentry/node";
+import * as Sentry7 from "@sentry/node";
 import Koa from "koa";
 import bodyParser from "koa-bodyparser";
 import koaLogger from "koa-logger";
@@ -420,7 +420,16 @@ var startWorkloop = (app, logger2, minBackoff2, maxBackoff2, maxWorkers) => {
 var workloop_default = startWorkloop;
 
 // src/api/execute.ts
-import * as Sentry3 from "@sentry/node";
+import * as Sentry4 from "@sentry/node";
+import {
+  JOB_COMPLETE as JOB_COMPLETE2,
+  JOB_START as JOB_START2,
+  WORKFLOW_COMPLETE as WORKFLOW_COMPLETE2,
+  WORKFLOW_LOG as WORKFLOW_LOG2,
+  WORKFLOW_START as WORKFLOW_START2,
+  JOB_ERROR as JOB_ERROR2,
+  WORKFLOW_ERROR as WORKFLOW_ERROR2
+} from "@openfn/engine-multi";
 
 // src/util/convert-lightning-plan.ts
 import crypto3 from "node:crypto";
@@ -491,6 +500,9 @@ var convert_lightning_plan_default = (run2, options = {}) => {
     if ("payload_limit_mb" in run2.options) {
       engineOpts.payloadLimitMb = run2.options.payload_limit_mb;
     }
+    if ("log_payload_limit_mb" in run2.options) {
+      engineOpts.logPayloadLimitMb = run2.options.log_payload_limit_mb;
+    }
     if ("run_memory_limit_mb" in run2.options) {
       engineOpts.memoryLimitMb = run2.options.run_memory_limit_mb;
     }
@@ -725,43 +737,59 @@ var stringify_default = (obj) => stringify(obj, (_key, value) => {
   return value;
 });
 
-// src/util/throttle.ts
-var createThrottler = () => {
-  const q = [];
-  let activePromise;
-  const add = (fn) => {
-    return (...args2) => (
-      // the wrapped function is a deferred promise
-      // that will reject or resolve as usual
-      new Promise((resolve5, reject) => {
-        q.push({ fn, args: args2, resolve: resolve5, reject });
-        shift();
-      })
-    );
-  };
-  const shift = () => {
-    if (activePromise) {
-      return;
-    }
-    const next = q.shift();
-    if (next) {
-      const { fn, args: args2, resolve: resolve5, reject } = next;
-      activePromise = fn(...args2).then(resolve5).catch(reject).finally(() => {
-        activePromise = void 0;
-        shift();
-      });
-    }
-  };
-  return add;
-};
-var throttle_default = createThrottler;
-
 // src/util/timestamp.ts
 var timeInMicroseconds = (time) => time && (BigInt(time) / BigInt(1e3)).toString();
 
 // src/events/run-start.ts
 import { timestamp } from "@openfn/logger";
 
+// src/events/run-log.ts
+async function onRunLog(context, events) {
+  if (!Array.isArray(events)) {
+    events = [events];
+  }
+  const { state, options } = context;
+  const { batchLogs } = options ?? {};
+  const logs = events.map((evt) => {
+    let message = evt.message;
+    if (evt.redacted) {
+      message = [
+        `(Log message redacted: exceeds ${options.logPayloadLimitMb ?? options.payloadLimitMb}mb memory limit)`
+      ];
+    } else if (typeof evt.message === "string") {
+      message = JSON.parse(evt.message);
+    }
+    const logLine = {
+      message,
+      source: evt.name,
+      level: evt.level,
+      timestamp: timeInMicroseconds(evt.time)
+    };
+    if (state.activeStep) {
+      logLine.step_id = state.activeStep;
+    }
+    return logLine;
+  });
+  if (batchLogs) {
+    const payload = {
+      run_id: `${state.plan.id}`,
+      logs
+    };
+    return sendEvent(context, RUN_LOG, payload);
+  } else {
+    return new Promise(async (resolve5) => {
+      for (const log of logs) {
+        const payload = {
+          run_id: `${state.plan.id}`,
+          ...log
+        };
+        await sendEvent(context, RUN_LOG, payload);
+      }
+      resolve5();
+    });
+  }
+}
+
 // src/util/versions.ts
 import { mainSymbols } from "figures";
 var { triangleRightSmall: t } = mainSymbols;
@@ -829,21 +857,25 @@ async function onRunStart(context, event) {
     timestamp: timeInMicroseconds(event.time)
   });
   if ("payloadLimitMb" in options) {
-    await onJobLog(versionLogContext, {
-      // use the fake time in the log
-      time,
-      message: [`Payload limit: ${options.payloadLimitMb}mb`],
-      level: "info",
-      name: "RTE"
-    });
+    await onRunLog(versionLogContext, [
+      {
+        // use the fake time in the log
+        time,
+        message: [`Payload limit: ${options.payloadLimitMb}mb`],
+        level: "info",
+        name: "RTE"
+      }
+    ]);
   }
   const versionMessage = versions_default(versions);
-  await onJobLog(versionLogContext, {
-    time,
-    message: [versionMessage],
-    level: "info",
-    name: "VER"
-  });
+  await onRunLog(versionLogContext, [
+    {
+      time,
+      message: [versionMessage],
+      level: "info",
+      name: "VER"
+    }
+  ]);
 }
 
 // src/events/step-complete.ts
@@ -912,14 +944,16 @@ async function onStepComplete(context, event, error) {
     state.withheldDataclips[dataclipId] = true;
     evt.output_dataclip_error = "DATACLIP_TOO_LARGE";
     const time = (timestamp2() - BigInt(1e7)).toString();
-    await onJobLog(context, {
-      time,
-      message: [
-        "Dataclip exceeds payload limit: output will not be sent back to the app."
-      ],
-      level: "info",
-      name: "R/T"
-    });
+    await onRunLog(context, [
+      {
+        time,
+        message: [
+          "Dataclip exceeds payload limit: output will not be sent back to the app."
+        ],
+        level: "info",
+        name: "R/T"
+      }
+    ]);
   } else {
     evt.output_dataclip_id = dataclipId;
     if (!options || options.outputDataclips !== false) {
@@ -956,20 +990,24 @@ import { timestamp as timestamp3 } from "@openfn/logger";
 var log_final_reason_default = async (context, reason) => {
   const time = (timestamp3() - BigInt(1e7)).toString();
   const statusMessage = `Run complete with status: ${reason.reason}`;
-  await onJobLog(context, {
-    time,
-    message: [statusMessage],
-    level: "info",
-    name: "R/T"
-  });
-  if (reason.reason !== "success") {
-    const errorMessage = `${reason.error_type}: ${reason.error_message || "unknown"}`;
-    await onJobLog(context, {
+  await onRunLog(context, [
+    {
       time,
-      message: [errorMessage],
+      message: [statusMessage],
       level: "info",
       name: "R/T"
-    });
+    }
+  ]);
+  if (reason.reason !== "success") {
+    const errorMessage = `${reason.error_type}: ${reason.error_message || "unknown"}`;
+    await onRunLog(context, [
+      {
+        time,
+        message: [errorMessage],
+        level: "info",
+        name: "R/T"
+      }
+    ]);
   }
 };
 
@@ -1014,15 +1052,137 @@ async function onRunError(context, event) {
   }
 }
 
-// src/api/execute.ts
-var enc = new TextDecoder("utf-8");
+// src/api/process-events.ts
+import * as Sentry3 from "@sentry/node";
+import {
+  JOB_COMPLETE,
+  JOB_ERROR,
+  JOB_START,
+  WORKFLOW_COMPLETE,
+  WORKFLOW_ERROR,
+  WORKFLOW_LOG,
+  WORKFLOW_START
+} from "@openfn/engine-multi";
+var DEFAULT_BATCH_LIMIT = 10;
+var DEFAULT_BATCH_INTERVAL = 10;
 var eventMap = {
-  "workflow-start": RUN_START,
-  "job-start": STEP_START,
-  "job-complete": STEP_COMPLETE,
-  "workflow-log": RUN_LOG,
-  "workflow-complete": RUN_COMPLETE
+  [WORKFLOW_START]: RUN_START,
+  [JOB_START]: STEP_START,
+  [JOB_COMPLETE]: STEP_COMPLETE,
+  [WORKFLOW_LOG]: RUN_LOG,
+  [WORKFLOW_COMPLETE]: RUN_COMPLETE
 };
+var allEngineEvents = [
+  WORKFLOW_START,
+  WORKFLOW_COMPLETE,
+  JOB_START,
+  JOB_COMPLETE,
+  WORKFLOW_LOG,
+  JOB_ERROR,
+  WORKFLOW_ERROR
+];
+function eventProcessor(engine, context, callbacks, options = {}) {
+  const { id: planId, logger: logger2 } = context;
+  const {
+    batchLimit: limit = DEFAULT_BATCH_LIMIT,
+    batchInterval: interval = DEFAULT_BATCH_INTERVAL
+  } = options;
+  const queue = [];
+  const next = async () => {
+    const evt = queue[0];
+    if (evt) {
+      await process2(evt.name, evt.event);
+      queue.shift();
+      next();
+    }
+  };
+  let activeBatch = null;
+  let batch = [];
+  let start = -1;
+  let batchTimeout;
+  const sendBatch = async (name) => {
+    clearTimeout(batchTimeout);
+    activeBatch = null;
+    await send(name, batch, batch.length);
+    batch = [];
+  };
+  const send = async (name, payload, batchSize) => {
+    const lightningEvent = eventMap[name] ?? name;
+    await callbacks[name](context, payload);
+    if (batchSize) {
+      logger2.info(
+        `${planId} :: sent ${lightningEvent} (${batchSize}):: OK :: ${Date.now() - start}ms`
+      );
+    } else {
+      logger2.info(
+        `${planId} :: sent ${lightningEvent} :: OK :: ${Date.now() - start}ms`
+      );
+    }
+  };
+  const process2 = async (name, event) => {
+    if (name !== "workflow-log") {
+      Sentry3.addBreadcrumb({
+        category: "event",
+        message: name,
+        level: "info"
+      });
+    }
+    if (name === activeBatch) {
+      batch.push(event);
+      if (batch.length >= limit) {
+        await sendBatch(name);
+      }
+      return;
+    } else if (activeBatch) {
+      await sendBatch(activeBatch);
+    }
+    if (name in callbacks) {
+      try {
+        start = Date.now();
+        if (options?.batch?.[name]) {
+          activeBatch = name;
+          batch.push(event);
+          while (queue.length > 1 && queue[1].name === name) {
+            const [nextBatchItem] = queue.splice(1, 1);
+            batch.push(nextBatchItem.event);
+            if (batch.length >= limit) {
+              return sendBatch(name);
+            }
+          }
+          if (!batchTimeout) {
+            const batchName = activeBatch;
+            batchTimeout = setTimeout(async () => {
+              sendBatch(batchName);
+            }, interval);
+          }
+          return;
+        }
+        await send(name, event);
+      } catch (e2) {
+        if (!e2.reportedToSentry) {
+          Sentry3.captureException(e2);
+          logger2.error(e2);
+        }
+      }
+    } else {
+      logger2.warn("no event bound for", name);
+    }
+  };
+  const enqueue = (name, event) => {
+    queue.push({ name, event });
+    if (queue.length == 1) {
+      next();
+    }
+  };
+  const e = allEngineEvents.reduce(
+    (obj, e2) => Object.assign(obj, { [e2]: (p) => enqueue(e2, p) }),
+    {}
+  );
+  engine.listen(planId, e);
+}
+
+// src/api/execute.ts
+var enc = new TextDecoder("utf-8");
 function execute(channel, engine, logger2, plan, input, options = {}, onFinish = (_result) => {
 }) {
   logger2.info("executing ", plan.id);
@@ -1036,8 +1196,8 @@ function execute(channel, engine, logger2, plan, input, options = {}, onFinish =
     options,
     onFinish
   };
-  Sentry3.withIsolationScope(async () => {
-    Sentry3.addBreadcrumb({
+  Sentry4.withIsolationScope(async () => {
+    Sentry4.addBreadcrumb({
       category: "run",
       message: "Executing run: loading metadata",
       level: "info",
@@ -1045,47 +1205,26 @@ function execute(channel, engine, logger2, plan, input, options = {}, onFinish =
         runId: plan.id
       }
     });
-    const throttle = throttle_default();
-    const addEvent = (eventName, handler) => {
-      const wrappedFn = async (event) => {
-        if (eventName !== "workflow-log") {
-          Sentry3.addBreadcrumb({
-            category: "event",
-            message: eventName,
-            level: "info"
-          });
-        }
-        const lightningEvent = eventMap[eventName] ?? eventName;
-        try {
-          let start = Date.now();
-          await handler(context, event);
-          logger2.info(
-            `${plan.id} :: sent ${lightningEvent} :: OK :: ${Date.now() - start}ms`
-          );
-        } catch (e) {
-          if (!e.reportedToSentry) {
-            Sentry3.captureException(e);
-            logger2.error(e);
-          }
-        }
-      };
-      return {
-        [eventName]: wrappedFn
-      };
-    };
-    const listeners = Object.assign(
-      {},
-      addEvent("workflow-start", throttle(onRunStart)),
-      addEvent("job-start", throttle(onStepStart)),
-      addEvent("job-complete", throttle(onStepComplete)),
-      addEvent("job-error", throttle(onJobError)),
-      addEvent("workflow-log", throttle(onJobLog)),
-      // This will also resolve the promise
-      addEvent("workflow-complete", throttle(onWorkflowComplete)),
-      addEvent("workflow-error", throttle(onRunError))
-      // TODO send autoinstall logs
+    eventProcessor(
+      engine,
+      context,
+      {
+        [WORKFLOW_START2]: onRunStart,
+        [JOB_START2]: onStepStart,
+        [JOB_COMPLETE2]: onStepComplete,
+        [JOB_ERROR2]: onJobError,
+        [WORKFLOW_LOG2]: onRunLog,
+        [WORKFLOW_COMPLETE2]: onWorkflowComplete,
+        [WORKFLOW_ERROR2]: onRunError
+      },
+      {
+        batch: options.batchLogs ? {
+          [WORKFLOW_LOG2]: true
+        } : {},
+        batchInterval: options.batchInterval,
+        batchLimit: options.batchLimit
+      }
     );
-    engine.listen(plan.id, listeners);
     const resolvers = {
       credential: (id) => loadCredential(context, id)
       // TODO not supported right now
@@ -1108,7 +1247,7 @@ function execute(channel, engine, logger2, plan, input, options = {}, onFinish =
         }
       }
       try {
-        Sentry3.addBreadcrumb({
+        Sentry4.addBreadcrumb({
           category: "run",
           message: "run metadata loaded: starting run",
           level: "info",
@@ -1118,7 +1257,7 @@ function execute(channel, engine, logger2, plan, input, options = {}, onFinish =
         });
         engine.execute(plan, loadedInput, { resolvers, ...options });
       } catch (e) {
-        Sentry3.addBreadcrumb({
+        Sentry4.addBreadcrumb({
           category: "run",
           message: "exception in run",
           level: "info",
@@ -1145,29 +1284,6 @@ function onJobError(context, event) {
     return onStepComplete(context, event, event.error);
   }
 }
-function onJobLog(context, event) {
-  const { state, options } = context;
-  let message = event.message;
-  if (event.redacted) {
-    message = [
-      `(Log message redacted: exceeds ${options.payloadLimitMb}mb memory limit)`
-    ];
-  } else if (typeof event.message === "string") {
-    message = JSON.parse(event.message);
-  }
-  const log = {
-    run_id: `${state.plan.id}`,
-    message,
-    source: event.name,
-    level: event.level,
-    // @ts-ignore
-    timestamp: timeInMicroseconds(event.time)
-  };
-  if (state.activeStep) {
-    log.step_id = state.activeStep;
-  }
-  return sendEvent(context, RUN_LOG, log);
-}
 async function loadDataclip(context, stateId) {
   const result = await sendEvent(context, GET_DATACLIP, {
     id: stateId
@@ -1185,7 +1301,7 @@ var healthcheck_default = (ctx) => {
 };
 
 // src/channels/run.ts
-import * as Sentry4 from "@sentry/node";
+import * as Sentry5 from "@sentry/node";
 var joinRunChannel = (socket, token, runId, logger2, timeout = 30) => {
   return new Promise((resolve5, reject) => {
     let didReceiveOk = false;
@@ -1204,12 +1320,12 @@ var joinRunChannel = (socket, token, runId, logger2, timeout = 30) => {
         resolve5({ channel, run: run2 });
       }
     }).receive("error", (err) => {
-      Sentry4.captureException(err);
+      Sentry5.captureException(err);
       logger2.error(`error connecting to ${channelName}`, err);
       channel?.leave();
       reject(err);
     }).receive("timeout", (err) => {
-      Sentry4.captureException(err);
+      Sentry5.captureException(err);
       logger2.error(`Timeout for ${channelName}`, err);
       channel?.leave();
       reject(err);
@@ -1226,7 +1342,7 @@ var run_default = joinRunChannel;
 
 // src/channels/worker-queue.ts
 import EventEmitter2 from "node:events";
-import * as Sentry5 from "@sentry/node";
+import * as Sentry6 from "@sentry/node";
 import { Socket as PhxSocket } from "phoenix";
 import { WebSocket } from "ws";
 import { API_VERSION } from "@openfn/lexicon/lightning";
@@ -1265,13 +1381,13 @@ var connectToWorkerQueue = (endpoint, serverId, secret, logger2, options) => {
     SocketConstructor = PhxSocket
   } = options;
   const events = new EventEmitter2();
-  Sentry5.addBreadcrumb({
+  Sentry6.addBreadcrumb({
     category: "lifecycle",
     message: "Connecting to worker queue",
     level: "info"
   });
   worker_token_default(secret, serverId, logger2).then(async (token) => {
-    Sentry5.addBreadcrumb({
+    Sentry6.addBreadcrumb({
       category: "lifecycle",
       message: "Worker token generated",
       level: "info"
@@ -1290,7 +1406,7 @@ var connectToWorkerQueue = (endpoint, serverId, secret, logger2, options) => {
     let didOpen = false;
     let shouldReportConnectionError = true;
     socket.onOpen(() => {
-      Sentry5.addBreadcrumb({
+      Sentry6.addBreadcrumb({
         category: "lifecycle",
         message: "Web socket connected",
         level: "info"
@@ -1317,7 +1433,7 @@ var connectToWorkerQueue = (endpoint, serverId, secret, logger2, options) => {
       events.emit("disconnect");
     });
     socket.onError((e) => {
-      Sentry5.addBreadcrumb({
+      Sentry6.addBreadcrumb({
         category: "lifecycle",
         message: "Error in web socket connection",
         level: "info"
@@ -1325,7 +1441,7 @@ var connectToWorkerQueue = (endpoint, serverId, secret, logger2, options) => {
       if (shouldReportConnectionError) {
         logger2.debug("Reporting connection error to sentry");
         shouldReportConnectionError = false;
-        Sentry5.captureException(e);
+        Sentry6.captureException(e);
       }
       if (!didOpen) {
         events.emit("error", e.message);
@@ -1435,11 +1551,11 @@ function createServer(engine, options = {}) {
   app.events = new EventEmitter3();
   app.engine = engine;
   if (options.sentryDsn) {
-    Sentry6.init({
+    Sentry7.init({
       environment: options.sentryEnv,
       dsn: options.sentryDsn
     });
-    Sentry6.setupKoaErrorHandler(app);
+    Sentry7.setupKoaErrorHandler(app);
   }
   app.use(bodyParser());
   app.use(
@@ -1496,8 +1612,14 @@ function createServer(engine, options = {}) {
         if (!("payloadLimitMb" in options2)) {
           options2.payloadLimitMb = app.options.payloadLimitMb;
         }
+        if (!("logPayloadLimitMb" in options2)) {
+          options2.logPayloadLimitMb = app.options.logPayloadLimitMb;
+        }
         options2.timeoutRetryCount = app.options.timeoutRetryCount;
         options2.timeoutRetryDelay = app.options.timeoutRetryDelayMs ?? app.options.socketTimeoutSeconds;
+        options2.batchLogs = app.options.batchLogs;
+        options2.batchInterval = app.options.batchInterval;
+        options2.batchLimit = app.options.batchLimit;
         const onFinish = () => {
           const duration = (Date.now() - start) / 1e3;
           logger2.debug(
@@ -6450,6 +6572,9 @@ function parseArgs(argv) {
   const {
     OPENFN_ADAPTORS_REPO,
     WORKER_BACKOFF,
+    WORKER_BATCH_INTERVAL,
+    WORKER_BATCH_LIMIT,
+    WORKER_BATCH_LOGS,
     WORKER_CAPACITY,
     WORKER_CLAIM_TIMEOUT_SECONDS,
     WORKER_COLLECTIONS_URL,
@@ -6458,6 +6583,7 @@ function parseArgs(argv) {
     WORKER_LIGHTNING_SERVICE_URL,
     WORKER_LOG_LEVEL,
     WORKER_MAX_PAYLOAD_MB,
+    WORKER_MAX_LOG_PAYLOAD_MB,
     WORKER_MAX_RUN_DURATION_SECONDS,
     WORKER_MAX_RUN_MEMORY_MB,
     WORKER_MESSAGE_TIMEOUT_SECONDS,
@@ -6510,6 +6636,9 @@ function parseArgs(argv) {
     description: "Base64-encoded public key. Used to verify run tokens. Env: WORKER_LIGHTNING_PUBLIC_KEY"
   }).option("log", {
     description: "Set the log level for stdout (default to info, set to debug for verbose output). Env: WORKER_LOG_LEVEL"
+  }).option("log-payload-memory", {
+    description: "Maximum memory allocated to log payloads, in mb. Env: WORKER_MAX_LOG_PAYLOAD_MB",
+    type: "number"
   }).option("loop", {
     description: "Disable the claims loop",
     type: "boolean",
@@ -6518,6 +6647,15 @@ function parseArgs(argv) {
     description: "Use a mock runtime engine",
     type: "boolean",
     default: false
+  }).option("batch-logs", {
+    description: "Allow logs emitted from the server to be batched up. Env: WORKER_BATCH_LOGS",
+    type: "boolean"
+  }).option("batch-interval", {
+    description: "Interval for batching logs, in milliseconds. Env: WORKER_BATCH_INTERVAL",
+    type: "number"
+  }).option("batch-limit", {
+    description: "Maximum number of logs to batch before sending. Env: WORKER_BATCH_LIMIT",
+    type: "number"
   }).option("backoff", {
     description: "Claim backoff rules: min/max (in seconds). Env: WORKER_BACKOFF"
   }).option("capacity", {
@@ -6571,6 +6709,9 @@ function parseArgs(argv) {
       "ws://localhost:4000/worker"
     ),
     repoDir: setArg(args2.repoDir, WORKER_REPO_DIR),
+    batchLogs: setArg(args2.batchLogs, WORKER_BATCH_LOGS, false),
+    batchInterval: setArg(args2.batchInterval, WORKER_BATCH_INTERVAL, 10),
+    batchLimit: setArg(args2.batchLimit, WORKER_BATCH_LIMIT, 50),
     monorepoDir: setArg(args2.monorepoDir, OPENFN_ADAPTORS_REPO),
     secret: setArg(args2.secret, WORKER_SECRET),
     sentryDsn: setArg(args2.sentryDsn, WORKER_SENTRY_DSN),
@@ -6589,6 +6730,11 @@ function parseArgs(argv) {
     ),
     runMemory: setArg(args2.runMemory, WORKER_MAX_RUN_MEMORY_MB, 500),
     payloadMemory: setArg(args2.payloadMemory, WORKER_MAX_PAYLOAD_MB, 10),
+    logPayloadMemory: setArg(
+      args2.logPayloadMemory,
+      WORKER_MAX_LOG_PAYLOAD_MB,
+      1
+    ),
     maxRunDurationSeconds: setArg(
       args2.maxRunDurationSeconds,
       WORKER_MAX_RUN_DURATION_SECONDS,
@@ -6672,6 +6818,8 @@ function engineReady(engine) {
     },
     maxWorkflows: args.capacity,
     payloadLimitMb: args.payloadMemory,
+    logPayloadLimitMb: args.logPayloadMemory ?? 1,
+    // Default to 1MB
     collectionsVersion: args.collectionsVersion,
     collectionsUrl: args.collectionsUrl,
     monorepoDir: args.monorepoDir,
@@ -6680,7 +6828,10 @@ function engineReady(engine) {
     // deprecated!
     socketTimeoutSeconds: args.socketTimeoutSeconds,
     timeoutRetryCount: args.timeoutRetryCount,
-    timeoutRetryDelayMs: args.timeoutRetryDelayMs
+    timeoutRetryDelayMs: args.timeoutRetryDelayMs,
+    batchLogs: args.batchLogs,
+    batchInterval: args.batchInterval,
+    batchLimit: args.batchLimit
   };
   if (args.socketTimeoutSeconds) {
     logger.warn(

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openfn/ws-worker",
-  "version": "1.19.7",
+  "version": "1.20.1",
   "description": "A Websocket Worker to connect Lightning to a Runtime Engine",
   "main": "dist/index.js",
   "type": "module",
@@ -23,10 +23,10 @@
     "koa-logger": "^3.2.1",
     "phoenix": "1.7.10",
     "ws": "^8.18.3",
-    "@openfn/engine-multi": "1.8.5",
-    "@openfn/logger": "1.1.0",
-    "@openfn/runtime": "1.7.6",
-    "@openfn/lexicon": "^1.2.7"
+    "@openfn/engine-multi": "1.9.1",
+    "@openfn/lexicon": "^1.3.0",
+    "@openfn/logger": "1.1.1",
+    "@openfn/runtime": "1.7.7"
   },
   "devDependencies": {
     "@types/koa": "^2.15.0",
@@ -43,7 +43,7 @@
     "tsup": "^6.7.0",
     "typescript": "^4.9.5",
     "yargs": "^17.7.2",
-    "@openfn/lightning-mock": "2.3.8"
+    "@openfn/lightning-mock": "2.3.10"
   },
   "files": [
     "dist",