@openfn/ws-worker 1.21.4 → 1.22.0

package/dist/start.js

@@ -168,6 +168,7 @@ var GET_CREDENTIAL = "fetch:credential";
 var RUN_START = "run:start";
 var RUN_COMPLETE = "run:complete";
 var RUN_LOG = "run:log";
+var RUN_LOG_BATCH = "run:batch_logs";
 var STEP_START = "step:start";
 var STEP_COMPLETE = "step:complete";
 var INTERNAL_RUN_COMPLETE = "server:run-complete";
@@ -181,7 +182,9 @@ var destroy = async (app, logger2) => {
   await Promise.all([
     new Promise((resolve5) => {
       app.destroyed = true;
-      app.workloop?.stop("server closed");
+      for (const w of app.workloops) {
+        w.stop("server closed");
+      }
       app.server.close(async () => {
         resolve5();
       });
@@ -199,11 +202,11 @@ var destroy = async (app, logger2) => {
 var waitForRunsAndClaims = (app, logger2) => new Promise((resolve5) => {
   const log = () => {
     logger2.debug(
-      `Waiting for ${Object.keys(app.workflows).length} runs and ${Object.keys(app.openClaims).length} claims to complete...`
+      `Waiting for ${Object.keys(app.workflows).length} runs and ${app.pendingClaims()} claims to complete...`
     );
   };
   const checkAllClear = () => {
-    if (Object.keys(app.workflows).length + Object.keys(app.openClaims).length === 0) {
+    if (Object.keys(app.workflows).length + app.pendingClaims() === 0) {
       logger2.debug("All runs completed!");
       app.events.off(INTERNAL_RUN_COMPLETE, checkAllClear);
       app.events.off(INTERNAL_CLAIM_COMPLETE, checkAllClear);
@@ -212,7 +215,7 @@ var waitForRunsAndClaims = (app, logger2) => new Promise((resolve5) => {
       log();
     }
   };
-  if (Object.keys(app.workflows).length || Object.keys(app.openClaims).length) {
+  if (Object.keys(app.workflows).length || app.pendingClaims()) {
     log();
     app.events.on(INTERNAL_RUN_COMPLETE, checkAllClear);
     app.events.on(INTERNAL_CLAIM_COMPLETE, checkAllClear);
@@ -223,51 +226,6 @@ var waitForRunsAndClaims = (app, logger2) => new Promise((resolve5) => {
 });
 var destroy_default = destroy;
 
-// src/util/try-with-backoff.ts
-var BACKOFF_MULTIPLIER = 1.15;
-var tryWithBackoff = (fn, opts = {}) => {
-  const { min = 1e3, max = 1e4, maxRuns, runs = 1 } = opts;
-  let cancelled = false;
-  if (!opts.isCancelled) {
-    opts.isCancelled = () => cancelled;
-  }
-  const promise = new Promise(async (resolve5, reject) => {
-    try {
-      await fn();
-      resolve5();
-    } catch (e) {
-      if (e?.abort) {
-        cancelled = true;
-        return reject();
-      }
-      if (opts.isCancelled()) {
-        return resolve5();
-      }
-      if (!isNaN(maxRuns) && runs >= maxRuns) {
-        return reject(new Error("max runs exceeded"));
-      }
-      setTimeout(() => {
-        if (opts.isCancelled()) {
-          return resolve5();
-        }
-        const nextOpts = {
-          maxRuns,
-          runs: runs + 1,
-          min: Math.min(max, min * BACKOFF_MULTIPLIER),
-          max,
-          isCancelled: opts.isCancelled
-        };
-        tryWithBackoff(fn, nextOpts).then(resolve5).catch(reject);
-      }, min);
-    }
-  });
-  promise.cancel = () => {
-    cancelled = true;
-  };
-  return promise;
-};
-var try_with_backoff_default = tryWithBackoff;
-
 // src/api/claim.ts
 import v8 from "node:v8";
 import * as Sentry from "@sentry/node";
@@ -296,24 +254,26 @@ var ClaimError = class extends Error {
   }
 };
 var claimIdGen = 0;
-var claim = (app, logger2 = mockLogger, options = {}) => {
+var claim = (app, workloop, logger2 = mockLogger, options = {}) => {
   return new Promise((resolve5, reject) => {
-    app.openClaims ??= {};
-    const { maxWorkers = 5, demand = 1 } = options;
+    const { demand = 1 } = options;
     const podName = NAME ? `[${NAME}] ` : "";
-    const activeWorkers = Object.keys(app.workflows).length;
-    const pendingClaims = Object.values(app.openClaims).reduce(
+    const activeInWorkloop = workloop.activeRuns.size;
+    const capacity = workloop.capacity;
+    const pendingWorkloopClaims = Object.values(workloop.openClaims).reduce(
       (a, b) => a + b,
       0
     );
-    if (activeWorkers >= maxWorkers) {
-      app.workloop?.stop(`server at capacity (${activeWorkers}/${maxWorkers})`);
-      return reject(new ClaimError("Server at capacity"));
-    } else if (activeWorkers + pendingClaims >= maxWorkers) {
-      app.workloop?.stop(
-        `server at capacity (${activeWorkers}/${maxWorkers}, ${pendingClaims} pending)`
+    if (activeInWorkloop >= capacity) {
+      workloop.stop(
+        `workloop ${workloop.id} at capacity (${activeInWorkloop}/${capacity})`
+      );
+      return reject(new ClaimError("Workloop at capacity"));
+    } else if (activeInWorkloop + pendingWorkloopClaims >= capacity) {
+      workloop.stop(
+        `workloop ${workloop.id} at capacity (${activeInWorkloop}/${capacity}, ${pendingWorkloopClaims} pending)`
       );
-      return reject(new ClaimError("Server at capacity"));
+      return reject(new ClaimError("Workloop at capacity"));
     }
     if (!app.queueChannel) {
       logger2.warn("skipping claim attempt: websocket unavailable");
@@ -326,21 +286,22 @@ var claim = (app, logger2 = mockLogger, options = {}) => {
       return reject(e);
     }
     const claimId = ++claimIdGen;
-    app.openClaims[claimId] = demand;
+    workloop.openClaims[claimId] = demand;
     const { used_heap_size, heap_size_limit } = v8.getHeapStatistics();
     const usedHeapMb = Math.round(used_heap_size / 1024 / 1024);
     const totalHeapMb = Math.round(heap_size_limit / 1024 / 1024);
     const memPercent = Math.round(usedHeapMb / totalHeapMb * 100);
     logger2.debug(
-      `Claiming runs :: demand ${demand} | capacity ${activeWorkers}/${maxWorkers} | memory ${memPercent}% (${usedHeapMb}/${totalHeapMb}mb)`
+      `Claiming runs [${workloop.id}] :: demand ${demand} | capacity ${activeInWorkloop}/${capacity} | memory ${memPercent}% (${usedHeapMb}/${totalHeapMb}mb)`
     );
     app.events.emit(INTERNAL_CLAIM_START);
     const start = Date.now();
     app.queueChannel.push(CLAIM, {
       demand,
-      worker_name: NAME || null
+      worker_name: NAME || null,
+      queues: workloop.queues
     }).receive("ok", async ({ runs }) => {
-      delete app.openClaims[claimId];
+      delete workloop.openClaims[claimId];
       const duration = Date.now() - start;
       logger2.debug(
         `${podName}claimed ${runs.length} runs in ${duration}ms (${runs.length ? runs.map((r) => r.id).join(",") : "-"})`
@@ -364,17 +325,19 @@ var claim = (app, logger2 = mockLogger, options = {}) => {
         } else {
           logger2.debug("skipping run token validation for", run2.id);
         }
+        workloop.activeRuns.add(run2.id);
+        app.runWorkloopMap[run2.id] = workloop;
         logger2.debug(`${podName} starting run ${run2.id}`);
         app.execute(run2);
       }
       resolve5();
       app.events.emit(INTERNAL_CLAIM_COMPLETE, { runs });
     }).receive("error", (e) => {
-      delete app.openClaims[claimId];
+      delete workloop.openClaims[claimId];
       logger2.error("Error on claim", e);
       reject(new Error("claim error"));
     }).receive("timeout", () => {
-      delete app.openClaims[claimId];
+      delete workloop.openClaims[claimId];
       logger2.error("TIMEOUT on claim. Runs may be lost.");
       reject(new Error("timeout"));
     });
@@ -382,43 +345,6 @@ var claim = (app, logger2 = mockLogger, options = {}) => {
 };
 var claim_default = claim;
 
-// src/api/workloop.ts
-var startWorkloop = (app, logger2, minBackoff2, maxBackoff2, maxWorkers) => {
-  let promise;
-  let cancelled = false;
-  const workLoop = () => {
-    if (!cancelled) {
-      promise = try_with_backoff_default(
-        () => claim_default(app, logger2, {
-          maxWorkers
-        }),
-        {
-          min: minBackoff2,
-          max: maxBackoff2
-        }
-      );
-      promise.then(() => {
-        if (!cancelled) {
-          setTimeout(workLoop, minBackoff2);
-        }
-      }).catch(() => {
-      });
-    }
-  };
-  workLoop();
-  return {
-    stop: (reason = "reason unknown") => {
-      if (!cancelled) {
-        logger2.info(`cancelling workloop: ${reason}`);
-        cancelled = true;
-        promise.cancel();
-      }
-    },
-    isStopped: () => cancelled
-  };
-};
-var workloop_default = startWorkloop;
-
 // src/api/execute.ts
 import * as Sentry4 from "@sentry/node";
 import {
@@ -723,6 +649,51 @@ var stringify_default = (obj) => stringify(obj, (_key, value) => {
   return value;
 });
 
+// src/util/try-with-backoff.ts
+var BACKOFF_MULTIPLIER = 1.15;
+var tryWithBackoff = (fn, opts = {}) => {
+  const { min = 1e3, max = 1e4, maxRuns, runs = 1 } = opts;
+  let cancelled = false;
+  if (!opts.isCancelled) {
+    opts.isCancelled = () => cancelled;
+  }
+  const promise = new Promise(async (resolve5, reject) => {
+    try {
+      await fn();
+      resolve5();
+    } catch (e) {
+      if (e?.abort) {
+        cancelled = true;
+        return reject();
+      }
+      if (opts.isCancelled()) {
+        return resolve5();
+      }
+      if (!isNaN(maxRuns) && runs >= maxRuns) {
+        return reject(new Error("max runs exceeded"));
+      }
+      setTimeout(() => {
+        if (opts.isCancelled()) {
+          return resolve5();
+        }
+        const nextOpts = {
+          maxRuns,
+          runs: runs + 1,
+          min: Math.min(max, min * BACKOFF_MULTIPLIER),
+          max,
+          isCancelled: opts.isCancelled
+        };
+        tryWithBackoff(fn, nextOpts).then(resolve5).catch(reject);
+      }, min);
+    }
+  });
+  promise.cancel = () => {
+    cancelled = true;
+  };
+  return promise;
+};
+var try_with_backoff_default = tryWithBackoff;
+
 // src/util/timestamp.ts
 var timeInMicroseconds = (time) => time && (BigInt(time) / BigInt(1e3)).toString();
 
@@ -761,7 +732,7 @@ async function onRunLog(context, events) {
       run_id: `${state.plan.id}`,
       logs
     };
-    return sendEvent(context, RUN_LOG, payload);
+    return sendEvent(context, RUN_LOG_BATCH, payload);
   } else {
     return new Promise(async (resolve5) => {
       for (const log of logs) {
@@ -1383,6 +1354,7 @@ var connectToWorkerQueue = (endpoint, serverId, secret, logger2, options) => {
     messageTimeout = DEFAULT_MESSAGE_TIMEOUT_SECONDS,
     claimTimeout = DEFAULT_CLAIM_TIMEOUT_SECONDS,
     capacity,
+    queues,
     SocketConstructor = PhxSocket
   } = options;
   const events = new EventEmitter2();
@@ -1419,6 +1391,9 @@ var connectToWorkerQueue = (endpoint, serverId, secret, logger2, options) => {
       didOpen = true;
       shouldReportConnectionError = true;
       const joinPayload = { capacity };
+      if (queues) {
+        joinPayload.queues = queues;
+      }
       const channel = socket.channel("worker:queue", joinPayload);
       channel.onMessage = (ev, load) => {
         events.emit("message", ev, load);
@@ -1459,6 +1434,140 @@ var connectToWorkerQueue = (endpoint, serverId, secret, logger2, options) => {
 };
 var worker_queue_default = connectToWorkerQueue;
 
+// src/api/workloop.ts
+var Workloop = class {
+  constructor({
+    id,
+    queues,
+    capacity
+  }) {
+    this.activeRuns = /* @__PURE__ */ new Set();
+    this.openClaims = {};
+    this.cancelled = true;
+    this.id = id;
+    this.queues = queues;
+    this.capacity = capacity;
+  }
+  hasCapacity() {
+    const pendingClaims = Object.values(this.openClaims).reduce(
+      (a, b) => a + b,
+      0
+    );
+    return this.activeRuns.size + pendingClaims < this.capacity;
+  }
+  start(app, logger2, minBackoff2, maxBackoff2) {
+    this.logger = logger2;
+    this.cancelled = false;
+    const loop = () => {
+      if (!this.cancelled) {
+        this.promise = try_with_backoff_default(() => claim_default(app, this, logger2), {
+          min: minBackoff2,
+          max: maxBackoff2
+        });
+        this.promise.then(() => {
+          if (!this.cancelled) {
+            setTimeout(loop, minBackoff2);
+          }
+        }).catch(() => {
+        });
+      }
+    };
+    loop();
+  }
+  stop(reason = "reason unknown") {
+    if (!this.cancelled) {
+      this.logger?.info(`cancelling workloop: ${reason}`);
+      this.cancelled = true;
+      this.promise?.cancel();
+    }
+  }
+  isStopped() {
+    return this.cancelled;
+  }
+};
+
+// src/util/parse-workloops.ts
+var WorkloopValidationError = class extends Error {
+  constructor(message) {
+    super(message);
+    this.name = "WorkloopValidationError";
+  }
+};
+var VALID_NAME = /^[a-zA-Z0-9_]+$/;
+function parseWorkloops(input) {
+  const trimmed = input.trim();
+  if (!trimmed) {
+    throw new WorkloopValidationError("Workloop configuration cannot be empty");
+  }
+  const tokens = trimmed.split(/\s+/);
+  const configs = tokens.map(parseToken);
+  const seenConfigs = /* @__PURE__ */ new Map();
+  for (let i = 0; i < configs.length; i++) {
+    const key = JSON.stringify(configs[i].queues);
+    if (seenConfigs.has(key)) {
+      const prevIndex = seenConfigs.get(key);
+      console.warn(
+        `Warning: workloops at positions ${prevIndex} and ${i} have identical queue configurations: ${tokens[prevIndex]} and ${tokens[i]}`
+      );
+    } else {
+      seenConfigs.set(key, i);
+    }
+  }
+  return configs;
+}
+function parseToken(token) {
+  const lastColon = token.lastIndexOf(":");
+  if (lastColon === -1) {
+    throw new WorkloopValidationError(
+      `Invalid token "${token}": missing :<count> suffix`
+    );
+  }
+  const prefStr = token.slice(0, lastColon);
+  const countStr = token.slice(lastColon + 1);
+  const count = Number(countStr);
+  if (!Number.isInteger(count) || countStr !== String(Math.floor(count))) {
+    throw new WorkloopValidationError(
+      `Invalid count "${countStr}" in token "${token}": must be a positive integer`
+    );
+  }
+  if (count < 1) {
+    throw new WorkloopValidationError(
+      `Invalid count "${countStr}" in token "${token}": must be >= 1`
+    );
+  }
+  const names = prefStr.split(">");
+  for (const name of names) {
+    if (name === "") {
+      throw new WorkloopValidationError(`Empty queue name in token "${token}"`);
+    }
+    if (name !== "*" && !VALID_NAME.test(name)) {
+      throw new WorkloopValidationError(
+        `Invalid queue name "${name}" in token "${token}": must match /^[a-zA-Z0-9_]+$/ or be "*"`
+      );
+    }
+  }
+  const nonWildcardNames = names.filter((n) => n !== "*");
+  const seen = /* @__PURE__ */ new Set();
+  for (const name of nonWildcardNames) {
+    if (seen.has(name)) {
+      console.warn(
+        `Warning: duplicate queue name "${name}" in token "${token}"`
+      );
+    }
+    seen.add(name);
+  }
+  const wildcardIndex = names.indexOf("*");
+  if (wildcardIndex !== -1 && wildcardIndex !== names.length - 1) {
+    throw new WorkloopValidationError(
+      `Wildcard "*" must be the last element in token "${token}"`
+    );
+  }
+  return new Workloop({ id: token, queues: names, capacity: count });
+}
+
+// src/util/get-default-workloop-config.ts
+var get_default_workloop_config_default = (capacity = 5) => `manual>*:${capacity}`;
+
 // src/server.ts
 var exec = promisify(_exec);
 var DEFAULT_PORT = 2222;
@@ -1486,8 +1595,10 @@ function connect(app, logger2, options = {}) {
     app.resumeWorkloop();
   };
   const onDisconnect = () => {
-    if (!app.workloop?.isStopped()) {
-      app.workloop?.stop("Socket disconnected unexpectedly");
+    for (const w of app.workloops) {
+      if (!w.isStopped()) {
+        w.stop("Socket disconnected unexpectedly");
+      }
     }
     if (!app.destroyed) {
       logger2.info("Connection to lightning lost");
@@ -1509,17 +1620,26 @@ function connect(app, logger2, options = {}) {
   const onMessage = (event) => {
     if (event === WORK_AVAILABLE) {
       if (!app.destroyed) {
-        claim_default(app, logger2, { maxWorkers: options.maxWorkflows }).catch(() => {
-        });
+        for (const w of app.workloops) {
+          if (w.hasCapacity()) {
+            claim_default(app, w, logger2).catch(() => {
+            });
+          }
+        }
       }
     }
   };
+  const queuesMap = {};
+  for (const w of app.workloops) {
+    queuesMap[w.queues.join(">")] = w.capacity;
+  }
   worker_queue_default(options.lightning, app.id, options.secret, logger2, {
     // TODO: options.socketTimeoutSeconds wins because this is what USED to be used
     // But it's deprecated and should be removed soon
     messageTimeout: options.socketTimeoutSeconds ?? options.messageTimeoutSeconds,
     claimTimeout: options.claimTimeoutSeconds,
-    capacity: options.maxWorkflows
+    capacity: options.maxWorkflows,
+    queues: queuesMap
   }).on("connect", onConnect).on("disconnect", onDisconnect).on("error", onError).on("message", onMessage);
 }
 async function setupCollections(options, logger2) {
@@ -1568,27 +1688,36 @@ function createServer(engine, options = {}) {
       logger2.debug(str);
     })
   );
-  app.openClaims = {};
   app.workflows = {};
   app.destroyed = false;
+  app.workloops = parseWorkloops(
+    options.workloopConfigs ?? get_default_workloop_config_default(options.maxWorkflows)
+  );
+  app.runWorkloopMap = {};
   app.server = app.listen(port);
   logger2.success(`Worker ${app.id} listening on ${port}`);
   process.send?.("READY");
   router.get("/livez", healthcheck_default);
   router.get("/", healthcheck_default);
   app.options = options;
-  app.resumeWorkloop = () => {
+  app.resumeWorkloop = (workloop) => {
     if (options.noLoop || app.destroyed) {
       return;
     }
-    if (!app.workloop || app.workloop?.isStopped()) {
-      logger2.info("Starting workloop");
-      app.workloop = workloop_default(
+    const targets = workloop ? [workloop] : app.workloops;
+    for (const w of targets) {
+      if (!w.hasCapacity()) {
+        continue;
+      }
+      if (!w.isStopped()) {
+        w.stop("restarting");
+      }
+      logger2.info(`Starting workloop for ${w.id}`);
+      w.start(
         app,
         logger2,
         options.backoff?.min || MIN_BACKOFF,
-        options.backoff?.max || MAX_BACKOFF,
-        options.maxWorkflows
+        options.backoff?.max || MAX_BACKOFF
       );
     }
   };
@@ -1633,8 +1762,16 @@ function createServer(engine, options = {}) {
           );
           delete app.workflows[id];
           runChannel.leave();
-          app.events.emit(INTERNAL_RUN_COMPLETE);
-          app.resumeWorkloop();
+          const owningWorkloop = app.runWorkloopMap[id];
+          if (owningWorkloop) {
+            owningWorkloop.activeRuns.delete(id);
+            delete app.runWorkloopMap[id];
+            app.events.emit(INTERNAL_RUN_COMPLETE);
+            app.resumeWorkloop(owningWorkloop);
+          } else {
+            app.events.emit(INTERNAL_RUN_COMPLETE);
+            app.resumeWorkloop();
+          }
         };
         const context = execute(
           runChannel,
@@ -1648,7 +1785,14 @@ function createServer(engine, options = {}) {
         app.workflows[id] = context;
       } catch (e) {
         delete app.workflows[id];
-        app.resumeWorkloop();
+        const owningWorkloop = app.runWorkloopMap[id];
+        if (owningWorkloop) {
+          owningWorkloop.activeRuns.delete(id);
+          delete app.runWorkloopMap[id];
+          app.resumeWorkloop(owningWorkloop);
+        } else {
+          app.resumeWorkloop();
+        }
         logger2.error(`Unexpected error executing ${id}`);
         logger2.error(e);
       }
@@ -1658,9 +1802,13 @@ function createServer(engine, options = {}) {
   };
   router.post("/claim", async (ctx) => {
     logger2.info("triggering claim from POST request");
-    return claim_default(app, logger2, {
-      maxWorkers: options.maxWorkflows
-    }).then(() => {
+    const promises = app.workloops.map((w) => {
+      if (w.hasCapacity()) {
+        return claim_default(app, w, logger2);
+      }
+      return Promise.reject(new Error("Workloop at capacity"));
+    });
+    return Promise.any(promises).then(() => {
       logger2.info("claim complete: 1 run claimed");
       ctx.body = "complete";
       ctx.status = 200;
@@ -1671,10 +1819,15 @@ function createServer(engine, options = {}) {
     });
   });
   app.claim = () => {
-    return claim_default(app, logger2, {
-      maxWorkers: options.maxWorkflows
+    const promises = app.workloops.map((w) => {
+      if (w.hasCapacity()) {
+        return claim_default(app, w, logger2);
+      }
+      return Promise.reject(new Error("Workloop at capacity"));
     });
+    return Promise.any(promises);
   };
+  app.pendingClaims = () => app.workloops.reduce((sum, w) => sum + Object.keys(w.openClaims).length, 0);
   app.destroy = () => destroy_default(app, logger2);
   app.use(router.routes());
   if (options.lightning) {
@@ -6604,6 +6757,7 @@ function parseArgs(argv) {
     WORKER_STATE_PROPS_TO_REMOVE,
     WORKER_TIMEOUT_RETRY_COUNT,
     WORKER_TIMEOUT_RETRY_DELAY_MS,
+    WORKER_WORKLOOPS,
     WORKER_VALIDATION_RETRIES,
     WORKER_VALIDATION_TIMEOUT_MS
   } = process.env;
@@ -6665,8 +6819,12 @@ function parseArgs(argv) {
   }).option("backoff", {
     description: "Claim backoff rules: min/max (in seconds). Env: WORKER_BACKOFF"
   }).option("capacity", {
-    description: `max concurrent workers. Default ${DEFAULT_WORKER_CAPACITY}. Env: WORKER_CAPACITY`,
+    description: `Sets the maximum concurrent workers - but only if workloops is not set. Default ${DEFAULT_WORKER_CAPACITY}. Env: WORKER_CAPACITY`,
     type: "number"
+  }).option("workloops", {
+    description: 'Configure workloops with a priorised queue list and a max capacity. Syntax: "<queues>:<capacity> ...". Mutually exclusive with --capacity. Env: WORKER_WORKLOOPS',
+    type: "string",
+    example: "fast_lane:1 manual>*:4"
   }).option("state-props-to-remove", {
     description: "A list of properties to remove from the final state returned by a job. Env: WORKER_STATE_PROPS_TO_REMOVE",
     type: "array"
@@ -6704,10 +6862,25 @@ function parseArgs(argv) {
   }).option("timeout-retry-delay", {
     description: "When a websocket event receives a timeout, this option sets how log to wait before retrying Default 30000. Env: WORKER_TIMEOUT_RETRY_DELAY_MS",
     type: "number"
-  });
+  }).example(
+    "start --queues *:5",
+    "Default start configuration: a single workloop with capacity 5, claiming from all queues"
+  ).example(
+    "start --queues manual>*:5",
+    "A single workloop, capacity 5, which claims across two queues. Runs in the manual queue will be picked first, else any other queue will be picked."
+  ).example(
+    "start --queues fast_lane:1 manual>*:4",
+    "production start configuration with 1 fast lane workloop (capacity 1) and a second workloop with capacity 4"
+  );
   const args2 = parser2.parse();
+  const resolvedWorkloops = setArg(args2.workloops, WORKER_WORKLOOPS);
+  const capacityExplicit = args2.capacity !== void 0 || WORKER_CAPACITY !== void 0;
+  if (resolvedWorkloops !== void 0 && capacityExplicit) {
+    throw new Error("--workloops and --capacity are mutually exclusive");
+  }
   return {
     ...args2,
+    workloops: resolvedWorkloops,
     port: setArg(args2.port, WORKER_PORT, DEFAULT_PORT2),
     lightning: setArg(
       args2.lightning,
@@ -6796,8 +6969,16 @@ function parseArgs(argv) {
 
 // src/start.ts
 var args = parseArgs(process.argv);
+var workloopConfigs = args.workloops ?? get_default_workloop_config_default(args.capacity);
+var effectiveCapacity = workloopConfigs.trim().split(/\s+/).reduce((sum, token) => sum + (parseInt(token.split(":").pop()) || 0), 0);
 var logger = createLogger("SRV", { level: args.log });
-logger.info("Starting worker server...");
+logger.info("Starting worker...");
+logger.info(
+  "Workloops:",
+  workloopConfigs,
+  "effective capacity:",
+  effectiveCapacity
+);
 if (args.lightning === "mock") {
   args.lightning = "ws://localhost:8888/worker";
   if (!args.secret) {
@@ -6822,7 +7003,8 @@ function engineReady(engine) {
       min: minBackoff,
       max: maxBackoff
     },
-    maxWorkflows: args.capacity,
+    maxWorkflows: effectiveCapacity,
+    workloopConfigs,
     payloadLimitMb: args.payloadMemory,
     logPayloadLimitMb: args.logPayloadMemory ?? 1,
     // Default to 1MB
@@ -6872,7 +7054,7 @@ if (args.mock) {
   const engineOptions = {
     repoDir: args.repoDir,
     memoryLimitMb: args.runMemory,
-    maxWorkers: args.capacity,
+    maxWorkers: effectiveCapacity,
     statePropsToRemove: args.statePropsToRemove,
     runTimeoutMs: args.maxRunDurationSeconds * 1e3,
     workerValidationTimeout: args.engineValidationTimeoutMs,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openfn/ws-worker",
-  "version": "1.21.4",
+  "version": "1.22.0",
   "description": "A Websocket Worker to connect Lightning to a Runtime Engine",
   "main": "dist/index.js",
   "type": "module",