@openfn/ws-worker 1.21.5 → 1.22.1

package/CHANGELOG.md

@@ -1,5 +1,25 @@
 # ws-worker
 
+## 1.22.1
+
+### Patch Changes
+
+- 6fd3942: Fix an issue where unhandled errors could trigger a worker crash after a compilation error.
+- 32b43cb: When reporting compilation errors, prefer the step name to the id
+- Updated dependencies [6fd3942]
+- Updated dependencies [32b43cb]
+  - @openfn/engine-multi@1.10.5
+
+## 1.22.0
+
+### Minor Changes
+
+- 07457d7: Fastlane support: multiple concurrent workloops, each with its own isolated capacity and backoff.
+
+  Claims also include a `queues` key, which specifies a prioritised list of Lightning work queues to claim from.
+
+  Configure workloops with the `--workloops` CLI option or env. By default the worker users `manual>*5`, which provides parity behaviour to prior production.
+
 ## 1.21.5
 
 ### Patch Changes

package/README.md

@@ -80,6 +80,85 @@ To use the monorepo adaptor version:
 pnpm start --collections-version local --collections-url http://localhost:4000/collections
 ```
 
+## Workloops
+
+By default, the worker runs a single workloop that claims runs from any
+queue, preferring the `manual` queue (used for manually-triggered and
+webhook runs). This is equivalent to `--workloops "manual>*:5"`.
+
+The `--workloops` option lets you split the worker's capacity into
+independent groups, each with its own queue preference chain and slot
+count. This is useful for dedicating capacity to latency-sensitive
+workloads (e.g., sync webhooks on a `fast_lane` queue) while letting
+remaining capacity serve general work.
+
+```
+--workloops "<queues>:<capacity> <queues>:<capacity> ..."
+```
+
+### Syntax
+
+| Element | Meaning |
+|---------|---------|
+| `>` | Queue preference separator (left = highest priority) |
+| `*` | Wildcard: accept runs from any queue (must be last) |
+| `:N` | Number of slots for this group |
+| ` ` (space) | Group separator |
+
+### Examples
+
+```bash
+# 1 slot pinned to fast_lane (strict), 4 slots preferring manual then anything
+--workloops "fast_lane:1 manual>*:4"
+
+# 5 generic slots (pure FIFO across all queues) — equivalent to --capacity 5
+--workloops "*:5"
+
+# 2 fast lane (strict), 3 with manual preference
+--workloops "fast_lane:2 manual>*:3"
+
+# 1 slot preferring fast_lane > manual > rest, 4 generic
+--workloops "fast_lane>manual>*:1 *:4"
+```
+
+A group **without** `*` in its queue list is strict — it will only
+claim runs from the named queues. A group **with** `*` will accept any
+run, but prefers queues listed before the wildcard.
+
+### Environment variable
+
+```
+WORKER_WORKLOOPS="fast_lane:1 manual>*:4"
+```
+
+### Relationship to --capacity
+
+`--workloops` and `--capacity` are mutually exclusive. If neither is
+provided, the default is `--capacity 5`, which internally creates a
+single `manual>*:5` group. The total capacity of the worker is always
+the sum of all group slot counts.
+
+### How it works
+
+Each group runs its own independent claim loop with its own backoff
+timer. When a run completes, only the owning group's workloop resumes.
+A `work-available` push from Lightning triggers a claim attempt on
+every group that has free slots.
+
+```
+  Main Process (ws-worker)
+ ├── Workloop 1 (manual>*:2)  ─┐
+ ├── Workloop 2 (fast_lane:1) ─┼── all run in the main process as async loops
+ ├── Workloop 3 (*:5)         ─┘
+ │
+ └── Engine (single instance, shared by all lanes)
+     └── Child Process Pool (capacity = sum of all lanes/slots)
+         ├── Child 1 (forked) → Worker Thread (per task)
+         ├── Child 2 (forked) → Worker Thread (per task)
+         ├── ...on demand, reused after each task
+         └── Child N
+```
+
 ## Architecture
 
 Lightning is expected to maintain a queue of runs. The Worker pulls those runs from the queue, via websocket, and sends them off to the Engine for execution.

package/dist/index.d.ts

@@ -1,6 +1,6 @@
 import { EventEmitter } from 'node:events';
 import Koa from 'koa';
-import { LogLevel, Logger } from '@openfn/logger';
+import { Logger, LogLevel } from '@openfn/logger';
 import * as l from '@openfn/lexicon/lightning';
 import { ClaimRun } from '@openfn/lexicon/lightning';
 import { ExecutionPlan, Lazy, State, UUID } from '@openfn/lexicon';
@@ -8,10 +8,25 @@ import { ExecuteOptions, RuntimeEngine } from '@openfn/engine-multi';
 import { Channel as Channel$1 } from 'phoenix';
 import { Server } from 'http';
 
-type Workloop = {
-    stop: (reason?: string) => void;
-    isStopped: () => boolean;
-};
+declare class Workloop {
+    id: string;
+    queues: string[];
+    capacity: number;
+    activeRuns: Set<string>;
+    openClaims: Record<string, number>;
+    private cancelled;
+    private promise?;
+    private logger?;
+    constructor({ id, queues, capacity, }: {
+        id: string;
+        queues: string[];
+        capacity: number;
+    });
+    hasCapacity(): boolean;
+    start(app: ServerApp, logger: Logger, minBackoff: number, maxBackoff: number): void;
+    stop(reason?: string): void;
+    isStopped(): boolean;
+}
 
 // Internal server state for each run
 type RunState = {
@@ -74,6 +89,7 @@ type ServerOptions = {
     batchInterval?: number;
     batchLimit?: number;
     maxWorkflows?: number;
+    workloopConfigs?: string;
     port?: number;
     lightning?: string;
     logger?: Logger;
@@ -102,16 +118,17 @@ interface ServerApp extends Koa {
     socket?: any;
     queueChannel?: Channel;
     workflows: Record<string, true | Context>;
-    openClaims: Record<string, number>;
     destroyed: boolean;
     events: EventEmitter;
     server: Server;
     engine: RuntimeEngine;
     options: ServerOptions;
-    workloop?: Workloop;
+    workloops: Workloop[];
+    runWorkloopMap: Record<string, Workloop>;
     execute: ({ id, token }: ClaimRun) => Promise<void>;
     destroy: () => void;
-    resumeWorkloop: () => void;
+    resumeWorkloop: (workloop?: Workloop) => void;
+    pendingClaims: () => number;
     claim: () => Promise<any>;
 }
 declare function createServer(engine: RuntimeEngine, options?: ServerOptions): ServerApp;