@blokjs/trigger-worker 0.4.0 → 0.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/__tests__/integration/nats-adapter.real-nats.test.ts +116 -0
- package/__tests__/integration/pgboss-adapter.real-pg.test.ts +164 -0
- package/__tests__/integration/rabbitmq-adapter.real-rabbitmq.test.ts +179 -0
- package/__tests__/integration/sqs-adapter.real-sqs.test.ts +228 -0
- package/dist/WorkerTrigger.d.ts +37 -1
- package/dist/WorkerTrigger.js +142 -21
- package/dist/adapters/InMemoryAdapter.js +10 -5
- package/dist/adapters/KafkaAdapter.d.ts +62 -0
- package/dist/adapters/KafkaAdapter.js +236 -0
- package/dist/adapters/NATSAdapter.js +3 -3
- package/dist/adapters/PgBossAdapter.d.ts +56 -0
- package/dist/adapters/PgBossAdapter.js +251 -0
- package/dist/adapters/RabbitMQAdapter.d.ts +51 -0
- package/dist/adapters/RabbitMQAdapter.js +241 -0
- package/dist/adapters/RedisStreamsAdapter.d.ts +64 -0
- package/dist/adapters/RedisStreamsAdapter.js +240 -0
- package/dist/adapters/SQSAdapter.d.ts +61 -0
- package/dist/adapters/SQSAdapter.js +269 -0
- package/dist/adapters/factory.d.ts +34 -0
- package/dist/adapters/factory.js +103 -0
- package/dist/index.d.ts +21 -4
- package/dist/index.js +24 -4
- package/package.json +23 -5
- package/src/WorkerTrigger.ts +153 -22
- package/src/adapters/InMemoryAdapter.ts +9 -5
- package/src/adapters/KafkaAdapter.ts +277 -0
- package/src/adapters/NATSAdapter.ts +4 -2
- package/src/adapters/PgBossAdapter.ts +293 -0
- package/src/adapters/RabbitMQAdapter.ts +285 -0
- package/src/adapters/RedisStreamsAdapter.ts +286 -0
- package/src/adapters/SQSAdapter.ts +306 -0
- package/src/adapters/factory.test.ts +89 -0
- package/src/adapters/factory.ts +111 -0
- package/src/adapters/new-adapters.test.ts +130 -0
- package/src/index.ts +30 -4
- package/template/package.json +6 -6
- package/template/src/workflows/jobs/process-job.ts +37 -35
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@blokjs/trigger-worker",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.6.2",
|
|
4
4
|
"description": "Worker-based trigger for Blok workflows - supports background job processing with concurrency, retries, and scheduling",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/index.js",
|
|
@@ -14,32 +14,50 @@
|
|
|
14
14
|
"author": "Deskree Technologies Inc.",
|
|
15
15
|
"license": "Apache-2.0",
|
|
16
16
|
"dependencies": {
|
|
17
|
-
"@blokjs/helper": "^0.
|
|
18
|
-
"@blokjs/runner": "^0.
|
|
19
|
-
"@blokjs/shared": "^0.
|
|
17
|
+
"@blokjs/helper": "^0.6.2",
|
|
18
|
+
"@blokjs/runner": "^0.6.2",
|
|
19
|
+
"@blokjs/shared": "^0.6.2",
|
|
20
20
|
"@opentelemetry/api": "^1.9.0",
|
|
21
21
|
"uuid": "^11.1.0"
|
|
22
22
|
},
|
|
23
23
|
"devDependencies": {
|
|
24
|
+
"@types/amqplib": "^0.10.8",
|
|
24
25
|
"@types/node": "^22.15.21",
|
|
25
26
|
"@types/uuid": "^11.0.0",
|
|
27
|
+
"pg-boss": "^10.0.0",
|
|
26
28
|
"typescript": "^5.8.3",
|
|
27
29
|
"vitest": "^4.0.18"
|
|
28
30
|
},
|
|
29
31
|
"peerDependencies": {
|
|
32
|
+
"@aws-sdk/client-sqs": "^3.0.0",
|
|
33
|
+
"amqplib": "^0.10.0",
|
|
30
34
|
"bullmq": "^5.67.2",
|
|
31
35
|
"ioredis": "^5.9.2",
|
|
32
|
-
"
|
|
36
|
+
"kafkajs": "^2.2.0",
|
|
37
|
+
"nats": "",
|
|
38
|
+
"pg-boss": ""
|
|
33
39
|
},
|
|
34
40
|
"peerDependenciesMeta": {
|
|
41
|
+
"@aws-sdk/client-sqs": {
|
|
42
|
+
"optional": true
|
|
43
|
+
},
|
|
44
|
+
"amqplib": {
|
|
45
|
+
"optional": true
|
|
46
|
+
},
|
|
35
47
|
"bullmq": {
|
|
36
48
|
"optional": true
|
|
37
49
|
},
|
|
38
50
|
"ioredis": {
|
|
39
51
|
"optional": true
|
|
40
52
|
},
|
|
53
|
+
"kafkajs": {
|
|
54
|
+
"optional": true
|
|
55
|
+
},
|
|
41
56
|
"nats": {
|
|
42
57
|
"optional": true
|
|
58
|
+
},
|
|
59
|
+
"pg-boss": {
|
|
60
|
+
"optional": true
|
|
43
61
|
}
|
|
44
62
|
},
|
|
45
63
|
"private": false,
|
package/src/WorkerTrigger.ts
CHANGED
|
@@ -24,6 +24,7 @@ import {
|
|
|
24
24
|
type BlokService,
|
|
25
25
|
ConcurrencyLimitError,
|
|
26
26
|
ConcurrencyMetrics,
|
|
27
|
+
DebounceCoordinator,
|
|
27
28
|
DefaultLogger,
|
|
28
29
|
DeferredDispatchSignal,
|
|
29
30
|
type GlobalOptions,
|
|
@@ -34,6 +35,7 @@ import {
|
|
|
34
35
|
TriggerBase,
|
|
35
36
|
type TriggerResponse,
|
|
36
37
|
createConcurrencyBackend,
|
|
38
|
+
createDebounceBackend,
|
|
37
39
|
} from "@blokjs/runner";
|
|
38
40
|
import type { Context, RequestContext } from "@blokjs/shared";
|
|
39
41
|
import { type Span, SpanStatusCode, metrics, trace } from "@opentelemetry/api";
|
|
@@ -172,11 +174,31 @@ export abstract class WorkerTrigger extends TriggerBase {
|
|
|
172
174
|
process.env.PROJECT_VERSION || "0.0.1",
|
|
173
175
|
);
|
|
174
176
|
protected readonly logger = new DefaultLogger();
|
|
175
|
-
|
|
177
|
+
/**
|
|
178
|
+
* v0.7 PR 5 — the "default" adapter, used when a workflow's
|
|
179
|
+
* `trigger.worker.provider` field is omitted AND the
|
|
180
|
+
* `BLOK_WORKER_ADAPTER` env var is unset. Subclasses MAY set this
|
|
181
|
+
* for back-compat with the pre-v0.7 single-adapter pattern
|
|
182
|
+
* (`class WorkerServer extends WorkerTrigger { protected adapter = new NATSWorkerAdapter() }`).
|
|
183
|
+
*
|
|
184
|
+
* When unset AND no per-workflow provider is specified, the factory
|
|
185
|
+
* falls back to `in-memory`. The factory pool (`adapters/factory.ts`)
|
|
186
|
+
* tracks one connected adapter per provider so multiple workflows
|
|
187
|
+
* with the same provider share a single broker connection.
|
|
188
|
+
*/
|
|
189
|
+
protected adapter?: WorkerAdapter;
|
|
176
190
|
|
|
177
191
|
/** Active queues being processed */
|
|
178
192
|
protected activeQueues: Set<string> = new Set();
|
|
179
193
|
|
|
194
|
+
/**
|
|
195
|
+
* v0.7 PR 5 — adapter pool, keyed by provider name. Populated lazily
|
|
196
|
+
* inside `listen()` as workflows are matched to providers. Each
|
|
197
|
+
* adapter is connected once and reused across workflows that share
|
|
198
|
+
* its provider. Drained in `stop()`.
|
|
199
|
+
*/
|
|
200
|
+
protected adapterPool: Map<string, WorkerAdapter> = new Map();
|
|
201
|
+
|
|
180
202
|
// Subclasses provide these
|
|
181
203
|
protected abstract nodes: Record<string, BlokService<unknown>>;
|
|
182
204
|
protected abstract workflows: Record<string, HelperResponse>;
|
|
@@ -239,6 +261,28 @@ export abstract class WorkerTrigger extends TriggerBase {
|
|
|
239
261
|
);
|
|
240
262
|
}
|
|
241
263
|
|
|
264
|
+
// Tier C #1 · install the cross-process debounce backend
|
|
265
|
+
// (NATS KV / Redis) when the operator opted in via
|
|
266
|
+
// `BLOK_DEBOUNCE_BACKEND`. Default unset = the existing
|
|
267
|
+
// in-process behavior is preserved. On connect failure, log +
|
|
268
|
+
// fall back to in-memory coordination.
|
|
269
|
+
try {
|
|
270
|
+
const debounceBackend = createDebounceBackend();
|
|
271
|
+
if (debounceBackend) {
|
|
272
|
+
const leaseRaw = process.env.BLOK_DEBOUNCE_OWNER_LEASE_MS;
|
|
273
|
+
if (leaseRaw && /^\d+$/.test(leaseRaw)) {
|
|
274
|
+
DebounceCoordinator.getInstance().setOwnerLeaseMs(Number(leaseRaw));
|
|
275
|
+
}
|
|
276
|
+
await debounceBackend.connect();
|
|
277
|
+
DebounceCoordinator.getInstance().setBackend(debounceBackend);
|
|
278
|
+
this.logger.log(`[debounce] backend installed: ${debounceBackend.name}`);
|
|
279
|
+
}
|
|
280
|
+
} catch (err) {
|
|
281
|
+
this.logger.error(
|
|
282
|
+
`[debounce] backend install failed: ${err instanceof Error ? err.message : String(err)}; falling back to in-memory coordination`,
|
|
283
|
+
);
|
|
284
|
+
}
|
|
285
|
+
|
|
242
286
|
// Tier 2 quick-wins follow-up · install crash handlers + recover
|
|
243
287
|
// orphaned runs from a previous (dead) process. Idempotent + opt-out
|
|
244
288
|
// via `BLOK_CRASH_AUTOFLIP_DISABLED=1`.
|
|
@@ -269,20 +313,6 @@ export abstract class WorkerTrigger extends TriggerBase {
|
|
|
269
313
|
this.logger.error(`[shutdown] setup failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
270
314
|
}
|
|
271
315
|
|
|
272
|
-
// Connect to job backend
|
|
273
|
-
await this.adapter.connect();
|
|
274
|
-
this.logger.log(`Connected to ${this.adapter.provider} worker system`);
|
|
275
|
-
|
|
276
|
-
// Register health dependency
|
|
277
|
-
this.registerHealthDependency(`worker-${this.adapter.provider}`, async () => {
|
|
278
|
-
const healthy = await this.adapter.healthCheck();
|
|
279
|
-
return {
|
|
280
|
-
status: healthy ? ("healthy" as const) : ("unhealthy" as const),
|
|
281
|
-
lastChecked: Date.now(),
|
|
282
|
-
message: healthy ? "Connected" : "Connection lost",
|
|
283
|
-
};
|
|
284
|
-
});
|
|
285
|
-
|
|
286
316
|
// Find all workflows with worker triggers
|
|
287
317
|
const workerWorkflows = this.getWorkerWorkflows();
|
|
288
318
|
|
|
@@ -291,16 +321,19 @@ export abstract class WorkerTrigger extends TriggerBase {
|
|
|
291
321
|
return this.endCounter(startTime);
|
|
292
322
|
}
|
|
293
323
|
|
|
294
|
-
// Start processing each queue
|
|
324
|
+
// Start processing each queue, dispatching to the right adapter
|
|
325
|
+
// based on the workflow's `provider` field (with back-compat
|
|
326
|
+
// fallback to `this.adapter` when subclasses still set it).
|
|
295
327
|
for (const workflow of workerWorkflows) {
|
|
296
328
|
const config = workflow.config.trigger?.worker as WorkerTriggerOpts;
|
|
329
|
+
const adapter = await this.resolveAdapterForWorkflow(config);
|
|
297
330
|
this.logger.log(
|
|
298
|
-
`Starting worker for queue: ${config.queue} (concurrency=${config.concurrency}, retries=${config.retries})`,
|
|
331
|
+
`Starting worker for queue: ${config.queue} via ${adapter.provider} (concurrency=${config.concurrency}, retries=${config.retries})`,
|
|
299
332
|
);
|
|
300
333
|
|
|
301
334
|
this.activeQueues.add(config.queue);
|
|
302
335
|
|
|
303
|
-
await
|
|
336
|
+
await adapter.process(config, async (job) => {
|
|
304
337
|
await this.handleJob(job, workflow, config);
|
|
305
338
|
});
|
|
306
339
|
}
|
|
@@ -323,12 +356,28 @@ export abstract class WorkerTrigger extends TriggerBase {
|
|
|
323
356
|
* Stop all workers and disconnect
|
|
324
357
|
*/
|
|
325
358
|
async stop(): Promise<void> {
|
|
359
|
+
// Stop each queue on its owning adapter — adapters are tracked
|
|
360
|
+
// in the pool so multi-provider workers all drain cleanly.
|
|
326
361
|
for (const queue of this.activeQueues) {
|
|
327
|
-
|
|
362
|
+
for (const adapter of this.adapterPool.values()) {
|
|
363
|
+
try {
|
|
364
|
+
await adapter.stopProcessing(queue);
|
|
365
|
+
} catch {
|
|
366
|
+
/* swallow — adapter may not own this queue */
|
|
367
|
+
}
|
|
368
|
+
}
|
|
328
369
|
this.logger.log(`Stopped processing queue: ${queue}`);
|
|
329
370
|
}
|
|
330
371
|
this.activeQueues.clear();
|
|
331
|
-
|
|
372
|
+
// Disconnect every adapter we ever connected.
|
|
373
|
+
for (const adapter of this.adapterPool.values()) {
|
|
374
|
+
try {
|
|
375
|
+
await adapter.disconnect();
|
|
376
|
+
} catch (err) {
|
|
377
|
+
this.logger.error(`[blok][worker] disconnect failed: ${(err as Error).message}`);
|
|
378
|
+
}
|
|
379
|
+
}
|
|
380
|
+
this.adapterPool.clear();
|
|
332
381
|
this.destroyMonitoring();
|
|
333
382
|
this.logger.log("Worker trigger stopped");
|
|
334
383
|
}
|
|
@@ -355,14 +404,33 @@ export abstract class WorkerTrigger extends TriggerBase {
|
|
|
355
404
|
jobId?: string;
|
|
356
405
|
},
|
|
357
406
|
): Promise<string> {
|
|
358
|
-
|
|
407
|
+
// Back-compat: when a subclass set `this.adapter`, use it.
|
|
408
|
+
// Otherwise dispatch via the first pool adapter — typically the
|
|
409
|
+
// only one when a process owns one trigger workflow.
|
|
410
|
+
const adapter =
|
|
411
|
+
this.adapter ??
|
|
412
|
+
(this.adapterPool.size > 0 ? (this.adapterPool.values().next().value as WorkerAdapter) : undefined);
|
|
413
|
+
if (!adapter) {
|
|
414
|
+
throw new Error(
|
|
415
|
+
"[blok][worker] dispatch() called before any adapter is connected. Call listen() first, or set this.adapter on the subclass.",
|
|
416
|
+
);
|
|
417
|
+
}
|
|
418
|
+
return adapter.addJob(queue, data, opts);
|
|
359
419
|
}
|
|
360
420
|
|
|
361
421
|
/**
|
|
362
422
|
* Get statistics for a queue
|
|
363
423
|
*/
|
|
364
424
|
async getQueueStats(queue: string): Promise<WorkerQueueStats> {
|
|
365
|
-
|
|
425
|
+
const adapter =
|
|
426
|
+
this.adapter ??
|
|
427
|
+
(this.adapterPool.size > 0 ? (this.adapterPool.values().next().value as WorkerAdapter) : undefined);
|
|
428
|
+
if (!adapter) {
|
|
429
|
+
throw new Error(
|
|
430
|
+
"[blok][worker] getQueueStats() called before any adapter is connected. Call listen() first, or set this.adapter on the subclass.",
|
|
431
|
+
);
|
|
432
|
+
}
|
|
433
|
+
return adapter.getQueueStats(queue);
|
|
366
434
|
}
|
|
367
435
|
|
|
368
436
|
/**
|
|
@@ -372,6 +440,60 @@ export abstract class WorkerTrigger extends TriggerBase {
|
|
|
372
440
|
return Array.from(this.activeQueues);
|
|
373
441
|
}
|
|
374
442
|
|
|
443
|
+
/**
|
|
444
|
+
* v0.7 PR 5 — pick the adapter for a workflow's `provider` field.
|
|
445
|
+
*
|
|
446
|
+
* Resolution order:
|
|
447
|
+
* 1. Subclass-set `this.adapter` (back-compat: pre-v0.7 pattern
|
|
448
|
+
* where one process binds to one adapter at construction time).
|
|
449
|
+
* 2. Per-workflow `provider` field, looked up via the factory.
|
|
450
|
+
* 3. `BLOK_WORKER_ADAPTER` env var.
|
|
451
|
+
* 4. `in-memory` fallback.
|
|
452
|
+
*
|
|
453
|
+
* Adapters are connected on first use and pooled per provider so
|
|
454
|
+
* multiple workflows sharing a provider share one broker
|
|
455
|
+
* connection. Health-dependency registration also happens here so
|
|
456
|
+
* each provider is tracked individually in `/health`.
|
|
457
|
+
*/
|
|
458
|
+
protected async resolveAdapterForWorkflow(config: WorkerTriggerOpts): Promise<WorkerAdapter> {
|
|
459
|
+
// Subclass override wins for back-compat.
|
|
460
|
+
if (this.adapter) {
|
|
461
|
+
if (!this.adapter.isConnected()) {
|
|
462
|
+
await this.adapter.connect();
|
|
463
|
+
this.logger.log(`Connected to ${this.adapter.provider} worker system (subclass adapter)`);
|
|
464
|
+
this.registerAdapterHealth(this.adapter);
|
|
465
|
+
}
|
|
466
|
+
// Pool-track so stop() can drain it.
|
|
467
|
+
this.adapterPool.set(this.adapter.provider, this.adapter);
|
|
468
|
+
return this.adapter;
|
|
469
|
+
}
|
|
470
|
+
|
|
471
|
+
// Lazy-import the factory so the worker package doesn't pull in
|
|
472
|
+
// every adapter on import — only the ones actually exercised.
|
|
473
|
+
const { resolveProvider, createWorkerAdapter } = await import("./adapters/factory");
|
|
474
|
+
const provider = resolveProvider(config.provider);
|
|
475
|
+
let adapter = this.adapterPool.get(provider);
|
|
476
|
+
if (!adapter) {
|
|
477
|
+
adapter = createWorkerAdapter(provider);
|
|
478
|
+
await adapter.connect();
|
|
479
|
+
this.logger.log(`Connected to ${adapter.provider} worker system`);
|
|
480
|
+
this.registerAdapterHealth(adapter);
|
|
481
|
+
this.adapterPool.set(provider, adapter);
|
|
482
|
+
}
|
|
483
|
+
return adapter;
|
|
484
|
+
}
|
|
485
|
+
|
|
486
|
+
private registerAdapterHealth(adapter: WorkerAdapter): void {
|
|
487
|
+
this.registerHealthDependency(`worker-${adapter.provider}`, async () => {
|
|
488
|
+
const healthy = await adapter.healthCheck();
|
|
489
|
+
return {
|
|
490
|
+
status: healthy ? ("healthy" as const) : ("unhealthy" as const),
|
|
491
|
+
lastChecked: Date.now(),
|
|
492
|
+
message: healthy ? "Connected" : "Connection lost",
|
|
493
|
+
};
|
|
494
|
+
});
|
|
495
|
+
}
|
|
496
|
+
|
|
375
497
|
/**
|
|
376
498
|
* Get all workflows that have worker triggers
|
|
377
499
|
*/
|
|
@@ -452,6 +574,15 @@ export abstract class WorkerTrigger extends TriggerBase {
|
|
|
452
574
|
`Processing job ${jobId} from ${config.queue} (attempt ${job.attempts + 1}/${job.maxRetries + 1})`,
|
|
453
575
|
);
|
|
454
576
|
|
|
577
|
+
// v0.6 · apply the merged middleware chain (process-global →
|
|
578
|
+
// workflow-level → trigger-level) on the same ctx the main
|
|
579
|
+
// workflow will see. State mutations from middleware
|
|
580
|
+
// (e.g. ctx.state.identity) carry forward. Middleware that
|
|
581
|
+
// throws (via `@blokjs/throw`) propagates to the outer
|
|
582
|
+
// catch and is routed through the worker's retry / DLQ
|
|
583
|
+
// logic exactly like a main-workflow error.
|
|
584
|
+
await this.applyMiddlewareChain(ctx, this.nodeMap);
|
|
585
|
+
|
|
455
586
|
// Execute workflow with timeout if configured
|
|
456
587
|
let response: TriggerResponse;
|
|
457
588
|
if (config.timeout && config.timeout > 0) {
|
|
@@ -122,8 +122,14 @@ export class InMemoryAdapter implements WorkerAdapter {
|
|
|
122
122
|
throw new Error("Not connected. Call connect() first.");
|
|
123
123
|
}
|
|
124
124
|
|
|
125
|
-
|
|
126
|
-
|
|
125
|
+
// Get-or-init the per-queue job list. Same pattern as `stats`
|
|
126
|
+
// below — the previous code used `set-if-absent` then `.get()!`,
|
|
127
|
+
// but that non-null assertion is what biome flags. Pulling the
|
|
128
|
+
// reference once and seeding when missing keeps the type exact.
|
|
129
|
+
let jobs = this.jobs.get(queue);
|
|
130
|
+
if (!jobs) {
|
|
131
|
+
jobs = [];
|
|
132
|
+
this.jobs.set(queue, jobs);
|
|
127
133
|
}
|
|
128
134
|
if (!this.stats.has(queue)) {
|
|
129
135
|
this.stats.set(queue, { completed: 0, failed: 0 });
|
|
@@ -146,8 +152,6 @@ export class InMemoryAdapter implements WorkerAdapter {
|
|
|
146
152
|
job.scheduledAt = new Date(Date.now() + job.delay);
|
|
147
153
|
}
|
|
148
154
|
|
|
149
|
-
const jobs = this.jobs.get(queue)!;
|
|
150
|
-
|
|
151
155
|
// Insert sorted by priority (higher first)
|
|
152
156
|
const insertIdx = jobs.findIndex((j) => j.status === "waiting" && j.priority < job.priority);
|
|
153
157
|
if (insertIdx >= 0) {
|
|
@@ -245,7 +249,7 @@ export class InMemoryAdapter implements WorkerAdapter {
|
|
|
245
249
|
|
|
246
250
|
if (requeue && internalJob.attempts < internalJob.maxRetries) {
|
|
247
251
|
// Requeue with backoff
|
|
248
|
-
const backoff = Math.min(1000 *
|
|
252
|
+
const backoff = Math.min(1000 * 2 ** internalJob.attempts, 30000);
|
|
249
253
|
internalJob.status = "delayed";
|
|
250
254
|
internalJob.scheduledAt = new Date(Date.now() + backoff);
|
|
251
255
|
} else {
|
|
@@ -0,0 +1,277 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* KafkaAdapter — v0.7 PR 5 — Worker adapter backed by Apache Kafka via
|
|
3
|
+
* `kafkajs`. Consumes from a topic (the `queue` field) with a
|
|
4
|
+
* consumer-group identifier; produces via the same client.
|
|
5
|
+
*
|
|
6
|
+
* Kafka is fundamentally a streaming platform — not a queue — so a
|
|
7
|
+
* few semantics differ from BullMQ/SQS/RabbitMQ:
|
|
8
|
+
*
|
|
9
|
+
* - **Ordering**: per-partition, not per-topic. Set the partition
|
|
10
|
+
* key via the `dedupId` field on `addJob` to keep related
|
|
11
|
+
* messages on the same partition.
|
|
12
|
+
* - **Retries**: Kafka doesn't have a broker-side retry concept.
|
|
13
|
+
* The adapter re-throws on handler failure; offset commit is
|
|
14
|
+
* suppressed so the consumer re-polls the message on the next
|
|
15
|
+
* cycle. For real retry semantics, layer a dead-letter topic.
|
|
16
|
+
* - **Stats**: KafkaJS exposes consumer-group lag via its admin
|
|
17
|
+
* client; the lag count is reported as `waiting`. Other stats
|
|
18
|
+
* are tracked locally per consumer.
|
|
19
|
+
*
|
|
20
|
+
* Requires `kafkajs` as a peer dependency:
|
|
21
|
+
*
|
|
22
|
+
* bun add kafkajs
|
|
23
|
+
*
|
|
24
|
+
* Environment variables (read at adapter construction):
|
|
25
|
+
* - `KAFKA_BROKERS` — comma-separated list (default `localhost:9092`).
|
|
26
|
+
* - `KAFKA_CLIENT_ID` — client.id (default `"blok-worker"`).
|
|
27
|
+
* - `KAFKA_SASL_USERNAME` — SASL/PLAIN username (optional).
|
|
28
|
+
* - `KAFKA_SASL_PASSWORD` — SASL/PLAIN password (optional).
|
|
29
|
+
* - `KAFKA_SSL` — when `"true"`, enable TLS.
|
|
30
|
+
*/
|
|
31
|
+
|
|
32
|
+
import type { WorkerTriggerOpts } from "@blokjs/helper";
|
|
33
|
+
import { v4 as uuid } from "uuid";
|
|
34
|
+
import type { WorkerAdapter, WorkerJob, WorkerQueueStats } from "../WorkerTrigger";
|
|
35
|
+
|
|
36
|
+
export interface KafkaConfig {
|
|
37
|
+
brokers: string[];
|
|
38
|
+
clientId: string;
|
|
39
|
+
saslUsername?: string;
|
|
40
|
+
saslPassword?: string;
|
|
41
|
+
ssl: boolean;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
interface KafkaJsHandle {
|
|
45
|
+
producer?: {
|
|
46
|
+
connect: () => Promise<void>;
|
|
47
|
+
disconnect: () => Promise<void>;
|
|
48
|
+
send: (args: unknown) => Promise<unknown>;
|
|
49
|
+
};
|
|
50
|
+
consumers: Map<
|
|
51
|
+
string,
|
|
52
|
+
{
|
|
53
|
+
disconnect: () => Promise<void>;
|
|
54
|
+
stop: () => Promise<void>;
|
|
55
|
+
run: (opts: unknown) => Promise<void>;
|
|
56
|
+
}
|
|
57
|
+
>;
|
|
58
|
+
admin?: {
|
|
59
|
+
connect: () => Promise<void>;
|
|
60
|
+
disconnect: () => Promise<void>;
|
|
61
|
+
fetchTopicOffsets: (topic: string) => Promise<Array<{ partition: number; offset: string }>>;
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
interface QueueStatsCounters {
|
|
66
|
+
completed: number;
|
|
67
|
+
failed: number;
|
|
68
|
+
active: number;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
export class KafkaAdapter implements WorkerAdapter {
|
|
72
|
+
readonly provider = "kafka" as const;
|
|
73
|
+
private readonly config: KafkaConfig;
|
|
74
|
+
// biome-ignore lint/suspicious/noExplicitAny: kafkajs's exported `Kafka` constructor is loosely typed.
|
|
75
|
+
private kafka: any = null;
|
|
76
|
+
private handle: KafkaJsHandle = { consumers: new Map() };
|
|
77
|
+
private connected = false;
|
|
78
|
+
private stats: Map<string, QueueStatsCounters> = new Map();
|
|
79
|
+
|
|
80
|
+
constructor(config?: Partial<KafkaConfig>) {
|
|
81
|
+
this.config = {
|
|
82
|
+
brokers: config?.brokers ?? (process.env.KAFKA_BROKERS ?? "localhost:9092").split(",").map((s) => s.trim()),
|
|
83
|
+
clientId: config?.clientId ?? process.env.KAFKA_CLIENT_ID ?? "blok-worker",
|
|
84
|
+
saslUsername: config?.saslUsername ?? process.env.KAFKA_SASL_USERNAME,
|
|
85
|
+
saslPassword: config?.saslPassword ?? process.env.KAFKA_SASL_PASSWORD,
|
|
86
|
+
ssl: config?.ssl ?? process.env.KAFKA_SSL === "true",
|
|
87
|
+
};
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
async connect(): Promise<void> {
|
|
91
|
+
if (this.connected) return;
|
|
92
|
+
try {
|
|
93
|
+
// biome-ignore lint/suspicious/noExplicitAny: kafkajs is a runtime-loaded peer dep.
|
|
94
|
+
const kafkajs: any = await import("kafkajs");
|
|
95
|
+
const sasl =
|
|
96
|
+
this.config.saslUsername && this.config.saslPassword
|
|
97
|
+
? { mechanism: "plain", username: this.config.saslUsername, password: this.config.saslPassword }
|
|
98
|
+
: undefined;
|
|
99
|
+
this.kafka = new kafkajs.Kafka({
|
|
100
|
+
clientId: this.config.clientId,
|
|
101
|
+
brokers: this.config.brokers,
|
|
102
|
+
ssl: this.config.ssl,
|
|
103
|
+
sasl,
|
|
104
|
+
});
|
|
105
|
+
this.handle.producer = this.kafka.producer();
|
|
106
|
+
await this.handle.producer?.connect();
|
|
107
|
+
this.handle.admin = this.kafka.admin();
|
|
108
|
+
await this.handle.admin?.connect();
|
|
109
|
+
this.connected = true;
|
|
110
|
+
} catch (err) {
|
|
111
|
+
throw new Error(
|
|
112
|
+
`[blok][kafka] connect failed: ${(err as Error).message}. Install kafkajs as a peer dependency: bun add kafkajs`,
|
|
113
|
+
);
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
async disconnect(): Promise<void> {
|
|
118
|
+
if (!this.connected) return;
|
|
119
|
+
for (const [, consumer] of this.handle.consumers) {
|
|
120
|
+
try {
|
|
121
|
+
await consumer.disconnect();
|
|
122
|
+
} catch {
|
|
123
|
+
/* ignore */
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
this.handle.consumers.clear();
|
|
127
|
+
try {
|
|
128
|
+
await this.handle.producer?.disconnect();
|
|
129
|
+
} catch {
|
|
130
|
+
/* ignore */
|
|
131
|
+
}
|
|
132
|
+
try {
|
|
133
|
+
await this.handle.admin?.disconnect();
|
|
134
|
+
} catch {
|
|
135
|
+
/* ignore */
|
|
136
|
+
}
|
|
137
|
+
this.connected = false;
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
async process(config: WorkerTriggerOpts, handler: (job: WorkerJob) => Promise<void>): Promise<void> {
|
|
141
|
+
if (!this.connected) throw new Error("[blok][kafka] not connected. Call connect() first.");
|
|
142
|
+
const groupId = config.consumerGroup ?? `${config.queue}-group`;
|
|
143
|
+
const consumer = this.kafka.consumer({ groupId });
|
|
144
|
+
await consumer.connect();
|
|
145
|
+
await consumer.subscribe({ topic: config.queue, fromBeginning: config.fromBeginning === true });
|
|
146
|
+
this.handle.consumers.set(config.queue, consumer);
|
|
147
|
+
this.stats.set(config.queue, { completed: 0, failed: 0, active: 0 });
|
|
148
|
+
const stats = this.stats.get(config.queue) as QueueStatsCounters;
|
|
149
|
+
|
|
150
|
+
await consumer.run({
|
|
151
|
+
autoCommit: config.ack !== false,
|
|
152
|
+
eachMessage: async ({
|
|
153
|
+
message,
|
|
154
|
+
}: {
|
|
155
|
+
message: { key?: Buffer; value?: Buffer; offset: string; timestamp: string; headers?: Record<string, Buffer> };
|
|
156
|
+
}) => {
|
|
157
|
+
const payloadString = message.value?.toString("utf8") ?? "";
|
|
158
|
+
let data: unknown;
|
|
159
|
+
try {
|
|
160
|
+
data = payloadString.length > 0 ? JSON.parse(payloadString) : null;
|
|
161
|
+
} catch {
|
|
162
|
+
data = payloadString;
|
|
163
|
+
}
|
|
164
|
+
const headers: Record<string, string> = {};
|
|
165
|
+
if (message.headers) {
|
|
166
|
+
for (const [k, v] of Object.entries(message.headers)) headers[k] = v?.toString("utf8") ?? "";
|
|
167
|
+
}
|
|
168
|
+
const job: WorkerJob = {
|
|
169
|
+
id: message.key?.toString("utf8") ?? `${config.queue}:${message.offset}`,
|
|
170
|
+
data,
|
|
171
|
+
headers,
|
|
172
|
+
queue: config.queue,
|
|
173
|
+
priority: config.priority ?? 0,
|
|
174
|
+
attempts: 0,
|
|
175
|
+
maxRetries: config.retries ?? 0,
|
|
176
|
+
createdAt: new Date(Number.parseInt(message.timestamp, 10)),
|
|
177
|
+
timeout: config.timeout,
|
|
178
|
+
raw: message,
|
|
179
|
+
complete: async () => {
|
|
180
|
+
stats.completed += 1;
|
|
181
|
+
},
|
|
182
|
+
fail: async (_err: Error) => {
|
|
183
|
+
stats.failed += 1;
|
|
184
|
+
throw _err;
|
|
185
|
+
},
|
|
186
|
+
};
|
|
187
|
+
stats.active += 1;
|
|
188
|
+
try {
|
|
189
|
+
await handler(job);
|
|
190
|
+
stats.completed += 1;
|
|
191
|
+
} catch (err) {
|
|
192
|
+
stats.failed += 1;
|
|
193
|
+
throw err;
|
|
194
|
+
} finally {
|
|
195
|
+
stats.active = Math.max(0, stats.active - 1);
|
|
196
|
+
}
|
|
197
|
+
},
|
|
198
|
+
});
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
async addJob(
|
|
202
|
+
queue: string,
|
|
203
|
+
data: unknown,
|
|
204
|
+
opts?: { priority?: number; delay?: number; retries?: number; timeout?: number; jobId?: string },
|
|
205
|
+
): Promise<string> {
|
|
206
|
+
if (!this.connected) throw new Error("[blok][kafka] not connected. Call connect() first.");
|
|
207
|
+
if (!this.handle.producer) throw new Error("[blok][kafka] producer not initialized");
|
|
208
|
+
const key = opts?.jobId ?? uuid();
|
|
209
|
+
const payload = typeof data === "string" ? data : JSON.stringify(data);
|
|
210
|
+
await this.handle.producer.send({
|
|
211
|
+
topic: queue,
|
|
212
|
+
messages: [
|
|
213
|
+
{
|
|
214
|
+
key,
|
|
215
|
+
value: payload,
|
|
216
|
+
headers: opts?.delay ? { "x-blok-delay-ms": String(opts.delay) } : undefined,
|
|
217
|
+
},
|
|
218
|
+
],
|
|
219
|
+
});
|
|
220
|
+
return key;
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
async stopProcessing(queue: string): Promise<void> {
|
|
224
|
+
const consumer = this.handle.consumers.get(queue);
|
|
225
|
+
if (consumer) {
|
|
226
|
+
try {
|
|
227
|
+
await consumer.stop();
|
|
228
|
+
} catch {
|
|
229
|
+
/* ignore */
|
|
230
|
+
}
|
|
231
|
+
try {
|
|
232
|
+
await consumer.disconnect();
|
|
233
|
+
} catch {
|
|
234
|
+
/* ignore */
|
|
235
|
+
}
|
|
236
|
+
this.handle.consumers.delete(queue);
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
isConnected(): boolean {
|
|
241
|
+
return this.connected;
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
async healthCheck(): Promise<boolean> {
|
|
245
|
+
if (!this.connected || !this.handle.admin) return false;
|
|
246
|
+
try {
|
|
247
|
+
await this.handle.admin.fetchTopicOffsets("__consumer_offsets");
|
|
248
|
+
return true;
|
|
249
|
+
} catch {
|
|
250
|
+
return false;
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
async getQueueStats(queue: string): Promise<WorkerQueueStats> {
|
|
255
|
+
const counters = this.stats.get(queue) ?? { completed: 0, failed: 0, active: 0 };
|
|
256
|
+
let waiting = 0;
|
|
257
|
+
if (this.handle.admin) {
|
|
258
|
+
try {
|
|
259
|
+
const offsets = await this.handle.admin.fetchTopicOffsets(queue);
|
|
260
|
+
// Approximate: total committed offsets across partitions. Real lag
|
|
261
|
+
// requires admin.fetchOffsets({ groupId }) — skipped here to keep
|
|
262
|
+
// the call cheap; production deployments should use Kafka's
|
|
263
|
+
// dedicated lag metrics anyway.
|
|
264
|
+
waiting = offsets.reduce((sum, p) => sum + Number.parseInt(p.offset, 10), 0);
|
|
265
|
+
} catch {
|
|
266
|
+
waiting = 0;
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
return {
|
|
270
|
+
waiting,
|
|
271
|
+
active: counters.active,
|
|
272
|
+
completed: counters.completed,
|
|
273
|
+
failed: counters.failed,
|
|
274
|
+
delayed: 0,
|
|
275
|
+
};
|
|
276
|
+
}
|
|
277
|
+
}
|
|
@@ -232,7 +232,9 @@ export class NATSWorkerAdapter implements WorkerAdapter {
|
|
|
232
232
|
priority,
|
|
233
233
|
attempts,
|
|
234
234
|
maxRetries,
|
|
235
|
-
createdAt: new Date(
|
|
235
|
+
createdAt: new Date(
|
|
236
|
+
info.timestampNanos ? Math.floor(Number(info.timestampNanos) / 1_000_000) : Date.now(),
|
|
237
|
+
),
|
|
236
238
|
delay: delay || undefined,
|
|
237
239
|
timeout: timeout || config.timeout || undefined,
|
|
238
240
|
raw: msg,
|
|
@@ -256,7 +258,7 @@ export class NATSWorkerAdapter implements WorkerAdapter {
|
|
|
256
258
|
// holding here. createdMs is the message's first-publish timestamp;
|
|
257
259
|
// hold until createdMs + delay. Single-process semantics — for
|
|
258
260
|
// long deferrals, prefer trigger-level `delay` (DeferredRunScheduler).
|
|
259
|
-
const createdMs = info.timestampNanos ? Number(info.timestampNanos /
|
|
261
|
+
const createdMs = info.timestampNanos ? Math.floor(Number(info.timestampNanos) / 1_000_000) : Date.now();
|
|
260
262
|
const waitMs = computeXDelayHoldMs(delay, createdMs, Date.now());
|
|
261
263
|
if (waitMs > 0) {
|
|
262
264
|
await new Promise<void>((resolve) => setTimeout(resolve, waitMs));
|