@blokjs/trigger-worker 0.4.0 → 0.6.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. package/__tests__/integration/nats-adapter.real-nats.test.ts +116 -0
  2. package/__tests__/integration/pgboss-adapter.real-pg.test.ts +164 -0
  3. package/__tests__/integration/rabbitmq-adapter.real-rabbitmq.test.ts +179 -0
  4. package/__tests__/integration/sqs-adapter.real-sqs.test.ts +228 -0
  5. package/dist/WorkerTrigger.d.ts +37 -1
  6. package/dist/WorkerTrigger.js +142 -21
  7. package/dist/adapters/InMemoryAdapter.js +10 -5
  8. package/dist/adapters/KafkaAdapter.d.ts +62 -0
  9. package/dist/adapters/KafkaAdapter.js +236 -0
  10. package/dist/adapters/NATSAdapter.js +3 -3
  11. package/dist/adapters/PgBossAdapter.d.ts +56 -0
  12. package/dist/adapters/PgBossAdapter.js +251 -0
  13. package/dist/adapters/RabbitMQAdapter.d.ts +51 -0
  14. package/dist/adapters/RabbitMQAdapter.js +241 -0
  15. package/dist/adapters/RedisStreamsAdapter.d.ts +64 -0
  16. package/dist/adapters/RedisStreamsAdapter.js +240 -0
  17. package/dist/adapters/SQSAdapter.d.ts +61 -0
  18. package/dist/adapters/SQSAdapter.js +269 -0
  19. package/dist/adapters/factory.d.ts +34 -0
  20. package/dist/adapters/factory.js +103 -0
  21. package/dist/index.d.ts +21 -4
  22. package/dist/index.js +24 -4
  23. package/package.json +23 -5
  24. package/src/WorkerTrigger.ts +153 -22
  25. package/src/adapters/InMemoryAdapter.ts +9 -5
  26. package/src/adapters/KafkaAdapter.ts +277 -0
  27. package/src/adapters/NATSAdapter.ts +4 -2
  28. package/src/adapters/PgBossAdapter.ts +293 -0
  29. package/src/adapters/RabbitMQAdapter.ts +285 -0
  30. package/src/adapters/RedisStreamsAdapter.ts +286 -0
  31. package/src/adapters/SQSAdapter.ts +306 -0
  32. package/src/adapters/factory.test.ts +89 -0
  33. package/src/adapters/factory.ts +111 -0
  34. package/src/adapters/new-adapters.test.ts +130 -0
  35. package/src/index.ts +30 -4
  36. package/template/package.json +6 -6
  37. package/template/src/workflows/jobs/process-job.ts +37 -35
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@blokjs/trigger-worker",
3
- "version": "0.4.0",
3
+ "version": "0.6.2",
4
4
  "description": "Worker-based trigger for Blok workflows - supports background job processing with concurrency, retries, and scheduling",
5
5
  "type": "module",
6
6
  "main": "dist/index.js",
@@ -14,32 +14,50 @@
14
14
  "author": "Deskree Technologies Inc.",
15
15
  "license": "Apache-2.0",
16
16
  "dependencies": {
17
- "@blokjs/helper": "^0.4.0",
18
- "@blokjs/runner": "^0.4.0",
19
- "@blokjs/shared": "^0.4.0",
17
+ "@blokjs/helper": "^0.6.2",
18
+ "@blokjs/runner": "^0.6.2",
19
+ "@blokjs/shared": "^0.6.2",
20
20
  "@opentelemetry/api": "^1.9.0",
21
21
  "uuid": "^11.1.0"
22
22
  },
23
23
  "devDependencies": {
24
+ "@types/amqplib": "^0.10.8",
24
25
  "@types/node": "^22.15.21",
25
26
  "@types/uuid": "^11.0.0",
27
+ "pg-boss": "^10.0.0",
26
28
  "typescript": "^5.8.3",
27
29
  "vitest": "^4.0.18"
28
30
  },
29
31
  "peerDependencies": {
32
+ "@aws-sdk/client-sqs": "^3.0.0",
33
+ "amqplib": "^0.10.0",
30
34
  "bullmq": "^5.67.2",
31
35
  "ioredis": "^5.9.2",
32
- "nats": ""
36
+ "kafkajs": "^2.2.0",
37
+ "nats": "",
38
+ "pg-boss": ""
33
39
  },
34
40
  "peerDependenciesMeta": {
41
+ "@aws-sdk/client-sqs": {
42
+ "optional": true
43
+ },
44
+ "amqplib": {
45
+ "optional": true
46
+ },
35
47
  "bullmq": {
36
48
  "optional": true
37
49
  },
38
50
  "ioredis": {
39
51
  "optional": true
40
52
  },
53
+ "kafkajs": {
54
+ "optional": true
55
+ },
41
56
  "nats": {
42
57
  "optional": true
58
+ },
59
+ "pg-boss": {
60
+ "optional": true
43
61
  }
44
62
  },
45
63
  "private": false,
@@ -24,6 +24,7 @@ import {
24
24
  type BlokService,
25
25
  ConcurrencyLimitError,
26
26
  ConcurrencyMetrics,
27
+ DebounceCoordinator,
27
28
  DefaultLogger,
28
29
  DeferredDispatchSignal,
29
30
  type GlobalOptions,
@@ -34,6 +35,7 @@ import {
34
35
  TriggerBase,
35
36
  type TriggerResponse,
36
37
  createConcurrencyBackend,
38
+ createDebounceBackend,
37
39
  } from "@blokjs/runner";
38
40
  import type { Context, RequestContext } from "@blokjs/shared";
39
41
  import { type Span, SpanStatusCode, metrics, trace } from "@opentelemetry/api";
@@ -172,11 +174,31 @@ export abstract class WorkerTrigger extends TriggerBase {
172
174
  process.env.PROJECT_VERSION || "0.0.1",
173
175
  );
174
176
  protected readonly logger = new DefaultLogger();
175
- protected abstract adapter: WorkerAdapter;
177
+ /**
178
+ * v0.7 PR 5 — the "default" adapter, used when a workflow's
179
+ * `trigger.worker.provider` field is omitted AND the
180
+ * `BLOK_WORKER_ADAPTER` env var is unset. Subclasses MAY set this
181
+ * for back-compat with the pre-v0.7 single-adapter pattern
182
+ * (`class WorkerServer extends WorkerTrigger { protected adapter = new NATSWorkerAdapter() }`).
183
+ *
184
+ * When unset AND no per-workflow provider is specified, the factory
185
+ * falls back to `in-memory`. The factory pool (`adapters/factory.ts`)
186
+ * tracks one connected adapter per provider so multiple workflows
187
+ * with the same provider share a single broker connection.
188
+ */
189
+ protected adapter?: WorkerAdapter;
176
190
 
177
191
  /** Active queues being processed */
178
192
  protected activeQueues: Set<string> = new Set();
179
193
 
194
+ /**
195
+ * v0.7 PR 5 — adapter pool, keyed by provider name. Populated lazily
196
+ * inside `listen()` as workflows are matched to providers. Each
197
+ * adapter is connected once and reused across workflows that share
198
+ * its provider. Drained in `stop()`.
199
+ */
200
+ protected adapterPool: Map<string, WorkerAdapter> = new Map();
201
+
180
202
  // Subclasses provide these
181
203
  protected abstract nodes: Record<string, BlokService<unknown>>;
182
204
  protected abstract workflows: Record<string, HelperResponse>;
@@ -239,6 +261,28 @@ export abstract class WorkerTrigger extends TriggerBase {
239
261
  );
240
262
  }
241
263
 
264
+ // Tier C #1 · install the cross-process debounce backend
265
+ // (NATS KV / Redis) when the operator opted in via
266
+ // `BLOK_DEBOUNCE_BACKEND`. Default unset = the existing
267
+ // in-process behavior is preserved. On connect failure, log +
268
+ // fall back to in-memory coordination.
269
+ try {
270
+ const debounceBackend = createDebounceBackend();
271
+ if (debounceBackend) {
272
+ const leaseRaw = process.env.BLOK_DEBOUNCE_OWNER_LEASE_MS;
273
+ if (leaseRaw && /^\d+$/.test(leaseRaw)) {
274
+ DebounceCoordinator.getInstance().setOwnerLeaseMs(Number(leaseRaw));
275
+ }
276
+ await debounceBackend.connect();
277
+ DebounceCoordinator.getInstance().setBackend(debounceBackend);
278
+ this.logger.log(`[debounce] backend installed: ${debounceBackend.name}`);
279
+ }
280
+ } catch (err) {
281
+ this.logger.error(
282
+ `[debounce] backend install failed: ${err instanceof Error ? err.message : String(err)}; falling back to in-memory coordination`,
283
+ );
284
+ }
285
+
242
286
  // Tier 2 quick-wins follow-up · install crash handlers + recover
243
287
  // orphaned runs from a previous (dead) process. Idempotent + opt-out
244
288
  // via `BLOK_CRASH_AUTOFLIP_DISABLED=1`.
@@ -269,20 +313,6 @@ export abstract class WorkerTrigger extends TriggerBase {
269
313
  this.logger.error(`[shutdown] setup failed: ${err instanceof Error ? err.message : String(err)}`);
270
314
  }
271
315
 
272
- // Connect to job backend
273
- await this.adapter.connect();
274
- this.logger.log(`Connected to ${this.adapter.provider} worker system`);
275
-
276
- // Register health dependency
277
- this.registerHealthDependency(`worker-${this.adapter.provider}`, async () => {
278
- const healthy = await this.adapter.healthCheck();
279
- return {
280
- status: healthy ? ("healthy" as const) : ("unhealthy" as const),
281
- lastChecked: Date.now(),
282
- message: healthy ? "Connected" : "Connection lost",
283
- };
284
- });
285
-
286
316
  // Find all workflows with worker triggers
287
317
  const workerWorkflows = this.getWorkerWorkflows();
288
318
 
@@ -291,16 +321,19 @@ export abstract class WorkerTrigger extends TriggerBase {
291
321
  return this.endCounter(startTime);
292
322
  }
293
323
 
294
- // Start processing each queue
324
+ // Start processing each queue, dispatching to the right adapter
325
+ // based on the workflow's `provider` field (with back-compat
326
+ // fallback to `this.adapter` when subclasses still set it).
295
327
  for (const workflow of workerWorkflows) {
296
328
  const config = workflow.config.trigger?.worker as WorkerTriggerOpts;
329
+ const adapter = await this.resolveAdapterForWorkflow(config);
297
330
  this.logger.log(
298
- `Starting worker for queue: ${config.queue} (concurrency=${config.concurrency}, retries=${config.retries})`,
331
+ `Starting worker for queue: ${config.queue} via ${adapter.provider} (concurrency=${config.concurrency}, retries=${config.retries})`,
299
332
  );
300
333
 
301
334
  this.activeQueues.add(config.queue);
302
335
 
303
- await this.adapter.process(config, async (job) => {
336
+ await adapter.process(config, async (job) => {
304
337
  await this.handleJob(job, workflow, config);
305
338
  });
306
339
  }
@@ -323,12 +356,28 @@ export abstract class WorkerTrigger extends TriggerBase {
323
356
  * Stop all workers and disconnect
324
357
  */
325
358
  async stop(): Promise<void> {
359
+ // Stop each queue on its owning adapter — adapters are tracked
360
+ // in the pool so multi-provider workers all drain cleanly.
326
361
  for (const queue of this.activeQueues) {
327
- await this.adapter.stopProcessing(queue);
362
+ for (const adapter of this.adapterPool.values()) {
363
+ try {
364
+ await adapter.stopProcessing(queue);
365
+ } catch {
366
+ /* swallow — adapter may not own this queue */
367
+ }
368
+ }
328
369
  this.logger.log(`Stopped processing queue: ${queue}`);
329
370
  }
330
371
  this.activeQueues.clear();
331
- await this.adapter.disconnect();
372
+ // Disconnect every adapter we ever connected.
373
+ for (const adapter of this.adapterPool.values()) {
374
+ try {
375
+ await adapter.disconnect();
376
+ } catch (err) {
377
+ this.logger.error(`[blok][worker] disconnect failed: ${(err as Error).message}`);
378
+ }
379
+ }
380
+ this.adapterPool.clear();
332
381
  this.destroyMonitoring();
333
382
  this.logger.log("Worker trigger stopped");
334
383
  }
@@ -355,14 +404,33 @@ export abstract class WorkerTrigger extends TriggerBase {
355
404
  jobId?: string;
356
405
  },
357
406
  ): Promise<string> {
358
- return this.adapter.addJob(queue, data, opts);
407
+ // Back-compat: when a subclass set `this.adapter`, use it.
408
+ // Otherwise dispatch via the first pool adapter — typically the
409
+ // only one when a process owns one trigger workflow.
410
+ const adapter =
411
+ this.adapter ??
412
+ (this.adapterPool.size > 0 ? (this.adapterPool.values().next().value as WorkerAdapter) : undefined);
413
+ if (!adapter) {
414
+ throw new Error(
415
+ "[blok][worker] dispatch() called before any adapter is connected. Call listen() first, or set this.adapter on the subclass.",
416
+ );
417
+ }
418
+ return adapter.addJob(queue, data, opts);
359
419
  }
360
420
 
361
421
  /**
362
422
  * Get statistics for a queue
363
423
  */
364
424
  async getQueueStats(queue: string): Promise<WorkerQueueStats> {
365
- return this.adapter.getQueueStats(queue);
425
+ const adapter =
426
+ this.adapter ??
427
+ (this.adapterPool.size > 0 ? (this.adapterPool.values().next().value as WorkerAdapter) : undefined);
428
+ if (!adapter) {
429
+ throw new Error(
430
+ "[blok][worker] getQueueStats() called before any adapter is connected. Call listen() first, or set this.adapter on the subclass.",
431
+ );
432
+ }
433
+ return adapter.getQueueStats(queue);
366
434
  }
367
435
 
368
436
  /**
@@ -372,6 +440,60 @@ export abstract class WorkerTrigger extends TriggerBase {
372
440
  return Array.from(this.activeQueues);
373
441
  }
374
442
 
443
+ /**
444
+ * v0.7 PR 5 — pick the adapter for a workflow's `provider` field.
445
+ *
446
+ * Resolution order:
447
+ * 1. Subclass-set `this.adapter` (back-compat: pre-v0.7 pattern
448
+ * where one process binds to one adapter at construction time).
449
+ * 2. Per-workflow `provider` field, looked up via the factory.
450
+ * 3. `BLOK_WORKER_ADAPTER` env var.
451
+ * 4. `in-memory` fallback.
452
+ *
453
+ * Adapters are connected on first use and pooled per provider so
454
+ * multiple workflows sharing a provider share one broker
455
+ * connection. Health-dependency registration also happens here so
456
+ * each provider is tracked individually in `/health`.
457
+ */
458
+ protected async resolveAdapterForWorkflow(config: WorkerTriggerOpts): Promise<WorkerAdapter> {
459
+ // Subclass override wins for back-compat.
460
+ if (this.adapter) {
461
+ if (!this.adapter.isConnected()) {
462
+ await this.adapter.connect();
463
+ this.logger.log(`Connected to ${this.adapter.provider} worker system (subclass adapter)`);
464
+ this.registerAdapterHealth(this.adapter);
465
+ }
466
+ // Pool-track so stop() can drain it.
467
+ this.adapterPool.set(this.adapter.provider, this.adapter);
468
+ return this.adapter;
469
+ }
470
+
471
+ // Lazy-import the factory so the worker package doesn't pull in
472
+ // every adapter on import — only the ones actually exercised.
473
+ const { resolveProvider, createWorkerAdapter } = await import("./adapters/factory");
474
+ const provider = resolveProvider(config.provider);
475
+ let adapter = this.adapterPool.get(provider);
476
+ if (!adapter) {
477
+ adapter = createWorkerAdapter(provider);
478
+ await adapter.connect();
479
+ this.logger.log(`Connected to ${adapter.provider} worker system`);
480
+ this.registerAdapterHealth(adapter);
481
+ this.adapterPool.set(provider, adapter);
482
+ }
483
+ return adapter;
484
+ }
485
+
486
+ private registerAdapterHealth(adapter: WorkerAdapter): void {
487
+ this.registerHealthDependency(`worker-${adapter.provider}`, async () => {
488
+ const healthy = await adapter.healthCheck();
489
+ return {
490
+ status: healthy ? ("healthy" as const) : ("unhealthy" as const),
491
+ lastChecked: Date.now(),
492
+ message: healthy ? "Connected" : "Connection lost",
493
+ };
494
+ });
495
+ }
496
+
375
497
  /**
376
498
  * Get all workflows that have worker triggers
377
499
  */
@@ -452,6 +574,15 @@ export abstract class WorkerTrigger extends TriggerBase {
452
574
  `Processing job ${jobId} from ${config.queue} (attempt ${job.attempts + 1}/${job.maxRetries + 1})`,
453
575
  );
454
576
 
577
+ // v0.6 · apply the merged middleware chain (process-global →
578
+ // workflow-level → trigger-level) on the same ctx the main
579
+ // workflow will see. State mutations from middleware
580
+ // (e.g. ctx.state.identity) carry forward. Middleware that
581
+ // throws (via `@blokjs/throw`) propagates to the outer
582
+ // catch and is routed through the worker's retry / DLQ
583
+ // logic exactly like a main-workflow error.
584
+ await this.applyMiddlewareChain(ctx, this.nodeMap);
585
+
455
586
  // Execute workflow with timeout if configured
456
587
  let response: TriggerResponse;
457
588
  if (config.timeout && config.timeout > 0) {
@@ -122,8 +122,14 @@ export class InMemoryAdapter implements WorkerAdapter {
122
122
  throw new Error("Not connected. Call connect() first.");
123
123
  }
124
124
 
125
- if (!this.jobs.has(queue)) {
126
- this.jobs.set(queue, []);
125
+ // Get-or-init the per-queue job list. Same pattern as `stats`
126
+ // below — the previous code used `set-if-absent` then `.get()!`,
127
+ // but that non-null assertion is what biome flags. Pulling the
128
+ // reference once and seeding when missing keeps the type exact.
129
+ let jobs = this.jobs.get(queue);
130
+ if (!jobs) {
131
+ jobs = [];
132
+ this.jobs.set(queue, jobs);
127
133
  }
128
134
  if (!this.stats.has(queue)) {
129
135
  this.stats.set(queue, { completed: 0, failed: 0 });
@@ -146,8 +152,6 @@ export class InMemoryAdapter implements WorkerAdapter {
146
152
  job.scheduledAt = new Date(Date.now() + job.delay);
147
153
  }
148
154
 
149
- const jobs = this.jobs.get(queue)!;
150
-
151
155
  // Insert sorted by priority (higher first)
152
156
  const insertIdx = jobs.findIndex((j) => j.status === "waiting" && j.priority < job.priority);
153
157
  if (insertIdx >= 0) {
@@ -245,7 +249,7 @@ export class InMemoryAdapter implements WorkerAdapter {
245
249
 
246
250
  if (requeue && internalJob.attempts < internalJob.maxRetries) {
247
251
  // Requeue with backoff
248
- const backoff = Math.min(1000 * Math.pow(2, internalJob.attempts), 30000);
252
+ const backoff = Math.min(1000 * 2 ** internalJob.attempts, 30000);
249
253
  internalJob.status = "delayed";
250
254
  internalJob.scheduledAt = new Date(Date.now() + backoff);
251
255
  } else {
@@ -0,0 +1,277 @@
1
+ /**
2
+ * KafkaAdapter — v0.7 PR 5 — Worker adapter backed by Apache Kafka via
3
+ * `kafkajs`. Consumes from a topic (the `queue` field) with a
4
+ * consumer-group identifier; produces via the same client.
5
+ *
6
+ * Kafka is fundamentally a streaming platform — not a queue — so a
7
+ * few semantics differ from BullMQ/SQS/RabbitMQ:
8
+ *
9
+ * - **Ordering**: per-partition, not per-topic. Set the partition
10
+ * key via the `dedupId` field on `addJob` to keep related
11
+ * messages on the same partition.
12
+ * - **Retries**: Kafka doesn't have a broker-side retry concept.
13
+ * The adapter re-throws on handler failure; offset commit is
14
+ * suppressed so the consumer re-polls the message on the next
15
+ * cycle. For real retry semantics, layer a dead-letter topic.
16
+ * - **Stats**: KafkaJS exposes consumer-group lag via its admin
17
+ * client; the lag count is reported as `waiting`. Other stats
18
+ * are tracked locally per consumer.
19
+ *
20
+ * Requires `kafkajs` as a peer dependency:
21
+ *
22
+ * bun add kafkajs
23
+ *
24
+ * Environment variables (read at adapter construction):
25
+ * - `KAFKA_BROKERS` — comma-separated list (default `localhost:9092`).
26
+ * - `KAFKA_CLIENT_ID` — client.id (default `"blok-worker"`).
27
+ * - `KAFKA_SASL_USERNAME` — SASL/PLAIN username (optional).
28
+ * - `KAFKA_SASL_PASSWORD` — SASL/PLAIN password (optional).
29
+ * - `KAFKA_SSL` — when `"true"`, enable TLS.
30
+ */
31
+
32
+ import type { WorkerTriggerOpts } from "@blokjs/helper";
33
+ import { v4 as uuid } from "uuid";
34
+ import type { WorkerAdapter, WorkerJob, WorkerQueueStats } from "../WorkerTrigger";
35
+
36
+ export interface KafkaConfig {
37
+ brokers: string[];
38
+ clientId: string;
39
+ saslUsername?: string;
40
+ saslPassword?: string;
41
+ ssl: boolean;
42
+ }
43
+
44
+ interface KafkaJsHandle {
45
+ producer?: {
46
+ connect: () => Promise<void>;
47
+ disconnect: () => Promise<void>;
48
+ send: (args: unknown) => Promise<unknown>;
49
+ };
50
+ consumers: Map<
51
+ string,
52
+ {
53
+ disconnect: () => Promise<void>;
54
+ stop: () => Promise<void>;
55
+ run: (opts: unknown) => Promise<void>;
56
+ }
57
+ >;
58
+ admin?: {
59
+ connect: () => Promise<void>;
60
+ disconnect: () => Promise<void>;
61
+ fetchTopicOffsets: (topic: string) => Promise<Array<{ partition: number; offset: string }>>;
62
+ };
63
+ }
64
+
65
+ interface QueueStatsCounters {
66
+ completed: number;
67
+ failed: number;
68
+ active: number;
69
+ }
70
+
71
+ export class KafkaAdapter implements WorkerAdapter {
72
+ readonly provider = "kafka" as const;
73
+ private readonly config: KafkaConfig;
74
+ // biome-ignore lint/suspicious/noExplicitAny: kafkajs's exported `Kafka` constructor is loosely typed.
75
+ private kafka: any = null;
76
+ private handle: KafkaJsHandle = { consumers: new Map() };
77
+ private connected = false;
78
+ private stats: Map<string, QueueStatsCounters> = new Map();
79
+
80
+ constructor(config?: Partial<KafkaConfig>) {
81
+ this.config = {
82
+ brokers: config?.brokers ?? (process.env.KAFKA_BROKERS ?? "localhost:9092").split(",").map((s) => s.trim()),
83
+ clientId: config?.clientId ?? process.env.KAFKA_CLIENT_ID ?? "blok-worker",
84
+ saslUsername: config?.saslUsername ?? process.env.KAFKA_SASL_USERNAME,
85
+ saslPassword: config?.saslPassword ?? process.env.KAFKA_SASL_PASSWORD,
86
+ ssl: config?.ssl ?? process.env.KAFKA_SSL === "true",
87
+ };
88
+ }
89
+
90
+ async connect(): Promise<void> {
91
+ if (this.connected) return;
92
+ try {
93
+ // biome-ignore lint/suspicious/noExplicitAny: kafkajs is a runtime-loaded peer dep.
94
+ const kafkajs: any = await import("kafkajs");
95
+ const sasl =
96
+ this.config.saslUsername && this.config.saslPassword
97
+ ? { mechanism: "plain", username: this.config.saslUsername, password: this.config.saslPassword }
98
+ : undefined;
99
+ this.kafka = new kafkajs.Kafka({
100
+ clientId: this.config.clientId,
101
+ brokers: this.config.brokers,
102
+ ssl: this.config.ssl,
103
+ sasl,
104
+ });
105
+ this.handle.producer = this.kafka.producer();
106
+ await this.handle.producer?.connect();
107
+ this.handle.admin = this.kafka.admin();
108
+ await this.handle.admin?.connect();
109
+ this.connected = true;
110
+ } catch (err) {
111
+ throw new Error(
112
+ `[blok][kafka] connect failed: ${(err as Error).message}. Install kafkajs as a peer dependency: bun add kafkajs`,
113
+ );
114
+ }
115
+ }
116
+
117
+ async disconnect(): Promise<void> {
118
+ if (!this.connected) return;
119
+ for (const [, consumer] of this.handle.consumers) {
120
+ try {
121
+ await consumer.disconnect();
122
+ } catch {
123
+ /* ignore */
124
+ }
125
+ }
126
+ this.handle.consumers.clear();
127
+ try {
128
+ await this.handle.producer?.disconnect();
129
+ } catch {
130
+ /* ignore */
131
+ }
132
+ try {
133
+ await this.handle.admin?.disconnect();
134
+ } catch {
135
+ /* ignore */
136
+ }
137
+ this.connected = false;
138
+ }
139
+
140
+ async process(config: WorkerTriggerOpts, handler: (job: WorkerJob) => Promise<void>): Promise<void> {
141
+ if (!this.connected) throw new Error("[blok][kafka] not connected. Call connect() first.");
142
+ const groupId = config.consumerGroup ?? `${config.queue}-group`;
143
+ const consumer = this.kafka.consumer({ groupId });
144
+ await consumer.connect();
145
+ await consumer.subscribe({ topic: config.queue, fromBeginning: config.fromBeginning === true });
146
+ this.handle.consumers.set(config.queue, consumer);
147
+ this.stats.set(config.queue, { completed: 0, failed: 0, active: 0 });
148
+ const stats = this.stats.get(config.queue) as QueueStatsCounters;
149
+
150
+ await consumer.run({
151
+ autoCommit: config.ack !== false,
152
+ eachMessage: async ({
153
+ message,
154
+ }: {
155
+ message: { key?: Buffer; value?: Buffer; offset: string; timestamp: string; headers?: Record<string, Buffer> };
156
+ }) => {
157
+ const payloadString = message.value?.toString("utf8") ?? "";
158
+ let data: unknown;
159
+ try {
160
+ data = payloadString.length > 0 ? JSON.parse(payloadString) : null;
161
+ } catch {
162
+ data = payloadString;
163
+ }
164
+ const headers: Record<string, string> = {};
165
+ if (message.headers) {
166
+ for (const [k, v] of Object.entries(message.headers)) headers[k] = v?.toString("utf8") ?? "";
167
+ }
168
+ const job: WorkerJob = {
169
+ id: message.key?.toString("utf8") ?? `${config.queue}:${message.offset}`,
170
+ data,
171
+ headers,
172
+ queue: config.queue,
173
+ priority: config.priority ?? 0,
174
+ attempts: 0,
175
+ maxRetries: config.retries ?? 0,
176
+ createdAt: new Date(Number.parseInt(message.timestamp, 10)),
177
+ timeout: config.timeout,
178
+ raw: message,
179
+ complete: async () => {
180
+ stats.completed += 1;
181
+ },
182
+ fail: async (_err: Error) => {
183
+ stats.failed += 1;
184
+ throw _err;
185
+ },
186
+ };
187
+ stats.active += 1;
188
+ try {
189
+ await handler(job);
190
+ stats.completed += 1;
191
+ } catch (err) {
192
+ stats.failed += 1;
193
+ throw err;
194
+ } finally {
195
+ stats.active = Math.max(0, stats.active - 1);
196
+ }
197
+ },
198
+ });
199
+ }
200
+
201
+ async addJob(
202
+ queue: string,
203
+ data: unknown,
204
+ opts?: { priority?: number; delay?: number; retries?: number; timeout?: number; jobId?: string },
205
+ ): Promise<string> {
206
+ if (!this.connected) throw new Error("[blok][kafka] not connected. Call connect() first.");
207
+ if (!this.handle.producer) throw new Error("[blok][kafka] producer not initialized");
208
+ const key = opts?.jobId ?? uuid();
209
+ const payload = typeof data === "string" ? data : JSON.stringify(data);
210
+ await this.handle.producer.send({
211
+ topic: queue,
212
+ messages: [
213
+ {
214
+ key,
215
+ value: payload,
216
+ headers: opts?.delay ? { "x-blok-delay-ms": String(opts.delay) } : undefined,
217
+ },
218
+ ],
219
+ });
220
+ return key;
221
+ }
222
+
223
+ async stopProcessing(queue: string): Promise<void> {
224
+ const consumer = this.handle.consumers.get(queue);
225
+ if (consumer) {
226
+ try {
227
+ await consumer.stop();
228
+ } catch {
229
+ /* ignore */
230
+ }
231
+ try {
232
+ await consumer.disconnect();
233
+ } catch {
234
+ /* ignore */
235
+ }
236
+ this.handle.consumers.delete(queue);
237
+ }
238
+ }
239
+
240
+ isConnected(): boolean {
241
+ return this.connected;
242
+ }
243
+
244
+ async healthCheck(): Promise<boolean> {
245
+ if (!this.connected || !this.handle.admin) return false;
246
+ try {
247
+ await this.handle.admin.fetchTopicOffsets("__consumer_offsets");
248
+ return true;
249
+ } catch {
250
+ return false;
251
+ }
252
+ }
253
+
254
+ async getQueueStats(queue: string): Promise<WorkerQueueStats> {
255
+ const counters = this.stats.get(queue) ?? { completed: 0, failed: 0, active: 0 };
256
+ let waiting = 0;
257
+ if (this.handle.admin) {
258
+ try {
259
+ const offsets = await this.handle.admin.fetchTopicOffsets(queue);
260
+ // Approximate: total committed offsets across partitions. Real lag
261
+ // requires admin.fetchOffsets({ groupId }) — skipped here to keep
262
+ // the call cheap; production deployments should use Kafka's
263
+ // dedicated lag metrics anyway.
264
+ waiting = offsets.reduce((sum, p) => sum + Number.parseInt(p.offset, 10), 0);
265
+ } catch {
266
+ waiting = 0;
267
+ }
268
+ }
269
+ return {
270
+ waiting,
271
+ active: counters.active,
272
+ completed: counters.completed,
273
+ failed: counters.failed,
274
+ delayed: 0,
275
+ };
276
+ }
277
+ }
@@ -232,7 +232,9 @@ export class NATSWorkerAdapter implements WorkerAdapter {
232
232
  priority,
233
233
  attempts,
234
234
  maxRetries,
235
- createdAt: new Date(info.timestampNanos ? Number(info.timestampNanos / BigInt(1_000_000)) : Date.now()),
235
+ createdAt: new Date(
236
+ info.timestampNanos ? Math.floor(Number(info.timestampNanos) / 1_000_000) : Date.now(),
237
+ ),
236
238
  delay: delay || undefined,
237
239
  timeout: timeout || config.timeout || undefined,
238
240
  raw: msg,
@@ -256,7 +258,7 @@ export class NATSWorkerAdapter implements WorkerAdapter {
256
258
  // holding here. createdMs is the message's first-publish timestamp;
257
259
  // hold until createdMs + delay. Single-process semantics — for
258
260
  // long deferrals, prefer trigger-level `delay` (DeferredRunScheduler).
259
- const createdMs = info.timestampNanos ? Number(info.timestampNanos / BigInt(1_000_000)) : Date.now();
261
+ const createdMs = info.timestampNanos ? Math.floor(Number(info.timestampNanos) / 1_000_000) : Date.now();
260
262
  const waitMs = computeXDelayHoldMs(delay, createdMs, Date.now());
261
263
  if (waitMs > 0) {
262
264
  await new Promise<void>((resolve) => setTimeout(resolve, waitMs));