@blokjs/trigger-pubsub 0.2.3 → 0.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. package/__tests__/integration/gcp-pubsub.real-emulator.test.ts +235 -0
  2. package/__tests__/integration/kafka-pubsub.real-kafka.test.ts +269 -0
  3. package/__tests__/integration/nats-pubsub.real-nats.test.ts +138 -0
  4. package/dist/PubSubTrigger.d.ts +43 -3
  5. package/dist/PubSubTrigger.js +70 -16
  6. package/dist/adapters/AWSSNSAdapter.d.ts +16 -0
  7. package/dist/adapters/AWSSNSAdapter.js +52 -9
  8. package/dist/adapters/AzureServiceBusAdapter.d.ts +15 -0
  9. package/dist/adapters/AzureServiceBusAdapter.js +44 -11
  10. package/dist/adapters/GCPPubSubAdapter.d.ts +16 -0
  11. package/dist/adapters/GCPPubSubAdapter.js +42 -8
  12. package/dist/adapters/KafkaPubSubAdapter.d.ts +53 -0
  13. package/dist/adapters/KafkaPubSubAdapter.js +168 -0
  14. package/dist/adapters/NATSPubSubAdapter.d.ts +52 -0
  15. package/dist/adapters/NATSPubSubAdapter.js +260 -0
  16. package/dist/adapters/RedisStreamsPubSubAdapter.d.ts +49 -0
  17. package/dist/adapters/RedisStreamsPubSubAdapter.js +193 -0
  18. package/dist/adapters/factory.d.ts +22 -0
  19. package/dist/adapters/factory.js +80 -0
  20. package/dist/index.d.ts +36 -45
  21. package/dist/index.js +39 -46
  22. package/package.json +22 -10
  23. package/src/PubSubTrigger.ts +84 -18
  24. package/src/adapters/AWSSNSAdapter.ts +76 -12
  25. package/src/adapters/AzureServiceBusAdapter.ts +57 -14
  26. package/src/adapters/GCPPubSubAdapter.ts +50 -10
  27. package/src/adapters/KafkaPubSubAdapter.ts +194 -0
  28. package/src/adapters/NATSPubSubAdapter.ts +326 -0
  29. package/src/adapters/RedisStreamsPubSubAdapter.ts +225 -0
  30. package/src/adapters/factory.test.ts +87 -0
  31. package/src/adapters/factory.ts +88 -0
  32. package/src/adapters/new-adapters.test.ts +108 -0
  33. package/src/index.ts +40 -41
  34. package/template/package.json +6 -6
  35. package/template/src/runner/PubSubServer.ts +2 -2
  36. package/template/src/workflows/messages/on-message.ts +38 -34
@@ -14,10 +14,27 @@
14
14
  * - SQS_MAX_MESSAGES: Max messages per receive (default: 10)
15
15
  */
16
16
 
17
+ import type { MessageAttributeValue, SQSClient } from "@aws-sdk/client-sqs";
17
18
  import type { PubSubTriggerOpts } from "@blokjs/helper";
18
19
  import { v4 as uuid } from "uuid";
19
20
  import type { PubSubAdapter, PubSubMessage } from "../PubSubTrigger";
20
21
 
22
+ /**
23
+ * Shape of an SNS notification when delivered to an SQS subscription.
24
+ * SNS-to-SQS subscriptions wrap the publisher's payload inside a JSON
25
+ * envelope; the receiver parses `msg.Body` as this envelope and reads
26
+ * `Message` (the actual payload) plus the SNS-side attributes.
27
+ */
28
+ interface SNSNotificationEnvelope {
29
+ Type?: string;
30
+ MessageId?: string;
31
+ TopicArn?: string;
32
+ Message?: string;
33
+ Subject?: string;
34
+ Timestamp?: string;
35
+ MessageAttributes?: Record<string, { Type?: string; Value?: string }>;
36
+ }
37
+
21
38
  /**
22
39
  * AWS SNS/SQS configuration
23
40
  */
@@ -33,12 +50,23 @@ export interface AWSSNSConfig {
33
50
  export class AWSSNSAdapter implements PubSubAdapter {
34
51
  readonly provider = "aws" as const;
35
52
 
36
- private sqsClient: any;
53
+ private sqsClient: SQSClient | undefined;
37
54
  private connected = false;
38
55
  private config: AWSSNSConfig;
39
56
  private pollingIntervals: Map<string, NodeJS.Timeout> = new Map();
40
57
  private shouldStop = false;
41
58
 
59
+ /**
60
+ * Type-narrowing accessor for `this.sqsClient`. Field is undefined
61
+ * until `connect()` runs.
62
+ */
63
+ private requireSqsClient(): SQSClient {
64
+ if (!this.sqsClient) {
65
+ throw new Error("[AWSSNSAdapter] SQS client is not initialised — call connect() first");
66
+ }
67
+ return this.sqsClient;
68
+ }
69
+
42
70
  constructor(config?: Partial<AWSSNSConfig>) {
43
71
  this.config = {
44
72
  region: config?.region || process.env.AWS_REGION || "us-east-1",
@@ -66,8 +94,7 @@ export class AWSSNSAdapter implements PubSubAdapter {
66
94
  console.log(`[AWSSNSAdapter] Connected to AWS SNS/SQS: ${this.config.region}`);
67
95
  } catch (error) {
68
96
  throw new Error(
69
- `Failed to connect to AWS: ${(error as Error).message}. ` +
70
- `Make sure @aws-sdk/client-sqs is installed: npm install @aws-sdk/client-sqs`,
97
+ `Failed to connect to AWS: ${(error as Error).message}. Make sure @aws-sdk/client-sqs is installed: npm install @aws-sdk/client-sqs`,
71
98
  );
72
99
  }
73
100
  }
@@ -99,7 +126,10 @@ export class AWSSNSAdapter implements PubSubAdapter {
99
126
  throw new Error("Not connected to AWS. Call connect() first.");
100
127
  }
101
128
 
102
- // In AWS, subscription is the SQS queue URL that's subscribed to the SNS topic
129
+ // In AWS, subscription is the SQS queue URL that's subscribed to the SNS topic.
130
+ if (!config.subscription) {
131
+ throw new Error("[AWSSNSAdapter] `subscription` is required — must be the SQS queue URL bound to the SNS topic.");
132
+ }
103
133
  const queueUrl = config.subscription;
104
134
 
105
135
  // Start polling the SQS queue
@@ -129,16 +159,16 @@ export class AWSSNSAdapter implements PubSubAdapter {
129
159
  AttributeNames: ["All"],
130
160
  });
131
161
 
132
- const response = await this.sqsClient.send(command);
162
+ const response = await this.requireSqsClient().send(command);
133
163
 
134
164
  if (response.Messages && response.Messages.length > 0) {
135
165
  for (const msg of response.Messages) {
136
166
  // Parse SNS message wrapper
137
- let snsMessage: any;
167
+ let snsMessage: SNSNotificationEnvelope;
138
168
  let body: unknown;
139
169
 
140
170
  try {
141
- snsMessage = JSON.parse(msg.Body || "{}");
171
+ snsMessage = JSON.parse(msg.Body || "{}") as SNSNotificationEnvelope;
142
172
  // SNS wraps the actual message in a "Message" field
143
173
  if (snsMessage.Type === "Notification" && snsMessage.Message) {
144
174
  try {
@@ -160,14 +190,15 @@ export class AWSSNSAdapter implements PubSubAdapter {
160
190
  // SQS message attributes
161
191
  if (msg.MessageAttributes) {
162
192
  for (const [key, attr] of Object.entries(msg.MessageAttributes)) {
163
- attributes[key] = (attr as any).StringValue || "";
193
+ const sqsAttr = attr as MessageAttributeValue;
194
+ attributes[key] = sqsAttr.StringValue || "";
164
195
  }
165
196
  }
166
197
 
167
198
  // SNS message attributes (if present)
168
199
  if (snsMessage.MessageAttributes) {
169
200
  for (const [key, attr] of Object.entries(snsMessage.MessageAttributes)) {
170
- attributes[`sns_${key}`] = (attr as any).Value || "";
201
+ attributes[`sns_${key}`] = attr.Value || "";
171
202
  }
172
203
  }
173
204
 
@@ -185,7 +216,7 @@ export class AWSSNSAdapter implements PubSubAdapter {
185
216
  QueueUrl: queueUrl,
186
217
  ReceiptHandle: msg.ReceiptHandle,
187
218
  });
188
- await this.sqsClient.send(deleteCommand);
219
+ await this.requireSqsClient().send(deleteCommand);
189
220
  },
190
221
  nack: async () => {
191
222
  // Let the visibility timeout expire to return the message
@@ -196,7 +227,7 @@ export class AWSSNSAdapter implements PubSubAdapter {
196
227
  ReceiptHandle: msg.ReceiptHandle,
197
228
  VisibilityTimeout: 0,
198
229
  });
199
- await this.sqsClient.send(changeCommand);
230
+ await this.requireSqsClient().send(changeCommand);
200
231
  },
201
232
  };
202
233
 
@@ -238,6 +269,39 @@ export class AWSSNSAdapter implements PubSubAdapter {
238
269
  return this.connected;
239
270
  }
240
271
 
272
+ /**
273
+ * v0.7 PR 6 — publish to an SNS topic.
274
+ *
275
+ * `topic` must be the SNS topic ARN. `partitionKey` /
276
+ * `orderingKey` map to the FIFO `MessageGroupId` field for
277
+ * `.fifo` topics; ignored otherwise.
278
+ */
279
+ async publish(
280
+ topic: string,
281
+ payload: unknown,
282
+ opts?: { partitionKey?: string; orderingKey?: string },
283
+ ): Promise<void> {
284
+ if (!this.connected) throw new Error("[blok][pubsub-aws] not connected. Call connect() first.");
285
+ const moduleName = "@aws-sdk/client-sns";
286
+ // biome-ignore lint/suspicious/noExplicitAny: SDK loaded at runtime as a peer dep.
287
+ const sns: any = await import(moduleName);
288
+ const client = new sns.SNSClient({ region: this.config.region });
289
+ const isFifo = topic.endsWith(".fifo");
290
+ const params: Record<string, unknown> = {
291
+ TopicArn: topic,
292
+ Message: typeof payload === "string" ? payload : JSON.stringify(payload),
293
+ };
294
+ if (isFifo) {
295
+ params.MessageGroupId = opts?.partitionKey ?? opts?.orderingKey ?? "default";
296
+ params.MessageDeduplicationId = `${Date.now()}-${Math.random().toString(36).slice(2)}`;
297
+ }
298
+ try {
299
+ await client.send(new sns.PublishCommand(params));
300
+ } finally {
301
+ client.destroy?.();
302
+ }
303
+ }
304
+
241
305
  /**
242
306
  * Health check - verify AWS connectivity
243
307
  */
@@ -247,7 +311,7 @@ export class AWSSNSAdapter implements PubSubAdapter {
247
311
  try {
248
312
  const { ListQueuesCommand } = await import("@aws-sdk/client-sqs");
249
313
  const command = new ListQueuesCommand({ MaxResults: 1 });
250
- await this.sqsClient.send(command);
314
+ await this.requireSqsClient().send(command);
251
315
  return true;
252
316
  } catch {
253
317
  return false;
@@ -9,6 +9,12 @@
9
9
  * - AZURE_SERVICE_BUS_FULLY_QUALIFIED_NAMESPACE: Fully qualified namespace (if using DefaultAzureCredential)
10
10
  */
11
11
 
12
+ import type {
13
+ ProcessErrorArgs,
14
+ ServiceBusClient,
15
+ ServiceBusReceivedMessage,
16
+ ServiceBusReceiver,
17
+ } from "@azure/service-bus";
12
18
  import type { PubSubTriggerOpts } from "@blokjs/helper";
13
19
  import { v4 as uuid } from "uuid";
14
20
  import type { PubSubAdapter, PubSubMessage } from "../PubSubTrigger";
@@ -27,11 +33,22 @@ export interface AzureServiceBusConfig {
27
33
  export class AzureServiceBusAdapter implements PubSubAdapter {
28
34
  readonly provider = "azure" as const;
29
35
 
30
- private client: any;
31
- private receivers: Map<string, any> = new Map();
36
+ private client: ServiceBusClient | undefined;
37
+ private receivers: Map<string, ServiceBusReceiver> = new Map();
32
38
  private connected = false;
33
39
  private config: AzureServiceBusConfig;
34
40
 
41
+ /**
42
+ * Type-narrowing accessor for `this.client`. Field is undefined until
43
+ * `connect()` runs.
44
+ */
45
+ private requireClient(): ServiceBusClient {
46
+ if (!this.client) {
47
+ throw new Error("[AzureServiceBusAdapter] client is not initialised — call connect() first");
48
+ }
49
+ return this.client;
50
+ }
51
+
35
52
  constructor(config?: AzureServiceBusConfig) {
36
53
  this.config = {
37
54
  connectionString: config?.connectionString || process.env.AZURE_SERVICE_BUS_CONNECTION_STRING,
@@ -63,8 +80,7 @@ export class AzureServiceBusAdapter implements PubSubAdapter {
63
80
  console.log("[AzureServiceBusAdapter] Connected to Azure Service Bus");
64
81
  } catch (error) {
65
82
  throw new Error(
66
- `Failed to connect to Azure Service Bus: ${(error as Error).message}. ` +
67
- `Make sure @azure/service-bus is installed: npm install @azure/service-bus`,
83
+ `Failed to connect to Azure Service Bus: ${(error as Error).message}. Make sure @azure/service-bus is installed: npm install @azure/service-bus`,
68
84
  );
69
85
  }
70
86
  }
@@ -82,7 +98,7 @@ export class AzureServiceBusAdapter implements PubSubAdapter {
82
98
  }
83
99
  this.receivers.clear();
84
100
 
85
- await this.client.close();
101
+ await this.requireClient().close();
86
102
  this.connected = false;
87
103
  console.log("[AzureServiceBusAdapter] Disconnected from Azure Service Bus");
88
104
  } catch (error) {
@@ -98,17 +114,18 @@ export class AzureServiceBusAdapter implements PubSubAdapter {
98
114
  throw new Error("Not connected to Azure Service Bus. Call connect() first.");
99
115
  }
100
116
 
101
- let receiver: any;
117
+ const client = this.requireClient();
118
+ let receiver: ServiceBusReceiver;
102
119
 
103
120
  // Determine if this is a topic subscription or a queue
104
121
  if (config.subscription && config.topic) {
105
122
  // Topic with subscription
106
- receiver = this.client.createReceiver(config.topic, config.subscription, {
123
+ receiver = client.createReceiver(config.topic, config.subscription, {
107
124
  receiveMode: config.ack !== false ? "peekLock" : "receiveAndDelete",
108
125
  });
109
126
  } else {
110
127
  // Queue
111
- receiver = this.client.createReceiver(config.subscription || config.topic, {
128
+ receiver = client.createReceiver(config.subscription || config.topic, {
112
129
  receiveMode: config.ack !== false ? "peekLock" : "receiveAndDelete",
113
130
  });
114
131
  }
@@ -116,7 +133,7 @@ export class AzureServiceBusAdapter implements PubSubAdapter {
116
133
  const subscriptionKey = `${config.topic}/${config.subscription}`;
117
134
 
118
135
  // Message handler
119
- const processMessage = async (sbMessage: any) => {
136
+ const processMessage = async (sbMessage: ServiceBusReceivedMessage) => {
120
137
  // Parse message body
121
138
  let body: unknown;
122
139
  try {
@@ -139,7 +156,8 @@ export class AzureServiceBusAdapter implements PubSubAdapter {
139
156
 
140
157
  // Create pub/sub message
141
158
  const pubsubMessage: PubSubMessage = {
142
- id: sbMessage.messageId || uuid(),
159
+ // `messageId` may be string | number | Buffer per Azure SDK; coerce to string.
160
+ id: sbMessage.messageId !== undefined ? String(sbMessage.messageId) : uuid(),
143
161
  body,
144
162
  attributes,
145
163
  raw: sbMessage,
@@ -162,9 +180,10 @@ export class AzureServiceBusAdapter implements PubSubAdapter {
162
180
  }
163
181
  };
164
182
 
165
- // Error handler
166
- const processError = async (error: Error) => {
167
- console.error(`[AzureServiceBusAdapter] Error: ${error.message}`);
183
+ // Error handler — Azure SDK passes a `ProcessErrorArgs` with the
184
+ // underlying error inside `args.error` plus contextual fields.
185
+ const processError = async (args: ProcessErrorArgs) => {
186
+ console.error(`[AzureServiceBusAdapter] Error: ${args.error.message}`);
168
187
  };
169
188
 
170
189
  // Subscribe to messages
@@ -197,6 +216,30 @@ export class AzureServiceBusAdapter implements PubSubAdapter {
197
216
  return this.connected;
198
217
  }
199
218
 
219
+ /**
220
+ * v0.7 PR 6 — publish to an Azure Service Bus topic.
221
+ *
222
+ * `partitionKey` maps to Service Bus's `partitionKey`; `orderingKey`
223
+ * maps to `sessionId` (for session-enabled topics).
224
+ */
225
+ async publish(
226
+ topic: string,
227
+ payload: unknown,
228
+ opts?: { partitionKey?: string; orderingKey?: string },
229
+ ): Promise<void> {
230
+ if (!this.connected) throw new Error("[blok][pubsub-azure] not connected. Call connect() first.");
231
+ const sender = this.requireClient().createSender(topic);
232
+ try {
233
+ await sender.sendMessages({
234
+ body: payload,
235
+ partitionKey: opts?.partitionKey,
236
+ sessionId: opts?.orderingKey,
237
+ });
238
+ } finally {
239
+ await sender.close();
240
+ }
241
+ }
242
+
200
243
  /**
201
244
  * Health check - verify Azure Service Bus connectivity
202
245
  */
@@ -205,7 +248,7 @@ export class AzureServiceBusAdapter implements PubSubAdapter {
205
248
 
206
249
  try {
207
250
  // Create a temporary receiver to test connectivity
208
- const testReceiver = this.client.createReceiver("$default", {
251
+ const testReceiver = this.requireClient().createReceiver("$default", {
209
252
  receiveMode: "peekLock",
210
253
  });
211
254
  await testReceiver.close();
@@ -11,6 +11,7 @@
11
11
  */
12
12
 
13
13
  import type { PubSubTriggerOpts } from "@blokjs/helper";
14
+ import type { Message, PubSub, Subscription } from "@google-cloud/pubsub";
14
15
  import { v4 as uuid } from "uuid";
15
16
  import type { PubSubAdapter, PubSubMessage } from "../PubSubTrigger";
16
17
 
@@ -31,11 +32,22 @@ export interface GCPPubSubConfig {
31
32
  export class GCPPubSubAdapter implements PubSubAdapter {
32
33
  readonly provider = "gcp" as const;
33
34
 
34
- private client: any;
35
- private subscriptions: Map<string, any> = new Map();
35
+ private client: PubSub | undefined;
36
+ private subscriptions: Map<string, Subscription> = new Map();
36
37
  private connected = false;
37
38
  private config: GCPPubSubConfig;
38
39
 
40
+ /**
41
+ * Type-narrowing accessor for `this.client`. Field is undefined until
42
+ * `connect()` runs.
43
+ */
44
+ private requireClient(): PubSub {
45
+ if (!this.client) {
46
+ throw new Error("[GCPPubSubAdapter] client is not initialised — call connect() first");
47
+ }
48
+ return this.client;
49
+ }
50
+
39
51
  constructor(config?: GCPPubSubConfig) {
40
52
  this.config = {
41
53
  projectId: config?.projectId || process.env.GOOGLE_CLOUD_PROJECT,
@@ -62,8 +74,7 @@ export class GCPPubSubAdapter implements PubSubAdapter {
62
74
  console.log(`[GCPPubSubAdapter] Connected to GCP Pub/Sub: ${this.config.projectId}`);
63
75
  } catch (error) {
64
76
  throw new Error(
65
- `Failed to connect to GCP Pub/Sub: ${(error as Error).message}. ` +
66
- `Make sure @google-cloud/pubsub is installed: npm install @google-cloud/pubsub`,
77
+ `Failed to connect to GCP Pub/Sub: ${(error as Error).message}. Make sure @google-cloud/pubsub is installed: npm install @google-cloud/pubsub`,
67
78
  );
68
79
  }
69
80
  }
@@ -81,7 +92,7 @@ export class GCPPubSubAdapter implements PubSubAdapter {
81
92
  }
82
93
  this.subscriptions.clear();
83
94
 
84
- await this.client.close();
95
+ await this.requireClient().close();
85
96
  this.connected = false;
86
97
  console.log("[GCPPubSubAdapter] Disconnected from GCP Pub/Sub");
87
98
  } catch (error) {
@@ -97,18 +108,26 @@ export class GCPPubSubAdapter implements PubSubAdapter {
97
108
  throw new Error("Not connected to GCP Pub/Sub. Call connect() first.");
98
109
  }
99
110
 
111
+ if (!config.subscription) {
112
+ throw new Error("[GCPPubSubAdapter] `subscription` is required — must be the GCP subscription name.");
113
+ }
100
114
  const subscriptionName = config.subscription;
101
115
 
102
- // Get the subscription
103
- const subscription = this.client.subscription(subscriptionName, {
116
+ // Get the subscription. The GCP SDK calls the per-subscription
117
+ // deadline `maxAckDeadline` (a `Duration`); our author-facing
118
+ // field is `ackDeadline` (seconds) for parity with other
119
+ // providers. `Duration` is the GCP SDK's tc39-Temporal-shaped
120
+ // shim — construct via `Duration.from({seconds})`.
121
+ const { Duration } = await import("@google-cloud/pubsub");
122
+ const subscription = this.requireClient().subscription(subscriptionName, {
104
123
  flowControl: {
105
124
  maxMessages: config.maxMessages || 10,
106
125
  },
107
- ackDeadline: config.ackDeadline || 30,
126
+ maxAckDeadline: Duration.from({ seconds: config.ackDeadline || 30 }),
108
127
  });
109
128
 
110
129
  // Message handler
111
- const messageHandler = async (gcpMessage: any) => {
130
+ const messageHandler = async (gcpMessage: Message) => {
112
131
  // Parse message data
113
132
  let body: unknown;
114
133
  try {
@@ -177,6 +196,27 @@ export class GCPPubSubAdapter implements PubSubAdapter {
177
196
  return this.connected;
178
197
  }
179
198
 
199
+ /**
200
+ * v0.7 PR 6 — publish to a GCP Pub/Sub topic.
201
+ *
202
+ * `partitionKey` / `orderingKey` map to GCP's `orderingKey` (the
203
+ * topic must have message ordering enabled — otherwise the field
204
+ * is ignored by the broker).
205
+ */
206
+ async publish(
207
+ topic: string,
208
+ payload: unknown,
209
+ opts?: { partitionKey?: string; orderingKey?: string },
210
+ ): Promise<void> {
211
+ if (!this.connected) throw new Error("[blok][pubsub-gcp] not connected. Call connect() first.");
212
+ const body = typeof payload === "string" ? payload : JSON.stringify(payload);
213
+ const t = this.requireClient().topic(topic);
214
+ await t.publishMessage({
215
+ data: Buffer.from(body),
216
+ orderingKey: opts?.orderingKey ?? opts?.partitionKey,
217
+ });
218
+ }
219
+
180
220
  /**
181
221
  * Health check - verify GCP Pub/Sub connectivity
182
222
  */
@@ -185,7 +225,7 @@ export class GCPPubSubAdapter implements PubSubAdapter {
185
225
 
186
226
  try {
187
227
  // List subscriptions as a health check
188
- const [subscriptions] = await this.client.getSubscriptions({ pageSize: 1 });
228
+ const [subscriptions] = await this.requireClient().getSubscriptions({ pageSize: 1 });
189
229
  return true;
190
230
  } catch {
191
231
  return false;
@@ -0,0 +1,194 @@
1
+ /**
2
+ * KafkaPubSubAdapter — v0.7 PR 6 — Pub/Sub adapter backed by Apache
3
+ * Kafka via `kafkajs`.
4
+ *
5
+ * Pub/Sub vs Worker semantics: this adapter uses **per-subscriber
6
+ * consumer groups** so multiple subscribers each receive every message
7
+ * (fan-out). When `consumerGroup` is explicitly set, all subscribers
8
+ * share that group and compete (1 of N gets each message).
9
+ *
10
+ * Replay cursors via `startFrom`:
11
+ * - `"earliest"` → `fromBeginning: true` on first subscribe.
12
+ * - `"latest"` (default) → only new messages.
13
+ * - `{seq: N}` / `{timestamp: ms}` — provider-specific. Cleanest
14
+ * long-term path is Kafka's `admin.seek({offset|timestamp})` post-
15
+ * subscribe; v1 honors `earliest` / `latest` and `{seq}` via
16
+ * `auto.offset.reset` only.
17
+ *
18
+ * Requires `kafkajs` as a peer dependency.
19
+ *
20
+ * Environment variables:
21
+ * - `KAFKA_BROKERS` — comma-separated (default `localhost:9092`).
22
+ * - `KAFKA_CLIENT_ID` — default `"blok-pubsub"`.
23
+ * - `KAFKA_SASL_USERNAME` / `KAFKA_SASL_PASSWORD` — SASL/PLAIN.
24
+ * - `KAFKA_SSL` — `"true"` enables TLS.
25
+ */
26
+
27
+ import type { PubSubTriggerOpts } from "@blokjs/helper";
28
+ import { v4 as uuid } from "uuid";
29
+ import type { PubSubAdapter, PubSubMessage } from "../PubSubTrigger";
30
+
31
+ export interface KafkaPubSubConfig {
32
+ brokers: string[];
33
+ clientId: string;
34
+ saslUsername?: string;
35
+ saslPassword?: string;
36
+ ssl: boolean;
37
+ }
38
+
39
+ interface KafkaConsumerHandle {
40
+ disconnect: () => Promise<void>;
41
+ stop: () => Promise<void>;
42
+ }
43
+
44
+ export class KafkaPubSubAdapter implements PubSubAdapter {
45
+ readonly provider = "kafka" as const;
46
+ private readonly config: KafkaPubSubConfig;
47
+ // biome-ignore lint/suspicious/noExplicitAny: kafkajs's exported `Kafka` constructor is loosely typed.
48
+ private kafka: any = null;
49
+ // biome-ignore lint/suspicious/noExplicitAny: producer is created lazily and typed loosely.
50
+ private producer: any = null;
51
+ private consumers: Map<string, KafkaConsumerHandle> = new Map();
52
+ private connected = false;
53
+
54
+ constructor(config?: Partial<KafkaPubSubConfig>) {
55
+ this.config = {
56
+ brokers: config?.brokers ?? (process.env.KAFKA_BROKERS ?? "localhost:9092").split(",").map((s) => s.trim()),
57
+ clientId: config?.clientId ?? process.env.KAFKA_CLIENT_ID ?? "blok-pubsub",
58
+ saslUsername: config?.saslUsername ?? process.env.KAFKA_SASL_USERNAME,
59
+ saslPassword: config?.saslPassword ?? process.env.KAFKA_SASL_PASSWORD,
60
+ ssl: config?.ssl ?? process.env.KAFKA_SSL === "true",
61
+ };
62
+ }
63
+
64
+ async connect(): Promise<void> {
65
+ if (this.connected) return;
66
+ try {
67
+ // biome-ignore lint/suspicious/noExplicitAny: kafkajs is a runtime peer dep.
68
+ const kafkajs: any = await import("kafkajs");
69
+ const sasl =
70
+ this.config.saslUsername && this.config.saslPassword
71
+ ? { mechanism: "plain", username: this.config.saslUsername, password: this.config.saslPassword }
72
+ : undefined;
73
+ this.kafka = new kafkajs.Kafka({
74
+ clientId: this.config.clientId,
75
+ brokers: this.config.brokers,
76
+ ssl: this.config.ssl,
77
+ sasl,
78
+ });
79
+ this.producer = this.kafka.producer();
80
+ await this.producer.connect();
81
+ this.connected = true;
82
+ } catch (err) {
83
+ throw new Error(
84
+ `[blok][pubsub-kafka] connect failed: ${(err as Error).message}. Install kafkajs as a peer dependency: bun add kafkajs`,
85
+ );
86
+ }
87
+ }
88
+
89
+ async disconnect(): Promise<void> {
90
+ if (!this.connected) return;
91
+ for (const consumer of this.consumers.values()) {
92
+ try {
93
+ await consumer.disconnect();
94
+ } catch {
95
+ /* ignore */
96
+ }
97
+ }
98
+ this.consumers.clear();
99
+ try {
100
+ await this.producer?.disconnect();
101
+ } catch {
102
+ /* ignore */
103
+ }
104
+ this.producer = null;
105
+ this.connected = false;
106
+ }
107
+
108
+ async subscribe(config: PubSubTriggerOpts, handler: (message: PubSubMessage) => Promise<void>): Promise<void> {
109
+ if (!this.connected) throw new Error("[blok][pubsub-kafka] not connected. Call connect() first.");
110
+ // Fan-out: distinct group id per subscriber instance. Competing-
111
+ // consumer: explicit consumerGroup shared across all subscribers.
112
+ const groupId =
113
+ config.consumerGroup ?? `blok-fanout-${uuid().slice(0, 8)}-${config.topic.replace(/[^a-zA-Z0-9_]/g, "_")}`;
114
+ const consumer = this.kafka.consumer({ groupId });
115
+ await consumer.connect();
116
+ const fromBeginning = config.startFrom === "earliest";
117
+ await consumer.subscribe({ topic: config.topic, fromBeginning });
118
+ this.consumers.set(`${config.topic}#${groupId}`, consumer);
119
+
120
+ await consumer.run({
121
+ autoCommit: config.ack !== false,
122
+ eachMessage: async ({
123
+ message,
124
+ }: {
125
+ message: { key?: Buffer; value?: Buffer; offset: string; timestamp: string; headers?: Record<string, Buffer> };
126
+ }) => {
127
+ const text = message.value?.toString("utf8") ?? "";
128
+ let body: unknown = text;
129
+ try {
130
+ body = text.length > 0 ? JSON.parse(text) : null;
131
+ } catch {
132
+ /* leave as text */
133
+ }
134
+ const attributes: Record<string, string> = {};
135
+ if (message.headers) {
136
+ for (const [k, v] of Object.entries(message.headers)) attributes[k] = v?.toString("utf8") ?? "";
137
+ }
138
+ const msg: PubSubMessage = {
139
+ id: `${config.topic}:${message.offset}`,
140
+ body,
141
+ attributes,
142
+ raw: message,
143
+ topic: config.topic,
144
+ subscription: groupId,
145
+ publishTime: new Date(Number.parseInt(message.timestamp, 10)),
146
+ ack: async () => {
147
+ /* autoCommit handles ack */
148
+ },
149
+ nack: async () => {
150
+ /* kafkajs has no per-message nack; the offset commit is suppressed by throwing */
151
+ },
152
+ };
153
+ // On handler failure: re-throw so kafkajs suppresses the
154
+ // offset commit; the consumer will re-poll the message
155
+ // on the next cycle. The throw is intentional.
156
+ await handler(msg);
157
+ },
158
+ });
159
+ }
160
+
161
+ async unsubscribe(subscription: string): Promise<void> {
162
+ const consumer = this.consumers.get(subscription);
163
+ if (!consumer) return;
164
+ try {
165
+ await consumer.stop();
166
+ await consumer.disconnect();
167
+ } catch {
168
+ /* ignore */
169
+ }
170
+ this.consumers.delete(subscription);
171
+ }
172
+
173
+ async publish(
174
+ topic: string,
175
+ payload: unknown,
176
+ opts?: { partitionKey?: string; orderingKey?: string },
177
+ ): Promise<void> {
178
+ if (!this.connected || !this.producer) throw new Error("[blok][pubsub-kafka] not connected. Call connect() first.");
179
+ const body = typeof payload === "string" ? payload : JSON.stringify(payload);
180
+ const key = opts?.partitionKey ?? opts?.orderingKey;
181
+ await this.producer.send({
182
+ topic,
183
+ messages: [{ key, value: body }],
184
+ });
185
+ }
186
+
187
+ isConnected(): boolean {
188
+ return this.connected;
189
+ }
190
+
191
+ async healthCheck(): Promise<boolean> {
192
+ return this.connected;
193
+ }
194
+ }