@blokjs/trigger-pubsub 0.2.3 → 0.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/__tests__/integration/gcp-pubsub.real-emulator.test.ts +235 -0
- package/__tests__/integration/kafka-pubsub.real-kafka.test.ts +269 -0
- package/__tests__/integration/nats-pubsub.real-nats.test.ts +138 -0
- package/dist/PubSubTrigger.d.ts +43 -3
- package/dist/PubSubTrigger.js +70 -16
- package/dist/adapters/AWSSNSAdapter.d.ts +16 -0
- package/dist/adapters/AWSSNSAdapter.js +52 -9
- package/dist/adapters/AzureServiceBusAdapter.d.ts +15 -0
- package/dist/adapters/AzureServiceBusAdapter.js +44 -11
- package/dist/adapters/GCPPubSubAdapter.d.ts +16 -0
- package/dist/adapters/GCPPubSubAdapter.js +42 -8
- package/dist/adapters/KafkaPubSubAdapter.d.ts +53 -0
- package/dist/adapters/KafkaPubSubAdapter.js +168 -0
- package/dist/adapters/NATSPubSubAdapter.d.ts +52 -0
- package/dist/adapters/NATSPubSubAdapter.js +260 -0
- package/dist/adapters/RedisStreamsPubSubAdapter.d.ts +49 -0
- package/dist/adapters/RedisStreamsPubSubAdapter.js +193 -0
- package/dist/adapters/factory.d.ts +22 -0
- package/dist/adapters/factory.js +80 -0
- package/dist/index.d.ts +36 -45
- package/dist/index.js +39 -46
- package/package.json +22 -10
- package/src/PubSubTrigger.ts +84 -18
- package/src/adapters/AWSSNSAdapter.ts +76 -12
- package/src/adapters/AzureServiceBusAdapter.ts +57 -14
- package/src/adapters/GCPPubSubAdapter.ts +50 -10
- package/src/adapters/KafkaPubSubAdapter.ts +194 -0
- package/src/adapters/NATSPubSubAdapter.ts +326 -0
- package/src/adapters/RedisStreamsPubSubAdapter.ts +225 -0
- package/src/adapters/factory.test.ts +87 -0
- package/src/adapters/factory.ts +88 -0
- package/src/adapters/new-adapters.test.ts +108 -0
- package/src/index.ts +40 -41
- package/template/package.json +6 -6
- package/template/src/runner/PubSubServer.ts +2 -2
- package/template/src/workflows/messages/on-message.ts +38 -34
|
@@ -14,10 +14,27 @@
|
|
|
14
14
|
* - SQS_MAX_MESSAGES: Max messages per receive (default: 10)
|
|
15
15
|
*/
|
|
16
16
|
|
|
17
|
+
import type { MessageAttributeValue, SQSClient } from "@aws-sdk/client-sqs";
|
|
17
18
|
import type { PubSubTriggerOpts } from "@blokjs/helper";
|
|
18
19
|
import { v4 as uuid } from "uuid";
|
|
19
20
|
import type { PubSubAdapter, PubSubMessage } from "../PubSubTrigger";
|
|
20
21
|
|
|
22
|
+
/**
|
|
23
|
+
* Shape of an SNS notification when delivered to an SQS subscription.
|
|
24
|
+
* SNS-to-SQS subscriptions wrap the publisher's payload inside a JSON
|
|
25
|
+
* envelope; the receiver parses `msg.Body` as this envelope and reads
|
|
26
|
+
* `Message` (the actual payload) plus the SNS-side attributes.
|
|
27
|
+
*/
|
|
28
|
+
interface SNSNotificationEnvelope {
|
|
29
|
+
Type?: string;
|
|
30
|
+
MessageId?: string;
|
|
31
|
+
TopicArn?: string;
|
|
32
|
+
Message?: string;
|
|
33
|
+
Subject?: string;
|
|
34
|
+
Timestamp?: string;
|
|
35
|
+
MessageAttributes?: Record<string, { Type?: string; Value?: string }>;
|
|
36
|
+
}
|
|
37
|
+
|
|
21
38
|
/**
|
|
22
39
|
* AWS SNS/SQS configuration
|
|
23
40
|
*/
|
|
@@ -33,12 +50,23 @@ export interface AWSSNSConfig {
|
|
|
33
50
|
export class AWSSNSAdapter implements PubSubAdapter {
|
|
34
51
|
readonly provider = "aws" as const;
|
|
35
52
|
|
|
36
|
-
private sqsClient:
|
|
53
|
+
private sqsClient: SQSClient | undefined;
|
|
37
54
|
private connected = false;
|
|
38
55
|
private config: AWSSNSConfig;
|
|
39
56
|
private pollingIntervals: Map<string, NodeJS.Timeout> = new Map();
|
|
40
57
|
private shouldStop = false;
|
|
41
58
|
|
|
59
|
+
/**
|
|
60
|
+
* Type-narrowing accessor for `this.sqsClient`. Field is undefined
|
|
61
|
+
* until `connect()` runs.
|
|
62
|
+
*/
|
|
63
|
+
private requireSqsClient(): SQSClient {
|
|
64
|
+
if (!this.sqsClient) {
|
|
65
|
+
throw new Error("[AWSSNSAdapter] SQS client is not initialised — call connect() first");
|
|
66
|
+
}
|
|
67
|
+
return this.sqsClient;
|
|
68
|
+
}
|
|
69
|
+
|
|
42
70
|
constructor(config?: Partial<AWSSNSConfig>) {
|
|
43
71
|
this.config = {
|
|
44
72
|
region: config?.region || process.env.AWS_REGION || "us-east-1",
|
|
@@ -66,8 +94,7 @@ export class AWSSNSAdapter implements PubSubAdapter {
|
|
|
66
94
|
console.log(`[AWSSNSAdapter] Connected to AWS SNS/SQS: ${this.config.region}`);
|
|
67
95
|
} catch (error) {
|
|
68
96
|
throw new Error(
|
|
69
|
-
`Failed to connect to AWS: ${(error as Error).message}.
|
|
70
|
-
`Make sure @aws-sdk/client-sqs is installed: npm install @aws-sdk/client-sqs`,
|
|
97
|
+
`Failed to connect to AWS: ${(error as Error).message}. Make sure @aws-sdk/client-sqs is installed: npm install @aws-sdk/client-sqs`,
|
|
71
98
|
);
|
|
72
99
|
}
|
|
73
100
|
}
|
|
@@ -99,7 +126,10 @@ export class AWSSNSAdapter implements PubSubAdapter {
|
|
|
99
126
|
throw new Error("Not connected to AWS. Call connect() first.");
|
|
100
127
|
}
|
|
101
128
|
|
|
102
|
-
// In AWS, subscription is the SQS queue URL that's subscribed to the SNS topic
|
|
129
|
+
// In AWS, subscription is the SQS queue URL that's subscribed to the SNS topic.
|
|
130
|
+
if (!config.subscription) {
|
|
131
|
+
throw new Error("[AWSSNSAdapter] `subscription` is required — must be the SQS queue URL bound to the SNS topic.");
|
|
132
|
+
}
|
|
103
133
|
const queueUrl = config.subscription;
|
|
104
134
|
|
|
105
135
|
// Start polling the SQS queue
|
|
@@ -129,16 +159,16 @@ export class AWSSNSAdapter implements PubSubAdapter {
|
|
|
129
159
|
AttributeNames: ["All"],
|
|
130
160
|
});
|
|
131
161
|
|
|
132
|
-
const response = await this.
|
|
162
|
+
const response = await this.requireSqsClient().send(command);
|
|
133
163
|
|
|
134
164
|
if (response.Messages && response.Messages.length > 0) {
|
|
135
165
|
for (const msg of response.Messages) {
|
|
136
166
|
// Parse SNS message wrapper
|
|
137
|
-
let snsMessage:
|
|
167
|
+
let snsMessage: SNSNotificationEnvelope;
|
|
138
168
|
let body: unknown;
|
|
139
169
|
|
|
140
170
|
try {
|
|
141
|
-
snsMessage = JSON.parse(msg.Body || "{}");
|
|
171
|
+
snsMessage = JSON.parse(msg.Body || "{}") as SNSNotificationEnvelope;
|
|
142
172
|
// SNS wraps the actual message in a "Message" field
|
|
143
173
|
if (snsMessage.Type === "Notification" && snsMessage.Message) {
|
|
144
174
|
try {
|
|
@@ -160,14 +190,15 @@ export class AWSSNSAdapter implements PubSubAdapter {
|
|
|
160
190
|
// SQS message attributes
|
|
161
191
|
if (msg.MessageAttributes) {
|
|
162
192
|
for (const [key, attr] of Object.entries(msg.MessageAttributes)) {
|
|
163
|
-
|
|
193
|
+
const sqsAttr = attr as MessageAttributeValue;
|
|
194
|
+
attributes[key] = sqsAttr.StringValue || "";
|
|
164
195
|
}
|
|
165
196
|
}
|
|
166
197
|
|
|
167
198
|
// SNS message attributes (if present)
|
|
168
199
|
if (snsMessage.MessageAttributes) {
|
|
169
200
|
for (const [key, attr] of Object.entries(snsMessage.MessageAttributes)) {
|
|
170
|
-
attributes[`sns_${key}`] =
|
|
201
|
+
attributes[`sns_${key}`] = attr.Value || "";
|
|
171
202
|
}
|
|
172
203
|
}
|
|
173
204
|
|
|
@@ -185,7 +216,7 @@ export class AWSSNSAdapter implements PubSubAdapter {
|
|
|
185
216
|
QueueUrl: queueUrl,
|
|
186
217
|
ReceiptHandle: msg.ReceiptHandle,
|
|
187
218
|
});
|
|
188
|
-
await this.
|
|
219
|
+
await this.requireSqsClient().send(deleteCommand);
|
|
189
220
|
},
|
|
190
221
|
nack: async () => {
|
|
191
222
|
// Let the visibility timeout expire to return the message
|
|
@@ -196,7 +227,7 @@ export class AWSSNSAdapter implements PubSubAdapter {
|
|
|
196
227
|
ReceiptHandle: msg.ReceiptHandle,
|
|
197
228
|
VisibilityTimeout: 0,
|
|
198
229
|
});
|
|
199
|
-
await this.
|
|
230
|
+
await this.requireSqsClient().send(changeCommand);
|
|
200
231
|
},
|
|
201
232
|
};
|
|
202
233
|
|
|
@@ -238,6 +269,39 @@ export class AWSSNSAdapter implements PubSubAdapter {
|
|
|
238
269
|
return this.connected;
|
|
239
270
|
}
|
|
240
271
|
|
|
272
|
+
/**
|
|
273
|
+
* v0.7 PR 6 — publish to an SNS topic.
|
|
274
|
+
*
|
|
275
|
+
* `topic` must be the SNS topic ARN. `partitionKey` /
|
|
276
|
+
* `orderingKey` map to the FIFO `MessageGroupId` field for
|
|
277
|
+
* `.fifo` topics; ignored otherwise.
|
|
278
|
+
*/
|
|
279
|
+
async publish(
|
|
280
|
+
topic: string,
|
|
281
|
+
payload: unknown,
|
|
282
|
+
opts?: { partitionKey?: string; orderingKey?: string },
|
|
283
|
+
): Promise<void> {
|
|
284
|
+
if (!this.connected) throw new Error("[blok][pubsub-aws] not connected. Call connect() first.");
|
|
285
|
+
const moduleName = "@aws-sdk/client-sns";
|
|
286
|
+
// biome-ignore lint/suspicious/noExplicitAny: SDK loaded at runtime as a peer dep.
|
|
287
|
+
const sns: any = await import(moduleName);
|
|
288
|
+
const client = new sns.SNSClient({ region: this.config.region });
|
|
289
|
+
const isFifo = topic.endsWith(".fifo");
|
|
290
|
+
const params: Record<string, unknown> = {
|
|
291
|
+
TopicArn: topic,
|
|
292
|
+
Message: typeof payload === "string" ? payload : JSON.stringify(payload),
|
|
293
|
+
};
|
|
294
|
+
if (isFifo) {
|
|
295
|
+
params.MessageGroupId = opts?.partitionKey ?? opts?.orderingKey ?? "default";
|
|
296
|
+
params.MessageDeduplicationId = `${Date.now()}-${Math.random().toString(36).slice(2)}`;
|
|
297
|
+
}
|
|
298
|
+
try {
|
|
299
|
+
await client.send(new sns.PublishCommand(params));
|
|
300
|
+
} finally {
|
|
301
|
+
client.destroy?.();
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
|
|
241
305
|
/**
|
|
242
306
|
* Health check - verify AWS connectivity
|
|
243
307
|
*/
|
|
@@ -247,7 +311,7 @@ export class AWSSNSAdapter implements PubSubAdapter {
|
|
|
247
311
|
try {
|
|
248
312
|
const { ListQueuesCommand } = await import("@aws-sdk/client-sqs");
|
|
249
313
|
const command = new ListQueuesCommand({ MaxResults: 1 });
|
|
250
|
-
await this.
|
|
314
|
+
await this.requireSqsClient().send(command);
|
|
251
315
|
return true;
|
|
252
316
|
} catch {
|
|
253
317
|
return false;
|
|
@@ -9,6 +9,12 @@
|
|
|
9
9
|
* - AZURE_SERVICE_BUS_FULLY_QUALIFIED_NAMESPACE: Fully qualified namespace (if using DefaultAzureCredential)
|
|
10
10
|
*/
|
|
11
11
|
|
|
12
|
+
import type {
|
|
13
|
+
ProcessErrorArgs,
|
|
14
|
+
ServiceBusClient,
|
|
15
|
+
ServiceBusReceivedMessage,
|
|
16
|
+
ServiceBusReceiver,
|
|
17
|
+
} from "@azure/service-bus";
|
|
12
18
|
import type { PubSubTriggerOpts } from "@blokjs/helper";
|
|
13
19
|
import { v4 as uuid } from "uuid";
|
|
14
20
|
import type { PubSubAdapter, PubSubMessage } from "../PubSubTrigger";
|
|
@@ -27,11 +33,22 @@ export interface AzureServiceBusConfig {
|
|
|
27
33
|
export class AzureServiceBusAdapter implements PubSubAdapter {
|
|
28
34
|
readonly provider = "azure" as const;
|
|
29
35
|
|
|
30
|
-
private client:
|
|
31
|
-
private receivers: Map<string,
|
|
36
|
+
private client: ServiceBusClient | undefined;
|
|
37
|
+
private receivers: Map<string, ServiceBusReceiver> = new Map();
|
|
32
38
|
private connected = false;
|
|
33
39
|
private config: AzureServiceBusConfig;
|
|
34
40
|
|
|
41
|
+
/**
|
|
42
|
+
* Type-narrowing accessor for `this.client`. Field is undefined until
|
|
43
|
+
* `connect()` runs.
|
|
44
|
+
*/
|
|
45
|
+
private requireClient(): ServiceBusClient {
|
|
46
|
+
if (!this.client) {
|
|
47
|
+
throw new Error("[AzureServiceBusAdapter] client is not initialised — call connect() first");
|
|
48
|
+
}
|
|
49
|
+
return this.client;
|
|
50
|
+
}
|
|
51
|
+
|
|
35
52
|
constructor(config?: AzureServiceBusConfig) {
|
|
36
53
|
this.config = {
|
|
37
54
|
connectionString: config?.connectionString || process.env.AZURE_SERVICE_BUS_CONNECTION_STRING,
|
|
@@ -63,8 +80,7 @@ export class AzureServiceBusAdapter implements PubSubAdapter {
|
|
|
63
80
|
console.log("[AzureServiceBusAdapter] Connected to Azure Service Bus");
|
|
64
81
|
} catch (error) {
|
|
65
82
|
throw new Error(
|
|
66
|
-
`Failed to connect to Azure Service Bus: ${(error as Error).message}.
|
|
67
|
-
`Make sure @azure/service-bus is installed: npm install @azure/service-bus`,
|
|
83
|
+
`Failed to connect to Azure Service Bus: ${(error as Error).message}. Make sure @azure/service-bus is installed: npm install @azure/service-bus`,
|
|
68
84
|
);
|
|
69
85
|
}
|
|
70
86
|
}
|
|
@@ -82,7 +98,7 @@ export class AzureServiceBusAdapter implements PubSubAdapter {
|
|
|
82
98
|
}
|
|
83
99
|
this.receivers.clear();
|
|
84
100
|
|
|
85
|
-
await this.
|
|
101
|
+
await this.requireClient().close();
|
|
86
102
|
this.connected = false;
|
|
87
103
|
console.log("[AzureServiceBusAdapter] Disconnected from Azure Service Bus");
|
|
88
104
|
} catch (error) {
|
|
@@ -98,17 +114,18 @@ export class AzureServiceBusAdapter implements PubSubAdapter {
|
|
|
98
114
|
throw new Error("Not connected to Azure Service Bus. Call connect() first.");
|
|
99
115
|
}
|
|
100
116
|
|
|
101
|
-
|
|
117
|
+
const client = this.requireClient();
|
|
118
|
+
let receiver: ServiceBusReceiver;
|
|
102
119
|
|
|
103
120
|
// Determine if this is a topic subscription or a queue
|
|
104
121
|
if (config.subscription && config.topic) {
|
|
105
122
|
// Topic with subscription
|
|
106
|
-
receiver =
|
|
123
|
+
receiver = client.createReceiver(config.topic, config.subscription, {
|
|
107
124
|
receiveMode: config.ack !== false ? "peekLock" : "receiveAndDelete",
|
|
108
125
|
});
|
|
109
126
|
} else {
|
|
110
127
|
// Queue
|
|
111
|
-
receiver =
|
|
128
|
+
receiver = client.createReceiver(config.subscription || config.topic, {
|
|
112
129
|
receiveMode: config.ack !== false ? "peekLock" : "receiveAndDelete",
|
|
113
130
|
});
|
|
114
131
|
}
|
|
@@ -116,7 +133,7 @@ export class AzureServiceBusAdapter implements PubSubAdapter {
|
|
|
116
133
|
const subscriptionKey = `${config.topic}/${config.subscription}`;
|
|
117
134
|
|
|
118
135
|
// Message handler
|
|
119
|
-
const processMessage = async (sbMessage:
|
|
136
|
+
const processMessage = async (sbMessage: ServiceBusReceivedMessage) => {
|
|
120
137
|
// Parse message body
|
|
121
138
|
let body: unknown;
|
|
122
139
|
try {
|
|
@@ -139,7 +156,8 @@ export class AzureServiceBusAdapter implements PubSubAdapter {
|
|
|
139
156
|
|
|
140
157
|
// Create pub/sub message
|
|
141
158
|
const pubsubMessage: PubSubMessage = {
|
|
142
|
-
|
|
159
|
+
// `messageId` may be string | number | Buffer per Azure SDK; coerce to string.
|
|
160
|
+
id: sbMessage.messageId !== undefined ? String(sbMessage.messageId) : uuid(),
|
|
143
161
|
body,
|
|
144
162
|
attributes,
|
|
145
163
|
raw: sbMessage,
|
|
@@ -162,9 +180,10 @@ export class AzureServiceBusAdapter implements PubSubAdapter {
|
|
|
162
180
|
}
|
|
163
181
|
};
|
|
164
182
|
|
|
165
|
-
// Error handler
|
|
166
|
-
|
|
167
|
-
|
|
183
|
+
// Error handler — Azure SDK passes a `ProcessErrorArgs` with the
|
|
184
|
+
// underlying error inside `args.error` plus contextual fields.
|
|
185
|
+
const processError = async (args: ProcessErrorArgs) => {
|
|
186
|
+
console.error(`[AzureServiceBusAdapter] Error: ${args.error.message}`);
|
|
168
187
|
};
|
|
169
188
|
|
|
170
189
|
// Subscribe to messages
|
|
@@ -197,6 +216,30 @@ export class AzureServiceBusAdapter implements PubSubAdapter {
|
|
|
197
216
|
return this.connected;
|
|
198
217
|
}
|
|
199
218
|
|
|
219
|
+
/**
|
|
220
|
+
* v0.7 PR 6 — publish to an Azure Service Bus topic.
|
|
221
|
+
*
|
|
222
|
+
* `partitionKey` maps to Service Bus's `partitionKey`; `orderingKey`
|
|
223
|
+
* maps to `sessionId` (for session-enabled topics).
|
|
224
|
+
*/
|
|
225
|
+
async publish(
|
|
226
|
+
topic: string,
|
|
227
|
+
payload: unknown,
|
|
228
|
+
opts?: { partitionKey?: string; orderingKey?: string },
|
|
229
|
+
): Promise<void> {
|
|
230
|
+
if (!this.connected) throw new Error("[blok][pubsub-azure] not connected. Call connect() first.");
|
|
231
|
+
const sender = this.requireClient().createSender(topic);
|
|
232
|
+
try {
|
|
233
|
+
await sender.sendMessages({
|
|
234
|
+
body: payload,
|
|
235
|
+
partitionKey: opts?.partitionKey,
|
|
236
|
+
sessionId: opts?.orderingKey,
|
|
237
|
+
});
|
|
238
|
+
} finally {
|
|
239
|
+
await sender.close();
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
|
|
200
243
|
/**
|
|
201
244
|
* Health check - verify Azure Service Bus connectivity
|
|
202
245
|
*/
|
|
@@ -205,7 +248,7 @@ export class AzureServiceBusAdapter implements PubSubAdapter {
|
|
|
205
248
|
|
|
206
249
|
try {
|
|
207
250
|
// Create a temporary receiver to test connectivity
|
|
208
|
-
const testReceiver = this.
|
|
251
|
+
const testReceiver = this.requireClient().createReceiver("$default", {
|
|
209
252
|
receiveMode: "peekLock",
|
|
210
253
|
});
|
|
211
254
|
await testReceiver.close();
|
|
@@ -11,6 +11,7 @@
|
|
|
11
11
|
*/
|
|
12
12
|
|
|
13
13
|
import type { PubSubTriggerOpts } from "@blokjs/helper";
|
|
14
|
+
import type { Message, PubSub, Subscription } from "@google-cloud/pubsub";
|
|
14
15
|
import { v4 as uuid } from "uuid";
|
|
15
16
|
import type { PubSubAdapter, PubSubMessage } from "../PubSubTrigger";
|
|
16
17
|
|
|
@@ -31,11 +32,22 @@ export interface GCPPubSubConfig {
|
|
|
31
32
|
export class GCPPubSubAdapter implements PubSubAdapter {
|
|
32
33
|
readonly provider = "gcp" as const;
|
|
33
34
|
|
|
34
|
-
private client:
|
|
35
|
-
private subscriptions: Map<string,
|
|
35
|
+
private client: PubSub | undefined;
|
|
36
|
+
private subscriptions: Map<string, Subscription> = new Map();
|
|
36
37
|
private connected = false;
|
|
37
38
|
private config: GCPPubSubConfig;
|
|
38
39
|
|
|
40
|
+
/**
|
|
41
|
+
* Type-narrowing accessor for `this.client`. Field is undefined until
|
|
42
|
+
* `connect()` runs.
|
|
43
|
+
*/
|
|
44
|
+
private requireClient(): PubSub {
|
|
45
|
+
if (!this.client) {
|
|
46
|
+
throw new Error("[GCPPubSubAdapter] client is not initialised — call connect() first");
|
|
47
|
+
}
|
|
48
|
+
return this.client;
|
|
49
|
+
}
|
|
50
|
+
|
|
39
51
|
constructor(config?: GCPPubSubConfig) {
|
|
40
52
|
this.config = {
|
|
41
53
|
projectId: config?.projectId || process.env.GOOGLE_CLOUD_PROJECT,
|
|
@@ -62,8 +74,7 @@ export class GCPPubSubAdapter implements PubSubAdapter {
|
|
|
62
74
|
console.log(`[GCPPubSubAdapter] Connected to GCP Pub/Sub: ${this.config.projectId}`);
|
|
63
75
|
} catch (error) {
|
|
64
76
|
throw new Error(
|
|
65
|
-
`Failed to connect to GCP Pub/Sub: ${(error as Error).message}.
|
|
66
|
-
`Make sure @google-cloud/pubsub is installed: npm install @google-cloud/pubsub`,
|
|
77
|
+
`Failed to connect to GCP Pub/Sub: ${(error as Error).message}. Make sure @google-cloud/pubsub is installed: npm install @google-cloud/pubsub`,
|
|
67
78
|
);
|
|
68
79
|
}
|
|
69
80
|
}
|
|
@@ -81,7 +92,7 @@ export class GCPPubSubAdapter implements PubSubAdapter {
|
|
|
81
92
|
}
|
|
82
93
|
this.subscriptions.clear();
|
|
83
94
|
|
|
84
|
-
await this.
|
|
95
|
+
await this.requireClient().close();
|
|
85
96
|
this.connected = false;
|
|
86
97
|
console.log("[GCPPubSubAdapter] Disconnected from GCP Pub/Sub");
|
|
87
98
|
} catch (error) {
|
|
@@ -97,18 +108,26 @@ export class GCPPubSubAdapter implements PubSubAdapter {
|
|
|
97
108
|
throw new Error("Not connected to GCP Pub/Sub. Call connect() first.");
|
|
98
109
|
}
|
|
99
110
|
|
|
111
|
+
if (!config.subscription) {
|
|
112
|
+
throw new Error("[GCPPubSubAdapter] `subscription` is required — must be the GCP subscription name.");
|
|
113
|
+
}
|
|
100
114
|
const subscriptionName = config.subscription;
|
|
101
115
|
|
|
102
|
-
// Get the subscription
|
|
103
|
-
|
|
116
|
+
// Get the subscription. The GCP SDK calls the per-subscription
|
|
117
|
+
// deadline `maxAckDeadline` (a `Duration`); our author-facing
|
|
118
|
+
// field is `ackDeadline` (seconds) for parity with other
|
|
119
|
+
// providers. `Duration` is the GCP SDK's tc39-Temporal-shaped
|
|
120
|
+
// shim — construct via `Duration.from({seconds})`.
|
|
121
|
+
const { Duration } = await import("@google-cloud/pubsub");
|
|
122
|
+
const subscription = this.requireClient().subscription(subscriptionName, {
|
|
104
123
|
flowControl: {
|
|
105
124
|
maxMessages: config.maxMessages || 10,
|
|
106
125
|
},
|
|
107
|
-
|
|
126
|
+
maxAckDeadline: Duration.from({ seconds: config.ackDeadline || 30 }),
|
|
108
127
|
});
|
|
109
128
|
|
|
110
129
|
// Message handler
|
|
111
|
-
const messageHandler = async (gcpMessage:
|
|
130
|
+
const messageHandler = async (gcpMessage: Message) => {
|
|
112
131
|
// Parse message data
|
|
113
132
|
let body: unknown;
|
|
114
133
|
try {
|
|
@@ -177,6 +196,27 @@ export class GCPPubSubAdapter implements PubSubAdapter {
|
|
|
177
196
|
return this.connected;
|
|
178
197
|
}
|
|
179
198
|
|
|
199
|
+
/**
|
|
200
|
+
* v0.7 PR 6 — publish to a GCP Pub/Sub topic.
|
|
201
|
+
*
|
|
202
|
+
* `partitionKey` / `orderingKey` map to GCP's `orderingKey` (the
|
|
203
|
+
* topic must have message ordering enabled — otherwise the field
|
|
204
|
+
* is ignored by the broker).
|
|
205
|
+
*/
|
|
206
|
+
async publish(
|
|
207
|
+
topic: string,
|
|
208
|
+
payload: unknown,
|
|
209
|
+
opts?: { partitionKey?: string; orderingKey?: string },
|
|
210
|
+
): Promise<void> {
|
|
211
|
+
if (!this.connected) throw new Error("[blok][pubsub-gcp] not connected. Call connect() first.");
|
|
212
|
+
const body = typeof payload === "string" ? payload : JSON.stringify(payload);
|
|
213
|
+
const t = this.requireClient().topic(topic);
|
|
214
|
+
await t.publishMessage({
|
|
215
|
+
data: Buffer.from(body),
|
|
216
|
+
orderingKey: opts?.orderingKey ?? opts?.partitionKey,
|
|
217
|
+
});
|
|
218
|
+
}
|
|
219
|
+
|
|
180
220
|
/**
|
|
181
221
|
* Health check - verify GCP Pub/Sub connectivity
|
|
182
222
|
*/
|
|
@@ -185,7 +225,7 @@ export class GCPPubSubAdapter implements PubSubAdapter {
|
|
|
185
225
|
|
|
186
226
|
try {
|
|
187
227
|
// List subscriptions as a health check
|
|
188
|
-
const [subscriptions] = await this.
|
|
228
|
+
const [subscriptions] = await this.requireClient().getSubscriptions({ pageSize: 1 });
|
|
189
229
|
return true;
|
|
190
230
|
} catch {
|
|
191
231
|
return false;
|
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* KafkaPubSubAdapter — v0.7 PR 6 — Pub/Sub adapter backed by Apache
|
|
3
|
+
* Kafka via `kafkajs`.
|
|
4
|
+
*
|
|
5
|
+
* Pub/Sub vs Worker semantics: this adapter uses **per-subscriber
|
|
6
|
+
* consumer groups** so multiple subscribers each receive every message
|
|
7
|
+
* (fan-out). When `consumerGroup` is explicitly set, all subscribers
|
|
8
|
+
* share that group and compete (1 of N gets each message).
|
|
9
|
+
*
|
|
10
|
+
* Replay cursors via `startFrom`:
|
|
11
|
+
* - `"earliest"` → `fromBeginning: true` on first subscribe.
|
|
12
|
+
* - `"latest"` (default) → only new messages.
|
|
13
|
+
* - `{seq: N}` / `{timestamp: ms}` — provider-specific. Cleanest
|
|
14
|
+
* long-term path is Kafka's `admin.seek({offset|timestamp})` post-
|
|
15
|
+
* subscribe; v1 honors `earliest` / `latest` and `{seq}` via
|
|
16
|
+
* `auto.offset.reset` only.
|
|
17
|
+
*
|
|
18
|
+
* Requires `kafkajs` as a peer dependency.
|
|
19
|
+
*
|
|
20
|
+
* Environment variables:
|
|
21
|
+
* - `KAFKA_BROKERS` — comma-separated (default `localhost:9092`).
|
|
22
|
+
* - `KAFKA_CLIENT_ID` — default `"blok-pubsub"`.
|
|
23
|
+
* - `KAFKA_SASL_USERNAME` / `KAFKA_SASL_PASSWORD` — SASL/PLAIN.
|
|
24
|
+
* - `KAFKA_SSL` — `"true"` enables TLS.
|
|
25
|
+
*/
|
|
26
|
+
|
|
27
|
+
import type { PubSubTriggerOpts } from "@blokjs/helper";
|
|
28
|
+
import { v4 as uuid } from "uuid";
|
|
29
|
+
import type { PubSubAdapter, PubSubMessage } from "../PubSubTrigger";
|
|
30
|
+
|
|
31
|
+
export interface KafkaPubSubConfig {
|
|
32
|
+
brokers: string[];
|
|
33
|
+
clientId: string;
|
|
34
|
+
saslUsername?: string;
|
|
35
|
+
saslPassword?: string;
|
|
36
|
+
ssl: boolean;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
interface KafkaConsumerHandle {
|
|
40
|
+
disconnect: () => Promise<void>;
|
|
41
|
+
stop: () => Promise<void>;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export class KafkaPubSubAdapter implements PubSubAdapter {
|
|
45
|
+
readonly provider = "kafka" as const;
|
|
46
|
+
private readonly config: KafkaPubSubConfig;
|
|
47
|
+
// biome-ignore lint/suspicious/noExplicitAny: kafkajs's exported `Kafka` constructor is loosely typed.
|
|
48
|
+
private kafka: any = null;
|
|
49
|
+
// biome-ignore lint/suspicious/noExplicitAny: producer is created lazily and typed loosely.
|
|
50
|
+
private producer: any = null;
|
|
51
|
+
private consumers: Map<string, KafkaConsumerHandle> = new Map();
|
|
52
|
+
private connected = false;
|
|
53
|
+
|
|
54
|
+
constructor(config?: Partial<KafkaPubSubConfig>) {
|
|
55
|
+
this.config = {
|
|
56
|
+
brokers: config?.brokers ?? (process.env.KAFKA_BROKERS ?? "localhost:9092").split(",").map((s) => s.trim()),
|
|
57
|
+
clientId: config?.clientId ?? process.env.KAFKA_CLIENT_ID ?? "blok-pubsub",
|
|
58
|
+
saslUsername: config?.saslUsername ?? process.env.KAFKA_SASL_USERNAME,
|
|
59
|
+
saslPassword: config?.saslPassword ?? process.env.KAFKA_SASL_PASSWORD,
|
|
60
|
+
ssl: config?.ssl ?? process.env.KAFKA_SSL === "true",
|
|
61
|
+
};
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
async connect(): Promise<void> {
|
|
65
|
+
if (this.connected) return;
|
|
66
|
+
try {
|
|
67
|
+
// biome-ignore lint/suspicious/noExplicitAny: kafkajs is a runtime peer dep.
|
|
68
|
+
const kafkajs: any = await import("kafkajs");
|
|
69
|
+
const sasl =
|
|
70
|
+
this.config.saslUsername && this.config.saslPassword
|
|
71
|
+
? { mechanism: "plain", username: this.config.saslUsername, password: this.config.saslPassword }
|
|
72
|
+
: undefined;
|
|
73
|
+
this.kafka = new kafkajs.Kafka({
|
|
74
|
+
clientId: this.config.clientId,
|
|
75
|
+
brokers: this.config.brokers,
|
|
76
|
+
ssl: this.config.ssl,
|
|
77
|
+
sasl,
|
|
78
|
+
});
|
|
79
|
+
this.producer = this.kafka.producer();
|
|
80
|
+
await this.producer.connect();
|
|
81
|
+
this.connected = true;
|
|
82
|
+
} catch (err) {
|
|
83
|
+
throw new Error(
|
|
84
|
+
`[blok][pubsub-kafka] connect failed: ${(err as Error).message}. Install kafkajs as a peer dependency: bun add kafkajs`,
|
|
85
|
+
);
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
async disconnect(): Promise<void> {
|
|
90
|
+
if (!this.connected) return;
|
|
91
|
+
for (const consumer of this.consumers.values()) {
|
|
92
|
+
try {
|
|
93
|
+
await consumer.disconnect();
|
|
94
|
+
} catch {
|
|
95
|
+
/* ignore */
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
this.consumers.clear();
|
|
99
|
+
try {
|
|
100
|
+
await this.producer?.disconnect();
|
|
101
|
+
} catch {
|
|
102
|
+
/* ignore */
|
|
103
|
+
}
|
|
104
|
+
this.producer = null;
|
|
105
|
+
this.connected = false;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
async subscribe(config: PubSubTriggerOpts, handler: (message: PubSubMessage) => Promise<void>): Promise<void> {
|
|
109
|
+
if (!this.connected) throw new Error("[blok][pubsub-kafka] not connected. Call connect() first.");
|
|
110
|
+
// Fan-out: distinct group id per subscriber instance. Competing-
|
|
111
|
+
// consumer: explicit consumerGroup shared across all subscribers.
|
|
112
|
+
const groupId =
|
|
113
|
+
config.consumerGroup ?? `blok-fanout-${uuid().slice(0, 8)}-${config.topic.replace(/[^a-zA-Z0-9_]/g, "_")}`;
|
|
114
|
+
const consumer = this.kafka.consumer({ groupId });
|
|
115
|
+
await consumer.connect();
|
|
116
|
+
const fromBeginning = config.startFrom === "earliest";
|
|
117
|
+
await consumer.subscribe({ topic: config.topic, fromBeginning });
|
|
118
|
+
this.consumers.set(`${config.topic}#${groupId}`, consumer);
|
|
119
|
+
|
|
120
|
+
await consumer.run({
|
|
121
|
+
autoCommit: config.ack !== false,
|
|
122
|
+
eachMessage: async ({
|
|
123
|
+
message,
|
|
124
|
+
}: {
|
|
125
|
+
message: { key?: Buffer; value?: Buffer; offset: string; timestamp: string; headers?: Record<string, Buffer> };
|
|
126
|
+
}) => {
|
|
127
|
+
const text = message.value?.toString("utf8") ?? "";
|
|
128
|
+
let body: unknown = text;
|
|
129
|
+
try {
|
|
130
|
+
body = text.length > 0 ? JSON.parse(text) : null;
|
|
131
|
+
} catch {
|
|
132
|
+
/* leave as text */
|
|
133
|
+
}
|
|
134
|
+
const attributes: Record<string, string> = {};
|
|
135
|
+
if (message.headers) {
|
|
136
|
+
for (const [k, v] of Object.entries(message.headers)) attributes[k] = v?.toString("utf8") ?? "";
|
|
137
|
+
}
|
|
138
|
+
const msg: PubSubMessage = {
|
|
139
|
+
id: `${config.topic}:${message.offset}`,
|
|
140
|
+
body,
|
|
141
|
+
attributes,
|
|
142
|
+
raw: message,
|
|
143
|
+
topic: config.topic,
|
|
144
|
+
subscription: groupId,
|
|
145
|
+
publishTime: new Date(Number.parseInt(message.timestamp, 10)),
|
|
146
|
+
ack: async () => {
|
|
147
|
+
/* autoCommit handles ack */
|
|
148
|
+
},
|
|
149
|
+
nack: async () => {
|
|
150
|
+
/* kafkajs has no per-message nack; the offset commit is suppressed by throwing */
|
|
151
|
+
},
|
|
152
|
+
};
|
|
153
|
+
// On handler failure: re-throw so kafkajs suppresses the
|
|
154
|
+
// offset commit; the consumer will re-poll the message
|
|
155
|
+
// on the next cycle. The throw is intentional.
|
|
156
|
+
await handler(msg);
|
|
157
|
+
},
|
|
158
|
+
});
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
async unsubscribe(subscription: string): Promise<void> {
|
|
162
|
+
const consumer = this.consumers.get(subscription);
|
|
163
|
+
if (!consumer) return;
|
|
164
|
+
try {
|
|
165
|
+
await consumer.stop();
|
|
166
|
+
await consumer.disconnect();
|
|
167
|
+
} catch {
|
|
168
|
+
/* ignore */
|
|
169
|
+
}
|
|
170
|
+
this.consumers.delete(subscription);
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
async publish(
|
|
174
|
+
topic: string,
|
|
175
|
+
payload: unknown,
|
|
176
|
+
opts?: { partitionKey?: string; orderingKey?: string },
|
|
177
|
+
): Promise<void> {
|
|
178
|
+
if (!this.connected || !this.producer) throw new Error("[blok][pubsub-kafka] not connected. Call connect() first.");
|
|
179
|
+
const body = typeof payload === "string" ? payload : JSON.stringify(payload);
|
|
180
|
+
const key = opts?.partitionKey ?? opts?.orderingKey;
|
|
181
|
+
await this.producer.send({
|
|
182
|
+
topic,
|
|
183
|
+
messages: [{ key, value: body }],
|
|
184
|
+
});
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
isConnected(): boolean {
|
|
188
|
+
return this.connected;
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
async healthCheck(): Promise<boolean> {
|
|
192
|
+
return this.connected;
|
|
193
|
+
}
|
|
194
|
+
}
|