@blokjs/trigger-pubsub 0.2.3 → 0.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/__tests__/integration/gcp-pubsub.real-emulator.test.ts +235 -0
- package/__tests__/integration/kafka-pubsub.real-kafka.test.ts +269 -0
- package/__tests__/integration/nats-pubsub.real-nats.test.ts +138 -0
- package/dist/PubSubTrigger.d.ts +43 -3
- package/dist/PubSubTrigger.js +70 -16
- package/dist/adapters/AWSSNSAdapter.d.ts +16 -0
- package/dist/adapters/AWSSNSAdapter.js +52 -9
- package/dist/adapters/AzureServiceBusAdapter.d.ts +15 -0
- package/dist/adapters/AzureServiceBusAdapter.js +44 -11
- package/dist/adapters/GCPPubSubAdapter.d.ts +16 -0
- package/dist/adapters/GCPPubSubAdapter.js +42 -8
- package/dist/adapters/KafkaPubSubAdapter.d.ts +53 -0
- package/dist/adapters/KafkaPubSubAdapter.js +168 -0
- package/dist/adapters/NATSPubSubAdapter.d.ts +52 -0
- package/dist/adapters/NATSPubSubAdapter.js +260 -0
- package/dist/adapters/RedisStreamsPubSubAdapter.d.ts +49 -0
- package/dist/adapters/RedisStreamsPubSubAdapter.js +193 -0
- package/dist/adapters/factory.d.ts +22 -0
- package/dist/adapters/factory.js +80 -0
- package/dist/index.d.ts +36 -45
- package/dist/index.js +39 -46
- package/package.json +22 -10
- package/src/PubSubTrigger.ts +84 -18
- package/src/adapters/AWSSNSAdapter.ts +76 -12
- package/src/adapters/AzureServiceBusAdapter.ts +57 -14
- package/src/adapters/GCPPubSubAdapter.ts +50 -10
- package/src/adapters/KafkaPubSubAdapter.ts +194 -0
- package/src/adapters/NATSPubSubAdapter.ts +326 -0
- package/src/adapters/RedisStreamsPubSubAdapter.ts +225 -0
- package/src/adapters/factory.test.ts +87 -0
- package/src/adapters/factory.ts +88 -0
- package/src/adapters/new-adapters.test.ts +108 -0
- package/src/index.ts +40 -41
- package/template/package.json +6 -6
- package/template/src/runner/PubSubServer.ts +2 -2
- package/template/src/workflows/messages/on-message.ts +38 -34
|
@@ -0,0 +1,235 @@
|
|
|
1
|
+
import type { PubSubMessage } from "@blokjs/runner";
|
|
2
|
+
import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";
|
|
3
|
+
import { GCPPubSubAdapter } from "../../src/adapters/GCPPubSubAdapter";
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Real-GCP-emulator integration test for `GCPPubSubAdapter` (closes
|
|
7
|
+
* Phase 2.1 broker-adapter test debt deferred from PR #91).
|
|
8
|
+
*
|
|
9
|
+
* Gated on `BLOK_INTEGRATION_GCP_PUBSUB_ENDPOINT`. Skipped when unset.
|
|
10
|
+
*
|
|
11
|
+
* Bring up the emulator via:
|
|
12
|
+
* docker compose -f infra/testing/docker-compose.yml up -d pubsub-emulator
|
|
13
|
+
*
|
|
14
|
+
* Then run:
|
|
15
|
+
* BLOK_INTEGRATION_GCP_PUBSUB_ENDPOINT=localhost:8086 \
|
|
16
|
+
* PUBSUB_EMULATOR_HOST=localhost:8086 \
|
|
17
|
+
* PUBSUB_PROJECT_ID=blok-test \
|
|
18
|
+
* bun run test
|
|
19
|
+
*
|
|
20
|
+
* Note: the GCP SDK's `PubSub` client only routes to the emulator when
|
|
21
|
+
* `PUBSUB_EMULATOR_HOST` is set BEFORE import. We set it in beforeAll
|
|
22
|
+
* to keep developers from having to remember the dance.
|
|
23
|
+
*/
|
|
24
|
+
|
|
25
|
+
const GCP_PUBSUB_ENDPOINT = process.env.BLOK_INTEGRATION_GCP_PUBSUB_ENDPOINT;
|
|
26
|
+
const d = GCP_PUBSUB_ENDPOINT ? describe : describe.skip;
|
|
27
|
+
|
|
28
|
+
const TEST_TIMEOUT_MS = 30_000;
|
|
29
|
+
const TEST_PROJECT = "blok-test";
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* Narrow shape for the bits of `@google-cloud/pubsub` we drive directly
|
|
33
|
+
* from this test (topic + subscription creation). Keeps us off `any`
|
|
34
|
+
* per the repo's no-`any`-in-tests rule.
|
|
35
|
+
*/
|
|
36
|
+
interface GcpTopic {
|
|
37
|
+
createSubscription(name: string): Promise<unknown>;
|
|
38
|
+
delete(): Promise<unknown>;
|
|
39
|
+
get(opts?: { autoCreate?: boolean }): Promise<unknown>;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
interface GcpSubscription {
|
|
43
|
+
delete(): Promise<unknown>;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
interface GcpPubSubClient {
|
|
47
|
+
topic(name: string): GcpTopic;
|
|
48
|
+
subscription(name: string): GcpSubscription;
|
|
49
|
+
createTopic(name: string): Promise<[GcpTopic, unknown]>;
|
|
50
|
+
close(): Promise<void>;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
interface GcpPubSubModule {
|
|
54
|
+
PubSub: new (opts: { projectId: string }) => GcpPubSubClient;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
d("GCPPubSubAdapter — real GCP Pub/Sub emulator", () => {
|
|
58
|
+
let adapter: GCPPubSubAdapter;
|
|
59
|
+
let testClient: GcpPubSubClient | null = null;
|
|
60
|
+
const createdTopics: string[] = [];
|
|
61
|
+
const createdSubscriptions: string[] = [];
|
|
62
|
+
|
|
63
|
+
beforeAll(async () => {
|
|
64
|
+
// Both the SDK and the adapter look at PUBSUB_EMULATOR_HOST — set
|
|
65
|
+
// it BEFORE importing to make sure the gRPC client routes to the
|
|
66
|
+
// emulator instead of trying production endpoints (and waiting
|
|
67
|
+
// for credentials we don't have).
|
|
68
|
+
process.env.PUBSUB_EMULATOR_HOST = GCP_PUBSUB_ENDPOINT;
|
|
69
|
+
process.env.PUBSUB_PROJECT_ID = TEST_PROJECT;
|
|
70
|
+
process.env.GOOGLE_CLOUD_PROJECT = TEST_PROJECT;
|
|
71
|
+
|
|
72
|
+
adapter = new GCPPubSubAdapter({ projectId: TEST_PROJECT });
|
|
73
|
+
await adapter.connect();
|
|
74
|
+
|
|
75
|
+
// Direct SDK client used only to create / tear down topics +
|
|
76
|
+
// subscriptions. The adapter doesn't expose these (production
|
|
77
|
+
// users provision them ahead of time via Terraform / gcloud).
|
|
78
|
+
const sdk = (await import("@google-cloud/pubsub")) as unknown as GcpPubSubModule;
|
|
79
|
+
testClient = new sdk.PubSub({ projectId: TEST_PROJECT });
|
|
80
|
+
}, TEST_TIMEOUT_MS);
|
|
81
|
+
|
|
82
|
+
async function createTopicAndSub(topicName: string, subscriptionName: string): Promise<void> {
|
|
83
|
+
if (!testClient) throw new Error("test client not initialised — beforeAll didn't run");
|
|
84
|
+
const [topic] = await testClient.createTopic(topicName);
|
|
85
|
+
createdTopics.push(topicName);
|
|
86
|
+
await topic.createSubscription(subscriptionName);
|
|
87
|
+
createdSubscriptions.push(subscriptionName);
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
afterEach(async () => {
|
|
91
|
+
// Tear down per-test resources so a flaky test doesn't poison
|
|
92
|
+
// the next one with stale messages or shared subscriptions.
|
|
93
|
+
for (const name of createdSubscriptions.splice(0)) {
|
|
94
|
+
try {
|
|
95
|
+
await testClient?.subscription(name).delete();
|
|
96
|
+
} catch {
|
|
97
|
+
/* ignore — best-effort */
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
for (const name of createdTopics.splice(0)) {
|
|
101
|
+
try {
|
|
102
|
+
await testClient?.topic(name).delete();
|
|
103
|
+
} catch {
|
|
104
|
+
/* ignore */
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
});
|
|
108
|
+
|
|
109
|
+
afterAll(async () => {
|
|
110
|
+
await adapter.disconnect();
|
|
111
|
+
try {
|
|
112
|
+
await testClient?.close();
|
|
113
|
+
} catch {
|
|
114
|
+
/* ignore */
|
|
115
|
+
}
|
|
116
|
+
});
|
|
117
|
+
|
|
118
|
+
it(
|
|
119
|
+
"publishes a message and the subscriber receives it",
|
|
120
|
+
async () => {
|
|
121
|
+
const topic = `blok-test-gcp-publish-${Math.random().toString(36).slice(2)}`;
|
|
122
|
+
const subscription = `${topic}-sub`;
|
|
123
|
+
await createTopicAndSub(topic, subscription);
|
|
124
|
+
|
|
125
|
+
const received: PubSubMessage[] = [];
|
|
126
|
+
await adapter.subscribe({ topic, subscription, durable: true }, async (msg) => {
|
|
127
|
+
received.push(msg);
|
|
128
|
+
await msg.ack();
|
|
129
|
+
});
|
|
130
|
+
|
|
131
|
+
// Pub/Sub subscriptions can take a moment to register the
|
|
132
|
+
// streaming pull connection. Give it a short warm-up so the
|
|
133
|
+
// first publish lands after the subscriber is actively pulling.
|
|
134
|
+
await new Promise((r) => setTimeout(r, 500));
|
|
135
|
+
|
|
136
|
+
await adapter.publish(topic, { hello: "gcp", n: 1 });
|
|
137
|
+
|
|
138
|
+
await waitFor(() => received.length === 1, TEST_TIMEOUT_MS - 5_000);
|
|
139
|
+
|
|
140
|
+
expect(received[0].body).toEqual({ hello: "gcp", n: 1 });
|
|
141
|
+
expect(received[0].topic).toBe(topic);
|
|
142
|
+
expect(received[0].subscription).toBe(subscription);
|
|
143
|
+
|
|
144
|
+
await adapter.unsubscribe(subscription);
|
|
145
|
+
},
|
|
146
|
+
TEST_TIMEOUT_MS,
|
|
147
|
+
);
|
|
148
|
+
|
|
149
|
+
it(
|
|
150
|
+
"delivers multiple messages to a single subscriber in publish order",
|
|
151
|
+
async () => {
|
|
152
|
+
const topic = `blok-test-gcp-order-${Math.random().toString(36).slice(2)}`;
|
|
153
|
+
const subscription = `${topic}-sub`;
|
|
154
|
+
await createTopicAndSub(topic, subscription);
|
|
155
|
+
|
|
156
|
+
const received: number[] = [];
|
|
157
|
+
await adapter.subscribe({ topic, subscription, durable: true }, async (msg) => {
|
|
158
|
+
const body = msg.body as { n: number };
|
|
159
|
+
received.push(body.n);
|
|
160
|
+
await msg.ack();
|
|
161
|
+
});
|
|
162
|
+
await new Promise((r) => setTimeout(r, 500));
|
|
163
|
+
|
|
164
|
+
for (let i = 0; i < 5; i++) {
|
|
165
|
+
await adapter.publish(topic, { n: i });
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
await waitFor(() => received.length === 5, TEST_TIMEOUT_MS - 5_000);
|
|
169
|
+
|
|
170
|
+
// GCP Pub/Sub doesn't guarantee order across a topic without
|
|
171
|
+
// `messageOrdering: true` + an `orderingKey`. Assert the set
|
|
172
|
+
// of values, not the sequence.
|
|
173
|
+
const seen = new Set(received);
|
|
174
|
+
expect(seen.size).toBe(5);
|
|
175
|
+
for (let i = 0; i < 5; i++) {
|
|
176
|
+
expect(seen.has(i)).toBe(true);
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
await adapter.unsubscribe(subscription);
|
|
180
|
+
},
|
|
181
|
+
TEST_TIMEOUT_MS,
|
|
182
|
+
);
|
|
183
|
+
|
|
184
|
+
it(
|
|
185
|
+
"isolates subscriptions across topics",
|
|
186
|
+
async () => {
|
|
187
|
+
const topicA = `blok-test-gcp-iso-a-${Math.random().toString(36).slice(2)}`;
|
|
188
|
+
const topicB = `blok-test-gcp-iso-b-${Math.random().toString(36).slice(2)}`;
|
|
189
|
+
const subA = `${topicA}-sub`;
|
|
190
|
+
const subB = `${topicB}-sub`;
|
|
191
|
+
await createTopicAndSub(topicA, subA);
|
|
192
|
+
await createTopicAndSub(topicB, subB);
|
|
193
|
+
|
|
194
|
+
const receivedA: PubSubMessage[] = [];
|
|
195
|
+
const receivedB: PubSubMessage[] = [];
|
|
196
|
+
|
|
197
|
+
await adapter.subscribe({ topic: topicA, subscription: subA, durable: true }, async (msg) => {
|
|
198
|
+
receivedA.push(msg);
|
|
199
|
+
await msg.ack();
|
|
200
|
+
});
|
|
201
|
+
await adapter.subscribe({ topic: topicB, subscription: subB, durable: true }, async (msg) => {
|
|
202
|
+
receivedB.push(msg);
|
|
203
|
+
await msg.ack();
|
|
204
|
+
});
|
|
205
|
+
await new Promise((r) => setTimeout(r, 500));
|
|
206
|
+
|
|
207
|
+
await adapter.publish(topicA, { from: "A" });
|
|
208
|
+
await adapter.publish(topicB, { from: "B" });
|
|
209
|
+
|
|
210
|
+
await waitFor(() => receivedA.length === 1 && receivedB.length === 1, TEST_TIMEOUT_MS - 5_000);
|
|
211
|
+
|
|
212
|
+
expect(receivedA[0].body).toEqual({ from: "A" });
|
|
213
|
+
expect(receivedB[0].body).toEqual({ from: "B" });
|
|
214
|
+
|
|
215
|
+
await adapter.unsubscribe(subA);
|
|
216
|
+
await adapter.unsubscribe(subB);
|
|
217
|
+
},
|
|
218
|
+
TEST_TIMEOUT_MS,
|
|
219
|
+
);
|
|
220
|
+
|
|
221
|
+
it("isConnected + healthCheck reflect the live state", async () => {
|
|
222
|
+
expect(adapter.isConnected()).toBe(true);
|
|
223
|
+
const healthy = await adapter.healthCheck();
|
|
224
|
+
expect(healthy).toBe(true);
|
|
225
|
+
});
|
|
226
|
+
});
|
|
227
|
+
|
|
228
|
+
async function waitFor(predicate: () => boolean, timeoutMs: number): Promise<void> {
|
|
229
|
+
const start = Date.now();
|
|
230
|
+
while (Date.now() - start < timeoutMs) {
|
|
231
|
+
if (predicate()) return;
|
|
232
|
+
await new Promise((r) => setTimeout(r, 100));
|
|
233
|
+
}
|
|
234
|
+
throw new Error(`waitFor timed out after ${timeoutMs}ms`);
|
|
235
|
+
}
|
|
@@ -0,0 +1,269 @@
|
|
|
1
|
+
import type { PubSubMessage } from "@blokjs/runner";
|
|
2
|
+
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
|
3
|
+
import { KafkaPubSubAdapter } from "../../src/adapters/KafkaPubSubAdapter";
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Narrow shapes for the bits of `kafkajs` we touch in this test. The
|
|
7
|
+
* runtime types in the package are loosely-typed (the adapter itself
|
|
8
|
+
* pins `any` behind biome-ignore comments); these test-local interfaces
|
|
9
|
+
* keep us on the safe `as unknown as <T>` boundary-cast path required
|
|
10
|
+
* by the repo's no-`any`-in-tests rule.
|
|
11
|
+
*/
|
|
12
|
+
interface KafkaAdminClient {
|
|
13
|
+
connect(): Promise<void>;
|
|
14
|
+
disconnect(): Promise<void>;
|
|
15
|
+
createTopics(opts: {
|
|
16
|
+
waitForLeaders?: boolean;
|
|
17
|
+
topics: Array<{ topic: string; numPartitions: number; replicationFactor: number }>;
|
|
18
|
+
}): Promise<boolean>;
|
|
19
|
+
fetchTopicMetadata(opts: { topics: string[] }): Promise<unknown>;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
interface KafkaConsumerForWarmup {
|
|
23
|
+
connect(): Promise<void>;
|
|
24
|
+
subscribe(opts: { topic: string; fromBeginning?: boolean }): Promise<void>;
|
|
25
|
+
run(opts: { eachMessage: () => Promise<void> }): Promise<void>;
|
|
26
|
+
disconnect(): Promise<void>;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
interface KafkaClient {
|
|
30
|
+
admin(): KafkaAdminClient;
|
|
31
|
+
consumer(opts: { groupId: string }): KafkaConsumerForWarmup;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
interface KafkaJsModule {
|
|
35
|
+
Kafka: new (opts: { clientId: string; brokers: string[] }) => KafkaClient;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* Real-Kafka integration test for `KafkaPubSubAdapter` (closes Phase 2.1
|
|
40
|
+
* broker-adapter test debt deferred from PR #91).
|
|
41
|
+
*
|
|
42
|
+
* Exercises both the **fan-out** (no `consumerGroup` — adapter generates
|
|
43
|
+
* per-subscriber group ids) and **competing-consumer** (explicit shared
|
|
44
|
+
* `consumerGroup`) delivery patterns documented in the adapter.
|
|
45
|
+
*
|
|
46
|
+
* Gated on `BLOK_INTEGRATION_KAFKA_BROKERS`. Skipped when unset, so the
|
|
47
|
+
* regular unit test run on a developer laptop without docker-compose
|
|
48
|
+
* doesn't break.
|
|
49
|
+
*
|
|
50
|
+
* Bring up the test fixtures via:
|
|
51
|
+
* docker compose -f infra/testing/docker-compose.yml up -d kafka
|
|
52
|
+
*
|
|
53
|
+
* Then run:
|
|
54
|
+
* BLOK_INTEGRATION_KAFKA_BROKERS=localhost:9094 bun run test
|
|
55
|
+
*/
|
|
56
|
+
|
|
57
|
+
const KAFKA_BROKERS = process.env.BLOK_INTEGRATION_KAFKA_BROKERS;
|
|
58
|
+
const d = KAFKA_BROKERS ? describe : describe.skip;
|
|
59
|
+
|
|
60
|
+
// CI runners are slower than local — Kafka group coordinator assignment
|
|
61
|
+
// alone takes a few seconds for a fresh consumer, and the first message
|
|
62
|
+
// after subscribe usually waits one heartbeat. Bump the per-test cap so
|
|
63
|
+
// CI doesn't flake on the assignment + first-poll round-trip.
|
|
64
|
+
const TEST_TIMEOUT_MS = 45_000;
|
|
65
|
+
const SUBSCRIPTION_WARMUP_MS = 2_000;
|
|
66
|
+
|
|
67
|
+
d("KafkaPubSubAdapter — real Kafka", () => {
|
|
68
|
+
let producer: KafkaPubSubAdapter;
|
|
69
|
+
let consumerA: KafkaPubSubAdapter;
|
|
70
|
+
let consumerB: KafkaPubSubAdapter;
|
|
71
|
+
let admin: KafkaAdminClient | null = null;
|
|
72
|
+
|
|
73
|
+
beforeAll(async () => {
|
|
74
|
+
const brokers = KAFKA_BROKERS?.split(",").map((s) => s.trim()) ?? [];
|
|
75
|
+
producer = new KafkaPubSubAdapter({ brokers, clientId: "blok-test-pubsub-producer" });
|
|
76
|
+
await producer.connect();
|
|
77
|
+
|
|
78
|
+
consumerA = new KafkaPubSubAdapter({ brokers, clientId: "blok-test-pubsub-consumer-a" });
|
|
79
|
+
await consumerA.connect();
|
|
80
|
+
|
|
81
|
+
consumerB = new KafkaPubSubAdapter({ brokers, clientId: "blok-test-pubsub-consumer-b" });
|
|
82
|
+
await consumerB.connect();
|
|
83
|
+
|
|
84
|
+
// Admin client used only by this test to pre-create topics
|
|
85
|
+
// before consumers subscribe — eliminates the metadata-propagation
|
|
86
|
+
// race where `auto.create.topics.enable` lets a subscribe succeed
|
|
87
|
+
// against a topic the producer can't immediately see (kafkajs
|
|
88
|
+
// reports "This server does not host this topic-partition" on the
|
|
89
|
+
// first publish attempt). The adapter itself relies on auto-create
|
|
90
|
+
// in production (Kafka's durable-log model means orphan publishes
|
|
91
|
+
// are valid); this guard is purely a test-hygiene measure.
|
|
92
|
+
const kafkajs = (await import("kafkajs")) as unknown as KafkaJsModule;
|
|
93
|
+
const adminKafka = new kafkajs.Kafka({ clientId: "blok-test-pubsub-admin", brokers });
|
|
94
|
+
admin = adminKafka.admin();
|
|
95
|
+
await admin.connect();
|
|
96
|
+
|
|
97
|
+
// Cold-start warmup. On a freshly booted broker (the CI case),
|
|
98
|
+
// Kafka's `__consumer_offsets` topic and the group-coordinator
|
|
99
|
+
// election aren't ready immediately after the broker accepts
|
|
100
|
+
// admin connections. The first `consumer.run()` that triggers
|
|
101
|
+
// `findCoordinator` hits the 5-retry budget before the offsets
|
|
102
|
+
// topic is fully initialised and throws
|
|
103
|
+
// `KafkaJSNumberOfRetriesExceeded: This is not the correct
|
|
104
|
+
// coordinator for this group`.
|
|
105
|
+
//
|
|
106
|
+
// Drive that initialisation explicitly here: create a throwaway
|
|
107
|
+
// topic, run a brief consumer against it (which forces group-
|
|
108
|
+
// coordinator election), then disconnect. Subsequent real test
|
|
109
|
+
// subscribes find a warm coordinator and skip the race.
|
|
110
|
+
const warmupTopic = `blok-test-pubsub-warmup-${Math.random().toString(36).slice(2)}`;
|
|
111
|
+
await admin.createTopics({
|
|
112
|
+
waitForLeaders: true,
|
|
113
|
+
topics: [{ topic: warmupTopic, numPartitions: 1, replicationFactor: 1 }],
|
|
114
|
+
});
|
|
115
|
+
const warmupConsumer = adminKafka.consumer({ groupId: `blok-test-warmup-${Math.random().toString(36).slice(2)}` });
|
|
116
|
+
await warmupConsumer.connect();
|
|
117
|
+
await warmupConsumer.subscribe({ topic: warmupTopic, fromBeginning: true });
|
|
118
|
+
await warmupConsumer.run({
|
|
119
|
+
eachMessage: async () => {
|
|
120
|
+
/* never fires — topic stays empty */
|
|
121
|
+
},
|
|
122
|
+
});
|
|
123
|
+
// Give the group coordinator a beat to finalise the join+sync
|
|
124
|
+
// for the warmup group. Without this, the next test's subscribe
|
|
125
|
+
// can still race on a brand-new group's coordinator lookup.
|
|
126
|
+
await new Promise((r) => setTimeout(r, 2_000));
|
|
127
|
+
await warmupConsumer.disconnect();
|
|
128
|
+
}, TEST_TIMEOUT_MS);
|
|
129
|
+
|
|
130
|
+
afterAll(async () => {
|
|
131
|
+
// Best-effort cleanup. A failed test can leave a consumer in a
|
|
132
|
+
// state where `disconnect()` hangs (kafkajs internal retry loop
|
|
133
|
+
// still running). Wrap each call in a 5s timeout so a single
|
|
134
|
+
// stuck consumer doesn't trip vitest's default 10s afterAll
|
|
135
|
+
// cap and cascade-mask the real failure.
|
|
136
|
+
const safeDisconnect = async (label: string, fn: () => Promise<void>): Promise<void> => {
|
|
137
|
+
try {
|
|
138
|
+
await Promise.race([
|
|
139
|
+
fn(),
|
|
140
|
+
new Promise((_, reject) => setTimeout(() => reject(new Error(`${label} disconnect timed out`)), 5_000)),
|
|
141
|
+
]);
|
|
142
|
+
} catch {
|
|
143
|
+
/* ignore — afterAll is best-effort */
|
|
144
|
+
}
|
|
145
|
+
};
|
|
146
|
+
await safeDisconnect("consumerA", () => consumerA.disconnect());
|
|
147
|
+
await safeDisconnect("consumerB", () => consumerB.disconnect());
|
|
148
|
+
await safeDisconnect("producer", () => producer.disconnect());
|
|
149
|
+
if (admin) {
|
|
150
|
+
await safeDisconnect("admin", () => admin?.disconnect() ?? Promise.resolve());
|
|
151
|
+
}
|
|
152
|
+
}, 30_000);
|
|
153
|
+
|
|
154
|
+
async function createTopic(topic: string, numPartitions = 1): Promise<void> {
|
|
155
|
+
if (!admin) throw new Error("admin client not initialised — beforeAll didn't run");
|
|
156
|
+
await admin.createTopics({
|
|
157
|
+
waitForLeaders: true,
|
|
158
|
+
topics: [{ topic, numPartitions, replicationFactor: 1 }],
|
|
159
|
+
});
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
it(
|
|
163
|
+
"fan-out: every subscriber without an explicit consumerGroup receives every message",
|
|
164
|
+
async () => {
|
|
165
|
+
// Topic name varies per run so the consumer-group offset state
|
|
166
|
+
// from a prior run doesn't bleed into this one. Pre-create via
|
|
167
|
+
// admin so the producer's metadata is fresh before the first
|
|
168
|
+
// publish — bypasses the auto-create-on-publish race that
|
|
169
|
+
// shows up as "This server does not host this topic-partition".
|
|
170
|
+
const topic = `blok-test-pubsub-fanout-${Math.random().toString(36).slice(2)}`;
|
|
171
|
+
await createTopic(topic, 1);
|
|
172
|
+
const receivedA: PubSubMessage[] = [];
|
|
173
|
+
const receivedB: PubSubMessage[] = [];
|
|
174
|
+
|
|
175
|
+
await consumerA.subscribe({ topic, durable: false, startFrom: "earliest" }, async (msg) => {
|
|
176
|
+
receivedA.push(msg);
|
|
177
|
+
});
|
|
178
|
+
await consumerB.subscribe({ topic, durable: false, startFrom: "earliest" }, async (msg) => {
|
|
179
|
+
receivedB.push(msg);
|
|
180
|
+
});
|
|
181
|
+
|
|
182
|
+
// Kafka group-coordinator assignment is asynchronous; the
|
|
183
|
+
// `consumer.run` call above returns once the consumer LOOP has
|
|
184
|
+
// started, but the first heartbeat-driven assignment isn't done
|
|
185
|
+
// yet. Without this warm-up we routinely lose the first publish
|
|
186
|
+
// to "no assigned partition" on cold-CI runs.
|
|
187
|
+
await new Promise((r) => setTimeout(r, SUBSCRIPTION_WARMUP_MS));
|
|
188
|
+
|
|
189
|
+
await producer.publish(topic, { hello: "kafka", n: 1 });
|
|
190
|
+
|
|
191
|
+
await waitFor(() => receivedA.length === 1 && receivedB.length === 1, TEST_TIMEOUT_MS - 10_000);
|
|
192
|
+
|
|
193
|
+
expect(receivedA[0].body).toEqual({ hello: "kafka", n: 1 });
|
|
194
|
+
expect(receivedB[0].body).toEqual({ hello: "kafka", n: 1 });
|
|
195
|
+
expect(receivedA[0].topic).toBe(topic);
|
|
196
|
+
expect(receivedB[0].topic).toBe(topic);
|
|
197
|
+
},
|
|
198
|
+
TEST_TIMEOUT_MS,
|
|
199
|
+
);
|
|
200
|
+
|
|
201
|
+
it(
|
|
202
|
+
"competing-consumer: explicit consumerGroup means each message goes to exactly one subscriber",
|
|
203
|
+
async () => {
|
|
204
|
+
// Three partitions so each consumer in the shared group gets at
|
|
205
|
+
// least one assignment — exercises the actual competing-consumer
|
|
206
|
+
// load-balance instead of routing all 10 messages to whichever
|
|
207
|
+
// consumer happened to win the single-partition leadership.
|
|
208
|
+
const topic = `blok-test-pubsub-competing-${Math.random().toString(36).slice(2)}`;
|
|
209
|
+
await createTopic(topic, 3);
|
|
210
|
+
const group = `blok-test-workers-${Math.random().toString(36).slice(2)}`;
|
|
211
|
+
const receivedA: PubSubMessage[] = [];
|
|
212
|
+
const receivedB: PubSubMessage[] = [];
|
|
213
|
+
|
|
214
|
+
await consumerA.subscribe({ topic, consumerGroup: group, durable: false, startFrom: "earliest" }, async (msg) => {
|
|
215
|
+
receivedA.push(msg);
|
|
216
|
+
});
|
|
217
|
+
await consumerB.subscribe({ topic, consumerGroup: group, durable: false, startFrom: "earliest" }, async (msg) => {
|
|
218
|
+
receivedB.push(msg);
|
|
219
|
+
});
|
|
220
|
+
|
|
221
|
+
await new Promise((r) => setTimeout(r, SUBSCRIPTION_WARMUP_MS));
|
|
222
|
+
|
|
223
|
+
// Publish 10 messages with distinct partition keys so the
|
|
224
|
+
// broker spreads them across the topic's partitions. The
|
|
225
|
+
// auto-created topic defaults to 1 partition, which means BOTH
|
|
226
|
+
// consumers join the same group but only ONE gets the partition
|
|
227
|
+
// assignment — and therefore all 10 messages. We can't assert a
|
|
228
|
+
// 50/50 split, but we CAN assert exactly-once delivery across
|
|
229
|
+
// the union.
|
|
230
|
+
for (let i = 0; i < 10; i++) {
|
|
231
|
+
await producer.publish(topic, { n: i }, { partitionKey: String(i) });
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
await waitFor(() => receivedA.length + receivedB.length === 10, TEST_TIMEOUT_MS - 10_000);
|
|
235
|
+
|
|
236
|
+
expect(receivedA.length + receivedB.length).toBe(10);
|
|
237
|
+
const seenN = new Set<number>();
|
|
238
|
+
for (const m of [...receivedA, ...receivedB]) {
|
|
239
|
+
const n = (m.body as { n: number }).n;
|
|
240
|
+
expect(seenN.has(n)).toBe(false);
|
|
241
|
+
seenN.add(n);
|
|
242
|
+
}
|
|
243
|
+
expect(seenN.size).toBe(10);
|
|
244
|
+
},
|
|
245
|
+
TEST_TIMEOUT_MS,
|
|
246
|
+
);
|
|
247
|
+
|
|
248
|
+
it(
|
|
249
|
+
"publish to a topic with no subscribers does not throw",
|
|
250
|
+
async () => {
|
|
251
|
+
// Note: kafkajs's auto.create.topics.enable is true on the test
|
|
252
|
+
// broker, so this also exercises the on-publish topic-create
|
|
253
|
+
// path. The publish should succeed even if no consumer is
|
|
254
|
+
// listening — that's the durable-log model Kafka guarantees.
|
|
255
|
+
const topic = `blok-test-pubsub-orphan-${Math.random().toString(36).slice(2)}`;
|
|
256
|
+
await expect(producer.publish(topic, { dropped: true })).resolves.toBeUndefined();
|
|
257
|
+
},
|
|
258
|
+
TEST_TIMEOUT_MS,
|
|
259
|
+
);
|
|
260
|
+
});
|
|
261
|
+
|
|
262
|
+
async function waitFor(predicate: () => boolean, timeoutMs: number): Promise<void> {
|
|
263
|
+
const start = Date.now();
|
|
264
|
+
while (Date.now() - start < timeoutMs) {
|
|
265
|
+
if (predicate()) return;
|
|
266
|
+
await new Promise((r) => setTimeout(r, 50));
|
|
267
|
+
}
|
|
268
|
+
throw new Error(`waitFor timed out after ${timeoutMs}ms`);
|
|
269
|
+
}
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
import type { PubSubMessage } from "@blokjs/runner";
|
|
2
|
+
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
|
3
|
+
import { NATSPubSubAdapter } from "../../src/adapters/NATSPubSubAdapter";
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Real-NATS integration test for `NATSPubSubAdapter` (closes the
|
|
7
|
+
* integration test debt from PR #87).
|
|
8
|
+
*
|
|
9
|
+
* Exercises both the **fan-out** (no `consumerGroup`) and
|
|
10
|
+
* **competing-consumer** (with `consumerGroup`) delivery patterns
|
|
11
|
+
* documented in the trigger.
|
|
12
|
+
*
|
|
13
|
+
* Gated on `BLOK_INTEGRATION_NATS_SERVERS`. Skipped when unset.
|
|
14
|
+
*
|
|
15
|
+
* Bring up the test fixtures via:
|
|
16
|
+
* docker compose -f infra/testing/docker-compose.yml up -d nats
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
const NATS_SERVERS = process.env.BLOK_INTEGRATION_NATS_SERVERS;
|
|
20
|
+
const d = NATS_SERVERS ? describe : describe.skip;
|
|
21
|
+
|
|
22
|
+
// CI runners are slower than local — JetStream stream + consumer creation
|
|
23
|
+
// + cross-instance subscription propagation can take several seconds on
|
|
24
|
+
// a cold container. Override the default 5s timeout so CI doesn't flake.
|
|
25
|
+
const TEST_TIMEOUT_MS = 30_000;
|
|
26
|
+
const SUBSCRIPTION_WARMUP_MS = 500;
|
|
27
|
+
|
|
28
|
+
d("NATSPubSubAdapter — real NATS", () => {
|
|
29
|
+
let producer: NATSPubSubAdapter;
|
|
30
|
+
let consumerA: NATSPubSubAdapter;
|
|
31
|
+
let consumerB: NATSPubSubAdapter;
|
|
32
|
+
|
|
33
|
+
beforeAll(async () => {
|
|
34
|
+
producer = new NATSPubSubAdapter({
|
|
35
|
+
servers: NATS_SERVERS?.split(",").map((s) => s.trim()) ?? [],
|
|
36
|
+
});
|
|
37
|
+
await producer.connect();
|
|
38
|
+
|
|
39
|
+
consumerA = new NATSPubSubAdapter({
|
|
40
|
+
servers: NATS_SERVERS?.split(",").map((s) => s.trim()) ?? [],
|
|
41
|
+
});
|
|
42
|
+
await consumerA.connect();
|
|
43
|
+
|
|
44
|
+
consumerB = new NATSPubSubAdapter({
|
|
45
|
+
servers: NATS_SERVERS?.split(",").map((s) => s.trim()) ?? [],
|
|
46
|
+
});
|
|
47
|
+
await consumerB.connect();
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
afterAll(async () => {
|
|
51
|
+
await consumerA.disconnect();
|
|
52
|
+
await consumerB.disconnect();
|
|
53
|
+
await producer.disconnect();
|
|
54
|
+
});
|
|
55
|
+
|
|
56
|
+
it(
|
|
57
|
+
"fan-out: every subscriber without consumerGroup receives every message",
|
|
58
|
+
async () => {
|
|
59
|
+
const topic = `blok-test-pubsub-fanout-${Math.random().toString(36).slice(2)}`;
|
|
60
|
+
const receivedA: PubSubMessage[] = [];
|
|
61
|
+
const receivedB: PubSubMessage[] = [];
|
|
62
|
+
|
|
63
|
+
await consumerA.subscribe({ topic, durable: false }, async (msg) => {
|
|
64
|
+
receivedA.push(msg);
|
|
65
|
+
});
|
|
66
|
+
await consumerB.subscribe({ topic, durable: false }, async (msg) => {
|
|
67
|
+
receivedB.push(msg);
|
|
68
|
+
});
|
|
69
|
+
|
|
70
|
+
// JetStream subscription registration is asynchronous — give the
|
|
71
|
+
// consumer a moment to install before publishing.
|
|
72
|
+
await new Promise((r) => setTimeout(r, SUBSCRIPTION_WARMUP_MS));
|
|
73
|
+
|
|
74
|
+
await producer.publish(topic, { hello: "world", n: 1 });
|
|
75
|
+
|
|
76
|
+
await waitFor(() => receivedA.length === 1 && receivedB.length === 1, TEST_TIMEOUT_MS - 5_000);
|
|
77
|
+
|
|
78
|
+
expect(receivedA[0].body).toEqual({ hello: "world", n: 1 });
|
|
79
|
+
expect(receivedB[0].body).toEqual({ hello: "world", n: 1 });
|
|
80
|
+
},
|
|
81
|
+
TEST_TIMEOUT_MS,
|
|
82
|
+
);
|
|
83
|
+
|
|
84
|
+
it(
|
|
85
|
+
"competing-consumer: consumerGroup means each message goes to one subscriber",
|
|
86
|
+
async () => {
|
|
87
|
+
const topic = `blok-test-pubsub-competing-${Math.random().toString(36).slice(2)}`;
|
|
88
|
+
const group = "workers";
|
|
89
|
+
const receivedA: PubSubMessage[] = [];
|
|
90
|
+
const receivedB: PubSubMessage[] = [];
|
|
91
|
+
|
|
92
|
+
await consumerA.subscribe({ topic, consumerGroup: group, durable: false }, async (msg) => {
|
|
93
|
+
receivedA.push(msg);
|
|
94
|
+
});
|
|
95
|
+
await consumerB.subscribe({ topic, consumerGroup: group, durable: false }, async (msg) => {
|
|
96
|
+
receivedB.push(msg);
|
|
97
|
+
});
|
|
98
|
+
|
|
99
|
+
await new Promise((r) => setTimeout(r, SUBSCRIPTION_WARMUP_MS));
|
|
100
|
+
|
|
101
|
+
// Publish 10 messages — should split across A + B (not exact 50/50
|
|
102
|
+
// but each message reaches exactly one).
|
|
103
|
+
for (let i = 0; i < 10; i++) {
|
|
104
|
+
await producer.publish(topic, { n: i });
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
await waitFor(() => receivedA.length + receivedB.length === 10, TEST_TIMEOUT_MS - 5_000);
|
|
108
|
+
|
|
109
|
+
expect(receivedA.length + receivedB.length).toBe(10);
|
|
110
|
+
// No duplicate keys across both lists.
|
|
111
|
+
const seenN = new Set<number>();
|
|
112
|
+
for (const m of [...receivedA, ...receivedB]) {
|
|
113
|
+
const n = (m.body as { n: number }).n;
|
|
114
|
+
expect(seenN.has(n)).toBe(false);
|
|
115
|
+
seenN.add(n);
|
|
116
|
+
}
|
|
117
|
+
},
|
|
118
|
+
TEST_TIMEOUT_MS,
|
|
119
|
+
);
|
|
120
|
+
|
|
121
|
+
it(
|
|
122
|
+
"publish to an unsubscribed topic does not throw",
|
|
123
|
+
async () => {
|
|
124
|
+
const topic = `blok-test-pubsub-orphan-${Math.random().toString(36).slice(2)}`;
|
|
125
|
+
await expect(producer.publish(topic, { dropped: true })).resolves.toBeUndefined();
|
|
126
|
+
},
|
|
127
|
+
TEST_TIMEOUT_MS,
|
|
128
|
+
);
|
|
129
|
+
});
|
|
130
|
+
|
|
131
|
+
async function waitFor(predicate: () => boolean, timeoutMs: number): Promise<void> {
|
|
132
|
+
const start = Date.now();
|
|
133
|
+
while (Date.now() - start < timeoutMs) {
|
|
134
|
+
if (predicate()) return;
|
|
135
|
+
await new Promise((r) => setTimeout(r, 50));
|
|
136
|
+
}
|
|
137
|
+
throw new Error(`waitFor timed out after ${timeoutMs}ms`);
|
|
138
|
+
}
|