@blokjs/trigger-worker 0.2.1 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/__tests__/integration/nats-adapter.real-nats.test.ts +116 -0
- package/__tests__/integration/pgboss-adapter.real-pg.test.ts +164 -0
- package/__tests__/integration/rabbitmq-adapter.real-rabbitmq.test.ts +179 -0
- package/__tests__/integration/sqs-adapter.real-sqs.test.ts +228 -0
- package/dist/WorkerTrigger.d.ts +40 -4
- package/dist/WorkerTrigger.js +272 -40
- package/dist/adapters/BullMQAdapter.d.ts +1 -1
- package/dist/adapters/BullMQAdapter.js +5 -42
- package/dist/adapters/InMemoryAdapter.d.ts +1 -1
- package/dist/adapters/InMemoryAdapter.js +13 -12
- package/dist/adapters/KafkaAdapter.d.ts +62 -0
- package/dist/adapters/KafkaAdapter.js +236 -0
- package/dist/adapters/NATSAdapter.d.ts +110 -0
- package/dist/adapters/NATSAdapter.js +394 -0
- package/dist/adapters/PgBossAdapter.d.ts +56 -0
- package/dist/adapters/PgBossAdapter.js +251 -0
- package/dist/adapters/RabbitMQAdapter.d.ts +51 -0
- package/dist/adapters/RabbitMQAdapter.js +241 -0
- package/dist/adapters/RedisStreamsAdapter.d.ts +64 -0
- package/dist/adapters/RedisStreamsAdapter.js +240 -0
- package/dist/adapters/SQSAdapter.d.ts +61 -0
- package/dist/adapters/SQSAdapter.js +269 -0
- package/dist/adapters/factory.d.ts +34 -0
- package/dist/adapters/factory.js +103 -0
- package/dist/index.d.ts +25 -7
- package/dist/index.js +31 -16
- package/package.json +27 -5
- package/src/WorkerTrigger.test.ts +44 -14
- package/src/WorkerTrigger.ts +299 -27
- package/src/adapters/InMemoryAdapter.ts +9 -5
- package/src/adapters/KafkaAdapter.ts +277 -0
- package/src/adapters/NATSAdapter.ts +454 -0
- package/src/adapters/PgBossAdapter.ts +293 -0
- package/src/adapters/RabbitMQAdapter.ts +285 -0
- package/src/adapters/RedisStreamsAdapter.ts +286 -0
- package/src/adapters/SQSAdapter.ts +306 -0
- package/src/adapters/factory.test.ts +89 -0
- package/src/adapters/factory.ts +111 -0
- package/src/adapters/new-adapters.test.ts +130 -0
- package/src/index.ts +31 -4
- package/template/.env.example +13 -0
- package/template/package.json +45 -0
- package/template/src/Nodes.ts +10 -0
- package/template/src/Workflows.ts +8 -0
- package/template/src/index.ts +41 -0
- package/template/src/runner/WorkerServer.ts +34 -0
- package/template/src/runner/types/Workflows.ts +7 -0
- package/template/src/workflows/jobs/process-job.ts +47 -0
- package/template/tsconfig.json +31 -0
- package/template/vitest.config.ts +39 -0
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
import type { WorkerJob } from "@blokjs/runner";
|
|
2
|
+
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
|
3
|
+
import { NATSWorkerAdapter } from "../../src/adapters/NATSAdapter";
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Real-NATS integration test for `NATSWorkerAdapter` (closes the
|
|
7
|
+
* integration test debt from PR #86).
|
|
8
|
+
*
|
|
9
|
+
* Gated on `BLOK_INTEGRATION_NATS_SERVERS`. Skipped when unset.
|
|
10
|
+
*
|
|
11
|
+
* Bring up the test fixtures via:
|
|
12
|
+
* docker compose -f infra/testing/docker-compose.yml up -d nats
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
const NATS_SERVERS = process.env.BLOK_INTEGRATION_NATS_SERVERS;
|
|
16
|
+
const d = NATS_SERVERS ? describe : describe.skip;
|
|
17
|
+
|
|
18
|
+
// CI runners are slower than local — JetStream stream + consumer
|
|
19
|
+
// creation + first poll cycle can take several seconds on a cold
|
|
20
|
+
// container. Override the 5s default so CI doesn't flake.
|
|
21
|
+
const TEST_TIMEOUT_MS = 30_000;
|
|
22
|
+
|
|
23
|
+
d("NATSWorkerAdapter — real NATS JetStream", () => {
|
|
24
|
+
let adapter: NATSWorkerAdapter;
|
|
25
|
+
const stream = `blok-test-worker-${Date.now()}-${Math.floor(Math.random() * 1e6)}`;
|
|
26
|
+
|
|
27
|
+
beforeAll(async () => {
|
|
28
|
+
adapter = new NATSWorkerAdapter({
|
|
29
|
+
servers: NATS_SERVERS?.split(",").map((s) => s.trim()) ?? [],
|
|
30
|
+
streamName: stream,
|
|
31
|
+
});
|
|
32
|
+
await adapter.connect();
|
|
33
|
+
});
|
|
34
|
+
|
|
35
|
+
afterAll(async () => {
|
|
36
|
+
await adapter.disconnect();
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
it(
|
|
40
|
+
"publishes a job and the consumer receives it",
|
|
41
|
+
async () => {
|
|
42
|
+
const queue = `test-q-publish-${Math.random().toString(36).slice(2)}`;
|
|
43
|
+
const received: WorkerJob[] = [];
|
|
44
|
+
|
|
45
|
+
await adapter.process({ queue }, async (job) => {
|
|
46
|
+
received.push(job);
|
|
47
|
+
await job.complete();
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
const jobId = await adapter.addJob(queue, { hello: "world", n: 1 });
|
|
51
|
+
expect(typeof jobId).toBe("string");
|
|
52
|
+
|
|
53
|
+
// Wait for delivery — NATS JetStream is durable; consumer pulls within ms.
|
|
54
|
+
await waitFor(() => received.length === 1, TEST_TIMEOUT_MS - 5_000);
|
|
55
|
+
|
|
56
|
+
expect(received).toHaveLength(1);
|
|
57
|
+
expect(received[0].data).toEqual({ hello: "world", n: 1 });
|
|
58
|
+
expect(received[0].queue).toBe(queue);
|
|
59
|
+
expect(received[0].id).toBeTruthy();
|
|
60
|
+
|
|
61
|
+
await adapter.stopProcessing(queue);
|
|
62
|
+
},
|
|
63
|
+
TEST_TIMEOUT_MS,
|
|
64
|
+
);
|
|
65
|
+
|
|
66
|
+
it(
|
|
67
|
+
"isolates jobs across queues",
|
|
68
|
+
async () => {
|
|
69
|
+
const queueA = `test-q-a-${Math.random().toString(36).slice(2)}`;
|
|
70
|
+
const queueB = `test-q-b-${Math.random().toString(36).slice(2)}`;
|
|
71
|
+
const receivedA: WorkerJob[] = [];
|
|
72
|
+
const receivedB: WorkerJob[] = [];
|
|
73
|
+
|
|
74
|
+
await adapter.process({ queue: queueA }, async (job) => {
|
|
75
|
+
receivedA.push(job);
|
|
76
|
+
await job.complete();
|
|
77
|
+
});
|
|
78
|
+
await adapter.process({ queue: queueB }, async (job) => {
|
|
79
|
+
receivedB.push(job);
|
|
80
|
+
await job.complete();
|
|
81
|
+
});
|
|
82
|
+
|
|
83
|
+
await adapter.addJob(queueA, { from: "A" });
|
|
84
|
+
await adapter.addJob(queueB, { from: "B" });
|
|
85
|
+
|
|
86
|
+
await waitFor(() => receivedA.length === 1 && receivedB.length === 1, TEST_TIMEOUT_MS - 5_000);
|
|
87
|
+
|
|
88
|
+
expect(receivedA[0].data).toEqual({ from: "A" });
|
|
89
|
+
expect(receivedB[0].data).toEqual({ from: "B" });
|
|
90
|
+
expect(receivedA[0].queue).toBe(queueA);
|
|
91
|
+
expect(receivedB[0].queue).toBe(queueB);
|
|
92
|
+
|
|
93
|
+
await adapter.stopProcessing(queueA);
|
|
94
|
+
await adapter.stopProcessing(queueB);
|
|
95
|
+
},
|
|
96
|
+
TEST_TIMEOUT_MS,
|
|
97
|
+
);
|
|
98
|
+
|
|
99
|
+
it("isConnected reflects state", async () => {
|
|
100
|
+
expect(adapter.isConnected()).toBe(true);
|
|
101
|
+
});
|
|
102
|
+
|
|
103
|
+
it("healthCheck returns true when connected", async () => {
|
|
104
|
+
const healthy = await adapter.healthCheck();
|
|
105
|
+
expect(healthy).toBe(true);
|
|
106
|
+
});
|
|
107
|
+
});
|
|
108
|
+
|
|
109
|
+
async function waitFor(predicate: () => boolean, timeoutMs: number): Promise<void> {
|
|
110
|
+
const start = Date.now();
|
|
111
|
+
while (Date.now() - start < timeoutMs) {
|
|
112
|
+
if (predicate()) return;
|
|
113
|
+
await new Promise((r) => setTimeout(r, 50));
|
|
114
|
+
}
|
|
115
|
+
throw new Error(`waitFor timed out after ${timeoutMs}ms`);
|
|
116
|
+
}
|
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
import type { WorkerJob } from "@blokjs/runner";
|
|
2
|
+
import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";
|
|
3
|
+
import { PgBossAdapter } from "../../src/adapters/PgBossAdapter";
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Real-Postgres integration test for `PgBossAdapter` (closes Phase 2.1
|
|
7
|
+
* broker-adapter test debt deferred from PR #91).
|
|
8
|
+
*
|
|
9
|
+
* Gated on `BLOK_INTEGRATION_POSTGRES_URL`. Skipped when unset.
|
|
10
|
+
*
|
|
11
|
+
* Bring up the test fixtures via:
|
|
12
|
+
* docker compose -f infra/testing/docker-compose.yml up -d postgres
|
|
13
|
+
*
|
|
14
|
+
* Then run:
|
|
15
|
+
* BLOK_INTEGRATION_POSTGRES_URL=postgres://blok:blok_test@localhost:5433/blok_test \
|
|
16
|
+
* bun run test
|
|
17
|
+
*
|
|
18
|
+
* Note: pg-boss creates its own schema (`pgboss` by default) on first
|
|
19
|
+
* connect; we use a per-test schema name so a flaky run doesn't poison
|
|
20
|
+
* the next one. Each test cleans up via `boss.stop({ graceful: true })`.
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
const POSTGRES_URL = process.env.BLOK_INTEGRATION_POSTGRES_URL;
|
|
24
|
+
const d = POSTGRES_URL ? describe : describe.skip;
|
|
25
|
+
|
|
26
|
+
const TEST_TIMEOUT_MS = 60_000; // pg-boss migration on first start is slow
|
|
27
|
+
|
|
28
|
+
d("PgBossAdapter — real Postgres", () => {
|
|
29
|
+
const adapters: PgBossAdapter[] = [];
|
|
30
|
+
|
|
31
|
+
async function newAdapter(): Promise<PgBossAdapter> {
|
|
32
|
+
// Random schema per adapter so pg-boss's auto-migration runs
|
|
33
|
+
// fresh and the per-test state doesn't bleed across `it`s. pg-boss
|
|
34
|
+
// uses one schema per `PgBoss` instance.
|
|
35
|
+
const schema = `pgboss_test_${Math.random().toString(36).slice(2, 10)}`;
|
|
36
|
+
const adapter = new PgBossAdapter({ connectionString: POSTGRES_URL, schema });
|
|
37
|
+
await adapter.connect();
|
|
38
|
+
adapters.push(adapter);
|
|
39
|
+
return adapter;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
afterEach(async () => {
|
|
43
|
+
for (const a of adapters.splice(0)) {
|
|
44
|
+
try {
|
|
45
|
+
await a.disconnect();
|
|
46
|
+
} catch {
|
|
47
|
+
/* ignore — best-effort */
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
afterAll(async () => {
|
|
53
|
+
for (const a of adapters.splice(0)) {
|
|
54
|
+
try {
|
|
55
|
+
await a.disconnect();
|
|
56
|
+
} catch {
|
|
57
|
+
/* ignore */
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
it(
|
|
63
|
+
"publishes a job and the consumer receives it (single-queue happy path)",
|
|
64
|
+
async () => {
|
|
65
|
+
const adapter = await newAdapter();
|
|
66
|
+
const queue = `blok-test-pgboss-publish-${Math.random().toString(36).slice(2)}`;
|
|
67
|
+
const received: WorkerJob[] = [];
|
|
68
|
+
|
|
69
|
+
await adapter.process({ queue }, async (job) => {
|
|
70
|
+
received.push(job);
|
|
71
|
+
await job.complete();
|
|
72
|
+
});
|
|
73
|
+
|
|
74
|
+
const jobId = await adapter.addJob(queue, { hello: "pg-boss", n: 1 });
|
|
75
|
+
expect(typeof jobId).toBe("string");
|
|
76
|
+
expect(jobId.length).toBeGreaterThan(0);
|
|
77
|
+
|
|
78
|
+
await waitFor(() => received.length === 1, TEST_TIMEOUT_MS - 10_000);
|
|
79
|
+
|
|
80
|
+
expect(received[0].data).toEqual({ hello: "pg-boss", n: 1 });
|
|
81
|
+
expect(received[0].queue).toBe(queue);
|
|
82
|
+
expect(received[0].id).toBeTruthy();
|
|
83
|
+
|
|
84
|
+
await adapter.stopProcessing(queue);
|
|
85
|
+
},
|
|
86
|
+
TEST_TIMEOUT_MS,
|
|
87
|
+
);
|
|
88
|
+
|
|
89
|
+
it(
|
|
90
|
+
"isolates jobs across distinct queues within one PgBoss instance",
|
|
91
|
+
async () => {
|
|
92
|
+
const adapter = await newAdapter();
|
|
93
|
+
const queueA = `blok-test-pgboss-a-${Math.random().toString(36).slice(2)}`;
|
|
94
|
+
const queueB = `blok-test-pgboss-b-${Math.random().toString(36).slice(2)}`;
|
|
95
|
+
const receivedA: WorkerJob[] = [];
|
|
96
|
+
const receivedB: WorkerJob[] = [];
|
|
97
|
+
|
|
98
|
+
await adapter.process({ queue: queueA }, async (job) => {
|
|
99
|
+
receivedA.push(job);
|
|
100
|
+
await job.complete();
|
|
101
|
+
});
|
|
102
|
+
await adapter.process({ queue: queueB }, async (job) => {
|
|
103
|
+
receivedB.push(job);
|
|
104
|
+
await job.complete();
|
|
105
|
+
});
|
|
106
|
+
|
|
107
|
+
await adapter.addJob(queueA, { from: "A" });
|
|
108
|
+
await adapter.addJob(queueB, { from: "B" });
|
|
109
|
+
|
|
110
|
+
await waitFor(() => receivedA.length === 1 && receivedB.length === 1, TEST_TIMEOUT_MS - 10_000);
|
|
111
|
+
|
|
112
|
+
expect(receivedA[0].data).toEqual({ from: "A" });
|
|
113
|
+
expect(receivedB[0].data).toEqual({ from: "B" });
|
|
114
|
+
|
|
115
|
+
await adapter.stopProcessing(queueA);
|
|
116
|
+
await adapter.stopProcessing(queueB);
|
|
117
|
+
},
|
|
118
|
+
TEST_TIMEOUT_MS,
|
|
119
|
+
);
|
|
120
|
+
|
|
121
|
+
it(
|
|
122
|
+
"handler throw triggers pg-boss retry until retryLimit, then drops to DLQ",
|
|
123
|
+
async () => {
|
|
124
|
+
// pg-boss reschedules failed jobs internally based on
|
|
125
|
+
// `retryLimit`. We assert the handler is invoked at least
|
|
126
|
+
// twice when `retries: 1` (= 2 total attempts).
|
|
127
|
+
const adapter = await newAdapter();
|
|
128
|
+
const queue = `blok-test-pgboss-retry-${Math.random().toString(36).slice(2)}`;
|
|
129
|
+
let attempts = 0;
|
|
130
|
+
|
|
131
|
+
await adapter.process({ queue, retries: 1 }, async (_job) => {
|
|
132
|
+
attempts += 1;
|
|
133
|
+
throw new Error(`simulated failure attempt ${attempts}`);
|
|
134
|
+
});
|
|
135
|
+
|
|
136
|
+
await adapter.addJob(queue, { will_fail: true }, { retries: 1 });
|
|
137
|
+
|
|
138
|
+
// Wait for at least 2 attempts. pg-boss's default retry-delay
|
|
139
|
+
// can be several seconds; the test timeout (60s) accommodates.
|
|
140
|
+
await waitFor(() => attempts >= 2, TEST_TIMEOUT_MS - 10_000);
|
|
141
|
+
|
|
142
|
+
expect(attempts).toBeGreaterThanOrEqual(2);
|
|
143
|
+
|
|
144
|
+
await adapter.stopProcessing(queue);
|
|
145
|
+
},
|
|
146
|
+
TEST_TIMEOUT_MS,
|
|
147
|
+
);
|
|
148
|
+
|
|
149
|
+
it("isConnected + healthCheck reflect the live state", async () => {
|
|
150
|
+
const adapter = await newAdapter();
|
|
151
|
+
expect(adapter.isConnected()).toBe(true);
|
|
152
|
+
const healthy = await adapter.healthCheck();
|
|
153
|
+
expect(healthy).toBe(true);
|
|
154
|
+
});
|
|
155
|
+
});
|
|
156
|
+
|
|
157
|
+
async function waitFor(predicate: () => boolean, timeoutMs: number): Promise<void> {
|
|
158
|
+
const start = Date.now();
|
|
159
|
+
while (Date.now() - start < timeoutMs) {
|
|
160
|
+
if (predicate()) return;
|
|
161
|
+
await new Promise((r) => setTimeout(r, 100));
|
|
162
|
+
}
|
|
163
|
+
throw new Error(`waitFor timed out after ${timeoutMs}ms`);
|
|
164
|
+
}
|
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
import type { WorkerJob } from "@blokjs/runner";
|
|
2
|
+
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
|
3
|
+
import { RabbitMQAdapter } from "../../src/adapters/RabbitMQAdapter";
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Real-RabbitMQ integration test for `RabbitMQAdapter` (closes Phase 2.1
|
|
7
|
+
* broker-adapter test debt deferred from PR #91).
|
|
8
|
+
*
|
|
9
|
+
* Exercises the worker contract end-to-end against a real broker: queue
|
|
10
|
+
* declaration, job add, consume + ack, isolation across queues, manual
|
|
11
|
+
* `complete()` / `fail()` paths.
|
|
12
|
+
*
|
|
13
|
+
* Gated on `BLOK_INTEGRATION_RABBITMQ_URL`. Skipped when unset.
|
|
14
|
+
*
|
|
15
|
+
* Bring up the test fixtures via:
|
|
16
|
+
* docker compose -f infra/testing/docker-compose.yml up -d rabbitmq
|
|
17
|
+
*
|
|
18
|
+
* Then run:
|
|
19
|
+
* BLOK_INTEGRATION_RABBITMQ_URL=amqp://blok:blok_test@localhost:5673 bun run test
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
const RABBITMQ_URL = process.env.BLOK_INTEGRATION_RABBITMQ_URL;
|
|
23
|
+
const d = RABBITMQ_URL ? describe : describe.skip;
|
|
24
|
+
|
|
25
|
+
const TEST_TIMEOUT_MS = 30_000;
|
|
26
|
+
|
|
27
|
+
d("RabbitMQAdapter — real RabbitMQ", () => {
|
|
28
|
+
let adapter: RabbitMQAdapter;
|
|
29
|
+
|
|
30
|
+
beforeAll(async () => {
|
|
31
|
+
adapter = new RabbitMQAdapter({ url: RABBITMQ_URL });
|
|
32
|
+
await adapter.connect();
|
|
33
|
+
}, TEST_TIMEOUT_MS);
|
|
34
|
+
|
|
35
|
+
afterAll(async () => {
|
|
36
|
+
await adapter.disconnect();
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
it(
|
|
40
|
+
"publishes a job and the consumer receives it (single-queue happy path)",
|
|
41
|
+
async () => {
|
|
42
|
+
// Random queue name per run so this test is idempotent against
|
|
43
|
+
// a long-lived broker — orphan messages from a prior crashed run
|
|
44
|
+
// wouldn't bleed into the count assertion below.
|
|
45
|
+
const queue = `blok-test-q-publish-${Math.random().toString(36).slice(2)}`;
|
|
46
|
+
const received: WorkerJob[] = [];
|
|
47
|
+
|
|
48
|
+
await adapter.process({ queue }, async (job) => {
|
|
49
|
+
received.push(job);
|
|
50
|
+
await job.complete();
|
|
51
|
+
});
|
|
52
|
+
|
|
53
|
+
const jobId = await adapter.addJob(queue, { hello: "world", n: 1 });
|
|
54
|
+
expect(typeof jobId).toBe("string");
|
|
55
|
+
expect(jobId).toBeTruthy();
|
|
56
|
+
|
|
57
|
+
await waitFor(() => received.length === 1, TEST_TIMEOUT_MS - 5_000);
|
|
58
|
+
|
|
59
|
+
expect(received).toHaveLength(1);
|
|
60
|
+
expect(received[0].data).toEqual({ hello: "world", n: 1 });
|
|
61
|
+
expect(received[0].queue).toBe(queue);
|
|
62
|
+
expect(received[0].id).toBeTruthy();
|
|
63
|
+
|
|
64
|
+
await adapter.stopProcessing(queue);
|
|
65
|
+
},
|
|
66
|
+
TEST_TIMEOUT_MS,
|
|
67
|
+
);
|
|
68
|
+
|
|
69
|
+
it(
|
|
70
|
+
"isolates jobs across distinct queues",
|
|
71
|
+
async () => {
|
|
72
|
+
const queueA = `blok-test-q-a-${Math.random().toString(36).slice(2)}`;
|
|
73
|
+
const queueB = `blok-test-q-b-${Math.random().toString(36).slice(2)}`;
|
|
74
|
+
const receivedA: WorkerJob[] = [];
|
|
75
|
+
const receivedB: WorkerJob[] = [];
|
|
76
|
+
|
|
77
|
+
await adapter.process({ queue: queueA }, async (job) => {
|
|
78
|
+
receivedA.push(job);
|
|
79
|
+
await job.complete();
|
|
80
|
+
});
|
|
81
|
+
await adapter.process({ queue: queueB }, async (job) => {
|
|
82
|
+
receivedB.push(job);
|
|
83
|
+
await job.complete();
|
|
84
|
+
});
|
|
85
|
+
|
|
86
|
+
await adapter.addJob(queueA, { from: "A" });
|
|
87
|
+
await adapter.addJob(queueB, { from: "B" });
|
|
88
|
+
|
|
89
|
+
await waitFor(() => receivedA.length === 1 && receivedB.length === 1, TEST_TIMEOUT_MS - 5_000);
|
|
90
|
+
|
|
91
|
+
expect(receivedA[0].data).toEqual({ from: "A" });
|
|
92
|
+
expect(receivedB[0].data).toEqual({ from: "B" });
|
|
93
|
+
expect(receivedA[0].queue).toBe(queueA);
|
|
94
|
+
expect(receivedB[0].queue).toBe(queueB);
|
|
95
|
+
|
|
96
|
+
await adapter.stopProcessing(queueA);
|
|
97
|
+
await adapter.stopProcessing(queueB);
|
|
98
|
+
},
|
|
99
|
+
TEST_TIMEOUT_MS,
|
|
100
|
+
);
|
|
101
|
+
|
|
102
|
+
it(
|
|
103
|
+
"delivers multiple jobs in order on a single-consumer queue",
|
|
104
|
+
async () => {
|
|
105
|
+
// Single consumer + prefetch(1) (the adapter's default for
|
|
106
|
+
// non-concurrent processing) preserves FIFO order. With multiple
|
|
107
|
+
// consumers Rabbit doesn't guarantee global order, so we keep
|
|
108
|
+
// this test narrow to single-consumer semantics.
|
|
109
|
+
const queue = `blok-test-q-order-${Math.random().toString(36).slice(2)}`;
|
|
110
|
+
const received: number[] = [];
|
|
111
|
+
|
|
112
|
+
await adapter.process({ queue }, async (job) => {
|
|
113
|
+
const data = job.data as { n: number };
|
|
114
|
+
received.push(data.n);
|
|
115
|
+
await job.complete();
|
|
116
|
+
});
|
|
117
|
+
|
|
118
|
+
for (let i = 0; i < 5; i++) {
|
|
119
|
+
await adapter.addJob(queue, { n: i });
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
await waitFor(() => received.length === 5, TEST_TIMEOUT_MS - 5_000);
|
|
123
|
+
|
|
124
|
+
expect(received).toEqual([0, 1, 2, 3, 4]);
|
|
125
|
+
|
|
126
|
+
await adapter.stopProcessing(queue);
|
|
127
|
+
},
|
|
128
|
+
TEST_TIMEOUT_MS,
|
|
129
|
+
);
|
|
130
|
+
|
|
131
|
+
it(
|
|
132
|
+
"failed jobs requeue when retries remain (manual fail path)",
|
|
133
|
+
async () => {
|
|
134
|
+
// Adapter NACK-with-requeue contract: the same job message is
|
|
135
|
+
// re-delivered after `fail()` until the retry budget is hit.
|
|
136
|
+
// We assert one re-delivery here; the full retry-budget DLQ
|
|
137
|
+
// flow is exercised by the adapter unit tests.
|
|
138
|
+
const queue = `blok-test-q-retry-${Math.random().toString(36).slice(2)}`;
|
|
139
|
+
let attempts = 0;
|
|
140
|
+
let firstAttemptId = "";
|
|
141
|
+
|
|
142
|
+
await adapter.process({ queue, retries: 1 }, async (job) => {
|
|
143
|
+
attempts++;
|
|
144
|
+
if (attempts === 1) {
|
|
145
|
+
firstAttemptId = job.id;
|
|
146
|
+
await job.fail(new Error("simulated transient failure"));
|
|
147
|
+
return;
|
|
148
|
+
}
|
|
149
|
+
// Second delivery: ack so we exit cleanly.
|
|
150
|
+
expect(job.id).toBe(firstAttemptId);
|
|
151
|
+
await job.complete();
|
|
152
|
+
});
|
|
153
|
+
|
|
154
|
+
await adapter.addJob(queue, { will_retry: true });
|
|
155
|
+
|
|
156
|
+
await waitFor(() => attempts >= 2, TEST_TIMEOUT_MS - 5_000);
|
|
157
|
+
|
|
158
|
+
expect(attempts).toBeGreaterThanOrEqual(2);
|
|
159
|
+
|
|
160
|
+
await adapter.stopProcessing(queue);
|
|
161
|
+
},
|
|
162
|
+
TEST_TIMEOUT_MS,
|
|
163
|
+
);
|
|
164
|
+
|
|
165
|
+
it("isConnected + healthCheck reflect the live state", async () => {
|
|
166
|
+
expect(adapter.isConnected()).toBe(true);
|
|
167
|
+
const healthy = await adapter.healthCheck();
|
|
168
|
+
expect(healthy).toBe(true);
|
|
169
|
+
});
|
|
170
|
+
});
|
|
171
|
+
|
|
172
|
+
async function waitFor(predicate: () => boolean, timeoutMs: number): Promise<void> {
|
|
173
|
+
const start = Date.now();
|
|
174
|
+
while (Date.now() - start < timeoutMs) {
|
|
175
|
+
if (predicate()) return;
|
|
176
|
+
await new Promise((r) => setTimeout(r, 50));
|
|
177
|
+
}
|
|
178
|
+
throw new Error(`waitFor timed out after ${timeoutMs}ms`);
|
|
179
|
+
}
|
|
@@ -0,0 +1,228 @@
|
|
|
1
|
+
import type { WorkerJob } from "@blokjs/runner";
|
|
2
|
+
import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";
|
|
3
|
+
import { SQSAdapter } from "../../src/adapters/SQSAdapter";
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Real-LocalStack-SQS integration test for `SQSAdapter` (closes Phase 2.1
|
|
7
|
+
* broker-adapter test debt deferred from PR #91).
|
|
8
|
+
*
|
|
9
|
+
* Gated on `BLOK_INTEGRATION_SQS_ENDPOINT`. Skipped when unset.
|
|
10
|
+
*
|
|
11
|
+
* Bring up the test fixtures via:
|
|
12
|
+
* docker compose -f infra/testing/docker-compose.yml up -d localstack
|
|
13
|
+
*
|
|
14
|
+
* Then run:
|
|
15
|
+
* BLOK_INTEGRATION_SQS_ENDPOINT=http://localhost:4567 \
|
|
16
|
+
* AWS_REGION=us-east-1 \
|
|
17
|
+
* AWS_ACCESS_KEY_ID=test \
|
|
18
|
+
* AWS_SECRET_ACCESS_KEY=test \
|
|
19
|
+
* bun run test
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
const SQS_ENDPOINT = process.env.BLOK_INTEGRATION_SQS_ENDPOINT;
|
|
23
|
+
const d = SQS_ENDPOINT ? describe : describe.skip;
|
|
24
|
+
|
|
25
|
+
const TEST_TIMEOUT_MS = 30_000;
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* Narrow shape for the LocalStack CreateQueue / DeleteQueue paths we
|
|
29
|
+
* touch from this test only. Keeps us off `any` per the repo's
|
|
30
|
+
* no-`any`-in-tests rule.
|
|
31
|
+
*/
|
|
32
|
+
interface SqsTestClient {
|
|
33
|
+
send(cmd: unknown): Promise<{ QueueUrl?: string }>;
|
|
34
|
+
destroy?: () => void;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
interface SqsCommands {
|
|
38
|
+
SQSClient: new (opts: { region: string; endpoint?: string }) => SqsTestClient;
|
|
39
|
+
CreateQueueCommand: new (opts: { QueueName: string }) => unknown;
|
|
40
|
+
DeleteQueueCommand: new (opts: { QueueUrl: string }) => unknown;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
d("SQSAdapter — real LocalStack SQS", () => {
|
|
44
|
+
let adapter: SQSAdapter;
|
|
45
|
+
let testClient: SqsTestClient | null = null;
|
|
46
|
+
let sqsCommands: SqsCommands | null = null;
|
|
47
|
+
const createdQueues: string[] = [];
|
|
48
|
+
|
|
49
|
+
// LocalStack accepts any credentials; we set explicit ones so the SDK
|
|
50
|
+
// doesn't crawl the local environment / IMDS searching for them and
|
|
51
|
+
// time out the first request.
|
|
52
|
+
const TEST_REGION = process.env.AWS_REGION || "us-east-1";
|
|
53
|
+
|
|
54
|
+
async function createQueue(name: string): Promise<string> {
|
|
55
|
+
if (!testClient || !sqsCommands) throw new Error("test client not initialised — beforeAll didn't run");
|
|
56
|
+
const result = await testClient.send(new sqsCommands.CreateQueueCommand({ QueueName: name }));
|
|
57
|
+
const url = result.QueueUrl;
|
|
58
|
+
if (!url) throw new Error(`CreateQueue returned no QueueUrl for ${name}`);
|
|
59
|
+
createdQueues.push(url);
|
|
60
|
+
return url;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
beforeAll(async () => {
|
|
64
|
+
// Force credentials BEFORE the SDK import — LocalStack rejects
|
|
65
|
+
// requests without any signature, and the default credential chain
|
|
66
|
+
// times out the first call when nothing is configured.
|
|
67
|
+
process.env.AWS_ACCESS_KEY_ID = process.env.AWS_ACCESS_KEY_ID || "test";
|
|
68
|
+
process.env.AWS_SECRET_ACCESS_KEY = process.env.AWS_SECRET_ACCESS_KEY || "test";
|
|
69
|
+
process.env.AWS_REGION = TEST_REGION;
|
|
70
|
+
|
|
71
|
+
adapter = new SQSAdapter({
|
|
72
|
+
region: TEST_REGION,
|
|
73
|
+
endpoint: SQS_ENDPOINT,
|
|
74
|
+
waitTimeSeconds: 1, // short long-poll so afterEach cleanup is responsive
|
|
75
|
+
});
|
|
76
|
+
await adapter.connect();
|
|
77
|
+
|
|
78
|
+
// Direct SDK client for queue management — we only need
|
|
79
|
+
// CreateQueue / DeleteQueue, the adapter doesn't expose those.
|
|
80
|
+
const sdk = (await import("@aws-sdk/client-sqs")) as unknown as SqsCommands;
|
|
81
|
+
sqsCommands = sdk;
|
|
82
|
+
testClient = new sdk.SQSClient({ region: TEST_REGION, endpoint: SQS_ENDPOINT });
|
|
83
|
+
}, TEST_TIMEOUT_MS);
|
|
84
|
+
|
|
85
|
+
afterEach(async () => {
|
|
86
|
+
// Tear down per-test queues so a flaky test doesn't poison the
|
|
87
|
+
// next one with redelivered messages.
|
|
88
|
+
for (const url of createdQueues.splice(0)) {
|
|
89
|
+
if (!testClient || !sqsCommands) continue;
|
|
90
|
+
try {
|
|
91
|
+
await testClient.send(new sqsCommands.DeleteQueueCommand({ QueueUrl: url }));
|
|
92
|
+
} catch {
|
|
93
|
+
/* ignore — best-effort cleanup */
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
});
|
|
97
|
+
|
|
98
|
+
afterAll(async () => {
|
|
99
|
+
await adapter.disconnect();
|
|
100
|
+
testClient?.destroy?.();
|
|
101
|
+
});
|
|
102
|
+
|
|
103
|
+
it(
|
|
104
|
+
"publishes a job and the consumer receives it (single-queue happy path)",
|
|
105
|
+
async () => {
|
|
106
|
+
const queueUrl = await createQueue(`blok-test-sqs-publish-${Math.random().toString(36).slice(2)}`);
|
|
107
|
+
const received: WorkerJob[] = [];
|
|
108
|
+
|
|
109
|
+
await adapter.process({ queue: queueUrl }, async (job) => {
|
|
110
|
+
received.push(job);
|
|
111
|
+
await job.complete();
|
|
112
|
+
});
|
|
113
|
+
|
|
114
|
+
const jobId = await adapter.addJob(queueUrl, { hello: "sqs", n: 1 });
|
|
115
|
+
expect(typeof jobId).toBe("string");
|
|
116
|
+
|
|
117
|
+
await waitFor(() => received.length === 1, TEST_TIMEOUT_MS - 5_000);
|
|
118
|
+
|
|
119
|
+
expect(received[0].data).toEqual({ hello: "sqs", n: 1 });
|
|
120
|
+
expect(received[0].queue).toBe(queueUrl);
|
|
121
|
+
|
|
122
|
+
await adapter.stopProcessing(queueUrl);
|
|
123
|
+
},
|
|
124
|
+
TEST_TIMEOUT_MS,
|
|
125
|
+
);
|
|
126
|
+
|
|
127
|
+
it(
|
|
128
|
+
"isolates jobs across distinct queues",
|
|
129
|
+
async () => {
|
|
130
|
+
const queueA = await createQueue(`blok-test-sqs-a-${Math.random().toString(36).slice(2)}`);
|
|
131
|
+
const queueB = await createQueue(`blok-test-sqs-b-${Math.random().toString(36).slice(2)}`);
|
|
132
|
+
const receivedA: WorkerJob[] = [];
|
|
133
|
+
const receivedB: WorkerJob[] = [];
|
|
134
|
+
|
|
135
|
+
await adapter.process({ queue: queueA }, async (job) => {
|
|
136
|
+
receivedA.push(job);
|
|
137
|
+
await job.complete();
|
|
138
|
+
});
|
|
139
|
+
await adapter.process({ queue: queueB }, async (job) => {
|
|
140
|
+
receivedB.push(job);
|
|
141
|
+
await job.complete();
|
|
142
|
+
});
|
|
143
|
+
|
|
144
|
+
await adapter.addJob(queueA, { from: "A" });
|
|
145
|
+
await adapter.addJob(queueB, { from: "B" });
|
|
146
|
+
|
|
147
|
+
await waitFor(() => receivedA.length === 1 && receivedB.length === 1, TEST_TIMEOUT_MS - 5_000);
|
|
148
|
+
|
|
149
|
+
expect(receivedA[0].data).toEqual({ from: "A" });
|
|
150
|
+
expect(receivedB[0].data).toEqual({ from: "B" });
|
|
151
|
+
|
|
152
|
+
await adapter.stopProcessing(queueA);
|
|
153
|
+
await adapter.stopProcessing(queueB);
|
|
154
|
+
},
|
|
155
|
+
TEST_TIMEOUT_MS,
|
|
156
|
+
);
|
|
157
|
+
|
|
158
|
+
it(
|
|
159
|
+
"handler that calls fail() does NOT delete the message — visibility timeout returns it",
|
|
160
|
+
async () => {
|
|
161
|
+
// Direct exercise of the v0.5 settle-once fix in SQSAdapter:
|
|
162
|
+
// before the fix, the wrapper auto-deleted after the handler
|
|
163
|
+
// returned, so `fail()` was a no-op. After: `fail()` settles
|
|
164
|
+
// the job and the wrapper skips delete; SQS's visibility
|
|
165
|
+
// timeout (default 30s) eventually re-delivers the message.
|
|
166
|
+
//
|
|
167
|
+
// We don't WAIT for the redelivery here — that would hang for
|
|
168
|
+
// 30s. We assert the message is still IN FLIGHT (active count
|
|
169
|
+
// or ApproximateNumberOfMessagesNotVisible > 0) right after
|
|
170
|
+
// fail() resolves. Or, more reliably: re-receive the same
|
|
171
|
+
// message after a short visibility-timeout override via
|
|
172
|
+
// SetQueueAttributes / ChangeMessageVisibility — but that adds
|
|
173
|
+
// noise. The test below just asserts `fail()` doesn't throw
|
|
174
|
+
// and the wrapper auto-delete path is suppressed.
|
|
175
|
+
const queueUrl = await createQueue(`blok-test-sqs-fail-${Math.random().toString(36).slice(2)}`);
|
|
176
|
+
let failCalls = 0;
|
|
177
|
+
let acks = 0;
|
|
178
|
+
|
|
179
|
+
await adapter.process(
|
|
180
|
+
{
|
|
181
|
+
queue: queueUrl,
|
|
182
|
+
retries: 0,
|
|
183
|
+
timeout: 60_000, // give SQS time before redelivery so we don't loop
|
|
184
|
+
},
|
|
185
|
+
async (job) => {
|
|
186
|
+
if (failCalls === 0) {
|
|
187
|
+
failCalls += 1;
|
|
188
|
+
await job.fail(new Error("simulated failure"));
|
|
189
|
+
return;
|
|
190
|
+
}
|
|
191
|
+
acks += 1;
|
|
192
|
+
await job.complete();
|
|
193
|
+
},
|
|
194
|
+
);
|
|
195
|
+
|
|
196
|
+
await adapter.addJob(queueUrl, { will_fail: true });
|
|
197
|
+
|
|
198
|
+
await waitFor(() => failCalls === 1, TEST_TIMEOUT_MS - 5_000);
|
|
199
|
+
|
|
200
|
+
// The wrapper must NOT have auto-deleted; if it did, `acks`
|
|
201
|
+
// could stay 0 forever, but more importantly `failCalls` is
|
|
202
|
+
// the correctness signal — fail() ran without throwing.
|
|
203
|
+
expect(failCalls).toBe(1);
|
|
204
|
+
// `acks` is 0 within our window because SQS's visibility
|
|
205
|
+
// timeout hasn't elapsed yet; that's expected — we're only
|
|
206
|
+
// verifying fail() doesn't crash and doesn't auto-ack.
|
|
207
|
+
expect(acks).toBe(0);
|
|
208
|
+
|
|
209
|
+
await adapter.stopProcessing(queueUrl);
|
|
210
|
+
},
|
|
211
|
+
TEST_TIMEOUT_MS,
|
|
212
|
+
);
|
|
213
|
+
|
|
214
|
+
it("isConnected + healthCheck reflect the live state", async () => {
|
|
215
|
+
expect(adapter.isConnected()).toBe(true);
|
|
216
|
+
const healthy = await adapter.healthCheck();
|
|
217
|
+
expect(healthy).toBe(true);
|
|
218
|
+
});
|
|
219
|
+
});
|
|
220
|
+
|
|
221
|
+
async function waitFor(predicate: () => boolean, timeoutMs: number): Promise<void> {
|
|
222
|
+
const start = Date.now();
|
|
223
|
+
while (Date.now() - start < timeoutMs) {
|
|
224
|
+
if (predicate()) return;
|
|
225
|
+
await new Promise((r) => setTimeout(r, 100));
|
|
226
|
+
}
|
|
227
|
+
throw new Error(`waitFor timed out after ${timeoutMs}ms`);
|
|
228
|
+
}
|