@blokjs/trigger-worker 0.6.18 → 0.6.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/WorkerTrigger.d.ts +27 -3
- package/dist/WorkerTrigger.js +168 -26
- package/dist/adapters/KafkaAdapter.d.ts +5 -0
- package/dist/adapters/KafkaAdapter.js +12 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.js +2 -2
- package/package.json +5 -4
- package/CHANGELOG.md +0 -22
- package/__tests__/integration/nats-adapter.real-nats.test.ts +0 -116
- package/__tests__/integration/pgboss-adapter.real-pg.test.ts +0 -164
- package/__tests__/integration/rabbitmq-adapter.real-rabbitmq.test.ts +0 -179
- package/__tests__/integration/sqs-adapter.real-sqs.test.ts +0 -228
- package/src/WorkerTrigger.test.ts +0 -540
- package/src/WorkerTrigger.ts +0 -784
- package/src/adapters/BullMQAdapter.ts +0 -296
- package/src/adapters/InMemoryAdapter.ts +0 -280
- package/src/adapters/KafkaAdapter.ts +0 -277
- package/src/adapters/NATSAdapter.ts +0 -454
- package/src/adapters/PgBossAdapter.ts +0 -293
- package/src/adapters/RabbitMQAdapter.ts +0 -285
- package/src/adapters/RedisStreamsAdapter.ts +0 -286
- package/src/adapters/SQSAdapter.ts +0 -306
- package/src/adapters/factory.test.ts +0 -89
- package/src/adapters/factory.ts +0 -111
- package/src/adapters/new-adapters.test.ts +0 -130
- package/src/index.ts +0 -94
- package/template/.env.example +0 -13
- package/template/package.json +0 -45
- package/template/src/Nodes.ts +0 -10
- package/template/src/Workflows.ts +0 -8
- package/template/src/index.ts +0 -41
- package/template/src/runner/WorkerServer.ts +0 -34
- package/template/src/runner/types/Workflows.ts +0 -7
- package/template/src/workflows/jobs/process-job.ts +0 -47
- package/template/tsconfig.json +0 -31
- package/template/vitest.config.ts +0 -39
- package/tsconfig.json +0 -32
|
@@ -1,286 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* RedisStreamsAdapter — v0.7 PR 5 — Worker adapter backed by Redis
|
|
3
|
-
* Streams via `ioredis`. Consumes from a stream (the `queue` field)
|
|
4
|
-
* with a consumer group (`consumerGroup`); produces via `XADD`.
|
|
5
|
-
*
|
|
6
|
-
* Semantics:
|
|
7
|
-
* - **Consumer groups**: required. `consumerGroup` defaults to
|
|
8
|
-
* `"${queue}-group"`. The group is auto-created (`MKSTREAM` on the
|
|
9
|
-
* stream too) on first `process()` call.
|
|
10
|
-
* - **Consumer name**: per-process uuid, so multiple instances of
|
|
11
|
-
* the same worker process don't share pending entries.
|
|
12
|
-
* - **Retries**: pending entries are XACK'd on success / left
|
|
13
|
-
* unacked on failure (visible in `XPENDING`). A redrive loop
|
|
14
|
-
* reads pending entries older than `timeout` and re-delivers
|
|
15
|
-
* to the current consumer.
|
|
16
|
-
* - **Auto-claim**: skipped in v1 — operators should run XAUTOCLAIM
|
|
17
|
-
* periodically out of band.
|
|
18
|
-
*
|
|
19
|
-
* Requires `ioredis` as a peer dependency:
|
|
20
|
-
*
|
|
21
|
-
* bun add ioredis
|
|
22
|
-
*
|
|
23
|
-
* Environment variables:
|
|
24
|
-
* - `REDIS_HOST` (default `localhost`)
|
|
25
|
-
* - `REDIS_PORT` (default `6379`)
|
|
26
|
-
* - `REDIS_PASSWORD`
|
|
27
|
-
* - `REDIS_DB`
|
|
28
|
-
*/
|
|
29
|
-
|
|
30
|
-
import type { WorkerTriggerOpts } from "@blokjs/helper";
|
|
31
|
-
import { v4 as uuid } from "uuid";
|
|
32
|
-
import type { WorkerAdapter, WorkerJob, WorkerQueueStats } from "../WorkerTrigger";
|
|
33
|
-
|
|
34
|
-
export interface RedisStreamsConfig {
|
|
35
|
-
host: string;
|
|
36
|
-
port: number;
|
|
37
|
-
password?: string;
|
|
38
|
-
db?: number;
|
|
39
|
-
blockMs: number;
|
|
40
|
-
count: number;
|
|
41
|
-
}
|
|
42
|
-
|
|
43
|
-
interface RedisClient {
|
|
44
|
-
xadd(stream: string, ...args: string[]): Promise<string>;
|
|
45
|
-
xreadgroup(...args: string[]): Promise<Array<[string, Array<[string, string[]]>]> | null>;
|
|
46
|
-
xgroup(...args: string[]): Promise<string>;
|
|
47
|
-
xack(stream: string, group: string, ...ids: string[]): Promise<number>;
|
|
48
|
-
xlen(stream: string): Promise<number>;
|
|
49
|
-
xpending(stream: string, group: string): Promise<unknown>;
|
|
50
|
-
ping(): Promise<string>;
|
|
51
|
-
quit(): Promise<string>;
|
|
52
|
-
}
|
|
53
|
-
|
|
54
|
-
interface QueueRunner {
|
|
55
|
-
stop: boolean;
|
|
56
|
-
loops: number;
|
|
57
|
-
}
|
|
58
|
-
|
|
59
|
-
interface QueueStatsCounters {
|
|
60
|
-
completed: number;
|
|
61
|
-
failed: number;
|
|
62
|
-
active: number;
|
|
63
|
-
}
|
|
64
|
-
|
|
65
|
-
export class RedisStreamsAdapter implements WorkerAdapter {
|
|
66
|
-
readonly provider = "redis" as const;
|
|
67
|
-
private readonly config: RedisStreamsConfig;
|
|
68
|
-
private client: RedisClient | null = null;
|
|
69
|
-
private runners: Map<string, QueueRunner> = new Map();
|
|
70
|
-
private connected = false;
|
|
71
|
-
private stats: Map<string, QueueStatsCounters> = new Map();
|
|
72
|
-
private consumerName = `blok-${uuid().slice(0, 8)}`;
|
|
73
|
-
|
|
74
|
-
constructor(config?: Partial<RedisStreamsConfig>) {
|
|
75
|
-
this.config = {
|
|
76
|
-
host: config?.host ?? process.env.REDIS_HOST ?? "localhost",
|
|
77
|
-
port: config?.port ?? Number.parseInt(process.env.REDIS_PORT ?? "6379", 10),
|
|
78
|
-
password: config?.password ?? process.env.REDIS_PASSWORD,
|
|
79
|
-
db: config?.db ?? Number.parseInt(process.env.REDIS_DB ?? "0", 10),
|
|
80
|
-
blockMs: config?.blockMs ?? 5000,
|
|
81
|
-
count: config?.count ?? 10,
|
|
82
|
-
};
|
|
83
|
-
}
|
|
84
|
-
|
|
85
|
-
async connect(): Promise<void> {
|
|
86
|
-
if (this.connected) return;
|
|
87
|
-
try {
|
|
88
|
-
// biome-ignore lint/suspicious/noExplicitAny: ioredis is a runtime peer dep.
|
|
89
|
-
const ioredis: any = await import("ioredis");
|
|
90
|
-
const IORedis = ioredis.default ?? ioredis.Redis ?? ioredis;
|
|
91
|
-
this.client = new IORedis({
|
|
92
|
-
host: this.config.host,
|
|
93
|
-
port: this.config.port,
|
|
94
|
-
password: this.config.password,
|
|
95
|
-
db: this.config.db,
|
|
96
|
-
}) as RedisClient;
|
|
97
|
-
await this.client.ping();
|
|
98
|
-
this.connected = true;
|
|
99
|
-
} catch (err) {
|
|
100
|
-
throw new Error(
|
|
101
|
-
`[blok][redis] connect failed: ${(err as Error).message}. Install ioredis as a peer dependency: bun add ioredis`,
|
|
102
|
-
);
|
|
103
|
-
}
|
|
104
|
-
}
|
|
105
|
-
|
|
106
|
-
async disconnect(): Promise<void> {
|
|
107
|
-
if (!this.connected) return;
|
|
108
|
-
for (const runner of this.runners.values()) runner.stop = true;
|
|
109
|
-
// Wait for in-flight loops to drain.
|
|
110
|
-
const drainDeadline = Date.now() + 2000;
|
|
111
|
-
while (Date.now() < drainDeadline) {
|
|
112
|
-
let active = 0;
|
|
113
|
-
for (const r of this.runners.values()) active += r.loops;
|
|
114
|
-
if (active === 0) break;
|
|
115
|
-
await new Promise((r) => setTimeout(r, 50));
|
|
116
|
-
}
|
|
117
|
-
this.runners.clear();
|
|
118
|
-
try {
|
|
119
|
-
await this.client?.quit();
|
|
120
|
-
} catch {
|
|
121
|
-
/* ignore */
|
|
122
|
-
}
|
|
123
|
-
this.client = null;
|
|
124
|
-
this.connected = false;
|
|
125
|
-
}
|
|
126
|
-
|
|
127
|
-
async process(config: WorkerTriggerOpts, handler: (job: WorkerJob) => Promise<void>): Promise<void> {
|
|
128
|
-
if (!this.connected || !this.client) throw new Error("[blok][redis] not connected. Call connect() first.");
|
|
129
|
-
const stream = config.queue;
|
|
130
|
-
const group = config.consumerGroup ?? `${stream}-group`;
|
|
131
|
-
// Create group + stream if missing. XGROUP CREATE with MKSTREAM
|
|
132
|
-
// errors with "BUSYGROUP" if the group already exists — swallow it.
|
|
133
|
-
try {
|
|
134
|
-
await this.client.xgroup("CREATE", stream, group, "$", "MKSTREAM");
|
|
135
|
-
} catch (err) {
|
|
136
|
-
if (!/BUSYGROUP/i.test((err as Error).message)) throw err;
|
|
137
|
-
}
|
|
138
|
-
|
|
139
|
-
const runner: QueueRunner = { stop: false, loops: 0 };
|
|
140
|
-
this.runners.set(stream, runner);
|
|
141
|
-
this.stats.set(stream, { completed: 0, failed: 0, active: 0 });
|
|
142
|
-
const stats = this.stats.get(stream) as QueueStatsCounters;
|
|
143
|
-
|
|
144
|
-
const concurrency = Math.max(1, config.concurrency ?? 1);
|
|
145
|
-
for (let i = 0; i < concurrency; i += 1) {
|
|
146
|
-
void this.runConsumerLoop(stream, group, config, handler, runner, stats);
|
|
147
|
-
}
|
|
148
|
-
}
|
|
149
|
-
|
|
150
|
-
private async runConsumerLoop(
|
|
151
|
-
stream: string,
|
|
152
|
-
group: string,
|
|
153
|
-
config: WorkerTriggerOpts,
|
|
154
|
-
handler: (job: WorkerJob) => Promise<void>,
|
|
155
|
-
runner: QueueRunner,
|
|
156
|
-
stats: QueueStatsCounters,
|
|
157
|
-
): Promise<void> {
|
|
158
|
-
if (!this.client) return;
|
|
159
|
-
runner.loops += 1;
|
|
160
|
-
try {
|
|
161
|
-
while (!runner.stop) {
|
|
162
|
-
let entries: Array<[string, Array<[string, string[]]>]> | null = null;
|
|
163
|
-
try {
|
|
164
|
-
entries = await this.client.xreadgroup(
|
|
165
|
-
"GROUP",
|
|
166
|
-
group,
|
|
167
|
-
this.consumerName,
|
|
168
|
-
"COUNT",
|
|
169
|
-
String(this.config.count),
|
|
170
|
-
"BLOCK",
|
|
171
|
-
String(this.config.blockMs),
|
|
172
|
-
"STREAMS",
|
|
173
|
-
stream,
|
|
174
|
-
">",
|
|
175
|
-
);
|
|
176
|
-
} catch (err) {
|
|
177
|
-
await new Promise((r) => setTimeout(r, 1000));
|
|
178
|
-
continue;
|
|
179
|
-
}
|
|
180
|
-
if (!entries) continue;
|
|
181
|
-
for (const [, msgs] of entries) {
|
|
182
|
-
for (const [id, fields] of msgs) {
|
|
183
|
-
if (runner.stop) break;
|
|
184
|
-
stats.active += 1;
|
|
185
|
-
try {
|
|
186
|
-
const payload = this.fieldsToObject(fields);
|
|
187
|
-
const dataString = typeof payload.data === "string" ? payload.data : "";
|
|
188
|
-
let data: unknown;
|
|
189
|
-
try {
|
|
190
|
-
data = dataString.length > 0 ? JSON.parse(dataString) : null;
|
|
191
|
-
} catch {
|
|
192
|
-
data = dataString;
|
|
193
|
-
}
|
|
194
|
-
const job: WorkerJob = {
|
|
195
|
-
id,
|
|
196
|
-
data,
|
|
197
|
-
headers: {},
|
|
198
|
-
queue: stream,
|
|
199
|
-
priority: config.priority ?? 0,
|
|
200
|
-
attempts: 0,
|
|
201
|
-
maxRetries: config.retries ?? 0,
|
|
202
|
-
createdAt: new Date(Number.parseInt(id.split("-")[0] ?? String(Date.now()), 10)),
|
|
203
|
-
timeout: config.timeout,
|
|
204
|
-
raw: { id, fields },
|
|
205
|
-
complete: async () => {
|
|
206
|
-
await this.client?.xack(stream, group, id);
|
|
207
|
-
stats.completed += 1;
|
|
208
|
-
},
|
|
209
|
-
fail: async (_err: Error) => {
|
|
210
|
-
stats.failed += 1;
|
|
211
|
-
},
|
|
212
|
-
};
|
|
213
|
-
await handler(job);
|
|
214
|
-
if (config.ack !== false) await this.client?.xack(stream, group, id);
|
|
215
|
-
stats.completed += 1;
|
|
216
|
-
} catch {
|
|
217
|
-
stats.failed += 1;
|
|
218
|
-
// Pending entry left unacked — picked up by XAUTOCLAIM.
|
|
219
|
-
} finally {
|
|
220
|
-
stats.active = Math.max(0, stats.active - 1);
|
|
221
|
-
}
|
|
222
|
-
}
|
|
223
|
-
}
|
|
224
|
-
}
|
|
225
|
-
} finally {
|
|
226
|
-
runner.loops -= 1;
|
|
227
|
-
}
|
|
228
|
-
}
|
|
229
|
-
|
|
230
|
-
private fieldsToObject(fields: string[]): Record<string, string> {
|
|
231
|
-
const out: Record<string, string> = {};
|
|
232
|
-
for (let i = 0; i < fields.length; i += 2) {
|
|
233
|
-
out[fields[i]] = fields[i + 1];
|
|
234
|
-
}
|
|
235
|
-
return out;
|
|
236
|
-
}
|
|
237
|
-
|
|
238
|
-
async addJob(
|
|
239
|
-
queue: string,
|
|
240
|
-
data: unknown,
|
|
241
|
-
opts?: { priority?: number; delay?: number; retries?: number; timeout?: number; jobId?: string },
|
|
242
|
-
): Promise<string> {
|
|
243
|
-
if (!this.connected || !this.client) throw new Error("[blok][redis] not connected. Call connect() first.");
|
|
244
|
-
const payload = typeof data === "string" ? data : JSON.stringify(data);
|
|
245
|
-
const args: string[] = [];
|
|
246
|
-
if (opts?.jobId) args.push("NOMKSTREAM");
|
|
247
|
-
const id = await this.client.xadd(queue, "*", "data", payload, "jobId", opts?.jobId ?? "");
|
|
248
|
-
return id;
|
|
249
|
-
}
|
|
250
|
-
|
|
251
|
-
async stopProcessing(queue: string): Promise<void> {
|
|
252
|
-
const runner = this.runners.get(queue);
|
|
253
|
-
if (runner) runner.stop = true;
|
|
254
|
-
}
|
|
255
|
-
|
|
256
|
-
isConnected(): boolean {
|
|
257
|
-
return this.connected;
|
|
258
|
-
}
|
|
259
|
-
|
|
260
|
-
async healthCheck(): Promise<boolean> {
|
|
261
|
-
if (!this.connected || !this.client) return false;
|
|
262
|
-
try {
|
|
263
|
-
const pong = await this.client.ping();
|
|
264
|
-
return pong === "PONG";
|
|
265
|
-
} catch {
|
|
266
|
-
return false;
|
|
267
|
-
}
|
|
268
|
-
}
|
|
269
|
-
|
|
270
|
-
async getQueueStats(queue: string): Promise<WorkerQueueStats> {
|
|
271
|
-
const counters = this.stats.get(queue) ?? { completed: 0, failed: 0, active: 0 };
|
|
272
|
-
let waiting = 0;
|
|
273
|
-
try {
|
|
274
|
-
waiting = (await this.client?.xlen(queue)) ?? 0;
|
|
275
|
-
} catch {
|
|
276
|
-
/* ignore */
|
|
277
|
-
}
|
|
278
|
-
return {
|
|
279
|
-
waiting,
|
|
280
|
-
active: counters.active,
|
|
281
|
-
completed: counters.completed,
|
|
282
|
-
failed: counters.failed,
|
|
283
|
-
delayed: 0,
|
|
284
|
-
};
|
|
285
|
-
}
|
|
286
|
-
}
|
|
@@ -1,306 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* SQSAdapter — v0.7 PR 5 — Worker adapter backed by AWS SQS via
|
|
3
|
-
* `@aws-sdk/client-sqs`. Polls a queue URL (the `queue` field) with
|
|
4
|
-
* long-polling; processes messages with manual delete (ACK).
|
|
5
|
-
*
|
|
6
|
-
* Semantics:
|
|
7
|
-
* - **Long polling**: `WaitTimeSeconds=20` by default — minimises
|
|
8
|
-
* poll cost. `concurrency` controls how many parallel
|
|
9
|
-
* `ReceiveMessage` loops run.
|
|
10
|
-
* - **Visibility timeout**: configured via `timeout` (ms → s).
|
|
11
|
-
* Messages reappear after this if the worker doesn't delete them.
|
|
12
|
-
* - **Retries**: SQS handles retries automatically via redrive
|
|
13
|
-
* policy on the queue itself. The adapter doesn't simulate
|
|
14
|
-
* retries client-side — set `MaxReceiveCount` on the queue's
|
|
15
|
-
* redrive policy and a DLQ via `deadLetterQueue`.
|
|
16
|
-
* - **FIFO support**: when the queue URL ends with `.fifo`, the
|
|
17
|
-
* adapter passes `MessageGroupId` from `dedupId` or a default.
|
|
18
|
-
*
|
|
19
|
-
* Requires `@aws-sdk/client-sqs` as a peer dependency:
|
|
20
|
-
*
|
|
21
|
-
* bun add @aws-sdk/client-sqs
|
|
22
|
-
*
|
|
23
|
-
* Environment variables (standard AWS SDK):
|
|
24
|
-
* - `AWS_REGION` — default `us-east-1`.
|
|
25
|
-
* - `AWS_ACCESS_KEY_ID` — credentials (or use a profile).
|
|
26
|
-
* - `AWS_SECRET_ACCESS_KEY`
|
|
27
|
-
* - `SQS_ENDPOINT_URL` — for local SQS (LocalStack / ElasticMQ).
|
|
28
|
-
*/
|
|
29
|
-
|
|
30
|
-
import type { WorkerTriggerOpts } from "@blokjs/helper";
|
|
31
|
-
import { v4 as uuid } from "uuid";
|
|
32
|
-
import type { WorkerAdapter, WorkerJob, WorkerQueueStats } from "../WorkerTrigger";
|
|
33
|
-
|
|
34
|
-
export interface SQSConfig {
|
|
35
|
-
region: string;
|
|
36
|
-
endpoint?: string;
|
|
37
|
-
waitTimeSeconds: number;
|
|
38
|
-
maxNumberOfMessages: number;
|
|
39
|
-
}
|
|
40
|
-
|
|
41
|
-
interface SqsMessage {
|
|
42
|
-
MessageId?: string;
|
|
43
|
-
ReceiptHandle?: string;
|
|
44
|
-
Body?: string;
|
|
45
|
-
Attributes?: Record<string, string>;
|
|
46
|
-
MessageAttributes?: Record<string, { StringValue?: string }>;
|
|
47
|
-
}
|
|
48
|
-
|
|
49
|
-
interface QueueRunner {
|
|
50
|
-
stop: boolean;
|
|
51
|
-
loops: number;
|
|
52
|
-
}
|
|
53
|
-
|
|
54
|
-
interface QueueStatsCounters {
|
|
55
|
-
completed: number;
|
|
56
|
-
failed: number;
|
|
57
|
-
active: number;
|
|
58
|
-
}
|
|
59
|
-
|
|
60
|
-
export class SQSAdapter implements WorkerAdapter {
|
|
61
|
-
readonly provider = "sqs" as const;
|
|
62
|
-
private readonly config: SQSConfig;
|
|
63
|
-
// biome-ignore lint/suspicious/noExplicitAny: @aws-sdk/client-sqs client + command types are loose.
|
|
64
|
-
private client: any = null;
|
|
65
|
-
// biome-ignore lint/suspicious/noExplicitAny: same as above.
|
|
66
|
-
private commands: any = null;
|
|
67
|
-
private runners: Map<string, QueueRunner> = new Map();
|
|
68
|
-
private connected = false;
|
|
69
|
-
private stats: Map<string, QueueStatsCounters> = new Map();
|
|
70
|
-
|
|
71
|
-
constructor(config?: Partial<SQSConfig>) {
|
|
72
|
-
this.config = {
|
|
73
|
-
region: config?.region ?? process.env.AWS_REGION ?? "us-east-1",
|
|
74
|
-
endpoint: config?.endpoint ?? process.env.SQS_ENDPOINT_URL,
|
|
75
|
-
waitTimeSeconds: config?.waitTimeSeconds ?? 20,
|
|
76
|
-
maxNumberOfMessages: config?.maxNumberOfMessages ?? 10,
|
|
77
|
-
};
|
|
78
|
-
}
|
|
79
|
-
|
|
80
|
-
async connect(): Promise<void> {
|
|
81
|
-
if (this.connected) return;
|
|
82
|
-
try {
|
|
83
|
-
// biome-ignore lint/suspicious/noExplicitAny: peer dep
|
|
84
|
-
const sdk: any = await import("@aws-sdk/client-sqs");
|
|
85
|
-
this.client = new sdk.SQSClient({ region: this.config.region, endpoint: this.config.endpoint });
|
|
86
|
-
this.commands = sdk;
|
|
87
|
-
this.connected = true;
|
|
88
|
-
} catch (err) {
|
|
89
|
-
throw new Error(
|
|
90
|
-
`[blok][sqs] connect failed: ${(err as Error).message}. Install @aws-sdk/client-sqs as a peer dependency: bun add @aws-sdk/client-sqs`,
|
|
91
|
-
);
|
|
92
|
-
}
|
|
93
|
-
}
|
|
94
|
-
|
|
95
|
-
async disconnect(): Promise<void> {
|
|
96
|
-
if (!this.connected) return;
|
|
97
|
-
for (const runner of this.runners.values()) runner.stop = true;
|
|
98
|
-
// Wait for in-flight loops to drain — up to 500ms each.
|
|
99
|
-
const drainDeadline = Date.now() + 2000;
|
|
100
|
-
while (Date.now() < drainDeadline) {
|
|
101
|
-
let active = 0;
|
|
102
|
-
for (const r of this.runners.values()) active += r.loops;
|
|
103
|
-
if (active === 0) break;
|
|
104
|
-
await new Promise((r) => setTimeout(r, 50));
|
|
105
|
-
}
|
|
106
|
-
this.runners.clear();
|
|
107
|
-
try {
|
|
108
|
-
this.client?.destroy?.();
|
|
109
|
-
} catch {
|
|
110
|
-
/* ignore */
|
|
111
|
-
}
|
|
112
|
-
this.client = null;
|
|
113
|
-
this.connected = false;
|
|
114
|
-
}
|
|
115
|
-
|
|
116
|
-
async process(config: WorkerTriggerOpts, handler: (job: WorkerJob) => Promise<void>): Promise<void> {
|
|
117
|
-
if (!this.connected) throw new Error("[blok][sqs] not connected. Call connect() first.");
|
|
118
|
-
const runner: QueueRunner = { stop: false, loops: 0 };
|
|
119
|
-
this.runners.set(config.queue, runner);
|
|
120
|
-
this.stats.set(config.queue, { completed: 0, failed: 0, active: 0 });
|
|
121
|
-
const stats = this.stats.get(config.queue) as QueueStatsCounters;
|
|
122
|
-
|
|
123
|
-
const concurrency = Math.max(1, config.concurrency ?? 1);
|
|
124
|
-
for (let i = 0; i < concurrency; i += 1) {
|
|
125
|
-
void this.runConsumerLoop(config, handler, runner, stats);
|
|
126
|
-
}
|
|
127
|
-
}
|
|
128
|
-
|
|
129
|
-
private async runConsumerLoop(
|
|
130
|
-
config: WorkerTriggerOpts,
|
|
131
|
-
handler: (job: WorkerJob) => Promise<void>,
|
|
132
|
-
runner: QueueRunner,
|
|
133
|
-
stats: QueueStatsCounters,
|
|
134
|
-
): Promise<void> {
|
|
135
|
-
runner.loops += 1;
|
|
136
|
-
try {
|
|
137
|
-
while (!runner.stop) {
|
|
138
|
-
let response: { Messages?: SqsMessage[] } = {};
|
|
139
|
-
try {
|
|
140
|
-
response = await this.client.send(
|
|
141
|
-
new this.commands.ReceiveMessageCommand({
|
|
142
|
-
QueueUrl: config.queue,
|
|
143
|
-
MaxNumberOfMessages: Math.min(10, this.config.maxNumberOfMessages),
|
|
144
|
-
WaitTimeSeconds: this.config.waitTimeSeconds,
|
|
145
|
-
VisibilityTimeout: typeof config.timeout === "number" ? Math.ceil(config.timeout / 1000) : 30,
|
|
146
|
-
AttributeNames: ["All"],
|
|
147
|
-
MessageAttributeNames: ["All"],
|
|
148
|
-
}),
|
|
149
|
-
);
|
|
150
|
-
} catch (err) {
|
|
151
|
-
// Transient — back off briefly then retry.
|
|
152
|
-
await new Promise((r) => setTimeout(r, 1000));
|
|
153
|
-
continue;
|
|
154
|
-
}
|
|
155
|
-
const messages = response.Messages ?? [];
|
|
156
|
-
for (const m of messages) {
|
|
157
|
-
if (runner.stop) break;
|
|
158
|
-
stats.active += 1;
|
|
159
|
-
// Tracks whether the message has already been settled via
|
|
160
|
-
// the WorkerJob API (`complete` / `fail`). Declared OUTSIDE
|
|
161
|
-
// the try so the catch arm can read it and skip its own
|
|
162
|
-
// nack/fail bookkeeping when the handler explicitly
|
|
163
|
-
// settled. Without this flag we'd double-delete the
|
|
164
|
-
// receipt handle AND a `fail()` call would be silently
|
|
165
|
-
// overruled by the wrapper deleting the message anyway.
|
|
166
|
-
// Caught by the real-broker integration test in
|
|
167
|
-
// `__tests__/integration/sqs-adapter.real-sqs.test.ts`.
|
|
168
|
-
let settled = false;
|
|
169
|
-
try {
|
|
170
|
-
const payloadString = m.Body ?? "";
|
|
171
|
-
let data: unknown;
|
|
172
|
-
try {
|
|
173
|
-
data = payloadString.length > 0 ? JSON.parse(payloadString) : null;
|
|
174
|
-
} catch {
|
|
175
|
-
data = payloadString;
|
|
176
|
-
}
|
|
177
|
-
const headers: Record<string, string> = {};
|
|
178
|
-
for (const [k, v] of Object.entries(m.MessageAttributes ?? {})) {
|
|
179
|
-
if (typeof v.StringValue === "string") headers[k] = v.StringValue;
|
|
180
|
-
}
|
|
181
|
-
const job: WorkerJob = {
|
|
182
|
-
id: m.MessageId ?? `${config.queue}:${uuid()}`,
|
|
183
|
-
data,
|
|
184
|
-
headers,
|
|
185
|
-
queue: config.queue,
|
|
186
|
-
priority: config.priority ?? 0,
|
|
187
|
-
attempts: Number.parseInt(m.Attributes?.ApproximateReceiveCount ?? "1", 10) - 1,
|
|
188
|
-
maxRetries: config.retries ?? 0,
|
|
189
|
-
createdAt: new Date(),
|
|
190
|
-
timeout: config.timeout,
|
|
191
|
-
raw: m,
|
|
192
|
-
complete: async () => {
|
|
193
|
-
if (settled) return;
|
|
194
|
-
if (m.ReceiptHandle) {
|
|
195
|
-
await this.client.send(
|
|
196
|
-
new this.commands.DeleteMessageCommand({
|
|
197
|
-
QueueUrl: config.queue,
|
|
198
|
-
ReceiptHandle: m.ReceiptHandle,
|
|
199
|
-
}),
|
|
200
|
-
);
|
|
201
|
-
}
|
|
202
|
-
stats.completed += 1;
|
|
203
|
-
settled = true;
|
|
204
|
-
},
|
|
205
|
-
fail: async (_err: Error) => {
|
|
206
|
-
if (settled) return;
|
|
207
|
-
stats.failed += 1;
|
|
208
|
-
// No DeleteMessage call — visibility timeout
|
|
209
|
-
// expires and SQS returns the message to the
|
|
210
|
-
// queue automatically. DLQ takeover happens via
|
|
211
|
-
// the queue's RedrivePolicy + MaxReceiveCount.
|
|
212
|
-
settled = true;
|
|
213
|
-
},
|
|
214
|
-
};
|
|
215
|
-
await handler(job);
|
|
216
|
-
if (!settled && config.ack !== false && m.ReceiptHandle) {
|
|
217
|
-
await this.client.send(
|
|
218
|
-
new this.commands.DeleteMessageCommand({ QueueUrl: config.queue, ReceiptHandle: m.ReceiptHandle }),
|
|
219
|
-
);
|
|
220
|
-
stats.completed += 1;
|
|
221
|
-
settled = true;
|
|
222
|
-
}
|
|
223
|
-
} catch {
|
|
224
|
-
if (!settled) {
|
|
225
|
-
stats.failed += 1;
|
|
226
|
-
// Leave the message in flight — SQS visibility
|
|
227
|
-
// timeout expiry returns it to the queue.
|
|
228
|
-
}
|
|
229
|
-
} finally {
|
|
230
|
-
stats.active = Math.max(0, stats.active - 1);
|
|
231
|
-
}
|
|
232
|
-
}
|
|
233
|
-
}
|
|
234
|
-
} finally {
|
|
235
|
-
runner.loops -= 1;
|
|
236
|
-
}
|
|
237
|
-
}
|
|
238
|
-
|
|
239
|
-
async addJob(
|
|
240
|
-
queue: string,
|
|
241
|
-
data: unknown,
|
|
242
|
-
opts?: { priority?: number; delay?: number; retries?: number; timeout?: number; jobId?: string },
|
|
243
|
-
): Promise<string> {
|
|
244
|
-
if (!this.connected) throw new Error("[blok][sqs] not connected. Call connect() first.");
|
|
245
|
-
const messageId = opts?.jobId ?? uuid();
|
|
246
|
-
const isFifo = queue.endsWith(".fifo");
|
|
247
|
-
const params: Record<string, unknown> = {
|
|
248
|
-
QueueUrl: queue,
|
|
249
|
-
MessageBody: typeof data === "string" ? data : JSON.stringify(data),
|
|
250
|
-
};
|
|
251
|
-
if (isFifo) {
|
|
252
|
-
params.MessageGroupId = opts?.jobId ?? "default";
|
|
253
|
-
params.MessageDeduplicationId = messageId;
|
|
254
|
-
}
|
|
255
|
-
if (typeof opts?.delay === "number" && opts.delay > 0) {
|
|
256
|
-
params.DelaySeconds = Math.min(900, Math.ceil(opts.delay / 1000));
|
|
257
|
-
}
|
|
258
|
-
const result = await this.client.send(new this.commands.SendMessageCommand(params));
|
|
259
|
-
return (result.MessageId as string) ?? messageId;
|
|
260
|
-
}
|
|
261
|
-
|
|
262
|
-
async stopProcessing(queue: string): Promise<void> {
|
|
263
|
-
const runner = this.runners.get(queue);
|
|
264
|
-
if (runner) runner.stop = true;
|
|
265
|
-
}
|
|
266
|
-
|
|
267
|
-
isConnected(): boolean {
|
|
268
|
-
return this.connected;
|
|
269
|
-
}
|
|
270
|
-
|
|
271
|
-
async healthCheck(): Promise<boolean> {
|
|
272
|
-
if (!this.connected) return false;
|
|
273
|
-
try {
|
|
274
|
-
// ListQueues is a cheap permission probe.
|
|
275
|
-
await this.client.send(new this.commands.ListQueuesCommand({ MaxResults: 1 }));
|
|
276
|
-
return true;
|
|
277
|
-
} catch {
|
|
278
|
-
return false;
|
|
279
|
-
}
|
|
280
|
-
}
|
|
281
|
-
|
|
282
|
-
async getQueueStats(queue: string): Promise<WorkerQueueStats> {
|
|
283
|
-
const counters = this.stats.get(queue) ?? { completed: 0, failed: 0, active: 0 };
|
|
284
|
-
let waiting = 0;
|
|
285
|
-
let delayed = 0;
|
|
286
|
-
try {
|
|
287
|
-
const result = await this.client.send(
|
|
288
|
-
new this.commands.GetQueueAttributesCommand({
|
|
289
|
-
QueueUrl: queue,
|
|
290
|
-
AttributeNames: ["ApproximateNumberOfMessages", "ApproximateNumberOfMessagesDelayed"],
|
|
291
|
-
}),
|
|
292
|
-
);
|
|
293
|
-
waiting = Number.parseInt(result.Attributes?.ApproximateNumberOfMessages ?? "0", 10);
|
|
294
|
-
delayed = Number.parseInt(result.Attributes?.ApproximateNumberOfMessagesDelayed ?? "0", 10);
|
|
295
|
-
} catch {
|
|
296
|
-
/* ignore */
|
|
297
|
-
}
|
|
298
|
-
return {
|
|
299
|
-
waiting,
|
|
300
|
-
active: counters.active,
|
|
301
|
-
completed: counters.completed,
|
|
302
|
-
failed: counters.failed,
|
|
303
|
-
delayed,
|
|
304
|
-
};
|
|
305
|
-
}
|
|
306
|
-
}
|
|
@@ -1,89 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Adapter factory unit tests — v0.7 PR 5.
|
|
3
|
-
*
|
|
4
|
-
* Covers provider resolution order, the constructor lookup table,
|
|
5
|
-
* pool sharing, and the test-reset utility.
|
|
6
|
-
*/
|
|
7
|
-
|
|
8
|
-
import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
|
9
|
-
|
|
10
|
-
import { _resetAdapterPoolForTests, createWorkerAdapter, getOrCreateAdapter, resolveProvider } from "./factory";
|
|
11
|
-
|
|
12
|
-
describe("adapter factory — v0.7 PR 5", () => {
|
|
13
|
-
beforeEach(() => {
|
|
14
|
-
_resetAdapterPoolForTests();
|
|
15
|
-
process.env.BLOK_WORKER_ADAPTER = undefined;
|
|
16
|
-
});
|
|
17
|
-
|
|
18
|
-
afterEach(() => {
|
|
19
|
-
_resetAdapterPoolForTests();
|
|
20
|
-
process.env.BLOK_WORKER_ADAPTER = undefined;
|
|
21
|
-
});
|
|
22
|
-
|
|
23
|
-
describe("resolveProvider()", () => {
|
|
24
|
-
it("returns the explicit provider when set", () => {
|
|
25
|
-
expect(resolveProvider("kafka")).toBe("kafka");
|
|
26
|
-
});
|
|
27
|
-
|
|
28
|
-
it("falls back to BLOK_WORKER_ADAPTER env var", () => {
|
|
29
|
-
process.env.BLOK_WORKER_ADAPTER = "rabbitmq";
|
|
30
|
-
expect(resolveProvider(undefined)).toBe("rabbitmq");
|
|
31
|
-
});
|
|
32
|
-
|
|
33
|
-
it("falls back to in-memory when neither is set", () => {
|
|
34
|
-
expect(resolveProvider(undefined)).toBe("in-memory");
|
|
35
|
-
});
|
|
36
|
-
|
|
37
|
-
it("ignores invalid env values and falls back to in-memory", () => {
|
|
38
|
-
process.env.BLOK_WORKER_ADAPTER = "not-a-provider";
|
|
39
|
-
expect(resolveProvider(undefined)).toBe("in-memory");
|
|
40
|
-
});
|
|
41
|
-
|
|
42
|
-
it("explicit provider wins over the env var", () => {
|
|
43
|
-
process.env.BLOK_WORKER_ADAPTER = "bullmq";
|
|
44
|
-
expect(resolveProvider("kafka")).toBe("kafka");
|
|
45
|
-
});
|
|
46
|
-
});
|
|
47
|
-
|
|
48
|
-
describe("createWorkerAdapter()", () => {
|
|
49
|
-
it("returns the correct provider name for each built-in", () => {
|
|
50
|
-
expect(createWorkerAdapter("in-memory").provider).toBe("in-memory");
|
|
51
|
-
expect(createWorkerAdapter("nats").provider).toBe("nats");
|
|
52
|
-
expect(createWorkerAdapter("bullmq").provider).toBe("bullmq");
|
|
53
|
-
expect(createWorkerAdapter("kafka").provider).toBe("kafka");
|
|
54
|
-
expect(createWorkerAdapter("rabbitmq").provider).toBe("rabbitmq");
|
|
55
|
-
expect(createWorkerAdapter("sqs").provider).toBe("sqs");
|
|
56
|
-
expect(createWorkerAdapter("redis").provider).toBe("redis");
|
|
57
|
-
expect(createWorkerAdapter("pg-boss").provider).toBe("pg-boss");
|
|
58
|
-
});
|
|
59
|
-
|
|
60
|
-
it("each call returns a fresh instance", () => {
|
|
61
|
-
const a = createWorkerAdapter("in-memory");
|
|
62
|
-
const b = createWorkerAdapter("in-memory");
|
|
63
|
-
expect(a).not.toBe(b);
|
|
64
|
-
});
|
|
65
|
-
});
|
|
66
|
-
|
|
67
|
-
describe("getOrCreateAdapter()", () => {
|
|
68
|
-
it("returns the same instance on repeated calls for the same provider", () => {
|
|
69
|
-
const a = getOrCreateAdapter("in-memory");
|
|
70
|
-
const b = getOrCreateAdapter("in-memory");
|
|
71
|
-
expect(a).toBe(b);
|
|
72
|
-
});
|
|
73
|
-
|
|
74
|
-
it("returns different instances for different providers", () => {
|
|
75
|
-
const inMem = getOrCreateAdapter("in-memory");
|
|
76
|
-
const kafka = getOrCreateAdapter("kafka");
|
|
77
|
-
expect(inMem).not.toBe(kafka);
|
|
78
|
-
expect(inMem.provider).toBe("in-memory");
|
|
79
|
-
expect(kafka.provider).toBe("kafka");
|
|
80
|
-
});
|
|
81
|
-
|
|
82
|
-
it("_resetAdapterPoolForTests() drops cached instances", () => {
|
|
83
|
-
const first = getOrCreateAdapter("in-memory");
|
|
84
|
-
_resetAdapterPoolForTests();
|
|
85
|
-
const second = getOrCreateAdapter("in-memory");
|
|
86
|
-
expect(first).not.toBe(second);
|
|
87
|
-
});
|
|
88
|
-
});
|
|
89
|
-
});
|