@trigger.dev/redis-worker 4.5.9 → 4.5.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +15625 -16447
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +1902 -1887
- package/dist/index.d.ts +1902 -1887
- package/dist/index.js +15617 -16442
- package/dist/index.js.map +1 -1
- package/package.json +6 -5
package/dist/index.d.cts
CHANGED
|
@@ -1,295 +1,293 @@
|
|
|
1
|
-
import { Callback,
|
|
2
|
-
import { Logger } from
|
|
3
|
-
import { z } from
|
|
4
|
-
import {
|
|
5
|
-
import { RetryOptions as RetryOptions$1 } from
|
|
6
|
-
|
|
1
|
+
import { Callback, Redis, RedisOptions, Result } from "@internal/redis";
|
|
2
|
+
import { Logger } from "@trigger.dev/core/logger";
|
|
3
|
+
import { z } from "zod";
|
|
4
|
+
import { Attributes, Context, Counter, Histogram, Meter, ObservableGauge, Span, SpanKind, Tracer } from "@internal/tracing";
|
|
5
|
+
import { RetryOptions as RetryOptions$1 } from "@trigger.dev/core/v3/schemas";
|
|
6
|
+
//#region src/queue.d.ts
|
|
7
7
|
interface MessageCatalogSchema {
|
|
8
|
-
|
|
8
|
+
[key: string]: z.ZodFirstPartySchemaTypes | z.ZodDiscriminatedUnion<any, any>;
|
|
9
9
|
}
|
|
10
10
|
type MessageCatalogKey<TMessageCatalog extends MessageCatalogSchema> = keyof TMessageCatalog;
|
|
11
11
|
type MessageCatalogValue<TMessageCatalog extends MessageCatalogSchema, TKey extends MessageCatalogKey<TMessageCatalog>> = z.infer<TMessageCatalog[TKey]>;
|
|
12
12
|
type AnyMessageCatalog = MessageCatalogSchema;
|
|
13
13
|
type QueueItem<TMessageCatalog extends MessageCatalogSchema> = {
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
14
|
+
id: string;
|
|
15
|
+
job: MessageCatalogKey<TMessageCatalog>;
|
|
16
|
+
item: MessageCatalogValue<TMessageCatalog, MessageCatalogKey<TMessageCatalog>>;
|
|
17
|
+
visibilityTimeoutMs: number;
|
|
18
|
+
attempt: number;
|
|
19
|
+
timestamp: Date;
|
|
20
|
+
deduplicationKey?: string;
|
|
21
21
|
};
|
|
22
22
|
type AnyQueueItem = {
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
23
|
+
id: string;
|
|
24
|
+
job: string;
|
|
25
|
+
item: any;
|
|
26
|
+
visibilityTimeoutMs: number;
|
|
27
|
+
attempt: number;
|
|
28
|
+
timestamp: Date;
|
|
29
|
+
deduplicationKey?: string;
|
|
30
30
|
};
|
|
31
31
|
declare class SimpleQueue<TMessageCatalog extends MessageCatalogSchema> {
|
|
32
|
-
|
|
32
|
+
#private;
|
|
33
|
+
name: string;
|
|
34
|
+
private redis;
|
|
35
|
+
private schema;
|
|
36
|
+
private logger;
|
|
37
|
+
constructor({ name, schema, redisOptions, logger }: {
|
|
33
38
|
name: string;
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
getJob(id: string): Promise<QueueItem<TMessageCatalog> | null>;
|
|
84
|
-
moveToDeadLetterQueue(id: string, errorMessage: string): Promise<void>;
|
|
85
|
-
sizeOfDeadLetterQueue(): Promise<number>;
|
|
86
|
-
redriveFromDeadLetterQueue(id: string): Promise<void>;
|
|
87
|
-
close(): Promise<void>;
|
|
39
|
+
schema: TMessageCatalog;
|
|
40
|
+
redisOptions: RedisOptions;
|
|
41
|
+
logger?: Logger;
|
|
42
|
+
});
|
|
43
|
+
cancel(cancellationKey: string): Promise<void>;
|
|
44
|
+
enqueue({ id, job, item, attempt, availableAt, visibilityTimeoutMs, cancellationKey }: {
|
|
45
|
+
id?: string;
|
|
46
|
+
job: MessageCatalogKey<TMessageCatalog>;
|
|
47
|
+
item: MessageCatalogValue<TMessageCatalog, MessageCatalogKey<TMessageCatalog>>;
|
|
48
|
+
attempt?: number;
|
|
49
|
+
availableAt?: Date;
|
|
50
|
+
visibilityTimeoutMs: number;
|
|
51
|
+
cancellationKey?: string;
|
|
52
|
+
}): Promise<void>;
|
|
53
|
+
enqueueOnce({ id, job, item, attempt, availableAt, visibilityTimeoutMs }: {
|
|
54
|
+
id: string;
|
|
55
|
+
job: MessageCatalogKey<TMessageCatalog>;
|
|
56
|
+
item: MessageCatalogValue<TMessageCatalog, MessageCatalogKey<TMessageCatalog>>;
|
|
57
|
+
attempt?: number;
|
|
58
|
+
availableAt?: Date;
|
|
59
|
+
visibilityTimeoutMs: number;
|
|
60
|
+
}): Promise<boolean>;
|
|
61
|
+
dequeue(count?: number): Promise<Array<QueueItem<TMessageCatalog>>>;
|
|
62
|
+
ack(id: string, deduplicationKey?: string): Promise<void>;
|
|
63
|
+
reschedule(id: string, availableAt: Date): Promise<void>;
|
|
64
|
+
size({ includeFuture }?: {
|
|
65
|
+
includeFuture?: boolean;
|
|
66
|
+
}): Promise<number>;
|
|
67
|
+
/**
|
|
68
|
+
* Age (in ms) of the oldest *overdue* message — the oldest item whose scheduled
|
|
69
|
+
* time has already passed (score <= now). Returns 0 when the queue is empty or
|
|
70
|
+
* only holds future/delayed or in-flight (future-scored) items.
|
|
71
|
+
*
|
|
72
|
+
* Resolves the candidate against the `items` hash so orphaned `queue` entries
|
|
73
|
+
* (a member whose payload is missing — the same stale state `dequeueItems`
|
|
74
|
+
* cleans up) don't report a phantom stall for work that can't be dequeued. The
|
|
75
|
+
* Lua scans due items oldest-first and returns the first score whose payload
|
|
76
|
+
* still exists.
|
|
77
|
+
*
|
|
78
|
+
* This is the generic stall signal: it stays at 0 while a queue drains healthily
|
|
79
|
+
* and rises only when due work sits undrained (poison block, dead consumer,
|
|
80
|
+
* backpressure).
|
|
81
|
+
*/
|
|
82
|
+
oldestMessageAge(): Promise<number>;
|
|
83
|
+
getJob(id: string): Promise<QueueItem<TMessageCatalog> | null>;
|
|
84
|
+
moveToDeadLetterQueue(id: string, errorMessage: string): Promise<void>;
|
|
85
|
+
sizeOfDeadLetterQueue(): Promise<number>;
|
|
86
|
+
redriveFromDeadLetterQueue(id: string): Promise<void>;
|
|
87
|
+
close(): Promise<void>;
|
|
88
88
|
}
|
|
89
89
|
declare module "@internal/redis" {
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
}
|
|
102
|
-
|
|
90
|
+
interface RedisCommander<Context> {
|
|
91
|
+
enqueueItem(queue: string, items: string, id: string, score: number, serializedItem: string, callback?: Callback<number>): Result<number, Context>;
|
|
92
|
+
enqueueItemWithCancellationKey(queue: string, items: string, cancellationKey: string, id: string, score: number, serializedItem: string, callback?: Callback<number>): Result<number, Context>;
|
|
93
|
+
dequeueItems(queue: string, items: string, now: number, count: number, callback?: Callback<Array<[string, string, string]>>): Result<Array<[string, string, string]>, Context>;
|
|
94
|
+
ackItem(queue: string, items: string, id: string, deduplicationKey: string, callback?: Callback<number>): Result<number, Context>;
|
|
95
|
+
redriveFromDeadLetterQueue(queue: string, items: string, dlq: string, dlqItems: string, id: string, callback?: Callback<number>): Result<number, Context>;
|
|
96
|
+
moveToDeadLetterQueue(queue: string, items: string, dlq: string, dlqItems: string, id: string, errorMessage: string, callback?: Callback<number>): Result<number, Context>;
|
|
97
|
+
enqueueItemOnce(queue: string, items: string, id: string, score: number, serializedItem: string, callback?: Callback<number>): Result<number, Context>;
|
|
98
|
+
getJob(queue: string, items: string, id: string, callback?: Callback<[string, string, string] | null>): Result<[string, string, string] | null, Context>;
|
|
99
|
+
getOldestDueScore(queue: string, items: string, now: number, callback?: Callback<string | number>): Result<string | number, Context>;
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
//#endregion
|
|
103
|
+
//#region src/worker.d.ts
|
|
103
104
|
declare const CronSchema: z.ZodObject<{
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
105
|
+
cron: z.ZodString;
|
|
106
|
+
lastTimestamp: z.ZodOptional<z.ZodNumber>;
|
|
107
|
+
timestamp: z.ZodNumber;
|
|
107
108
|
}, "strip", z.ZodTypeAny, {
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
109
|
+
cron: string;
|
|
110
|
+
lastTimestamp?: number | undefined;
|
|
111
|
+
timestamp: number;
|
|
111
112
|
}, {
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
113
|
+
cron: string;
|
|
114
|
+
lastTimestamp?: number | undefined;
|
|
115
|
+
timestamp: number;
|
|
115
116
|
}>;
|
|
116
117
|
type CronSchema = z.infer<typeof CronSchema>;
|
|
117
118
|
type BatchConfig = {
|
|
118
|
-
|
|
119
|
-
|
|
119
|
+
maxSize: number;
|
|
120
|
+
maxWaitMs: number;
|
|
120
121
|
};
|
|
121
122
|
type WorkerCatalog = {
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
};
|
|
134
|
-
type QueueCatalogFromWorkerCatalog<Catalog extends WorkerCatalog> = {
|
|
135
|
-
[K in keyof Catalog]: Catalog[K]["schema"];
|
|
123
|
+
[key: string]: {
|
|
124
|
+
schema: z.ZodFirstPartySchemaTypes | z.ZodDiscriminatedUnion<any, any>;
|
|
125
|
+
visibilityTimeoutMs: number;
|
|
126
|
+
retry?: RetryOptions$1;
|
|
127
|
+
cron?: string;
|
|
128
|
+
jitterInMs?: number;
|
|
129
|
+
/** Defaults to true. If false, errors will not be logged. */
|
|
130
|
+
logErrors?: boolean;
|
|
131
|
+
/** When set, items are accumulated and delivered in batches to the handler. */
|
|
132
|
+
batch?: BatchConfig;
|
|
133
|
+
};
|
|
136
134
|
};
|
|
135
|
+
type QueueCatalogFromWorkerCatalog<Catalog extends WorkerCatalog> = { [K in keyof Catalog]: Catalog[K]["schema"]; };
|
|
137
136
|
type JobHandlerParams<Catalog extends WorkerCatalog, K extends keyof Catalog> = {
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
137
|
+
id: string;
|
|
138
|
+
payload: z.infer<Catalog[K]["schema"]>;
|
|
139
|
+
visibilityTimeoutMs: number;
|
|
140
|
+
attempt: number;
|
|
141
|
+
deduplicationKey?: string;
|
|
143
142
|
};
|
|
144
143
|
type JobHandler<Catalog extends WorkerCatalog, K extends keyof Catalog> = (params: JobHandlerParams<Catalog, K>) => Promise<void>;
|
|
145
144
|
type JobHandlerFor<Catalog extends WorkerCatalog, K extends keyof Catalog> = Catalog[K] extends {
|
|
146
|
-
|
|
145
|
+
batch: BatchConfig;
|
|
147
146
|
} ? (items: Array<JobHandlerParams<Catalog, K>>) => Promise<void> : (params: JobHandlerParams<Catalog, K>) => Promise<void>;
|
|
148
147
|
type WorkerConcurrencyOptions = {
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
148
|
+
workers?: number;
|
|
149
|
+
tasksPerWorker?: number;
|
|
150
|
+
limit?: number;
|
|
152
151
|
};
|
|
153
152
|
type WorkerOptions<TCatalog extends WorkerCatalog> = {
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
tracer?: Tracer;
|
|
166
|
-
meter?: Meter;
|
|
153
|
+
name: string;
|
|
154
|
+
redisOptions: RedisOptions;
|
|
155
|
+
catalog: TCatalog;
|
|
156
|
+
jobs: { [K in keyof TCatalog]: JobHandlerFor<TCatalog, K>; };
|
|
157
|
+
concurrency?: WorkerConcurrencyOptions;
|
|
158
|
+
pollIntervalMs?: number;
|
|
159
|
+
immediatePollIntervalMs?: number;
|
|
160
|
+
shutdownTimeoutMs?: number;
|
|
161
|
+
logger?: Logger;
|
|
162
|
+
tracer?: Tracer;
|
|
163
|
+
meter?: Meter;
|
|
167
164
|
};
|
|
168
165
|
declare class Worker<TCatalog extends WorkerCatalog> {
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
}
|
|
292
|
-
|
|
166
|
+
#private;
|
|
167
|
+
private options;
|
|
168
|
+
private subscriber;
|
|
169
|
+
private tracer;
|
|
170
|
+
private meter;
|
|
171
|
+
private metrics;
|
|
172
|
+
queue: SimpleQueue<QueueCatalogFromWorkerCatalog<TCatalog>>;
|
|
173
|
+
private jobs;
|
|
174
|
+
private logger;
|
|
175
|
+
private workerLoops;
|
|
176
|
+
private isShuttingDown;
|
|
177
|
+
private concurrency;
|
|
178
|
+
private shutdownTimeoutMs;
|
|
179
|
+
private limiters;
|
|
180
|
+
private batchAccumulators;
|
|
181
|
+
constructor(options: WorkerOptions<TCatalog>);
|
|
182
|
+
start(): this;
|
|
183
|
+
/**
|
|
184
|
+
* Enqueues a job for processing.
|
|
185
|
+
* @param options - The enqueue options.
|
|
186
|
+
* @param options.id - Optional unique identifier for the job. If not provided, one will be generated. It prevents duplication.
|
|
187
|
+
* @param options.job - The job type from the worker catalog.
|
|
188
|
+
* @param options.payload - The job payload that matches the schema defined in the catalog.
|
|
189
|
+
* @param options.visibilityTimeoutMs - Optional visibility timeout in milliseconds. Defaults to value from catalog.
|
|
190
|
+
* @param options.availableAt - Optional date when the job should become available for processing. Defaults to now.
|
|
191
|
+
* @returns A promise that resolves when the job is enqueued.
|
|
192
|
+
*/
|
|
193
|
+
enqueue<K extends keyof TCatalog>({ id, job, payload, visibilityTimeoutMs, availableAt, cancellationKey }: {
|
|
194
|
+
id?: string;
|
|
195
|
+
job: K;
|
|
196
|
+
payload: z.infer<TCatalog[K]["schema"]>;
|
|
197
|
+
visibilityTimeoutMs?: number;
|
|
198
|
+
availableAt?: Date;
|
|
199
|
+
cancellationKey?: string;
|
|
200
|
+
}): Promise<void>;
|
|
201
|
+
/**
|
|
202
|
+
* Enqueues a job for processing once. If the job is already in the queue, it will be ignored.
|
|
203
|
+
* @param options - The enqueue options.
|
|
204
|
+
* @param options.id - Required unique identifier for the job.
|
|
205
|
+
* @param options.job - The job type from the worker catalog.
|
|
206
|
+
* @param options.payload - The job payload that matches the schema defined in the catalog.
|
|
207
|
+
* @param options.visibilityTimeoutMs - Optional visibility timeout in milliseconds. Defaults to value from catalog.
|
|
208
|
+
* @param options.availableAt - Optional date when the job should become available for processing. Defaults to now.
|
|
209
|
+
* @returns A promise that resolves when the job is enqueued.
|
|
210
|
+
*/
|
|
211
|
+
enqueueOnce<K extends keyof TCatalog>({ id, job, payload, visibilityTimeoutMs, availableAt }: {
|
|
212
|
+
id: string;
|
|
213
|
+
job: K;
|
|
214
|
+
payload: z.infer<TCatalog[K]["schema"]>;
|
|
215
|
+
visibilityTimeoutMs?: number;
|
|
216
|
+
availableAt?: Date;
|
|
217
|
+
}): Promise<boolean>;
|
|
218
|
+
/**
|
|
219
|
+
* Reschedules an existing job to a new available date.
|
|
220
|
+
* If the job isn't in the queue, it will be ignored.
|
|
221
|
+
*/
|
|
222
|
+
reschedule(id: string, availableAt: Date): Promise<void>;
|
|
223
|
+
/**
|
|
224
|
+
* Cancels a job before it's enqueued.
|
|
225
|
+
* @param cancellationKey - The cancellation key to cancel.
|
|
226
|
+
* @returns A promise that resolves when the job is cancelled.
|
|
227
|
+
*
|
|
228
|
+
* Any jobs enqueued with the same cancellation key will be not be enqueued.
|
|
229
|
+
*/
|
|
230
|
+
cancel(cancellationKey: string): Promise<void>;
|
|
231
|
+
ack(id: string): Promise<void>;
|
|
232
|
+
getJob(id: string): Promise<QueueItem<QueueCatalogFromWorkerCatalog<TCatalog>> | null>;
|
|
233
|
+
/**
|
|
234
|
+
* Returns true if the given job type has batch config in the catalog.
|
|
235
|
+
*/
|
|
236
|
+
private isBatchJob;
|
|
237
|
+
/**
|
|
238
|
+
* Returns the batch config for a job type, or undefined if not batch-enabled.
|
|
239
|
+
*/
|
|
240
|
+
private getBatchConfig;
|
|
241
|
+
/**
|
|
242
|
+
* The max dequeue count: the largest batch maxSize across catalog entries,
|
|
243
|
+
* falling back to tasksPerWorker for non-batch catalogs.
|
|
244
|
+
*/
|
|
245
|
+
private get maxDequeueCount();
|
|
246
|
+
/**
|
|
247
|
+
* The main loop that each worker runs. It repeatedly polls for items,
|
|
248
|
+
* processes them, and then waits before the next iteration.
|
|
249
|
+
*/
|
|
250
|
+
private runWorkerLoop;
|
|
251
|
+
/**
|
|
252
|
+
* Adds an item to the batch accumulator for its job type.
|
|
253
|
+
*/
|
|
254
|
+
private addToAccumulator;
|
|
255
|
+
/**
|
|
256
|
+
* Flushes any batch accumulators that have exceeded their maxWaitMs.
|
|
257
|
+
*/
|
|
258
|
+
private flushTimedOutBatches;
|
|
259
|
+
/**
|
|
260
|
+
* Flushes all batch accumulators (used during shutdown).
|
|
261
|
+
*/
|
|
262
|
+
private flushAllBatches;
|
|
263
|
+
/**
|
|
264
|
+
* Flushes the batch accumulator for a specific job type.
|
|
265
|
+
* Removes items from the accumulator and submits them to the limiter as a single batch.
|
|
266
|
+
*/
|
|
267
|
+
private flushBatch;
|
|
268
|
+
/**
|
|
269
|
+
* Processes a batch of items for a batch-enabled job type.
|
|
270
|
+
*/
|
|
271
|
+
private processBatch;
|
|
272
|
+
/**
|
|
273
|
+
* Processes a single item.
|
|
274
|
+
*/
|
|
275
|
+
private processItem;
|
|
276
|
+
private withHistogram;
|
|
277
|
+
private static delay;
|
|
278
|
+
private setupCron;
|
|
279
|
+
private enqueueCronJob;
|
|
280
|
+
private rescheduleCronJob;
|
|
281
|
+
private calculateNextScheduledAt;
|
|
282
|
+
private timestampIdentifier;
|
|
283
|
+
private setupSubscriber;
|
|
284
|
+
private handleRedriveMessage;
|
|
285
|
+
private setupShutdownHandlers;
|
|
286
|
+
private shutdown;
|
|
287
|
+
stop(): Promise<void>;
|
|
288
|
+
}
|
|
289
|
+
//#endregion
|
|
290
|
+
//#region src/utils.d.ts
|
|
293
291
|
/**
|
|
294
292
|
* Check if an error is an AbortError.
|
|
295
293
|
*
|
|
@@ -298,24 +296,25 @@ declare class Worker<TCatalog extends WorkerCatalog> {
|
|
|
298
296
|
* - Native Node.js AbortError from timers/promises (sets .name)
|
|
299
297
|
*/
|
|
300
298
|
declare function isAbortError(error: unknown): boolean;
|
|
301
|
-
|
|
299
|
+
//#endregion
|
|
300
|
+
//#region src/fair-queue/retry.d.ts
|
|
302
301
|
/**
|
|
303
302
|
* RetryStrategy interface for pluggable retry logic.
|
|
304
303
|
*/
|
|
305
304
|
interface RetryStrategy {
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
305
|
+
/**
|
|
306
|
+
* Calculate the next retry delay in milliseconds.
|
|
307
|
+
* Return null to indicate the message should be sent to DLQ.
|
|
308
|
+
*
|
|
309
|
+
* @param attempt - Current attempt number (1-indexed)
|
|
310
|
+
* @param error - Optional error from the failed attempt
|
|
311
|
+
* @returns Delay in milliseconds, or null to send to DLQ
|
|
312
|
+
*/
|
|
313
|
+
getNextDelay(attempt: number, error?: Error): number | null;
|
|
314
|
+
/**
|
|
315
|
+
* Maximum number of attempts before moving to DLQ.
|
|
316
|
+
*/
|
|
317
|
+
maxAttempts: number;
|
|
319
318
|
}
|
|
320
319
|
/**
|
|
321
320
|
* Exponential backoff retry strategy.
|
|
@@ -323,10 +322,10 @@ interface RetryStrategy {
|
|
|
323
322
|
* Uses the same algorithm as @trigger.dev/core's calculateNextRetryDelay.
|
|
324
323
|
*/
|
|
325
324
|
declare class ExponentialBackoffRetry implements RetryStrategy {
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
325
|
+
readonly maxAttempts: number;
|
|
326
|
+
private options;
|
|
327
|
+
constructor(options?: Partial<RetryOptions$1>);
|
|
328
|
+
getNextDelay(attempt: number, _error?: Error): number | null;
|
|
330
329
|
}
|
|
331
330
|
/**
|
|
332
331
|
* Fixed delay retry strategy.
|
|
@@ -334,13 +333,13 @@ declare class ExponentialBackoffRetry implements RetryStrategy {
|
|
|
334
333
|
* Always waits the same amount of time between retries.
|
|
335
334
|
*/
|
|
336
335
|
declare class FixedDelayRetry implements RetryStrategy {
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
336
|
+
readonly maxAttempts: number;
|
|
337
|
+
private delayMs;
|
|
338
|
+
constructor(options: {
|
|
339
|
+
maxAttempts: number;
|
|
340
|
+
delayMs: number;
|
|
341
|
+
});
|
|
342
|
+
getNextDelay(attempt: number, _error?: Error): number | null;
|
|
344
343
|
}
|
|
345
344
|
/**
|
|
346
345
|
* Linear backoff retry strategy.
|
|
@@ -348,15 +347,15 @@ declare class FixedDelayRetry implements RetryStrategy {
|
|
|
348
347
|
* Delay increases linearly with each attempt.
|
|
349
348
|
*/
|
|
350
349
|
declare class LinearBackoffRetry implements RetryStrategy {
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
350
|
+
readonly maxAttempts: number;
|
|
351
|
+
private baseDelayMs;
|
|
352
|
+
private maxDelayMs;
|
|
353
|
+
constructor(options: {
|
|
354
|
+
maxAttempts: number;
|
|
355
|
+
baseDelayMs: number;
|
|
356
|
+
maxDelayMs?: number;
|
|
357
|
+
});
|
|
358
|
+
getNextDelay(attempt: number, _error?: Error): number | null;
|
|
360
359
|
}
|
|
361
360
|
/**
|
|
362
361
|
* No retry strategy.
|
|
@@ -364,8 +363,8 @@ declare class LinearBackoffRetry implements RetryStrategy {
|
|
|
364
363
|
* Messages go directly to DLQ on first failure.
|
|
365
364
|
*/
|
|
366
365
|
declare class NoRetry implements RetryStrategy {
|
|
367
|
-
|
|
368
|
-
|
|
366
|
+
readonly maxAttempts = 1;
|
|
367
|
+
getNextDelay(_attempt: number, _error?: Error): number | null;
|
|
369
368
|
}
|
|
370
369
|
/**
|
|
371
370
|
* Immediate retry strategy.
|
|
@@ -373,21 +372,21 @@ declare class NoRetry implements RetryStrategy {
|
|
|
373
372
|
* Retries immediately without any delay.
|
|
374
373
|
*/
|
|
375
374
|
declare class ImmediateRetry implements RetryStrategy {
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
375
|
+
readonly maxAttempts: number;
|
|
376
|
+
constructor(maxAttempts: number);
|
|
377
|
+
getNextDelay(attempt: number, _error?: Error): number | null;
|
|
379
378
|
}
|
|
380
379
|
/**
|
|
381
380
|
* Custom retry strategy that uses a user-provided function.
|
|
382
381
|
*/
|
|
383
382
|
declare class CustomRetry implements RetryStrategy {
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
383
|
+
readonly maxAttempts: number;
|
|
384
|
+
private calculateDelay;
|
|
385
|
+
constructor(options: {
|
|
386
|
+
maxAttempts: number;
|
|
387
|
+
calculateDelay: (attempt: number, error?: Error) => number | null;
|
|
388
|
+
});
|
|
389
|
+
getNextDelay(attempt: number, error?: Error): number | null;
|
|
391
390
|
}
|
|
392
391
|
/**
|
|
393
392
|
* Default retry options matching @trigger.dev/core defaults.
|
|
@@ -397,258 +396,259 @@ declare const defaultRetryOptions: RetryOptions$1;
|
|
|
397
396
|
* Create an exponential backoff retry strategy with default options.
|
|
398
397
|
*/
|
|
399
398
|
declare function createDefaultRetryStrategy(): RetryStrategy;
|
|
400
|
-
|
|
399
|
+
//#endregion
|
|
400
|
+
//#region src/fair-queue/types.d.ts
|
|
401
401
|
/**
|
|
402
402
|
* Interface for a global rate limiter that limits processing across all consumers.
|
|
403
403
|
* When configured, consumers will check this before processing each message.
|
|
404
404
|
*/
|
|
405
405
|
interface GlobalRateLimiter {
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
406
|
+
/**
|
|
407
|
+
* Check if processing is allowed under the rate limit.
|
|
408
|
+
* @returns Object with allowed flag and optional resetAt timestamp (ms since epoch)
|
|
409
|
+
*/
|
|
410
|
+
limit(): Promise<{
|
|
411
|
+
allowed: boolean;
|
|
412
|
+
resetAt?: number;
|
|
413
|
+
}>;
|
|
414
414
|
}
|
|
415
415
|
/**
|
|
416
416
|
* Descriptor for a queue in the fair queue system.
|
|
417
417
|
* Contains all the metadata needed to identify and route a queue.
|
|
418
418
|
*/
|
|
419
419
|
interface QueueDescriptor {
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
420
|
+
/** Unique queue identifier */
|
|
421
|
+
id: string;
|
|
422
|
+
/** Tenant this queue belongs to */
|
|
423
|
+
tenantId: string;
|
|
424
|
+
/** Additional metadata for concurrency group extraction */
|
|
425
|
+
metadata: Record<string, unknown>;
|
|
426
426
|
}
|
|
427
427
|
/**
|
|
428
428
|
* A message in the queue with its metadata.
|
|
429
429
|
*/
|
|
430
430
|
interface QueueMessage<TPayload = unknown> {
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
431
|
+
/** Unique message identifier */
|
|
432
|
+
id: string;
|
|
433
|
+
/** The queue this message belongs to */
|
|
434
|
+
queueId: string;
|
|
435
|
+
/** Message payload */
|
|
436
|
+
payload: TPayload;
|
|
437
|
+
/** Timestamp when message was enqueued */
|
|
438
|
+
timestamp: number;
|
|
439
|
+
/** Current attempt number (1-indexed, for retries) */
|
|
440
|
+
attempt: number;
|
|
441
|
+
/** Optional metadata */
|
|
442
|
+
metadata?: Record<string, unknown>;
|
|
443
443
|
}
|
|
444
444
|
/**
|
|
445
445
|
* Internal message format stored in Redis.
|
|
446
446
|
* Includes additional fields for tracking and routing.
|
|
447
447
|
*/
|
|
448
448
|
interface StoredMessage<TPayload = unknown> {
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
449
|
+
/** Message ID */
|
|
450
|
+
id: string;
|
|
451
|
+
/** Queue ID */
|
|
452
|
+
queueId: string;
|
|
453
|
+
/** Tenant ID */
|
|
454
|
+
tenantId: string;
|
|
455
|
+
/** Message payload */
|
|
456
|
+
payload: TPayload;
|
|
457
|
+
/** Timestamp when enqueued */
|
|
458
|
+
timestamp: number;
|
|
459
|
+
/** Current attempt number */
|
|
460
|
+
attempt: number;
|
|
461
|
+
/** Worker queue to route to */
|
|
462
|
+
workerQueue?: string;
|
|
463
|
+
/** Additional metadata */
|
|
464
|
+
metadata?: Record<string, unknown>;
|
|
465
465
|
}
|
|
466
466
|
/**
|
|
467
467
|
* Queue with its score (oldest message timestamp) from the master queue.
|
|
468
468
|
*/
|
|
469
469
|
interface QueueWithScore {
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
470
|
+
/** Queue identifier */
|
|
471
|
+
queueId: string;
|
|
472
|
+
/** Score (typically oldest message timestamp) */
|
|
473
|
+
score: number;
|
|
474
|
+
/** Tenant ID extracted from queue */
|
|
475
|
+
tenantId: string;
|
|
476
476
|
}
|
|
477
477
|
/**
|
|
478
478
|
* Configuration for a concurrency group.
|
|
479
479
|
* Allows defining arbitrary levels of concurrency (tenant, org, project, etc.)
|
|
480
480
|
*/
|
|
481
481
|
interface ConcurrencyGroupConfig {
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
482
|
+
/** Group name (e.g., "tenant", "organization", "project") */
|
|
483
|
+
name: string;
|
|
484
|
+
/** Extract the group ID from a queue descriptor */
|
|
485
|
+
extractGroupId: (queue: QueueDescriptor) => string;
|
|
486
|
+
/** Get the concurrency limit for a specific group ID */
|
|
487
|
+
getLimit: (groupId: string) => Promise<number>;
|
|
488
|
+
/** Default limit if not specified */
|
|
489
|
+
defaultLimit: number;
|
|
490
490
|
}
|
|
491
491
|
/**
|
|
492
492
|
* Current concurrency state for a group.
|
|
493
493
|
*/
|
|
494
494
|
interface ConcurrencyState {
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
495
|
+
/** Group name */
|
|
496
|
+
groupName: string;
|
|
497
|
+
/** Group ID */
|
|
498
|
+
groupId: string;
|
|
499
|
+
/** Current active count */
|
|
500
|
+
current: number;
|
|
501
|
+
/** Configured limit */
|
|
502
|
+
limit: number;
|
|
503
503
|
}
|
|
504
504
|
/**
|
|
505
505
|
* Result of a concurrency check.
|
|
506
506
|
*/
|
|
507
507
|
interface ConcurrencyCheckResult {
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
508
|
+
/** Whether processing is allowed */
|
|
509
|
+
allowed: boolean;
|
|
510
|
+
/** If not allowed, which group is blocking */
|
|
511
|
+
blockedBy?: ConcurrencyState;
|
|
512
512
|
}
|
|
513
513
|
/**
|
|
514
514
|
* Information about a reclaimed message from visibility timeout.
|
|
515
515
|
* Used to release concurrency after a message is returned to the queue.
|
|
516
516
|
*/
|
|
517
517
|
interface ReclaimedMessageInfo {
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
518
|
+
/** Message ID */
|
|
519
|
+
messageId: string;
|
|
520
|
+
/** Queue ID */
|
|
521
|
+
queueId: string;
|
|
522
|
+
/** Tenant ID for concurrency release */
|
|
523
|
+
tenantId: string;
|
|
524
|
+
/** Additional metadata for concurrency group extraction */
|
|
525
|
+
metadata?: Record<string, unknown>;
|
|
526
526
|
}
|
|
527
527
|
/**
|
|
528
528
|
* Queues grouped by tenant for the scheduler.
|
|
529
529
|
*/
|
|
530
530
|
interface TenantQueues {
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
531
|
+
/** Tenant identifier */
|
|
532
|
+
tenantId: string;
|
|
533
|
+
/** Queue IDs belonging to this tenant, in priority order */
|
|
534
|
+
queues: string[];
|
|
535
535
|
}
|
|
536
536
|
/**
|
|
537
537
|
* Context provided to the scheduler for making decisions.
|
|
538
538
|
*/
|
|
539
539
|
interface SchedulerContext {
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
540
|
+
/** Get current concurrency for a group */
|
|
541
|
+
getCurrentConcurrency(groupName: string, groupId: string): Promise<number>;
|
|
542
|
+
/** Get concurrency limit for a group */
|
|
543
|
+
getConcurrencyLimit(groupName: string, groupId: string): Promise<number>;
|
|
544
|
+
/** Check if a group is at capacity */
|
|
545
|
+
isAtCapacity(groupName: string, groupId: string): Promise<boolean>;
|
|
546
|
+
/** Get queue descriptor by ID */
|
|
547
|
+
getQueueDescriptor(queueId: string): QueueDescriptor;
|
|
548
548
|
}
|
|
549
549
|
/**
|
|
550
550
|
* Extended context for two-level dispatch scheduling.
|
|
551
551
|
*/
|
|
552
552
|
interface DispatchSchedulerContext extends SchedulerContext {
|
|
553
|
-
|
|
554
|
-
|
|
553
|
+
/** Get queues for a specific tenant from the per-tenant queue index (Level 2) */
|
|
554
|
+
getQueuesForTenant(tenantId: string, limit?: number): Promise<QueueWithScore[]>;
|
|
555
555
|
}
|
|
556
556
|
/**
|
|
557
557
|
* Pluggable scheduler interface for fair queue selection.
|
|
558
558
|
*/
|
|
559
559
|
interface FairScheduler {
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
560
|
+
/**
|
|
561
|
+
* Select queues for processing from a master queue shard.
|
|
562
|
+
* Returns queues grouped by tenant, ordered by the fairness algorithm.
|
|
563
|
+
*
|
|
564
|
+
* @param masterQueueShard - The master queue shard key
|
|
565
|
+
* @param consumerId - The consumer making the request
|
|
566
|
+
* @param context - Context for concurrency checks
|
|
567
|
+
* @returns Queues grouped by tenant in priority order
|
|
568
|
+
*/
|
|
569
|
+
selectQueues(masterQueueShard: string, consumerId: string, context: SchedulerContext): Promise<TenantQueues[]>;
|
|
570
|
+
/**
|
|
571
|
+
* Select queues using the two-level tenant dispatch index.
|
|
572
|
+
* Level 1: reads tenantIds from dispatch shard.
|
|
573
|
+
* Level 2: reads queueIds from per-tenant index.
|
|
574
|
+
* Optional - falls back to selectQueues with flat queue list if not implemented.
|
|
575
|
+
*/
|
|
576
|
+
selectQueuesFromDispatch?(dispatchShardKey: string, consumerId: string, context: DispatchSchedulerContext): Promise<TenantQueues[]>;
|
|
577
|
+
/**
|
|
578
|
+
* Called after processing a message to update scheduler state.
|
|
579
|
+
* Optional - not all schedulers need to track state.
|
|
580
|
+
*/
|
|
581
|
+
recordProcessed?(tenantId: string, queueId: string): Promise<void>;
|
|
582
|
+
/**
|
|
583
|
+
* Called after processing multiple messages to update scheduler state.
|
|
584
|
+
* Batch variant for efficiency - reduces Redis calls when processing multiple messages.
|
|
585
|
+
* Optional - falls back to calling recordProcessed multiple times if not implemented.
|
|
586
|
+
*/
|
|
587
|
+
recordProcessedBatch?(tenantId: string, queueId: string, count: number): Promise<void>;
|
|
588
|
+
/**
|
|
589
|
+
* Initialize the scheduler (called once on startup).
|
|
590
|
+
*/
|
|
591
|
+
initialize?(): Promise<void>;
|
|
592
|
+
/**
|
|
593
|
+
* Cleanup scheduler resources.
|
|
594
|
+
*/
|
|
595
|
+
close?(): Promise<void>;
|
|
596
596
|
}
|
|
597
597
|
/**
|
|
598
598
|
* An in-flight message being processed.
|
|
599
599
|
*/
|
|
600
600
|
interface InFlightMessage<TPayload = unknown> {
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
601
|
+
/** Message ID */
|
|
602
|
+
messageId: string;
|
|
603
|
+
/** Queue ID */
|
|
604
|
+
queueId: string;
|
|
605
|
+
/** Message payload */
|
|
606
|
+
payload: TPayload;
|
|
607
|
+
/** When visibility timeout expires */
|
|
608
|
+
deadline: number;
|
|
609
|
+
/** Consumer that claimed this message */
|
|
610
|
+
consumerId: string;
|
|
611
611
|
}
|
|
612
612
|
/**
|
|
613
613
|
* Result of claiming a message.
|
|
614
614
|
*/
|
|
615
615
|
interface ClaimResult<TPayload = unknown> {
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
616
|
+
/** Whether the claim was successful */
|
|
617
|
+
claimed: boolean;
|
|
618
|
+
/** The claimed message if successful */
|
|
619
|
+
message?: InFlightMessage<TPayload>;
|
|
620
620
|
}
|
|
621
621
|
/**
|
|
622
622
|
* Interface for generating Redis keys for the fair queue system.
|
|
623
623
|
* Implementations can customize key prefixes and structures.
|
|
624
624
|
*/
|
|
625
625
|
interface FairQueueKeyProducer {
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
626
|
+
/** Get the master queue key for a shard */
|
|
627
|
+
masterQueueKey(shardId: number): string;
|
|
628
|
+
/** Get the queue key for storing messages */
|
|
629
|
+
queueKey(queueId: string): string;
|
|
630
|
+
/** Get the queue items hash key */
|
|
631
|
+
queueItemsKey(queueId: string): string;
|
|
632
|
+
/** Get the concurrency set key for a group */
|
|
633
|
+
concurrencyKey(groupName: string, groupId: string): string;
|
|
634
|
+
/** Get the in-flight sorted set key for a shard */
|
|
635
|
+
inflightKey(shardId: number): string;
|
|
636
|
+
/** Get the in-flight message data hash key */
|
|
637
|
+
inflightDataKey(shardId: number): string;
|
|
638
|
+
/** Get the worker queue key for a consumer */
|
|
639
|
+
workerQueueKey(consumerId: string): string;
|
|
640
|
+
/** Get the dead letter queue key for a tenant */
|
|
641
|
+
deadLetterQueueKey(tenantId: string): string;
|
|
642
|
+
/** Get the dead letter queue data hash key for a tenant */
|
|
643
|
+
deadLetterQueueDataKey(tenantId: string): string;
|
|
644
|
+
/** Get the dispatch index key for a shard (Level 1: tenantIds with capacity) */
|
|
645
|
+
dispatchKey(shardId: number): string;
|
|
646
|
+
/** Get the per-tenant queue index key (Level 2: queueIds for a tenant) */
|
|
647
|
+
tenantQueueIndexKey(tenantId: string): string;
|
|
648
|
+
/** Extract tenant ID from a queue ID */
|
|
649
|
+
extractTenantId(queueId: string): string;
|
|
650
|
+
/** Extract a specific group ID from a queue ID */
|
|
651
|
+
extractGroupId(groupName: string, queueId: string): string;
|
|
652
652
|
}
|
|
653
653
|
/**
|
|
654
654
|
* Worker queue configuration options.
|
|
@@ -656,33 +656,33 @@ interface FairQueueKeyProducer {
|
|
|
656
656
|
* and external consumers are responsible for consuming from those queues.
|
|
657
657
|
*/
|
|
658
658
|
interface WorkerQueueOptions<TPayload = unknown> {
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
659
|
+
/**
|
|
660
|
+
* Function to resolve which worker queue a message should go to.
|
|
661
|
+
* This is called during the claim-and-push phase to determine the target queue.
|
|
662
|
+
*/
|
|
663
|
+
resolveWorkerQueue: (message: StoredMessage<TPayload>) => string;
|
|
664
664
|
}
|
|
665
665
|
/**
|
|
666
666
|
* Retry and dead letter queue configuration.
|
|
667
667
|
*/
|
|
668
668
|
interface RetryOptions {
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
669
|
+
/** Retry strategy for failed messages */
|
|
670
|
+
strategy: RetryStrategy;
|
|
671
|
+
/** Whether to enable dead letter queue (default: true) */
|
|
672
|
+
deadLetterQueue?: boolean;
|
|
673
673
|
}
|
|
674
674
|
/**
|
|
675
675
|
* Queue cooloff configuration to avoid repeatedly polling concurrency-limited queues.
|
|
676
676
|
*/
|
|
677
677
|
interface CooloffOptions {
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
678
|
+
/** Whether cooloff is enabled (default: true) */
|
|
679
|
+
enabled?: boolean;
|
|
680
|
+
/** Number of consecutive empty dequeues before entering cooloff (default: 10) */
|
|
681
|
+
threshold?: number;
|
|
682
|
+
/** Duration of cooloff period in milliseconds (default: 10000) */
|
|
683
|
+
periodMs?: number;
|
|
684
|
+
/** Maximum number of cooloff state entries before triggering cleanup (default: 1000) */
|
|
685
|
+
maxStatesSize?: number;
|
|
686
686
|
}
|
|
687
687
|
/**
|
|
688
688
|
* Options for creating a FairQueue instance.
|
|
@@ -690,90 +690,90 @@ interface CooloffOptions {
|
|
|
690
690
|
* @typeParam TPayloadSchema - Zod schema for message payload validation
|
|
691
691
|
*/
|
|
692
692
|
interface FairQueueOptions<TPayloadSchema extends z.ZodTypeAny = z.ZodUnknown> {
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
693
|
+
/** Redis connection options */
|
|
694
|
+
redis: RedisOptions;
|
|
695
|
+
/** Key producer for Redis keys */
|
|
696
|
+
keys: FairQueueKeyProducer;
|
|
697
|
+
/** Scheduler for fair queue selection */
|
|
698
|
+
scheduler: FairScheduler;
|
|
699
|
+
/** Zod schema for message payload validation */
|
|
700
|
+
payloadSchema?: TPayloadSchema;
|
|
701
|
+
/** Whether to validate payloads on enqueue (default: false) */
|
|
702
|
+
validateOnEnqueue?: boolean;
|
|
703
|
+
/** Number of master queue shards (default: 1) */
|
|
704
|
+
shardCount?: number;
|
|
705
|
+
/** Concurrency group configurations */
|
|
706
|
+
concurrencyGroups?: ConcurrencyGroupConfig[];
|
|
707
|
+
/**
|
|
708
|
+
* Worker queue configuration.
|
|
709
|
+
* FairQueue routes messages to worker queues; external consumers handle consumption.
|
|
710
|
+
*/
|
|
711
|
+
workerQueue: WorkerQueueOptions<z.infer<TPayloadSchema>>;
|
|
712
|
+
/** Retry and dead letter queue configuration */
|
|
713
|
+
retry?: RetryOptions;
|
|
714
|
+
/** Visibility timeout in milliseconds (default: 30000) */
|
|
715
|
+
visibilityTimeoutMs?: number;
|
|
716
|
+
/** Heartbeat interval in milliseconds (default: visibilityTimeoutMs / 3) */
|
|
717
|
+
heartbeatIntervalMs?: number;
|
|
718
|
+
/** Interval for reclaiming timed-out messages (default: 5000) */
|
|
719
|
+
reclaimIntervalMs?: number;
|
|
720
|
+
/** Number of consumer loops to run (default: 1) */
|
|
721
|
+
consumerCount?: number;
|
|
722
|
+
/** Interval between consumer iterations in milliseconds (default: 100) */
|
|
723
|
+
consumerIntervalMs?: number;
|
|
724
|
+
/** Whether to start consumers on initialization (default: true) */
|
|
725
|
+
startConsumers?: boolean;
|
|
726
|
+
/** Maximum number of messages to claim in a single batch operation (default: 10) */
|
|
727
|
+
batchClaimSize?: number;
|
|
728
|
+
/** Maximum iterations before starting a new trace span (default: 500) */
|
|
729
|
+
consumerTraceMaxIterations?: number;
|
|
730
|
+
/** Maximum seconds before starting a new trace span (default: 60) */
|
|
731
|
+
consumerTraceTimeoutSeconds?: number;
|
|
732
|
+
/** Queue cooloff configuration */
|
|
733
|
+
cooloff?: CooloffOptions;
|
|
734
|
+
/** Logger instance */
|
|
735
|
+
logger?: Logger;
|
|
736
|
+
/** OpenTelemetry tracer */
|
|
737
|
+
tracer?: Tracer;
|
|
738
|
+
/** OpenTelemetry meter */
|
|
739
|
+
meter?: Meter;
|
|
740
|
+
/** Name for metrics/tracing (default: "fairqueue") */
|
|
741
|
+
name?: string;
|
|
742
|
+
/**
|
|
743
|
+
* Maximum number of items allowed in a worker queue before claiming pauses.
|
|
744
|
+
* When set, the claim phase checks worker queue depth and skips claiming if
|
|
745
|
+
* the queue is at or above this limit. This prevents unbounded worker queue
|
|
746
|
+
* growth which could cause visibility timeouts (claimed messages have a
|
|
747
|
+
* visibility timeout that ticks while they sit in the worker queue).
|
|
748
|
+
* Requires `workerQueueDepthCheckId` to know which queue to check.
|
|
749
|
+
* Disabled by default (0 = no limit).
|
|
750
|
+
*/
|
|
751
|
+
workerQueueMaxDepth?: number;
|
|
752
|
+
/**
|
|
753
|
+
* The worker queue ID to check depth against when workerQueueMaxDepth is set.
|
|
754
|
+
* Required when workerQueueMaxDepth > 0 and the system uses a single shared worker queue.
|
|
755
|
+
* If not set, depth checking is disabled even if workerQueueMaxDepth is set.
|
|
756
|
+
*/
|
|
757
|
+
workerQueueDepthCheckId?: string;
|
|
758
758
|
}
|
|
759
759
|
/**
|
|
760
760
|
* Context passed to the message handler.
|
|
761
761
|
*/
|
|
762
762
|
interface MessageHandlerContext<TPayload = unknown> {
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
763
|
+
/** The message being processed */
|
|
764
|
+
message: QueueMessage<TPayload>;
|
|
765
|
+
/** Queue descriptor */
|
|
766
|
+
queue: QueueDescriptor;
|
|
767
|
+
/** Consumer ID processing this message */
|
|
768
|
+
consumerId: string;
|
|
769
|
+
/** Extend the visibility timeout */
|
|
770
|
+
heartbeat(): Promise<boolean>;
|
|
771
|
+
/** Mark message as successfully processed */
|
|
772
|
+
complete(): Promise<void>;
|
|
773
|
+
/** Release message back to the queue for retry */
|
|
774
|
+
release(): Promise<void>;
|
|
775
|
+
/** Mark message as failed (triggers retry or DLQ) */
|
|
776
|
+
fail(error?: Error): Promise<void>;
|
|
777
777
|
}
|
|
778
778
|
/**
|
|
779
779
|
* Handler function for processing messages.
|
|
@@ -783,137 +783,138 @@ type MessageHandler<TPayload = unknown> = (context: MessageHandlerContext<TPaylo
|
|
|
783
783
|
* A message in the dead letter queue.
|
|
784
784
|
*/
|
|
785
785
|
interface DeadLetterMessage<TPayload = unknown> {
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
786
|
+
/** Message ID */
|
|
787
|
+
id: string;
|
|
788
|
+
/** Original queue ID */
|
|
789
|
+
queueId: string;
|
|
790
|
+
/** Tenant ID */
|
|
791
|
+
tenantId: string;
|
|
792
|
+
/** Message payload */
|
|
793
|
+
payload: TPayload;
|
|
794
|
+
/** Timestamp when moved to DLQ */
|
|
795
|
+
deadLetteredAt: number;
|
|
796
|
+
/** Number of attempts before DLQ */
|
|
797
|
+
attempts: number;
|
|
798
|
+
/** Last error message if available */
|
|
799
|
+
lastError?: string;
|
|
800
|
+
/** Original message timestamp */
|
|
801
|
+
originalTimestamp: number;
|
|
802
802
|
}
|
|
803
803
|
/**
|
|
804
804
|
* Cooloff state for a queue.
|
|
805
805
|
*/
|
|
806
806
|
type QueueCooloffState = {
|
|
807
|
-
|
|
808
|
-
|
|
807
|
+
tag: "normal";
|
|
808
|
+
consecutiveFailures: number;
|
|
809
809
|
} | {
|
|
810
|
-
|
|
811
|
-
|
|
810
|
+
tag: "cooloff";
|
|
811
|
+
expiresAt: number;
|
|
812
812
|
};
|
|
813
813
|
/**
|
|
814
814
|
* Options for enqueueing a message.
|
|
815
815
|
*/
|
|
816
816
|
interface EnqueueOptions<TPayload = unknown> {
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
817
|
+
/** Queue to add the message to */
|
|
818
|
+
queueId: string;
|
|
819
|
+
/** Tenant ID for the queue */
|
|
820
|
+
tenantId: string;
|
|
821
|
+
/** Message payload */
|
|
822
|
+
payload: TPayload;
|
|
823
|
+
/** Optional message ID (auto-generated if not provided) */
|
|
824
|
+
messageId?: string;
|
|
825
|
+
/** Optional timestamp (defaults to now) */
|
|
826
|
+
timestamp?: number;
|
|
827
|
+
/** Optional metadata for concurrency group extraction */
|
|
828
|
+
metadata?: Record<string, string>;
|
|
829
829
|
}
|
|
830
830
|
/**
|
|
831
831
|
* Options for enqueueing multiple messages.
|
|
832
832
|
*/
|
|
833
833
|
interface EnqueueBatchOptions<TPayload = unknown> {
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
834
|
+
/** Queue to add messages to */
|
|
835
|
+
queueId: string;
|
|
836
|
+
/** Tenant ID for the queue */
|
|
837
|
+
tenantId: string;
|
|
838
|
+
/** Messages to enqueue */
|
|
839
|
+
messages: Array<{
|
|
840
|
+
payload: TPayload;
|
|
841
|
+
messageId?: string;
|
|
842
|
+
timestamp?: number;
|
|
843
|
+
}>;
|
|
844
|
+
/** Optional metadata for concurrency group extraction */
|
|
845
|
+
metadata?: Record<string, string>;
|
|
846
846
|
}
|
|
847
847
|
/**
|
|
848
848
|
* Configuration for the Deficit Round Robin scheduler.
|
|
849
849
|
*/
|
|
850
850
|
interface DRRSchedulerConfig {
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
851
|
+
/** Credits allocated per tenant per round */
|
|
852
|
+
quantum: number;
|
|
853
|
+
/** Maximum accumulated deficit (prevents starvation) */
|
|
854
|
+
maxDeficit: number;
|
|
855
|
+
/** Maximum queues to fetch from master queue (default: 1000) */
|
|
856
|
+
masterQueueLimit?: number;
|
|
857
|
+
/** Redis options for state storage */
|
|
858
|
+
redis: RedisOptions;
|
|
859
|
+
/** Key producer */
|
|
860
|
+
keys: FairQueueKeyProducer;
|
|
861
|
+
/** Optional logger */
|
|
862
|
+
logger?: {
|
|
863
|
+
debug: (message: string, context?: Record<string, unknown>) => void;
|
|
864
|
+
error: (message: string, context?: Record<string, unknown>) => void;
|
|
865
|
+
};
|
|
866
866
|
}
|
|
867
867
|
/**
|
|
868
868
|
* Bias configuration for weighted shuffle scheduler.
|
|
869
869
|
*/
|
|
870
870
|
interface WeightedSchedulerBiases {
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
871
|
+
/**
|
|
872
|
+
* How much to bias towards tenants with higher concurrency limits.
|
|
873
|
+
* 0 = no bias, 1 = full bias based on limit differences
|
|
874
|
+
*/
|
|
875
|
+
concurrencyLimitBias: number;
|
|
876
|
+
/**
|
|
877
|
+
* How much to bias towards tenants with more available capacity.
|
|
878
|
+
* 0 = no bias, 1 = full bias based on available capacity
|
|
879
|
+
*/
|
|
880
|
+
availableCapacityBias: number;
|
|
881
|
+
/**
|
|
882
|
+
* Controls randomization of queue ordering within tenants.
|
|
883
|
+
* 0 = strict age-based ordering (oldest first)
|
|
884
|
+
* 1 = completely random ordering
|
|
885
|
+
* Values between 0-1 blend between age-based and random ordering
|
|
886
|
+
*/
|
|
887
|
+
queueAgeRandomization: number;
|
|
888
888
|
}
|
|
889
889
|
/**
|
|
890
890
|
* Configuration for the weighted shuffle scheduler.
|
|
891
891
|
*/
|
|
892
892
|
interface WeightedSchedulerConfig {
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
}
|
|
912
|
-
|
|
893
|
+
/** Redis options */
|
|
894
|
+
redis: RedisOptions;
|
|
895
|
+
/** Key producer */
|
|
896
|
+
keys: FairQueueKeyProducer;
|
|
897
|
+
/** Default tenant concurrency limit */
|
|
898
|
+
defaultTenantConcurrencyLimit?: number;
|
|
899
|
+
/** Maximum queues to consider from master queue */
|
|
900
|
+
masterQueueLimit?: number;
|
|
901
|
+
/** Bias configuration */
|
|
902
|
+
biases?: WeightedSchedulerBiases;
|
|
903
|
+
/** Number of iterations to reuse a snapshot */
|
|
904
|
+
reuseSnapshotCount?: number;
|
|
905
|
+
/** Maximum number of tenants to consider */
|
|
906
|
+
maximumTenantCount?: number;
|
|
907
|
+
/** Random seed for reproducibility */
|
|
908
|
+
seed?: string;
|
|
909
|
+
/** Optional tracer */
|
|
910
|
+
tracer?: Tracer;
|
|
911
|
+
}
|
|
912
|
+
//#endregion
|
|
913
|
+
//#region src/fair-queue/concurrency.d.ts
|
|
913
914
|
interface ConcurrencyManagerOptions {
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
915
|
+
redis: RedisOptions;
|
|
916
|
+
keys: FairQueueKeyProducer;
|
|
917
|
+
groups: ConcurrencyGroupConfig[];
|
|
917
918
|
}
|
|
918
919
|
/**
|
|
919
920
|
* ConcurrencyManager handles multi-level concurrency tracking and limiting.
|
|
@@ -924,87 +925,88 @@ interface ConcurrencyManagerOptions {
|
|
|
924
925
|
* - Efficient batch checking of all groups
|
|
925
926
|
*/
|
|
926
927
|
declare class ConcurrencyManager {
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
928
|
+
#private;
|
|
929
|
+
private options;
|
|
930
|
+
private redis;
|
|
931
|
+
private keys;
|
|
932
|
+
private groups;
|
|
933
|
+
private groupsByName;
|
|
934
|
+
constructor(options: ConcurrencyManagerOptions);
|
|
935
|
+
/**
|
|
936
|
+
* Check if a message can be processed given all concurrency constraints.
|
|
937
|
+
* Checks all configured groups and returns the first one at capacity.
|
|
938
|
+
*/
|
|
939
|
+
canProcess(queue: QueueDescriptor): Promise<ConcurrencyCheckResult>;
|
|
940
|
+
/**
|
|
941
|
+
* Reserve concurrency slots for a message across all groups.
|
|
942
|
+
* Atomic - either all groups are reserved or none.
|
|
943
|
+
*
|
|
944
|
+
* @returns true if reservation successful, false if any group is at capacity
|
|
945
|
+
*/
|
|
946
|
+
reserve(queue: QueueDescriptor, messageId: string): Promise<boolean>;
|
|
947
|
+
/**
|
|
948
|
+
* Release concurrency slots for a message across all groups.
|
|
949
|
+
*/
|
|
950
|
+
release(queue: QueueDescriptor, messageId: string): Promise<void>;
|
|
951
|
+
/**
|
|
952
|
+
* Release concurrency slots for multiple messages in a single pipeline.
|
|
953
|
+
* More efficient than calling release() multiple times.
|
|
954
|
+
*/
|
|
955
|
+
releaseBatch(messages: Array<{
|
|
956
|
+
queue: QueueDescriptor;
|
|
957
|
+
messageId: string;
|
|
958
|
+
}>): Promise<void>;
|
|
959
|
+
/**
|
|
960
|
+
* Get current concurrency for a specific group.
|
|
961
|
+
*/
|
|
962
|
+
getCurrentConcurrency(groupName: string, groupId: string): Promise<number>;
|
|
963
|
+
/**
|
|
964
|
+
* Get available capacity for a queue across all concurrency groups.
|
|
965
|
+
* Returns the minimum available capacity across all groups.
|
|
966
|
+
*/
|
|
967
|
+
getAvailableCapacity(queue: QueueDescriptor): Promise<number>;
|
|
968
|
+
/**
|
|
969
|
+
* Get concurrency limit for a specific group.
|
|
970
|
+
*/
|
|
971
|
+
getConcurrencyLimit(groupName: string, groupId: string): Promise<number>;
|
|
972
|
+
/**
|
|
973
|
+
* Check if a group is at capacity.
|
|
974
|
+
*/
|
|
975
|
+
isAtCapacity(groupName: string, groupId: string): Promise<boolean>;
|
|
976
|
+
/**
|
|
977
|
+
* Get full state for a group.
|
|
978
|
+
*/
|
|
979
|
+
getState(groupName: string, groupId: string): Promise<ConcurrencyState>;
|
|
980
|
+
/**
|
|
981
|
+
* Get all active message IDs for a group.
|
|
982
|
+
*/
|
|
983
|
+
getActiveMessages(groupName: string, groupId: string): Promise<string[]>;
|
|
984
|
+
/**
|
|
985
|
+
* Force-clear concurrency for a group (use with caution).
|
|
986
|
+
* Useful for cleanup after crashes.
|
|
987
|
+
*/
|
|
988
|
+
clearGroup(groupName: string, groupId: string): Promise<void>;
|
|
989
|
+
/**
|
|
990
|
+
* Remove a specific message from concurrency tracking.
|
|
991
|
+
* Useful for cleanup.
|
|
992
|
+
*/
|
|
993
|
+
removeMessage(messageId: string, queue: QueueDescriptor): Promise<void>;
|
|
994
|
+
/**
|
|
995
|
+
* Get configured group names.
|
|
996
|
+
*/
|
|
997
|
+
getGroupNames(): string[];
|
|
998
|
+
/**
|
|
999
|
+
* Close the Redis connection.
|
|
1000
|
+
*/
|
|
1001
|
+
close(): Promise<void>;
|
|
1001
1002
|
}
|
|
1002
1003
|
declare module "@internal/redis" {
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1004
|
+
interface RedisCommander<Context> {
|
|
1005
|
+
reserveConcurrency(numKeys: number, keys: string[], messageId: string, ...limits: string[]): Promise<number>;
|
|
1006
|
+
}
|
|
1006
1007
|
}
|
|
1007
|
-
|
|
1008
|
+
//#endregion
|
|
1009
|
+
//#region src/fair-queue/keyProducer.d.ts
|
|
1008
1010
|
/**
|
|
1009
1011
|
* Default key producer for the fair queue system.
|
|
1010
1012
|
* Uses a configurable prefix and standard key structure.
|
|
@@ -1021,58 +1023,59 @@ declare module "@internal/redis" {
|
|
|
1021
1023
|
* - Worker queue: {prefix}:worker:{consumerId}
|
|
1022
1024
|
*/
|
|
1023
1025
|
declare class DefaultFairQueueKeyProducer implements FairQueueKeyProducer {
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1026
|
+
#private;
|
|
1027
|
+
private readonly prefix;
|
|
1028
|
+
private readonly separator;
|
|
1029
|
+
constructor(options?: {
|
|
1030
|
+
prefix?: string;
|
|
1031
|
+
separator?: string;
|
|
1032
|
+
});
|
|
1033
|
+
masterQueueKey(shardId: number): string;
|
|
1034
|
+
queueKey(queueId: string): string;
|
|
1035
|
+
queueItemsKey(queueId: string): string;
|
|
1036
|
+
concurrencyKey(groupName: string, groupId: string): string;
|
|
1037
|
+
inflightKey(shardId: number): string;
|
|
1038
|
+
inflightDataKey(shardId: number): string;
|
|
1039
|
+
workerQueueKey(consumerId: string): string;
|
|
1040
|
+
dispatchKey(shardId: number): string;
|
|
1041
|
+
tenantQueueIndexKey(tenantId: string): string;
|
|
1042
|
+
deadLetterQueueKey(tenantId: string): string;
|
|
1043
|
+
deadLetterQueueDataKey(tenantId: string): string;
|
|
1044
|
+
/**
|
|
1045
|
+
* Extract tenant ID from a queue ID.
|
|
1046
|
+
* Default implementation assumes queue IDs are formatted as: tenant:{tenantId}:...
|
|
1047
|
+
* Override this method for custom queue ID formats.
|
|
1048
|
+
*/
|
|
1049
|
+
extractTenantId(queueId: string): string;
|
|
1050
|
+
/**
|
|
1051
|
+
* Extract a group ID from a queue ID.
|
|
1052
|
+
* Default implementation looks for pattern: {groupName}:{groupId}:...
|
|
1053
|
+
* Override this method for custom queue ID formats.
|
|
1054
|
+
*/
|
|
1055
|
+
extractGroupId(groupName: string, queueId: string): string;
|
|
1054
1056
|
}
|
|
1055
1057
|
/**
|
|
1056
1058
|
* Key producer with custom extraction logic via callbacks.
|
|
1057
1059
|
* Useful when queue IDs don't follow a standard pattern.
|
|
1058
1060
|
*/
|
|
1059
1061
|
declare class CallbackFairQueueKeyProducer extends DefaultFairQueueKeyProducer {
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
}
|
|
1071
|
-
|
|
1062
|
+
private readonly tenantExtractor;
|
|
1063
|
+
private readonly groupExtractor;
|
|
1064
|
+
constructor(options: {
|
|
1065
|
+
prefix?: string;
|
|
1066
|
+
separator?: string;
|
|
1067
|
+
extractTenantId: (queueId: string) => string;
|
|
1068
|
+
extractGroupId: (groupName: string, queueId: string) => string;
|
|
1069
|
+
});
|
|
1070
|
+
extractTenantId(queueId: string): string;
|
|
1071
|
+
extractGroupId(groupName: string, queueId: string): string;
|
|
1072
|
+
}
|
|
1073
|
+
//#endregion
|
|
1074
|
+
//#region src/fair-queue/masterQueue.d.ts
|
|
1072
1075
|
interface MasterQueueOptions {
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
+
redis: RedisOptions;
|
|
1077
|
+
keys: FairQueueKeyProducer;
|
|
1078
|
+
shardCount: number;
|
|
1076
1079
|
}
|
|
1077
1080
|
/**
|
|
1078
1081
|
* Master queue manages the top-level queue of queues.
|
|
@@ -1083,141 +1086,143 @@ interface MasterQueueOptions {
|
|
|
1083
1086
|
* - Queues scored by oldest message timestamp
|
|
1084
1087
|
*/
|
|
1085
1088
|
declare class MasterQueue {
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
|
|
1152
|
-
|
|
1153
|
-
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
|
|
1089
|
+
#private;
|
|
1090
|
+
private options;
|
|
1091
|
+
private redis;
|
|
1092
|
+
private keys;
|
|
1093
|
+
private shardCount;
|
|
1094
|
+
constructor(options: MasterQueueOptions);
|
|
1095
|
+
/**
|
|
1096
|
+
* Get the shard ID for a queue.
|
|
1097
|
+
* Uses consistent hashing based on queue ID.
|
|
1098
|
+
*/
|
|
1099
|
+
getShardForQueue(queueId: string): number;
|
|
1100
|
+
/**
|
|
1101
|
+
* Add a queue to its master queue shard.
|
|
1102
|
+
* Updates the score to the oldest message timestamp.
|
|
1103
|
+
*
|
|
1104
|
+
* @param queueId - The queue identifier
|
|
1105
|
+
* @param oldestMessageTimestamp - Timestamp of the oldest message in the queue
|
|
1106
|
+
*/
|
|
1107
|
+
addQueue(queueId: string, oldestMessageTimestamp: number): Promise<void>;
|
|
1108
|
+
/**
|
|
1109
|
+
* Update a queue's score in the master queue.
|
|
1110
|
+
* This is typically called after dequeuing to update to the new oldest message.
|
|
1111
|
+
*
|
|
1112
|
+
* @param queueId - The queue identifier
|
|
1113
|
+
* @param newOldestTimestamp - New timestamp of the oldest message
|
|
1114
|
+
*/
|
|
1115
|
+
updateQueueScore(queueId: string, newOldestTimestamp: number): Promise<void>;
|
|
1116
|
+
/**
|
|
1117
|
+
* Remove a queue from its master queue shard.
|
|
1118
|
+
* Called when a queue becomes empty.
|
|
1119
|
+
*
|
|
1120
|
+
* @param queueId - The queue identifier
|
|
1121
|
+
*/
|
|
1122
|
+
removeQueue(queueId: string): Promise<void>;
|
|
1123
|
+
/**
|
|
1124
|
+
* Get queues from a shard, ordered by oldest message (lowest score first).
|
|
1125
|
+
*
|
|
1126
|
+
* @param shardId - The shard to query
|
|
1127
|
+
* @param limit - Maximum number of queues to return (default: 1000)
|
|
1128
|
+
* @param maxScore - Maximum score (timestamp) to include (default: now)
|
|
1129
|
+
*/
|
|
1130
|
+
getQueuesFromShard(shardId: number, limit?: number, maxScore?: number): Promise<QueueWithScore[]>;
|
|
1131
|
+
/**
|
|
1132
|
+
* Get the number of queues in a shard.
|
|
1133
|
+
*/
|
|
1134
|
+
getShardQueueCount(shardId: number): Promise<number>;
|
|
1135
|
+
/**
|
|
1136
|
+
* Get total queue count across all shards.
|
|
1137
|
+
*/
|
|
1138
|
+
getTotalQueueCount(): Promise<number>;
|
|
1139
|
+
/**
|
|
1140
|
+
* Atomically add a queue to master queue only if queue has messages.
|
|
1141
|
+
* Uses Lua script for atomicity.
|
|
1142
|
+
*
|
|
1143
|
+
* @param queueId - The queue identifier
|
|
1144
|
+
* @param queueKey - The actual queue sorted set key
|
|
1145
|
+
* @returns Whether the queue was added to the master queue
|
|
1146
|
+
*/
|
|
1147
|
+
addQueueIfNotEmpty(queueId: string, queueKey: string): Promise<boolean>;
|
|
1148
|
+
/**
|
|
1149
|
+
* Atomically remove a queue from master queue only if queue is empty.
|
|
1150
|
+
* Uses Lua script for atomicity.
|
|
1151
|
+
*
|
|
1152
|
+
* @param queueId - The queue identifier
|
|
1153
|
+
* @param queueKey - The actual queue sorted set key
|
|
1154
|
+
* @returns Whether the queue was removed from the master queue
|
|
1155
|
+
*/
|
|
1156
|
+
removeQueueIfEmpty(queueId: string, queueKey: string): Promise<boolean>;
|
|
1157
|
+
/**
|
|
1158
|
+
* Close the Redis connection.
|
|
1159
|
+
*/
|
|
1160
|
+
close(): Promise<void>;
|
|
1158
1161
|
}
|
|
1159
1162
|
declare module "@internal/redis" {
|
|
1160
|
-
|
|
1161
|
-
|
|
1162
|
-
|
|
1163
|
-
|
|
1163
|
+
interface RedisCommander<Context> {
|
|
1164
|
+
addQueueIfNotEmpty(masterKey: string, queueKey: string, queueId: string): Promise<number>;
|
|
1165
|
+
removeQueueIfEmpty(masterKey: string, queueKey: string, queueId: string): Promise<number>;
|
|
1166
|
+
}
|
|
1164
1167
|
}
|
|
1165
|
-
|
|
1168
|
+
//#endregion
|
|
1169
|
+
//#region src/fair-queue/scheduler.d.ts
|
|
1166
1170
|
/**
|
|
1167
1171
|
* Base class for scheduler implementations.
|
|
1168
1172
|
* Provides common utilities and default implementations.
|
|
1169
1173
|
*/
|
|
1170
1174
|
declare abstract class BaseScheduler implements FairScheduler {
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
|
|
1188
|
-
|
|
1189
|
-
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1175
|
+
/**
|
|
1176
|
+
* Select queues for processing from a master queue shard.
|
|
1177
|
+
* Must be implemented by subclasses.
|
|
1178
|
+
*/
|
|
1179
|
+
abstract selectQueues(masterQueueShard: string, consumerId: string, context: SchedulerContext): Promise<TenantQueues[]>;
|
|
1180
|
+
/**
|
|
1181
|
+
* Called after processing a message to update scheduler state.
|
|
1182
|
+
* Default implementation does nothing.
|
|
1183
|
+
*/
|
|
1184
|
+
recordProcessed(_tenantId: string, _queueId: string): Promise<void>;
|
|
1185
|
+
/**
|
|
1186
|
+
* Called after processing multiple messages to update scheduler state.
|
|
1187
|
+
* Batch variant for efficiency - reduces Redis calls when processing multiple messages.
|
|
1188
|
+
* Default implementation does nothing.
|
|
1189
|
+
*/
|
|
1190
|
+
recordProcessedBatch(_tenantId: string, _queueId: string, _count: number): Promise<void>;
|
|
1191
|
+
/**
|
|
1192
|
+
* Initialize the scheduler.
|
|
1193
|
+
* Default implementation does nothing.
|
|
1194
|
+
*/
|
|
1195
|
+
initialize(): Promise<void>;
|
|
1196
|
+
/**
|
|
1197
|
+
* Cleanup scheduler resources.
|
|
1198
|
+
* Default implementation does nothing.
|
|
1199
|
+
*/
|
|
1200
|
+
close(): Promise<void>;
|
|
1201
|
+
/**
|
|
1202
|
+
* Helper to group queues by tenant.
|
|
1203
|
+
*/
|
|
1204
|
+
protected groupQueuesByTenant(queues: Array<{
|
|
1205
|
+
queueId: string;
|
|
1206
|
+
tenantId: string;
|
|
1207
|
+
}>): Map<string, string[]>;
|
|
1208
|
+
/**
|
|
1209
|
+
* Helper to convert grouped queues to TenantQueues array.
|
|
1210
|
+
*/
|
|
1211
|
+
protected toTenantQueuesArray(grouped: Map<string, string[]>): TenantQueues[];
|
|
1212
|
+
/**
|
|
1213
|
+
* Helper to filter out tenants at capacity.
|
|
1214
|
+
*/
|
|
1215
|
+
protected filterAtCapacity(tenants: TenantQueues[], context: SchedulerContext, groupName?: string): Promise<TenantQueues[]>;
|
|
1212
1216
|
}
|
|
1213
1217
|
/**
|
|
1214
1218
|
* Simple noop scheduler that returns empty results.
|
|
1215
1219
|
* Useful for testing or disabling scheduling.
|
|
1216
1220
|
*/
|
|
1217
1221
|
declare class NoopScheduler extends BaseScheduler {
|
|
1218
|
-
|
|
1222
|
+
selectQueues(_masterQueueShard: string, _consumerId: string, _context: SchedulerContext): Promise<TenantQueues[]>;
|
|
1219
1223
|
}
|
|
1220
|
-
|
|
1224
|
+
//#endregion
|
|
1225
|
+
//#region src/fair-queue/schedulers/drr.d.ts
|
|
1221
1226
|
/**
|
|
1222
1227
|
* Deficit Round Robin (DRR) Scheduler.
|
|
1223
1228
|
*
|
|
@@ -1233,71 +1238,72 @@ declare class NoopScheduler extends BaseScheduler {
|
|
|
1233
1238
|
* - Automatic deficit cleanup for inactive tenants
|
|
1234
1239
|
*/
|
|
1235
1240
|
declare class DRRScheduler extends BaseScheduler {
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1241
|
+
#private;
|
|
1242
|
+
private config;
|
|
1243
|
+
private redis;
|
|
1244
|
+
private keys;
|
|
1245
|
+
private quantum;
|
|
1246
|
+
private maxDeficit;
|
|
1247
|
+
private masterQueueLimit;
|
|
1248
|
+
private logger;
|
|
1249
|
+
constructor(config: DRRSchedulerConfig);
|
|
1250
|
+
/**
|
|
1251
|
+
* Select queues for processing using DRR algorithm.
|
|
1252
|
+
*
|
|
1253
|
+
* Algorithm:
|
|
1254
|
+
* 1. Get all queues from the master shard
|
|
1255
|
+
* 2. Group by tenant
|
|
1256
|
+
* 3. Filter out tenants at concurrency capacity
|
|
1257
|
+
* 4. Add quantum to each tenant's deficit (atomically)
|
|
1258
|
+
* 5. Select queues from tenants with deficit >= 1
|
|
1259
|
+
* 6. Order tenants by deficit (highest first for fairness)
|
|
1260
|
+
*/
|
|
1261
|
+
selectQueues(masterQueueShard: string, consumerId: string, context: SchedulerContext): Promise<TenantQueues[]>;
|
|
1262
|
+
/**
|
|
1263
|
+
* Select queues using the two-level tenant dispatch index.
|
|
1264
|
+
*
|
|
1265
|
+
* Algorithm:
|
|
1266
|
+
* 1. ZRANGEBYSCORE on dispatch index (gets only tenants with queues - much smaller)
|
|
1267
|
+
* 2. Add quantum to each tenant's deficit (atomically)
|
|
1268
|
+
* 3. Check capacity as safety net (dispatch should only have tenants with capacity)
|
|
1269
|
+
* 4. Select tenants with deficit >= 1, sorted by deficit (highest first)
|
|
1270
|
+
* 5. For each tenant, fetch their queues from Level 2 index
|
|
1271
|
+
*/
|
|
1272
|
+
selectQueuesFromDispatch(dispatchShardKey: string, consumerId: string, context: DispatchSchedulerContext): Promise<TenantQueues[]>;
|
|
1273
|
+
/**
|
|
1274
|
+
* Record that a message was processed from a tenant.
|
|
1275
|
+
* Decrements the tenant's deficit.
|
|
1276
|
+
*/
|
|
1277
|
+
recordProcessed(tenantId: string, _queueId: string): Promise<void>;
|
|
1278
|
+
/**
|
|
1279
|
+
* Record that multiple messages were processed from a tenant.
|
|
1280
|
+
* Decrements the tenant's deficit by count atomically.
|
|
1281
|
+
*/
|
|
1282
|
+
recordProcessedBatch(tenantId: string, _queueId: string, count: number): Promise<void>;
|
|
1283
|
+
close(): Promise<void>;
|
|
1284
|
+
/**
|
|
1285
|
+
* Get the current deficit for a tenant.
|
|
1286
|
+
*/
|
|
1287
|
+
getDeficit(tenantId: string): Promise<number>;
|
|
1288
|
+
/**
|
|
1289
|
+
* Reset deficit for a tenant.
|
|
1290
|
+
* Used when a tenant has no more active queues.
|
|
1291
|
+
*/
|
|
1292
|
+
resetDeficit(tenantId: string): Promise<void>;
|
|
1293
|
+
/**
|
|
1294
|
+
* Get all tenant deficits.
|
|
1295
|
+
*/
|
|
1296
|
+
getAllDeficits(): Promise<Map<string, number>>;
|
|
1292
1297
|
}
|
|
1293
1298
|
declare module "@internal/redis" {
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
}
|
|
1300
|
-
|
|
1299
|
+
interface RedisCommander<Context> {
|
|
1300
|
+
drrAddQuantum(deficitKey: string, quantum: string, maxDeficit: string, ...tenantIds: string[]): Promise<string[]>;
|
|
1301
|
+
drrDecrementDeficit(deficitKey: string, tenantId: string): Promise<string>;
|
|
1302
|
+
drrDecrementDeficitBatch(deficitKey: string, tenantId: string, count: string): Promise<string>;
|
|
1303
|
+
}
|
|
1304
|
+
}
|
|
1305
|
+
//#endregion
|
|
1306
|
+
//#region src/fair-queue/schedulers/weighted.d.ts
|
|
1301
1307
|
/**
|
|
1302
1308
|
* Weighted Shuffle Scheduler.
|
|
1303
1309
|
*
|
|
@@ -1312,27 +1318,28 @@ declare module "@internal/redis" {
|
|
|
1312
1318
|
* - Maximum tenant count to limit iteration
|
|
1313
1319
|
*/
|
|
1314
1320
|
declare class WeightedScheduler extends BaseScheduler {
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
}
|
|
1330
|
-
|
|
1321
|
+
#private;
|
|
1322
|
+
private config;
|
|
1323
|
+
private redis;
|
|
1324
|
+
private keys;
|
|
1325
|
+
private rng;
|
|
1326
|
+
private biases;
|
|
1327
|
+
private defaultTenantLimit;
|
|
1328
|
+
private masterQueueLimit;
|
|
1329
|
+
private reuseSnapshotCount;
|
|
1330
|
+
private maximumTenantCount;
|
|
1331
|
+
private snapshotCache;
|
|
1332
|
+
constructor(config: WeightedSchedulerConfig);
|
|
1333
|
+
selectQueues(masterQueueShard: string, consumerId: string, context: SchedulerContext): Promise<TenantQueues[]>;
|
|
1334
|
+
close(): Promise<void>;
|
|
1335
|
+
}
|
|
1336
|
+
//#endregion
|
|
1337
|
+
//#region src/fair-queue/schedulers/roundRobin.d.ts
|
|
1331
1338
|
interface RoundRobinSchedulerConfig {
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1335
|
-
|
|
1339
|
+
redis: RedisOptions;
|
|
1340
|
+
keys: FairQueueKeyProducer;
|
|
1341
|
+
/** Maximum queues to fetch from master queue per iteration */
|
|
1342
|
+
masterQueueLimit?: number;
|
|
1336
1343
|
}
|
|
1337
1344
|
/**
|
|
1338
1345
|
* Round Robin Scheduler.
|
|
@@ -1346,65 +1353,66 @@ interface RoundRobinSchedulerConfig {
|
|
|
1346
1353
|
* - No weighting or bias
|
|
1347
1354
|
*/
|
|
1348
1355
|
declare class RoundRobinScheduler extends BaseScheduler {
|
|
1349
|
-
|
|
1350
|
-
|
|
1351
|
-
|
|
1352
|
-
|
|
1353
|
-
|
|
1354
|
-
|
|
1355
|
-
|
|
1356
|
-
|
|
1357
|
-
}
|
|
1358
|
-
|
|
1356
|
+
#private;
|
|
1357
|
+
private config;
|
|
1358
|
+
private redis;
|
|
1359
|
+
private keys;
|
|
1360
|
+
private masterQueueLimit;
|
|
1361
|
+
constructor(config: RoundRobinSchedulerConfig);
|
|
1362
|
+
selectQueues(masterQueueShard: string, consumerId: string, context: SchedulerContext): Promise<TenantQueues[]>;
|
|
1363
|
+
close(): Promise<void>;
|
|
1364
|
+
}
|
|
1365
|
+
//#endregion
|
|
1366
|
+
//#region src/fair-queue/telemetry.d.ts
|
|
1359
1367
|
/**
|
|
1360
1368
|
* Semantic attributes for fair queue messaging operations.
|
|
1361
1369
|
*/
|
|
1362
1370
|
declare const FairQueueAttributes: {
|
|
1363
|
-
|
|
1364
|
-
|
|
1365
|
-
|
|
1366
|
-
|
|
1367
|
-
|
|
1368
|
-
|
|
1369
|
-
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1371
|
+
readonly QUEUE_ID: "fairqueue.queue_id";
|
|
1372
|
+
readonly TENANT_ID: "fairqueue.tenant_id";
|
|
1373
|
+
readonly MESSAGE_ID: "fairqueue.message_id";
|
|
1374
|
+
readonly SHARD_ID: "fairqueue.shard_id";
|
|
1375
|
+
readonly WORKER_QUEUE: "fairqueue.worker_queue";
|
|
1376
|
+
readonly CONSUMER_ID: "fairqueue.consumer_id";
|
|
1377
|
+
readonly ATTEMPT: "fairqueue.attempt";
|
|
1378
|
+
readonly CONCURRENCY_GROUP: "fairqueue.concurrency_group";
|
|
1379
|
+
readonly MESSAGE_COUNT: "fairqueue.message_count";
|
|
1380
|
+
readonly RESULT: "fairqueue.result";
|
|
1373
1381
|
};
|
|
1374
1382
|
/**
|
|
1375
1383
|
* Standard messaging semantic attributes.
|
|
1376
1384
|
*/
|
|
1377
1385
|
declare const MessagingAttributes: {
|
|
1378
|
-
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
|
|
1386
|
+
readonly SYSTEM: "messaging.system";
|
|
1387
|
+
readonly OPERATION: "messaging.operation";
|
|
1388
|
+
readonly MESSAGE_ID: "messaging.message_id";
|
|
1389
|
+
readonly DESTINATION_NAME: "messaging.destination.name";
|
|
1382
1390
|
};
|
|
1383
1391
|
/**
|
|
1384
1392
|
* FairQueue metrics collection.
|
|
1385
1393
|
*/
|
|
1386
1394
|
interface FairQueueMetrics {
|
|
1387
|
-
|
|
1388
|
-
|
|
1389
|
-
|
|
1390
|
-
|
|
1391
|
-
|
|
1392
|
-
|
|
1393
|
-
|
|
1394
|
-
|
|
1395
|
-
|
|
1396
|
-
|
|
1397
|
-
|
|
1398
|
-
|
|
1395
|
+
messagesEnqueued: Counter;
|
|
1396
|
+
messagesCompleted: Counter;
|
|
1397
|
+
messagesFailed: Counter;
|
|
1398
|
+
messagesRetried: Counter;
|
|
1399
|
+
messagesToDLQ: Counter;
|
|
1400
|
+
processingTime: Histogram;
|
|
1401
|
+
queueTime: Histogram;
|
|
1402
|
+
queueLength: ObservableGauge;
|
|
1403
|
+
masterQueueLength: ObservableGauge;
|
|
1404
|
+
dispatchLength: ObservableGauge;
|
|
1405
|
+
inflightCount: ObservableGauge;
|
|
1406
|
+
dlqLength: ObservableGauge;
|
|
1399
1407
|
}
|
|
1400
1408
|
/**
|
|
1401
1409
|
* Options for creating FairQueue telemetry.
|
|
1402
1410
|
*/
|
|
1403
1411
|
interface TelemetryOptions {
|
|
1404
|
-
|
|
1405
|
-
|
|
1406
|
-
|
|
1407
|
-
|
|
1412
|
+
tracer?: Tracer;
|
|
1413
|
+
meter?: Meter;
|
|
1414
|
+
/** Custom name for metrics prefix */
|
|
1415
|
+
name?: string;
|
|
1408
1416
|
}
|
|
1409
1417
|
/**
|
|
1410
1418
|
* Telemetry helper for FairQueue.
|
|
@@ -1415,139 +1423,139 @@ interface TelemetryOptions {
|
|
|
1415
1423
|
* - Context propagation helpers
|
|
1416
1424
|
*/
|
|
1417
1425
|
declare class FairQueueTelemetry {
|
|
1418
|
-
|
|
1419
|
-
|
|
1420
|
-
|
|
1421
|
-
|
|
1422
|
-
|
|
1423
|
-
|
|
1424
|
-
|
|
1425
|
-
|
|
1426
|
-
|
|
1427
|
-
|
|
1428
|
-
|
|
1429
|
-
|
|
1430
|
-
|
|
1431
|
-
|
|
1432
|
-
|
|
1433
|
-
|
|
1434
|
-
|
|
1435
|
-
|
|
1436
|
-
|
|
1437
|
-
|
|
1438
|
-
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
|
|
1443
|
-
|
|
1444
|
-
|
|
1445
|
-
|
|
1446
|
-
|
|
1447
|
-
|
|
1448
|
-
|
|
1449
|
-
|
|
1450
|
-
|
|
1451
|
-
|
|
1452
|
-
|
|
1453
|
-
|
|
1454
|
-
|
|
1455
|
-
|
|
1456
|
-
|
|
1457
|
-
|
|
1458
|
-
|
|
1459
|
-
|
|
1460
|
-
|
|
1461
|
-
|
|
1462
|
-
|
|
1463
|
-
|
|
1464
|
-
|
|
1465
|
-
|
|
1466
|
-
|
|
1467
|
-
|
|
1468
|
-
|
|
1469
|
-
|
|
1470
|
-
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
|
|
1474
|
-
|
|
1475
|
-
|
|
1476
|
-
|
|
1477
|
-
|
|
1478
|
-
|
|
1479
|
-
|
|
1480
|
-
|
|
1481
|
-
|
|
1482
|
-
|
|
1483
|
-
|
|
1484
|
-
|
|
1485
|
-
|
|
1486
|
-
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
|
|
1496
|
-
|
|
1497
|
-
|
|
1498
|
-
|
|
1499
|
-
|
|
1500
|
-
|
|
1501
|
-
|
|
1502
|
-
|
|
1503
|
-
|
|
1504
|
-
|
|
1505
|
-
|
|
1506
|
-
|
|
1507
|
-
|
|
1508
|
-
|
|
1509
|
-
|
|
1510
|
-
|
|
1511
|
-
|
|
1512
|
-
|
|
1426
|
+
#private;
|
|
1427
|
+
private tracer?;
|
|
1428
|
+
private meter?;
|
|
1429
|
+
private metrics?;
|
|
1430
|
+
private name;
|
|
1431
|
+
constructor(options: TelemetryOptions);
|
|
1432
|
+
/**
|
|
1433
|
+
* Create a traced span for an operation.
|
|
1434
|
+
* Returns the result of the function, or throws any error after recording it.
|
|
1435
|
+
*/
|
|
1436
|
+
trace<T>(name: string, fn: (span: Span) => Promise<T>, options?: {
|
|
1437
|
+
kind?: SpanKind;
|
|
1438
|
+
attributes?: Attributes;
|
|
1439
|
+
}): Promise<T>;
|
|
1440
|
+
/**
|
|
1441
|
+
* Synchronous version of trace.
|
|
1442
|
+
*/
|
|
1443
|
+
traceSync<T>(name: string, fn: (span: Span) => T, options?: {
|
|
1444
|
+
kind?: SpanKind;
|
|
1445
|
+
attributes?: Attributes;
|
|
1446
|
+
}): T;
|
|
1447
|
+
/**
|
|
1448
|
+
* Record a message enqueued.
|
|
1449
|
+
*/
|
|
1450
|
+
recordEnqueue(attributes?: Attributes): void;
|
|
1451
|
+
/**
|
|
1452
|
+
* Record a batch of messages enqueued.
|
|
1453
|
+
*/
|
|
1454
|
+
recordEnqueueBatch(count: number, attributes?: Attributes): void;
|
|
1455
|
+
/**
|
|
1456
|
+
* Record a message completed successfully.
|
|
1457
|
+
*/
|
|
1458
|
+
recordComplete(attributes?: Attributes): void;
|
|
1459
|
+
/**
|
|
1460
|
+
* Record a message processing failure.
|
|
1461
|
+
*/
|
|
1462
|
+
recordFailure(attributes?: Attributes): void;
|
|
1463
|
+
/**
|
|
1464
|
+
* Record a message retry.
|
|
1465
|
+
*/
|
|
1466
|
+
recordRetry(attributes?: Attributes): void;
|
|
1467
|
+
/**
|
|
1468
|
+
* Record a message sent to DLQ.
|
|
1469
|
+
*/
|
|
1470
|
+
recordDLQ(attributes?: Attributes): void;
|
|
1471
|
+
/**
|
|
1472
|
+
* Record message processing time.
|
|
1473
|
+
*
|
|
1474
|
+
* @param durationMs - Processing duration in milliseconds
|
|
1475
|
+
*/
|
|
1476
|
+
recordProcessingTime(durationMs: number, attributes?: Attributes): void;
|
|
1477
|
+
/**
|
|
1478
|
+
* Record time a message spent waiting in queue.
|
|
1479
|
+
*
|
|
1480
|
+
* @param durationMs - Queue wait time in milliseconds
|
|
1481
|
+
*/
|
|
1482
|
+
recordQueueTime(durationMs: number, attributes?: Attributes): void;
|
|
1483
|
+
/**
|
|
1484
|
+
* Register observable gauge callbacks.
|
|
1485
|
+
* Call this after FairQueue is initialized to register the gauge callbacks.
|
|
1486
|
+
*/
|
|
1487
|
+
registerGaugeCallbacks(callbacks: {
|
|
1488
|
+
getQueueLength?: (queueId: string) => Promise<number>;
|
|
1489
|
+
getMasterQueueLength?: (shardId: number) => Promise<number>;
|
|
1490
|
+
getDispatchLength?: (shardId: number) => Promise<number>;
|
|
1491
|
+
getInflightCount?: (shardId: number) => Promise<number>;
|
|
1492
|
+
getDLQLength?: (tenantId: string) => Promise<number>;
|
|
1493
|
+
shardCount?: number;
|
|
1494
|
+
observedQueues?: string[];
|
|
1495
|
+
observedTenants?: string[];
|
|
1496
|
+
}): void;
|
|
1497
|
+
/**
|
|
1498
|
+
* Create standard attributes for a message operation (for spans/traces).
|
|
1499
|
+
* Use this for span attributes where high cardinality is acceptable.
|
|
1500
|
+
*/
|
|
1501
|
+
messageAttributes(params: {
|
|
1502
|
+
queueId?: string;
|
|
1503
|
+
tenantId?: string;
|
|
1504
|
+
messageId?: string;
|
|
1505
|
+
attempt?: number;
|
|
1506
|
+
workerQueue?: string;
|
|
1507
|
+
consumerId?: string;
|
|
1508
|
+
}): Attributes;
|
|
1509
|
+
/**
|
|
1510
|
+
* Check if telemetry is enabled.
|
|
1511
|
+
*/
|
|
1512
|
+
get isEnabled(): boolean;
|
|
1513
|
+
/**
|
|
1514
|
+
* Check if tracing is enabled.
|
|
1515
|
+
*/
|
|
1516
|
+
get hasTracer(): boolean;
|
|
1517
|
+
/**
|
|
1518
|
+
* Check if metrics are enabled.
|
|
1519
|
+
*/
|
|
1520
|
+
get hasMetrics(): boolean;
|
|
1513
1521
|
}
|
|
1514
1522
|
/**
|
|
1515
1523
|
* State for tracking a consumer loop's batched span.
|
|
1516
1524
|
*/
|
|
1517
1525
|
interface ConsumerLoopState {
|
|
1518
|
-
|
|
1519
|
-
|
|
1520
|
-
|
|
1521
|
-
|
|
1522
|
-
|
|
1523
|
-
|
|
1524
|
-
|
|
1525
|
-
|
|
1526
|
-
|
|
1527
|
-
|
|
1528
|
-
|
|
1529
|
-
|
|
1530
|
-
|
|
1531
|
-
|
|
1532
|
-
|
|
1533
|
-
|
|
1534
|
-
|
|
1535
|
-
|
|
1526
|
+
/** Countdown of iterations before starting a new span */
|
|
1527
|
+
perTraceCountdown: number;
|
|
1528
|
+
/** When the current trace started */
|
|
1529
|
+
traceStartedAt: Date;
|
|
1530
|
+
/** The current batched span */
|
|
1531
|
+
currentSpan?: Span;
|
|
1532
|
+
/** The context for the current batched span */
|
|
1533
|
+
currentSpanContext?: Context;
|
|
1534
|
+
/** Number of iterations in the current span */
|
|
1535
|
+
iterationsCount: number;
|
|
1536
|
+
/** Total iterations across all spans */
|
|
1537
|
+
totalIterationsCount: number;
|
|
1538
|
+
/** Running duration in milliseconds for the current span */
|
|
1539
|
+
runningDurationInMs: number;
|
|
1540
|
+
/** Stats counters for the current span */
|
|
1541
|
+
stats: Record<string, number>;
|
|
1542
|
+
/** Flag to force span end on next iteration */
|
|
1543
|
+
endSpanInNextIteration: boolean;
|
|
1536
1544
|
}
|
|
1537
1545
|
/**
|
|
1538
1546
|
* Configuration for the BatchedSpanManager.
|
|
1539
1547
|
*/
|
|
1540
1548
|
interface BatchedSpanManagerOptions {
|
|
1541
|
-
|
|
1542
|
-
|
|
1543
|
-
|
|
1544
|
-
|
|
1545
|
-
|
|
1546
|
-
|
|
1547
|
-
|
|
1548
|
-
|
|
1549
|
-
|
|
1550
|
-
|
|
1549
|
+
/** The tracer to use for creating spans */
|
|
1550
|
+
tracer?: Tracer;
|
|
1551
|
+
/** Name prefix for spans */
|
|
1552
|
+
name: string;
|
|
1553
|
+
/** Maximum iterations before rotating the span */
|
|
1554
|
+
maxIterations: number;
|
|
1555
|
+
/** Maximum seconds before rotating the span */
|
|
1556
|
+
timeoutSeconds: number;
|
|
1557
|
+
/** Optional callback to get dynamic attributes when starting a new batched span */
|
|
1558
|
+
getDynamicAttributes?: () => Attributes;
|
|
1551
1559
|
}
|
|
1552
1560
|
/**
|
|
1553
1561
|
* Manages batched spans for consumer loops.
|
|
@@ -1556,71 +1564,72 @@ interface BatchedSpanManagerOptions {
|
|
|
1556
1564
|
* reducing the volume of spans while maintaining observability.
|
|
1557
1565
|
*/
|
|
1558
1566
|
declare class BatchedSpanManager {
|
|
1559
|
-
|
|
1560
|
-
|
|
1561
|
-
|
|
1562
|
-
|
|
1563
|
-
|
|
1564
|
-
|
|
1565
|
-
|
|
1566
|
-
|
|
1567
|
-
|
|
1568
|
-
|
|
1569
|
-
|
|
1570
|
-
|
|
1571
|
-
|
|
1572
|
-
|
|
1573
|
-
|
|
1574
|
-
|
|
1575
|
-
|
|
1576
|
-
|
|
1577
|
-
|
|
1578
|
-
|
|
1579
|
-
|
|
1580
|
-
|
|
1581
|
-
|
|
1582
|
-
|
|
1583
|
-
|
|
1584
|
-
|
|
1585
|
-
|
|
1586
|
-
|
|
1587
|
-
|
|
1588
|
-
|
|
1589
|
-
|
|
1590
|
-
|
|
1591
|
-
|
|
1592
|
-
|
|
1593
|
-
|
|
1594
|
-
|
|
1595
|
-
|
|
1596
|
-
|
|
1597
|
-
|
|
1598
|
-
|
|
1599
|
-
|
|
1600
|
-
|
|
1601
|
-
|
|
1602
|
-
|
|
1603
|
-
|
|
1604
|
-
|
|
1605
|
-
|
|
1606
|
-
|
|
1607
|
-
|
|
1608
|
-
|
|
1609
|
-
|
|
1567
|
+
private tracer?;
|
|
1568
|
+
private name;
|
|
1569
|
+
private maxIterations;
|
|
1570
|
+
private timeoutSeconds;
|
|
1571
|
+
private loopStates;
|
|
1572
|
+
private getDynamicAttributes?;
|
|
1573
|
+
constructor(options: BatchedSpanManagerOptions);
|
|
1574
|
+
/**
|
|
1575
|
+
* Initialize state for a consumer loop.
|
|
1576
|
+
*/
|
|
1577
|
+
initializeLoop(loopId: string): void;
|
|
1578
|
+
/**
|
|
1579
|
+
* Get the state for a consumer loop.
|
|
1580
|
+
*/
|
|
1581
|
+
getState(loopId: string): ConsumerLoopState | undefined;
|
|
1582
|
+
/**
|
|
1583
|
+
* Increment a stat counter for a loop.
|
|
1584
|
+
*/
|
|
1585
|
+
incrementStat(loopId: string, statName: string, value?: number): void;
|
|
1586
|
+
/**
|
|
1587
|
+
* Mark that the span should end on the next iteration.
|
|
1588
|
+
*/
|
|
1589
|
+
markForRotation(loopId: string): void;
|
|
1590
|
+
/**
|
|
1591
|
+
* Check if the span should be rotated (ended and a new one started).
|
|
1592
|
+
*/
|
|
1593
|
+
shouldRotate(loopId: string): boolean;
|
|
1594
|
+
/**
|
|
1595
|
+
* End the current span for a loop and record stats.
|
|
1596
|
+
*/
|
|
1597
|
+
endCurrentSpan(loopId: string): void;
|
|
1598
|
+
/**
|
|
1599
|
+
* Start a new batched span for a loop.
|
|
1600
|
+
*/
|
|
1601
|
+
startNewSpan(loopId: string, attributes?: Attributes): void;
|
|
1602
|
+
/**
|
|
1603
|
+
* Execute a function within the batched span context.
|
|
1604
|
+
* Automatically handles span rotation and iteration tracking.
|
|
1605
|
+
*/
|
|
1606
|
+
withBatchedSpan<T>(loopId: string, fn: (span: Span) => Promise<T>, options?: {
|
|
1607
|
+
iterationSpanName?: string;
|
|
1608
|
+
attributes?: Attributes;
|
|
1609
|
+
}): Promise<T>;
|
|
1610
|
+
/**
|
|
1611
|
+
* Clean up state for a loop when it's stopped.
|
|
1612
|
+
*/
|
|
1613
|
+
cleanup(loopId: string): void;
|
|
1614
|
+
/**
|
|
1615
|
+
* Clean up all loop states.
|
|
1616
|
+
*/
|
|
1617
|
+
cleanupAll(): void;
|
|
1610
1618
|
}
|
|
1611
1619
|
/**
|
|
1612
1620
|
* No-op telemetry instance for when telemetry is disabled.
|
|
1613
1621
|
*/
|
|
1614
1622
|
declare const noopTelemetry: FairQueueTelemetry;
|
|
1615
|
-
|
|
1623
|
+
//#endregion
|
|
1624
|
+
//#region src/fair-queue/tenantDispatch.d.ts
|
|
1616
1625
|
interface TenantDispatchOptions {
|
|
1617
|
-
|
|
1618
|
-
|
|
1619
|
-
|
|
1626
|
+
redis: RedisOptions;
|
|
1627
|
+
keys: FairQueueKeyProducer;
|
|
1628
|
+
shardCount: number;
|
|
1620
1629
|
}
|
|
1621
1630
|
interface TenantWithScore {
|
|
1622
|
-
|
|
1623
|
-
|
|
1631
|
+
tenantId: string;
|
|
1632
|
+
score: number;
|
|
1624
1633
|
}
|
|
1625
1634
|
/**
|
|
1626
1635
|
* TenantDispatch manages the two-level tenant dispatch index.
|
|
@@ -1638,67 +1647,68 @@ interface TenantWithScore {
|
|
|
1638
1647
|
* queue backlog so the scheduler iterates tenants (not queues) at Level 1.
|
|
1639
1648
|
*/
|
|
1640
1649
|
declare class TenantDispatch {
|
|
1641
|
-
|
|
1642
|
-
|
|
1643
|
-
|
|
1644
|
-
|
|
1645
|
-
|
|
1646
|
-
|
|
1647
|
-
|
|
1648
|
-
|
|
1649
|
-
|
|
1650
|
-
|
|
1651
|
-
|
|
1652
|
-
|
|
1653
|
-
|
|
1654
|
-
|
|
1655
|
-
|
|
1656
|
-
|
|
1657
|
-
|
|
1658
|
-
|
|
1659
|
-
|
|
1660
|
-
|
|
1661
|
-
|
|
1662
|
-
|
|
1663
|
-
|
|
1664
|
-
|
|
1665
|
-
|
|
1666
|
-
|
|
1667
|
-
|
|
1668
|
-
|
|
1669
|
-
|
|
1670
|
-
|
|
1671
|
-
|
|
1672
|
-
|
|
1673
|
-
|
|
1674
|
-
|
|
1675
|
-
|
|
1676
|
-
|
|
1677
|
-
|
|
1678
|
-
|
|
1679
|
-
|
|
1680
|
-
|
|
1681
|
-
|
|
1682
|
-
|
|
1683
|
-
|
|
1684
|
-
|
|
1685
|
-
|
|
1686
|
-
|
|
1687
|
-
|
|
1688
|
-
|
|
1689
|
-
|
|
1690
|
-
|
|
1691
|
-
}
|
|
1692
|
-
|
|
1650
|
+
private options;
|
|
1651
|
+
private redis;
|
|
1652
|
+
private keys;
|
|
1653
|
+
private shardCount;
|
|
1654
|
+
constructor(options: TenantDispatchOptions);
|
|
1655
|
+
/**
|
|
1656
|
+
* Get the dispatch shard ID for a tenant.
|
|
1657
|
+
* Uses jump consistent hash on the tenant ID so each tenant
|
|
1658
|
+
* always maps to exactly one dispatch shard.
|
|
1659
|
+
*/
|
|
1660
|
+
getShardForTenant(tenantId: string): number;
|
|
1661
|
+
/**
|
|
1662
|
+
* Get eligible tenants from a dispatch shard (Level 1).
|
|
1663
|
+
* Returns tenants ordered by oldest message (lowest score first).
|
|
1664
|
+
*/
|
|
1665
|
+
getTenantsFromShard(shardId: number, limit?: number, maxScore?: number): Promise<TenantWithScore[]>;
|
|
1666
|
+
/**
|
|
1667
|
+
* Get queues for a specific tenant (Level 2).
|
|
1668
|
+
* Returns queues ordered by oldest message (lowest score first).
|
|
1669
|
+
*/
|
|
1670
|
+
getQueuesForTenant(tenantId: string, limit?: number, maxScore?: number): Promise<QueueWithScore[]>;
|
|
1671
|
+
/**
|
|
1672
|
+
* Get the number of tenants in a dispatch shard.
|
|
1673
|
+
*/
|
|
1674
|
+
getShardTenantCount(shardId: number): Promise<number>;
|
|
1675
|
+
/**
|
|
1676
|
+
* Get total tenant count across all dispatch shards.
|
|
1677
|
+
* Note: tenants may appear in multiple shards, so this may overcount.
|
|
1678
|
+
*/
|
|
1679
|
+
getTotalTenantCount(): Promise<number>;
|
|
1680
|
+
/**
|
|
1681
|
+
* Get the number of queues for a tenant.
|
|
1682
|
+
*/
|
|
1683
|
+
getTenantQueueCount(tenantId: string): Promise<number>;
|
|
1684
|
+
/**
|
|
1685
|
+
* Remove a tenant from a specific dispatch shard.
|
|
1686
|
+
*/
|
|
1687
|
+
removeTenantFromShard(shardId: number, tenantId: string): Promise<void>;
|
|
1688
|
+
/**
|
|
1689
|
+
* Add a tenant to a dispatch shard with the given score.
|
|
1690
|
+
*/
|
|
1691
|
+
addTenantToShard(shardId: number, tenantId: string, score: number): Promise<void>;
|
|
1692
|
+
/**
|
|
1693
|
+
* Remove a queue from a tenant's queue index.
|
|
1694
|
+
*/
|
|
1695
|
+
removeQueueFromTenant(tenantId: string, queueId: string): Promise<void>;
|
|
1696
|
+
/**
|
|
1697
|
+
* Close the Redis connection.
|
|
1698
|
+
*/
|
|
1699
|
+
close(): Promise<void>;
|
|
1700
|
+
}
|
|
1701
|
+
//#endregion
|
|
1702
|
+
//#region src/fair-queue/visibility.d.ts
|
|
1693
1703
|
interface VisibilityManagerOptions {
|
|
1694
|
-
|
|
1695
|
-
|
|
1696
|
-
|
|
1697
|
-
|
|
1698
|
-
|
|
1699
|
-
|
|
1700
|
-
|
|
1701
|
-
|
|
1704
|
+
redis: RedisOptions;
|
|
1705
|
+
keys: FairQueueKeyProducer;
|
|
1706
|
+
shardCount: number;
|
|
1707
|
+
defaultTimeoutMs: number;
|
|
1708
|
+
logger?: {
|
|
1709
|
+
debug: (message: string, context?: Record<string, unknown>) => void;
|
|
1710
|
+
error: (message: string, context?: Record<string, unknown>) => void;
|
|
1711
|
+
};
|
|
1702
1712
|
}
|
|
1703
1713
|
/**
|
|
1704
1714
|
* VisibilityManager handles message visibility timeouts for safe message processing.
|
|
@@ -1714,140 +1724,141 @@ interface VisibilityManagerOptions {
|
|
|
1714
1724
|
* - In-flight data hash: field = messageId, value = JSON message data
|
|
1715
1725
|
*/
|
|
1716
1726
|
declare class VisibilityManager {
|
|
1717
|
-
|
|
1718
|
-
|
|
1719
|
-
|
|
1720
|
-
|
|
1721
|
-
|
|
1722
|
-
|
|
1723
|
-
|
|
1724
|
-
|
|
1725
|
-
|
|
1726
|
-
|
|
1727
|
-
|
|
1728
|
-
|
|
1729
|
-
|
|
1730
|
-
|
|
1731
|
-
|
|
1732
|
-
|
|
1733
|
-
|
|
1734
|
-
|
|
1735
|
-
|
|
1736
|
-
|
|
1737
|
-
|
|
1738
|
-
|
|
1739
|
-
|
|
1740
|
-
|
|
1741
|
-
|
|
1742
|
-
|
|
1743
|
-
|
|
1744
|
-
|
|
1745
|
-
|
|
1746
|
-
|
|
1747
|
-
|
|
1748
|
-
|
|
1749
|
-
|
|
1750
|
-
|
|
1751
|
-
|
|
1752
|
-
|
|
1753
|
-
|
|
1754
|
-
|
|
1755
|
-
|
|
1756
|
-
|
|
1757
|
-
|
|
1758
|
-
|
|
1759
|
-
|
|
1760
|
-
|
|
1761
|
-
|
|
1762
|
-
|
|
1763
|
-
|
|
1764
|
-
|
|
1765
|
-
|
|
1766
|
-
|
|
1767
|
-
|
|
1768
|
-
|
|
1769
|
-
|
|
1770
|
-
|
|
1771
|
-
|
|
1772
|
-
|
|
1773
|
-
|
|
1774
|
-
|
|
1775
|
-
|
|
1776
|
-
|
|
1777
|
-
|
|
1778
|
-
|
|
1779
|
-
|
|
1780
|
-
|
|
1781
|
-
|
|
1782
|
-
|
|
1783
|
-
|
|
1784
|
-
|
|
1785
|
-
|
|
1786
|
-
|
|
1787
|
-
|
|
1788
|
-
|
|
1789
|
-
|
|
1790
|
-
|
|
1791
|
-
|
|
1792
|
-
|
|
1793
|
-
|
|
1794
|
-
|
|
1795
|
-
|
|
1796
|
-
|
|
1797
|
-
|
|
1798
|
-
|
|
1799
|
-
|
|
1800
|
-
|
|
1801
|
-
|
|
1802
|
-
|
|
1803
|
-
|
|
1804
|
-
|
|
1805
|
-
|
|
1806
|
-
|
|
1807
|
-
|
|
1808
|
-
|
|
1809
|
-
|
|
1810
|
-
|
|
1811
|
-
|
|
1812
|
-
|
|
1813
|
-
|
|
1814
|
-
|
|
1815
|
-
|
|
1816
|
-
|
|
1817
|
-
|
|
1818
|
-
|
|
1819
|
-
|
|
1820
|
-
|
|
1821
|
-
|
|
1822
|
-
|
|
1823
|
-
|
|
1824
|
-
|
|
1825
|
-
|
|
1826
|
-
|
|
1827
|
-
|
|
1828
|
-
|
|
1829
|
-
|
|
1830
|
-
|
|
1831
|
-
|
|
1832
|
-
|
|
1727
|
+
#private;
|
|
1728
|
+
private options;
|
|
1729
|
+
private redis;
|
|
1730
|
+
private keys;
|
|
1731
|
+
private shardCount;
|
|
1732
|
+
private defaultTimeoutMs;
|
|
1733
|
+
private logger;
|
|
1734
|
+
constructor(options: VisibilityManagerOptions);
|
|
1735
|
+
/**
|
|
1736
|
+
* Claim a message for processing.
|
|
1737
|
+
* Moves the message from its queue to the in-flight set with a visibility timeout.
|
|
1738
|
+
*
|
|
1739
|
+
* @param queueId - The queue to claim from
|
|
1740
|
+
* @param queueKey - The Redis key for the queue sorted set
|
|
1741
|
+
* @param queueItemsKey - The Redis key for the queue items hash
|
|
1742
|
+
* @param consumerId - ID of the consumer claiming the message
|
|
1743
|
+
* @param timeoutMs - Visibility timeout in milliseconds
|
|
1744
|
+
* @returns Claim result with the message if successful
|
|
1745
|
+
*/
|
|
1746
|
+
claim<TPayload = unknown>(queueId: string, queueKey: string, queueItemsKey: string, consumerId: string, timeoutMs?: number): Promise<ClaimResult<TPayload>>;
|
|
1747
|
+
/**
|
|
1748
|
+
* Claim multiple messages for processing (batch claim).
|
|
1749
|
+
* Moves up to maxCount messages from the queue to the in-flight set.
|
|
1750
|
+
*
|
|
1751
|
+
* @param queueId - The queue to claim from
|
|
1752
|
+
* @param queueKey - The Redis key for the queue sorted set
|
|
1753
|
+
* @param queueItemsKey - The Redis key for the queue items hash
|
|
1754
|
+
* @param consumerId - ID of the consumer claiming the messages
|
|
1755
|
+
* @param maxCount - Maximum number of messages to claim
|
|
1756
|
+
* @param timeoutMs - Visibility timeout in milliseconds
|
|
1757
|
+
* @returns Array of claimed messages
|
|
1758
|
+
*/
|
|
1759
|
+
claimBatch<TPayload = unknown>(queueId: string, queueKey: string, queueItemsKey: string, consumerId: string, maxCount: number, timeoutMs?: number): Promise<Array<InFlightMessage<TPayload>>>;
|
|
1760
|
+
/**
|
|
1761
|
+
* Extend the visibility timeout for a message (heartbeat).
|
|
1762
|
+
*
|
|
1763
|
+
* @param messageId - The message ID
|
|
1764
|
+
* @param queueId - The queue ID
|
|
1765
|
+
* @param extendMs - Additional milliseconds to add to the deadline
|
|
1766
|
+
* @returns true if the heartbeat was successful
|
|
1767
|
+
*/
|
|
1768
|
+
heartbeat(messageId: string, queueId: string, extendMs: number): Promise<boolean>;
|
|
1769
|
+
/**
|
|
1770
|
+
* Mark a message as successfully processed.
|
|
1771
|
+
* Removes the message from in-flight tracking.
|
|
1772
|
+
*
|
|
1773
|
+
* @param messageId - The message ID
|
|
1774
|
+
* @param queueId - The queue ID
|
|
1775
|
+
*/
|
|
1776
|
+
complete(messageId: string, queueId: string): Promise<void>;
|
|
1777
|
+
/**
|
|
1778
|
+
* Release a message back to its queue.
|
|
1779
|
+
* Used when processing fails or consumer wants to retry later.
|
|
1780
|
+
*
|
|
1781
|
+
* @param messageId - The message ID
|
|
1782
|
+
* @param queueId - The queue ID
|
|
1783
|
+
* @param queueKey - The Redis key for the queue
|
|
1784
|
+
* @param queueItemsKey - The Redis key for the queue items hash
|
|
1785
|
+
* @param tenantQueueIndexKey - The Redis key for the tenant queue index (Level 2)
|
|
1786
|
+
* @param dispatchKey - The Redis key for the dispatch index (Level 1)
|
|
1787
|
+
* @param tenantId - The tenant ID
|
|
1788
|
+
* @param score - Optional score for the message (defaults to now)
|
|
1789
|
+
*/
|
|
1790
|
+
release<_TPayload = unknown>(messageId: string, queueId: string, queueKey: string, queueItemsKey: string, tenantQueueIndexKey: string, dispatchKey: string, tenantId: string, score?: number, updatedData?: string): Promise<void>;
|
|
1791
|
+
/**
|
|
1792
|
+
* Release multiple messages back to their queue in a single operation.
|
|
1793
|
+
* Used when processing fails or consumer wants to retry later.
|
|
1794
|
+
* All messages must belong to the same queue.
|
|
1795
|
+
*
|
|
1796
|
+
* @param messages - Array of messages to release (must all have same queueId)
|
|
1797
|
+
* @param queueId - The queue ID
|
|
1798
|
+
* @param queueKey - The Redis key for the queue
|
|
1799
|
+
* @param queueItemsKey - The Redis key for the queue items hash
|
|
1800
|
+
* @param tenantQueueIndexKey - The Redis key for the tenant queue index (Level 2)
|
|
1801
|
+
* @param dispatchKey - The Redis key for the dispatch index (Level 1)
|
|
1802
|
+
* @param tenantId - The tenant ID
|
|
1803
|
+
* @param score - Optional score for the messages (defaults to now)
|
|
1804
|
+
*/
|
|
1805
|
+
releaseBatch(messages: Array<{
|
|
1806
|
+
messageId: string;
|
|
1807
|
+
}>, queueId: string, queueKey: string, queueItemsKey: string, tenantQueueIndexKey: string, dispatchKey: string, tenantId: string, score?: number): Promise<void>;
|
|
1808
|
+
/**
|
|
1809
|
+
* Reclaim timed-out messages from a shard.
|
|
1810
|
+
* Returns messages to their original queues.
|
|
1811
|
+
*
|
|
1812
|
+
* @param shardId - The shard to check
|
|
1813
|
+
* @param getQueueKeys - Function to get queue keys for a queue ID
|
|
1814
|
+
* @returns Array of reclaimed message info for concurrency release
|
|
1815
|
+
*/
|
|
1816
|
+
reclaimTimedOut(shardId: number, getQueueKeys: (queueId: string) => {
|
|
1817
|
+
queueKey: string;
|
|
1818
|
+
queueItemsKey: string;
|
|
1819
|
+
tenantQueueIndexKey: string;
|
|
1820
|
+
dispatchKey: string;
|
|
1821
|
+
tenantId: string;
|
|
1822
|
+
}): Promise<ReclaimedMessageInfo[]>;
|
|
1823
|
+
/**
|
|
1824
|
+
* Get all in-flight messages for a shard.
|
|
1825
|
+
*/
|
|
1826
|
+
getInflightMessages(shardId: number): Promise<Array<{
|
|
1827
|
+
messageId: string;
|
|
1828
|
+
queueId: string;
|
|
1829
|
+
deadline: number;
|
|
1830
|
+
}>>;
|
|
1831
|
+
/**
|
|
1832
|
+
* Get count of in-flight messages for a shard.
|
|
1833
|
+
*/
|
|
1834
|
+
getInflightCount(shardId: number): Promise<number>;
|
|
1835
|
+
/**
|
|
1836
|
+
* Get total in-flight count across all shards.
|
|
1837
|
+
*/
|
|
1838
|
+
getTotalInflightCount(): Promise<number>;
|
|
1839
|
+
/**
|
|
1840
|
+
* Close the Redis connection.
|
|
1841
|
+
*/
|
|
1842
|
+
close(): Promise<void>;
|
|
1833
1843
|
}
|
|
1834
1844
|
declare module "@internal/redis" {
|
|
1835
|
-
|
|
1836
|
-
|
|
1837
|
-
|
|
1838
|
-
|
|
1839
|
-
|
|
1840
|
-
|
|
1841
|
-
|
|
1842
|
-
}
|
|
1843
|
-
|
|
1845
|
+
interface RedisCommander<Context> {
|
|
1846
|
+
claimMessage(queueKey: string, queueItemsKey: string, inflightKey: string, inflightDataKey: string, queueId: string, consumerId: string, deadline: string): Promise<[string, string] | null>;
|
|
1847
|
+
claimMessageBatch(queueKey: string, queueItemsKey: string, inflightKey: string, inflightDataKey: string, queueId: string, deadline: string, maxCount: string): Promise<string[]>;
|
|
1848
|
+
releaseMessage(inflightKey: string, inflightDataKey: string, queueKey: string, queueItemsKey: string, tenantQueueIndexKey: string, dispatchKey: string, member: string, messageId: string, score: string, queueId: string, updatedData: string, tenantId: string): Promise<number>;
|
|
1849
|
+
releaseMessageBatch(inflightKey: string, inflightDataKey: string, queueKey: string, queueItemsKey: string, tenantQueueIndexKey: string, dispatchKey: string, score: string, queueId: string, tenantId: string, ...membersAndMessageIds: string[]): Promise<number>;
|
|
1850
|
+
heartbeatMessage(inflightKey: string, member: string, newDeadline: string): Promise<number>;
|
|
1851
|
+
}
|
|
1852
|
+
}
|
|
1853
|
+
//#endregion
|
|
1854
|
+
//#region src/fair-queue/workerQueue.d.ts
|
|
1844
1855
|
interface WorkerQueueManagerOptions {
|
|
1845
|
-
|
|
1846
|
-
|
|
1847
|
-
|
|
1848
|
-
|
|
1849
|
-
|
|
1850
|
-
|
|
1856
|
+
redis: RedisOptions;
|
|
1857
|
+
keys: FairQueueKeyProducer;
|
|
1858
|
+
logger?: {
|
|
1859
|
+
debug: (message: string, context?: Record<string, unknown>) => void;
|
|
1860
|
+
error: (message: string, context?: Record<string, unknown>) => void;
|
|
1861
|
+
};
|
|
1851
1862
|
}
|
|
1852
1863
|
/**
|
|
1853
1864
|
* WorkerQueueManager handles the intermediate worker queue layer.
|
|
@@ -1863,85 +1874,86 @@ interface WorkerQueueManagerOptions {
|
|
|
1863
1874
|
* 3. Worker queue consumer does blocking pop to receive message
|
|
1864
1875
|
*/
|
|
1865
1876
|
declare class WorkerQueueManager {
|
|
1866
|
-
|
|
1867
|
-
|
|
1868
|
-
|
|
1869
|
-
|
|
1870
|
-
|
|
1871
|
-
|
|
1872
|
-
|
|
1873
|
-
|
|
1874
|
-
|
|
1875
|
-
|
|
1876
|
-
|
|
1877
|
-
|
|
1878
|
-
|
|
1879
|
-
|
|
1880
|
-
|
|
1881
|
-
|
|
1882
|
-
|
|
1883
|
-
|
|
1884
|
-
|
|
1885
|
-
|
|
1886
|
-
|
|
1887
|
-
|
|
1888
|
-
|
|
1889
|
-
|
|
1890
|
-
|
|
1891
|
-
|
|
1892
|
-
|
|
1893
|
-
|
|
1894
|
-
|
|
1895
|
-
|
|
1896
|
-
|
|
1897
|
-
|
|
1898
|
-
|
|
1899
|
-
|
|
1900
|
-
|
|
1901
|
-
|
|
1902
|
-
|
|
1903
|
-
|
|
1904
|
-
|
|
1905
|
-
|
|
1906
|
-
|
|
1907
|
-
|
|
1908
|
-
|
|
1909
|
-
|
|
1910
|
-
|
|
1911
|
-
|
|
1912
|
-
|
|
1913
|
-
|
|
1914
|
-
|
|
1915
|
-
|
|
1916
|
-
|
|
1917
|
-
|
|
1918
|
-
|
|
1919
|
-
|
|
1920
|
-
|
|
1921
|
-
|
|
1922
|
-
|
|
1923
|
-
|
|
1924
|
-
|
|
1925
|
-
|
|
1926
|
-
|
|
1927
|
-
|
|
1928
|
-
|
|
1929
|
-
|
|
1930
|
-
|
|
1931
|
-
|
|
1932
|
-
|
|
1933
|
-
|
|
1934
|
-
|
|
1935
|
-
|
|
1936
|
-
|
|
1937
|
-
|
|
1877
|
+
#private;
|
|
1878
|
+
private options;
|
|
1879
|
+
private redis;
|
|
1880
|
+
private keys;
|
|
1881
|
+
private logger;
|
|
1882
|
+
constructor(options: WorkerQueueManagerOptions);
|
|
1883
|
+
/**
|
|
1884
|
+
* Push a message key to a worker queue.
|
|
1885
|
+
* Called after claiming a message from the message queue.
|
|
1886
|
+
*
|
|
1887
|
+
* @param workerQueueId - The worker queue identifier
|
|
1888
|
+
* @param messageKey - The message key to push (typically "messageId:queueId")
|
|
1889
|
+
*/
|
|
1890
|
+
push(workerQueueId: string, messageKey: string): Promise<void>;
|
|
1891
|
+
/**
|
|
1892
|
+
* Push multiple message keys to a worker queue.
|
|
1893
|
+
*
|
|
1894
|
+
* @param workerQueueId - The worker queue identifier
|
|
1895
|
+
* @param messageKeys - The message keys to push
|
|
1896
|
+
*/
|
|
1897
|
+
pushBatch(workerQueueId: string, messageKeys: string[]): Promise<void>;
|
|
1898
|
+
/**
|
|
1899
|
+
* Blocking pop from a worker queue.
|
|
1900
|
+
* Waits until a message is available or timeout expires.
|
|
1901
|
+
*
|
|
1902
|
+
* @param workerQueueId - The worker queue identifier
|
|
1903
|
+
* @param timeoutSeconds - Maximum time to wait (0 = wait forever)
|
|
1904
|
+
* @param signal - Optional abort signal to cancel waiting
|
|
1905
|
+
* @returns The message key, or null if timeout
|
|
1906
|
+
*/
|
|
1907
|
+
blockingPop(workerQueueId: string, timeoutSeconds: number, signal?: AbortSignal): Promise<string | null>;
|
|
1908
|
+
/**
|
|
1909
|
+
* Non-blocking pop from a worker queue.
|
|
1910
|
+
*
|
|
1911
|
+
* @param workerQueueId - The worker queue identifier
|
|
1912
|
+
* @returns The message key and queue length, or null if empty
|
|
1913
|
+
*/
|
|
1914
|
+
pop(workerQueueId: string): Promise<{
|
|
1915
|
+
messageKey: string;
|
|
1916
|
+
queueLength: number;
|
|
1917
|
+
} | null>;
|
|
1918
|
+
/**
|
|
1919
|
+
* Get the current length of a worker queue.
|
|
1920
|
+
*/
|
|
1921
|
+
getLength(workerQueueId: string): Promise<number>;
|
|
1922
|
+
/**
|
|
1923
|
+
* Peek at all messages in a worker queue without removing them.
|
|
1924
|
+
* Useful for debugging and tests.
|
|
1925
|
+
*/
|
|
1926
|
+
peek(workerQueueId: string): Promise<string[]>;
|
|
1927
|
+
/**
|
|
1928
|
+
* Remove a specific message from the worker queue.
|
|
1929
|
+
* Used when a message needs to be removed without processing.
|
|
1930
|
+
*
|
|
1931
|
+
* @param workerQueueId - The worker queue identifier
|
|
1932
|
+
* @param messageKey - The message key to remove
|
|
1933
|
+
* @returns Number of removed items
|
|
1934
|
+
*/
|
|
1935
|
+
remove(workerQueueId: string, messageKey: string): Promise<number>;
|
|
1936
|
+
/**
|
|
1937
|
+
* Clear all messages from a worker queue.
|
|
1938
|
+
*/
|
|
1939
|
+
clear(workerQueueId: string): Promise<void>;
|
|
1940
|
+
/**
|
|
1941
|
+
* Close the Redis connection.
|
|
1942
|
+
*/
|
|
1943
|
+
close(): Promise<void>;
|
|
1944
|
+
/**
|
|
1945
|
+
* Register custom commands on an external Redis client.
|
|
1946
|
+
* Use this when initializing FairQueue with worker queues.
|
|
1947
|
+
*/
|
|
1948
|
+
registerCommands(redis: Redis): void;
|
|
1938
1949
|
}
|
|
1939
1950
|
declare module "@internal/redis" {
|
|
1940
|
-
|
|
1941
|
-
|
|
1942
|
-
|
|
1951
|
+
interface RedisCommander<Context> {
|
|
1952
|
+
popWithLength(workerQueueKey: string): Promise<[string, string] | null>;
|
|
1953
|
+
}
|
|
1943
1954
|
}
|
|
1944
|
-
|
|
1955
|
+
//#endregion
|
|
1956
|
+
//#region src/fair-queue/index.d.ts
|
|
1945
1957
|
/**
|
|
1946
1958
|
* FairQueue is the main orchestrator for fair queue message routing.
|
|
1947
1959
|
*
|
|
@@ -1961,463 +1973,466 @@ declare module "@internal/redis" {
|
|
|
1961
1973
|
* @typeParam TPayloadSchema - Zod schema for message payload validation
|
|
1962
1974
|
*/
|
|
1963
1975
|
declare class FairQueue<TPayloadSchema extends z.ZodTypeAny = z.ZodUnknown> {
|
|
1964
|
-
|
|
1965
|
-
|
|
1966
|
-
|
|
1967
|
-
|
|
1968
|
-
|
|
1969
|
-
|
|
1970
|
-
|
|
1971
|
-
|
|
1972
|
-
|
|
1973
|
-
|
|
1974
|
-
|
|
1975
|
-
|
|
1976
|
-
|
|
1977
|
-
|
|
1978
|
-
|
|
1979
|
-
|
|
1980
|
-
|
|
1981
|
-
|
|
1982
|
-
|
|
1983
|
-
|
|
1984
|
-
|
|
1985
|
-
|
|
1986
|
-
|
|
1987
|
-
|
|
1988
|
-
|
|
1989
|
-
|
|
1990
|
-
|
|
1991
|
-
|
|
1992
|
-
|
|
1993
|
-
|
|
1994
|
-
|
|
1995
|
-
|
|
1996
|
-
|
|
1997
|
-
|
|
1998
|
-
|
|
1999
|
-
|
|
2000
|
-
|
|
2001
|
-
|
|
2002
|
-
|
|
2003
|
-
|
|
2004
|
-
|
|
2005
|
-
|
|
2006
|
-
|
|
2007
|
-
|
|
2008
|
-
|
|
2009
|
-
|
|
2010
|
-
|
|
2011
|
-
|
|
2012
|
-
|
|
2013
|
-
|
|
2014
|
-
|
|
2015
|
-
|
|
2016
|
-
|
|
2017
|
-
|
|
2018
|
-
|
|
2019
|
-
|
|
2020
|
-
|
|
2021
|
-
|
|
2022
|
-
|
|
2023
|
-
|
|
2024
|
-
|
|
2025
|
-
|
|
2026
|
-
|
|
2027
|
-
|
|
2028
|
-
|
|
2029
|
-
|
|
2030
|
-
|
|
2031
|
-
|
|
2032
|
-
|
|
2033
|
-
|
|
2034
|
-
|
|
2035
|
-
|
|
2036
|
-
|
|
2037
|
-
|
|
2038
|
-
|
|
2039
|
-
|
|
2040
|
-
|
|
2041
|
-
|
|
2042
|
-
|
|
2043
|
-
|
|
2044
|
-
|
|
2045
|
-
|
|
2046
|
-
|
|
2047
|
-
|
|
2048
|
-
|
|
2049
|
-
|
|
2050
|
-
|
|
2051
|
-
|
|
2052
|
-
|
|
2053
|
-
|
|
2054
|
-
|
|
2055
|
-
|
|
2056
|
-
|
|
2057
|
-
|
|
2058
|
-
|
|
2059
|
-
|
|
2060
|
-
|
|
2061
|
-
|
|
2062
|
-
|
|
2063
|
-
|
|
2064
|
-
|
|
2065
|
-
|
|
2066
|
-
|
|
2067
|
-
|
|
2068
|
-
|
|
2069
|
-
|
|
2070
|
-
|
|
2071
|
-
|
|
2072
|
-
|
|
2073
|
-
|
|
2074
|
-
|
|
2075
|
-
|
|
2076
|
-
|
|
2077
|
-
|
|
2078
|
-
|
|
2079
|
-
|
|
2080
|
-
|
|
2081
|
-
|
|
2082
|
-
|
|
2083
|
-
|
|
2084
|
-
|
|
2085
|
-
|
|
2086
|
-
|
|
2087
|
-
|
|
2088
|
-
|
|
2089
|
-
|
|
2090
|
-
|
|
2091
|
-
|
|
2092
|
-
|
|
2093
|
-
|
|
2094
|
-
|
|
2095
|
-
|
|
2096
|
-
|
|
2097
|
-
|
|
2098
|
-
|
|
2099
|
-
|
|
2100
|
-
|
|
2101
|
-
|
|
2102
|
-
|
|
2103
|
-
|
|
2104
|
-
|
|
2105
|
-
|
|
2106
|
-
|
|
2107
|
-
|
|
2108
|
-
|
|
2109
|
-
|
|
2110
|
-
|
|
2111
|
-
|
|
2112
|
-
|
|
2113
|
-
|
|
2114
|
-
|
|
2115
|
-
|
|
2116
|
-
|
|
2117
|
-
|
|
2118
|
-
|
|
2119
|
-
|
|
2120
|
-
|
|
2121
|
-
|
|
2122
|
-
|
|
2123
|
-
|
|
2124
|
-
|
|
2125
|
-
|
|
2126
|
-
|
|
2127
|
-
|
|
2128
|
-
|
|
2129
|
-
|
|
2130
|
-
|
|
2131
|
-
|
|
2132
|
-
|
|
2133
|
-
|
|
1976
|
+
#private;
|
|
1977
|
+
private options;
|
|
1978
|
+
private redis;
|
|
1979
|
+
private keys;
|
|
1980
|
+
private scheduler;
|
|
1981
|
+
private masterQueue;
|
|
1982
|
+
private concurrencyManager?;
|
|
1983
|
+
private visibilityManager;
|
|
1984
|
+
private workerQueueManager;
|
|
1985
|
+
private telemetry;
|
|
1986
|
+
private logger;
|
|
1987
|
+
private payloadSchema?;
|
|
1988
|
+
private validateOnEnqueue;
|
|
1989
|
+
private retryStrategy?;
|
|
1990
|
+
private deadLetterQueueEnabled;
|
|
1991
|
+
private shardCount;
|
|
1992
|
+
private consumerCount;
|
|
1993
|
+
private consumerIntervalMs;
|
|
1994
|
+
private visibilityTimeoutMs;
|
|
1995
|
+
private heartbeatIntervalMs;
|
|
1996
|
+
private reclaimIntervalMs;
|
|
1997
|
+
private workerQueueResolver;
|
|
1998
|
+
private batchClaimSize;
|
|
1999
|
+
private cooloffEnabled;
|
|
2000
|
+
private cooloffThreshold;
|
|
2001
|
+
private cooloffPeriodMs;
|
|
2002
|
+
private maxCooloffStatesSize;
|
|
2003
|
+
private queueCooloffStates;
|
|
2004
|
+
private workerQueueMaxDepth;
|
|
2005
|
+
private workerQueueDepthCheckId?;
|
|
2006
|
+
private consumerTraceMaxIterations;
|
|
2007
|
+
private consumerTraceTimeoutSeconds;
|
|
2008
|
+
private batchedSpanManager;
|
|
2009
|
+
private isRunning;
|
|
2010
|
+
private abortController;
|
|
2011
|
+
private masterQueueConsumerLoops;
|
|
2012
|
+
private reclaimLoop?;
|
|
2013
|
+
private queueDescriptorCache;
|
|
2014
|
+
private tenantDispatch;
|
|
2015
|
+
constructor(options: FairQueueOptions<TPayloadSchema>);
|
|
2016
|
+
/**
|
|
2017
|
+
* Register observable gauge callbacks for telemetry.
|
|
2018
|
+
* Call this after FairQueue is created to enable gauge metrics.
|
|
2019
|
+
*
|
|
2020
|
+
* @param options.observedTenants - List of tenant IDs to observe for DLQ metrics
|
|
2021
|
+
*/
|
|
2022
|
+
registerTelemetryGauges(options?: {
|
|
2023
|
+
observedTenants?: string[];
|
|
2024
|
+
}): void;
|
|
2025
|
+
/**
|
|
2026
|
+
* Enqueue a single message to a queue.
|
|
2027
|
+
*/
|
|
2028
|
+
enqueue(options: EnqueueOptions<z.infer<TPayloadSchema>>): Promise<string>;
|
|
2029
|
+
/**
|
|
2030
|
+
* Enqueue multiple messages to a queue.
|
|
2031
|
+
*/
|
|
2032
|
+
enqueueBatch(options: EnqueueBatchOptions<z.infer<TPayloadSchema>>): Promise<string[]>;
|
|
2033
|
+
/**
|
|
2034
|
+
* Get messages from the dead letter queue for a tenant.
|
|
2035
|
+
*/
|
|
2036
|
+
getDeadLetterMessages(tenantId: string, limit?: number): Promise<DeadLetterMessage<z.infer<TPayloadSchema>>[]>;
|
|
2037
|
+
/**
|
|
2038
|
+
* Redrive a message from DLQ back to its original queue.
|
|
2039
|
+
*/
|
|
2040
|
+
redriveMessage(tenantId: string, messageId: string): Promise<boolean>;
|
|
2041
|
+
/**
|
|
2042
|
+
* Redrive all messages from DLQ back to their original queues.
|
|
2043
|
+
*/
|
|
2044
|
+
redriveAll(tenantId: string): Promise<number>;
|
|
2045
|
+
/**
|
|
2046
|
+
* Purge all messages from a tenant's DLQ.
|
|
2047
|
+
*/
|
|
2048
|
+
purgeDeadLetterQueue(tenantId: string): Promise<number>;
|
|
2049
|
+
/**
|
|
2050
|
+
* Get the number of messages in a tenant's DLQ.
|
|
2051
|
+
*/
|
|
2052
|
+
getDeadLetterQueueLength(tenantId: string): Promise<number>;
|
|
2053
|
+
/**
|
|
2054
|
+
* Get the size of the in-memory queue descriptor cache.
|
|
2055
|
+
* This cache stores metadata for queues that have been enqueued.
|
|
2056
|
+
* The cache is cleaned up when queues are fully processed.
|
|
2057
|
+
*/
|
|
2058
|
+
getQueueDescriptorCacheSize(): number;
|
|
2059
|
+
/**
|
|
2060
|
+
* Get the size of the in-memory cooloff states cache.
|
|
2061
|
+
* This cache tracks queues that are in cooloff due to repeated failures.
|
|
2062
|
+
* The cache is cleaned up when queues are fully processed or cooloff expires.
|
|
2063
|
+
*/
|
|
2064
|
+
getQueueCooloffStatesSize(): number;
|
|
2065
|
+
/**
|
|
2066
|
+
* Get all in-memory cache sizes for monitoring.
|
|
2067
|
+
* Useful for adding as span attributes.
|
|
2068
|
+
*/
|
|
2069
|
+
getCacheSizes(): {
|
|
2070
|
+
descriptorCacheSize: number;
|
|
2071
|
+
cooloffStatesSize: number;
|
|
2072
|
+
};
|
|
2073
|
+
/**
|
|
2074
|
+
* Start the master queue consumer loops and reclaim loop.
|
|
2075
|
+
* FairQueue claims messages and pushes them to worker queues.
|
|
2076
|
+
* External consumers are responsible for consuming from worker queues.
|
|
2077
|
+
*/
|
|
2078
|
+
start(): void;
|
|
2079
|
+
/**
|
|
2080
|
+
* Stop the consumer loops gracefully.
|
|
2081
|
+
*/
|
|
2082
|
+
stop(): Promise<void>;
|
|
2083
|
+
/**
|
|
2084
|
+
* Close all resources.
|
|
2085
|
+
*/
|
|
2086
|
+
close(): Promise<void>;
|
|
2087
|
+
/**
|
|
2088
|
+
* Get the number of messages in a queue.
|
|
2089
|
+
*/
|
|
2090
|
+
getQueueLength(queueId: string): Promise<number>;
|
|
2091
|
+
/**
|
|
2092
|
+
* Get total tenant count across dispatch shards plus any legacy queues still draining.
|
|
2093
|
+
*/
|
|
2094
|
+
getTotalQueueCount(): Promise<number>;
|
|
2095
|
+
/**
|
|
2096
|
+
* Get total in-flight message count.
|
|
2097
|
+
*/
|
|
2098
|
+
getTotalInflightCount(): Promise<number>;
|
|
2099
|
+
/**
|
|
2100
|
+
* Get the shard ID for a queue.
|
|
2101
|
+
*/
|
|
2102
|
+
getShardForQueue(queueId: string): number;
|
|
2103
|
+
/**
|
|
2104
|
+
* Get message data from in-flight storage.
|
|
2105
|
+
* External consumers use this to retrieve the stored message after popping from worker queue.
|
|
2106
|
+
*
|
|
2107
|
+
* @param messageId - The ID of the message
|
|
2108
|
+
* @param queueId - The queue ID the message belongs to
|
|
2109
|
+
* @returns The stored message or null if not found
|
|
2110
|
+
*/
|
|
2111
|
+
getMessageData(messageId: string, queueId: string): Promise<StoredMessage<z.infer<TPayloadSchema>> | null>;
|
|
2112
|
+
/**
|
|
2113
|
+
* Extend the visibility timeout for a message.
|
|
2114
|
+
* External consumers should call this periodically during long-running processing.
|
|
2115
|
+
*
|
|
2116
|
+
* @param messageId - The ID of the message
|
|
2117
|
+
* @param queueId - The queue ID the message belongs to
|
|
2118
|
+
* @returns true if heartbeat was successful
|
|
2119
|
+
*/
|
|
2120
|
+
heartbeatMessage(messageId: string, queueId: string): Promise<boolean>;
|
|
2121
|
+
/**
|
|
2122
|
+
* Mark a message as successfully processed.
|
|
2123
|
+
* This removes the message from in-flight and releases concurrency.
|
|
2124
|
+
*
|
|
2125
|
+
* @param messageId - The ID of the message
|
|
2126
|
+
* @param queueId - The queue ID the message belongs to
|
|
2127
|
+
*/
|
|
2128
|
+
completeMessage(messageId: string, queueId: string): Promise<void>;
|
|
2129
|
+
/**
|
|
2130
|
+
* Release a message back to the queue for processing by another consumer.
|
|
2131
|
+
* The message is placed at the back of the queue.
|
|
2132
|
+
*
|
|
2133
|
+
* @param messageId - The ID of the message
|
|
2134
|
+
* @param queueId - The queue ID the message belongs to
|
|
2135
|
+
*/
|
|
2136
|
+
releaseMessage(messageId: string, queueId: string): Promise<void>;
|
|
2137
|
+
/**
|
|
2138
|
+
* Mark a message as failed. This will trigger retry logic if configured,
|
|
2139
|
+
* or move the message to the dead letter queue.
|
|
2140
|
+
*
|
|
2141
|
+
* @param messageId - The ID of the message
|
|
2142
|
+
* @param queueId - The queue ID the message belongs to
|
|
2143
|
+
* @param error - Optional error that caused the failure
|
|
2144
|
+
*/
|
|
2145
|
+
failMessage(messageId: string, queueId: string, error?: Error): Promise<void>;
|
|
2134
2146
|
}
|
|
2135
2147
|
declare module "@internal/redis" {
|
|
2136
|
-
|
|
2137
|
-
|
|
2138
|
-
|
|
2139
|
-
|
|
2140
|
-
|
|
2141
|
-
|
|
2142
|
-
|
|
2143
|
-
|
|
2144
|
-
}
|
|
2145
|
-
|
|
2148
|
+
interface RedisCommander<Context> {
|
|
2149
|
+
enqueueMessageAtomic(queueKey: string, queueItemsKey: string, masterQueueKey: string, queueId: string, messageId: string, timestamp: string, payload: string): Promise<number>;
|
|
2150
|
+
enqueueBatchAtomic(queueKey: string, queueItemsKey: string, masterQueueKey: string, queueId: string, ...args: string[]): Promise<number>;
|
|
2151
|
+
updateMasterQueueIfEmpty(masterQueueKey: string, queueKey: string, queueId: string): Promise<number>;
|
|
2152
|
+
enqueueMessageAtomicV2(queueKey: string, queueItemsKey: string, tenantQueueIndexKey: string, dispatchKey: string, queueId: string, messageId: string, timestamp: string, payload: string, tenantId: string): Promise<number>;
|
|
2153
|
+
enqueueBatchAtomicV2(queueKey: string, queueItemsKey: string, tenantQueueIndexKey: string, dispatchKey: string, queueId: string, tenantId: string, ...args: string[]): Promise<number>;
|
|
2154
|
+
updateDispatchIndexes(queueKey: string, tenantQueueIndexKey: string, dispatchKey: string, queueId: string, tenantId: string): Promise<number>;
|
|
2155
|
+
}
|
|
2156
|
+
}
|
|
2157
|
+
//#endregion
|
|
2158
|
+
//#region src/mollifier/schemas.d.ts
|
|
2146
2159
|
declare const BufferEntryStatus: z.ZodEnum<["QUEUED", "DRAINING", "FAILED"]>;
|
|
2147
2160
|
type BufferEntryStatus = z.infer<typeof BufferEntryStatus>;
|
|
2148
2161
|
declare const BufferEntryError: z.ZodObject<{
|
|
2149
|
-
|
|
2150
|
-
|
|
2162
|
+
code: z.ZodString;
|
|
2163
|
+
message: z.ZodString;
|
|
2151
2164
|
}, "strip", z.ZodTypeAny, {
|
|
2152
|
-
|
|
2153
|
-
|
|
2165
|
+
code: string;
|
|
2166
|
+
message: string;
|
|
2154
2167
|
}, {
|
|
2155
|
-
|
|
2156
|
-
|
|
2168
|
+
code: string;
|
|
2169
|
+
message: string;
|
|
2157
2170
|
}>;
|
|
2158
2171
|
type BufferEntryError = z.infer<typeof BufferEntryError>;
|
|
2159
2172
|
declare const BufferEntrySchema: z.ZodObject<{
|
|
2160
|
-
|
|
2161
|
-
|
|
2162
|
-
|
|
2163
|
-
|
|
2164
|
-
|
|
2165
|
-
|
|
2166
|
-
|
|
2167
|
-
|
|
2168
|
-
|
|
2169
|
-
|
|
2170
|
-
|
|
2171
|
-
|
|
2172
|
-
|
|
2173
|
-
|
|
2174
|
-
|
|
2173
|
+
runId: z.ZodString;
|
|
2174
|
+
envId: z.ZodString;
|
|
2175
|
+
orgId: z.ZodString;
|
|
2176
|
+
payload: z.ZodString;
|
|
2177
|
+
status: z.ZodEnum<["QUEUED", "DRAINING", "FAILED"]>;
|
|
2178
|
+
attempts: z.ZodEffects<z.ZodString, number, string>;
|
|
2179
|
+
createdAt: z.ZodEffects<z.ZodString, Date, string>;
|
|
2180
|
+
createdAtMicros: z.ZodDefault<z.ZodEffects<z.ZodString, number, string>>;
|
|
2181
|
+
materialised: z.ZodDefault<z.ZodEffects<z.ZodUnion<[z.ZodLiteral<"true">, z.ZodLiteral<"false">]>, boolean, "false" | "true">>;
|
|
2182
|
+
idempotencyLookupKey: z.ZodDefault<z.ZodOptional<z.ZodString>>;
|
|
2183
|
+
metadataVersion: z.ZodDefault<z.ZodEffects<z.ZodString, number, string>>;
|
|
2184
|
+
lastError: z.ZodOptional<z.ZodEffects<z.ZodString, {
|
|
2185
|
+
code: string;
|
|
2186
|
+
message: string;
|
|
2187
|
+
}, string>>;
|
|
2175
2188
|
}, "strip", z.ZodTypeAny, {
|
|
2176
|
-
|
|
2177
|
-
|
|
2178
|
-
|
|
2179
|
-
|
|
2180
|
-
|
|
2181
|
-
|
|
2182
|
-
|
|
2183
|
-
|
|
2184
|
-
|
|
2185
|
-
|
|
2186
|
-
|
|
2187
|
-
|
|
2188
|
-
|
|
2189
|
-
|
|
2190
|
-
|
|
2189
|
+
runId: string;
|
|
2190
|
+
envId: string;
|
|
2191
|
+
orgId: string;
|
|
2192
|
+
payload: string;
|
|
2193
|
+
status: "DRAINING" | "FAILED" | "QUEUED";
|
|
2194
|
+
attempts: number;
|
|
2195
|
+
createdAt: Date;
|
|
2196
|
+
createdAtMicros: number;
|
|
2197
|
+
materialised: boolean;
|
|
2198
|
+
idempotencyLookupKey: string;
|
|
2199
|
+
metadataVersion: number;
|
|
2200
|
+
lastError?: {
|
|
2201
|
+
code: string;
|
|
2202
|
+
message: string;
|
|
2203
|
+
} | undefined;
|
|
2191
2204
|
}, {
|
|
2192
|
-
|
|
2193
|
-
|
|
2194
|
-
|
|
2195
|
-
|
|
2196
|
-
|
|
2197
|
-
|
|
2198
|
-
|
|
2199
|
-
|
|
2200
|
-
|
|
2201
|
-
|
|
2202
|
-
|
|
2203
|
-
|
|
2205
|
+
runId: string;
|
|
2206
|
+
envId: string;
|
|
2207
|
+
orgId: string;
|
|
2208
|
+
payload: string;
|
|
2209
|
+
status: "DRAINING" | "FAILED" | "QUEUED";
|
|
2210
|
+
attempts: string;
|
|
2211
|
+
createdAt: string;
|
|
2212
|
+
createdAtMicros?: string | undefined;
|
|
2213
|
+
materialised?: "false" | "true" | undefined;
|
|
2214
|
+
idempotencyLookupKey?: string | undefined;
|
|
2215
|
+
metadataVersion?: string | undefined;
|
|
2216
|
+
lastError?: string | undefined;
|
|
2204
2217
|
}>;
|
|
2205
2218
|
type BufferEntry = z.infer<typeof BufferEntrySchema>;
|
|
2206
2219
|
declare function serialiseSnapshot(snapshot: unknown): string;
|
|
2207
2220
|
declare function deserialiseSnapshot<T = unknown>(serialised: string): T;
|
|
2208
|
-
|
|
2221
|
+
//#endregion
|
|
2222
|
+
//#region src/mollifier/buffer.d.ts
|
|
2209
2223
|
type MollifierBufferOptions = {
|
|
2210
|
-
|
|
2211
|
-
|
|
2212
|
-
|
|
2213
|
-
|
|
2214
|
-
|
|
2215
|
-
|
|
2224
|
+
redisOptions: RedisOptions;
|
|
2225
|
+
logger?: Logger;
|
|
2226
|
+
ackGraceTtlSeconds?: number;
|
|
2227
|
+
maxRetriesPerRequest?: number;
|
|
2228
|
+
reconnectStepMs?: number;
|
|
2229
|
+
reconnectMaxMs?: number;
|
|
2216
2230
|
};
|
|
2217
2231
|
type SnapshotPatch = {
|
|
2218
|
-
|
|
2219
|
-
|
|
2220
|
-
|
|
2232
|
+
type: "append_tags";
|
|
2233
|
+
tags: string[];
|
|
2234
|
+
maxTags?: number;
|
|
2221
2235
|
} | {
|
|
2222
|
-
|
|
2223
|
-
|
|
2224
|
-
|
|
2236
|
+
type: "set_metadata";
|
|
2237
|
+
metadata: string;
|
|
2238
|
+
metadataType: string;
|
|
2225
2239
|
} | {
|
|
2226
|
-
|
|
2227
|
-
|
|
2240
|
+
type: "set_delay";
|
|
2241
|
+
delayUntil: string;
|
|
2228
2242
|
} | {
|
|
2229
|
-
|
|
2230
|
-
|
|
2231
|
-
|
|
2243
|
+
type: "mark_cancelled";
|
|
2244
|
+
cancelledAt: string;
|
|
2245
|
+
cancelReason?: string;
|
|
2232
2246
|
};
|
|
2233
2247
|
type MutateSnapshotResult = "applied_to_snapshot" | "not_found" | "busy" | "limit_exceeded";
|
|
2234
2248
|
type CasSetMetadataResult = {
|
|
2235
|
-
|
|
2236
|
-
|
|
2249
|
+
kind: "applied";
|
|
2250
|
+
newVersion: number;
|
|
2237
2251
|
} | {
|
|
2238
|
-
|
|
2239
|
-
|
|
2252
|
+
kind: "version_conflict";
|
|
2253
|
+
currentVersion: number;
|
|
2240
2254
|
} | {
|
|
2241
|
-
|
|
2255
|
+
kind: "not_found";
|
|
2242
2256
|
} | {
|
|
2243
|
-
|
|
2257
|
+
kind: "busy";
|
|
2244
2258
|
};
|
|
2245
2259
|
type AcceptResult = {
|
|
2246
|
-
|
|
2260
|
+
kind: "accepted";
|
|
2247
2261
|
} | {
|
|
2248
|
-
|
|
2262
|
+
kind: "duplicate_run_id";
|
|
2249
2263
|
} | {
|
|
2250
|
-
|
|
2251
|
-
|
|
2264
|
+
kind: "duplicate_idempotency";
|
|
2265
|
+
existingRunId: string;
|
|
2252
2266
|
};
|
|
2253
2267
|
type IdempotencyLookupInput = {
|
|
2254
|
-
|
|
2255
|
-
|
|
2256
|
-
|
|
2268
|
+
envId: string;
|
|
2269
|
+
taskIdentifier: string;
|
|
2270
|
+
idempotencyKey: string;
|
|
2257
2271
|
};
|
|
2258
2272
|
declare function idempotencyLookupKeyFor(input: IdempotencyLookupInput): string;
|
|
2259
2273
|
declare function makeIdempotencyClaimKey(input: IdempotencyLookupInput): string;
|
|
2260
2274
|
type IdempotencyClaimResult = {
|
|
2261
|
-
|
|
2275
|
+
kind: "claimed";
|
|
2262
2276
|
} | {
|
|
2263
|
-
|
|
2277
|
+
kind: "pending";
|
|
2264
2278
|
} | {
|
|
2265
|
-
|
|
2266
|
-
|
|
2279
|
+
kind: "resolved";
|
|
2280
|
+
runId: string;
|
|
2267
2281
|
};
|
|
2268
2282
|
declare class MollifierBuffer {
|
|
2269
|
-
|
|
2270
|
-
|
|
2271
|
-
|
|
2272
|
-
|
|
2273
|
-
|
|
2274
|
-
|
|
2275
|
-
runId: string;
|
|
2276
|
-
envId: string;
|
|
2277
|
-
orgId: string;
|
|
2278
|
-
payload: string;
|
|
2279
|
-
idempotencyKey?: string;
|
|
2280
|
-
taskIdentifier?: string;
|
|
2281
|
-
}): Promise<AcceptResult>;
|
|
2282
|
-
pop(envId: string): Promise<BufferEntry | null>;
|
|
2283
|
-
getEntry(runId: string): Promise<BufferEntry | null>;
|
|
2284
|
-
listOrgs(): Promise<string[]>;
|
|
2285
|
-
listEnvsForOrg(orgId: string): Promise<string[]>;
|
|
2286
|
-
listEntriesForEnv(envId: string, maxCount: number): Promise<BufferEntry[]>;
|
|
2287
|
-
mutateSnapshot(runId: string, patch: SnapshotPatch): Promise<MutateSnapshotResult>;
|
|
2288
|
-
casSetMetadata(input: {
|
|
2289
|
-
runId: string;
|
|
2290
|
-
expectedVersion: number;
|
|
2291
|
-
newMetadata: string;
|
|
2292
|
-
newMetadataType: string;
|
|
2293
|
-
}): Promise<CasSetMetadataResult>;
|
|
2294
|
-
claimIdempotency(input: IdempotencyLookupInput & {
|
|
2295
|
-
token: string;
|
|
2296
|
-
ttlSeconds: number;
|
|
2297
|
-
}): Promise<IdempotencyClaimResult>;
|
|
2298
|
-
publishClaim(input: IdempotencyLookupInput & {
|
|
2299
|
-
token: string;
|
|
2300
|
-
runId: string;
|
|
2301
|
-
ttlSeconds: number;
|
|
2302
|
-
}): Promise<boolean>;
|
|
2303
|
-
releaseClaim(input: IdempotencyLookupInput & {
|
|
2304
|
-
token: string;
|
|
2305
|
-
}): Promise<void>;
|
|
2306
|
-
resetResolvedClaim(input: IdempotencyLookupInput & {
|
|
2307
|
-
expectedRunId: string;
|
|
2308
|
-
}): Promise<boolean>;
|
|
2309
|
-
readClaim(input: IdempotencyLookupInput): Promise<IdempotencyClaimResult | null>;
|
|
2310
|
-
lookupIdempotency(input: IdempotencyLookupInput): Promise<string | null>;
|
|
2311
|
-
resetIdempotency(input: IdempotencyLookupInput): Promise<{
|
|
2312
|
-
clearedRunId: string | null;
|
|
2313
|
-
}>;
|
|
2314
|
-
ack(runId: string): Promise<void>;
|
|
2315
|
-
requeue(runId: string): Promise<void>;
|
|
2316
|
-
fail(runId: string, error: {
|
|
2317
|
-
code: string;
|
|
2318
|
-
message: string;
|
|
2319
|
-
}): Promise<boolean>;
|
|
2320
|
-
getDrainingCount(): Promise<number>;
|
|
2321
|
-
listStaleDraining(olderThanMs: number, limit: number): Promise<string[]>;
|
|
2322
|
-
getEntryTtlSeconds(runId: string): Promise<number>;
|
|
2323
|
-
evaluateTrip(envId: string, options: {
|
|
2324
|
-
windowMs: number;
|
|
2325
|
-
threshold: number;
|
|
2326
|
-
holdMs: number;
|
|
2327
|
-
}): Promise<{
|
|
2328
|
-
tripped: boolean;
|
|
2329
|
-
count: number;
|
|
2330
|
-
}>;
|
|
2331
|
-
close(): Promise<void>;
|
|
2332
|
-
}
|
|
2333
|
-
declare module "@internal/redis" {
|
|
2334
|
-
interface RedisCommander<Context> {
|
|
2335
|
-
acceptMollifierEntry(entryKey: string, queueKey: string, orgsKey: string, runId: string, envId: string, orgId: string, payload: string, createdAt: string, createdAtMicros: string, orgEnvsPrefix: string, idempotencyLookupKey: string, entryPrefix: string, callback?: Callback<number | string>): Result<number | string, Context>;
|
|
2336
|
-
popAndMarkDraining(queueKey: string, orgsKey: string, drainingSetKey: string, entryPrefix: string, envId: string, orgEnvsPrefix: string, callback?: Callback<string | null>): Result<string | null, Context>;
|
|
2337
|
-
requeueMollifierEntry(entryKey: string, orgsKey: string, drainingSetKey: string, queuePrefix: string, runId: string, orgEnvsPrefix: string, callback?: Callback<number>): Result<number, Context>;
|
|
2338
|
-
mutateMollifierSnapshot(entryKey: string, patchJson: string, callback?: Callback<string>): Result<string, Context>;
|
|
2339
|
-
casSetMollifierMetadata(entryKey: string, expectedVersion: string, newMetadata: string, newMetadataType: string, callback?: Callback<string>): Result<string, Context>;
|
|
2340
|
-
resetMollifierIdempotency(lookupKey: string, entryPrefix: string, claimKey: string, callback?: Callback<string>): Result<string, Context>;
|
|
2341
|
-
claimMollifierIdempotency(claimKey: string, pendingMarker: string, pendingPrefix: string, ttlSeconds: string, callback?: Callback<string>): Result<string, Context>;
|
|
2342
|
-
publishMollifierClaim(claimKey: string, ownerMarker: string, runId: string, ttlSeconds: string, callback?: Callback<number>): Result<number, Context>;
|
|
2343
|
-
releaseMollifierClaim(claimKey: string, ownerMarker: string, callback?: Callback<number>): Result<number, Context>;
|
|
2344
|
-
ackMollifierEntry(entryKey: string, drainingSetKey: string, graceTtlSeconds: string, runId: string, callback?: Callback<number>): Result<number, Context>;
|
|
2345
|
-
failMollifierEntry(entryKey: string, drainingSetKey: string, errorPayload: string, runId: string, callback?: Callback<number>): Result<number, Context>;
|
|
2346
|
-
delMollifierKeyIfEquals(key: string, expected: string, callback?: Callback<number>): Result<number, Context>;
|
|
2347
|
-
mollifierEvaluateTrip(rateKey: string, trippedKey: string, windowMs: string, threshold: string, holdMs: string, callback?: Callback<[number, number]>): Result<[number, number], Context>;
|
|
2348
|
-
}
|
|
2349
|
-
}
|
|
2350
|
-
|
|
2351
|
-
type MollifierDrainerHandler<TPayload> = (input: {
|
|
2283
|
+
#private;
|
|
2284
|
+
private readonly redis;
|
|
2285
|
+
private readonly logger;
|
|
2286
|
+
private readonly ackGraceTtlSeconds;
|
|
2287
|
+
constructor(options: MollifierBufferOptions);
|
|
2288
|
+
accept(input: {
|
|
2352
2289
|
runId: string;
|
|
2353
2290
|
envId: string;
|
|
2354
2291
|
orgId: string;
|
|
2355
|
-
payload:
|
|
2356
|
-
|
|
2357
|
-
|
|
2292
|
+
payload: string;
|
|
2293
|
+
idempotencyKey?: string;
|
|
2294
|
+
taskIdentifier?: string;
|
|
2295
|
+
}): Promise<AcceptResult>;
|
|
2296
|
+
pop(envId: string): Promise<BufferEntry | null>;
|
|
2297
|
+
getEntry(runId: string): Promise<BufferEntry | null>;
|
|
2298
|
+
listOrgs(): Promise<string[]>;
|
|
2299
|
+
listEnvsForOrg(orgId: string): Promise<string[]>;
|
|
2300
|
+
listEntriesForEnv(envId: string, maxCount: number): Promise<BufferEntry[]>;
|
|
2301
|
+
mutateSnapshot(runId: string, patch: SnapshotPatch): Promise<MutateSnapshotResult>;
|
|
2302
|
+
casSetMetadata(input: {
|
|
2303
|
+
runId: string;
|
|
2304
|
+
expectedVersion: number;
|
|
2305
|
+
newMetadata: string;
|
|
2306
|
+
newMetadataType: string;
|
|
2307
|
+
}): Promise<CasSetMetadataResult>;
|
|
2308
|
+
claimIdempotency(input: IdempotencyLookupInput & {
|
|
2309
|
+
token: string;
|
|
2310
|
+
ttlSeconds: number;
|
|
2311
|
+
}): Promise<IdempotencyClaimResult>;
|
|
2312
|
+
publishClaim(input: IdempotencyLookupInput & {
|
|
2313
|
+
token: string;
|
|
2314
|
+
runId: string;
|
|
2315
|
+
ttlSeconds: number;
|
|
2316
|
+
}): Promise<boolean>;
|
|
2317
|
+
releaseClaim(input: IdempotencyLookupInput & {
|
|
2318
|
+
token: string;
|
|
2319
|
+
}): Promise<void>;
|
|
2320
|
+
resetResolvedClaim(input: IdempotencyLookupInput & {
|
|
2321
|
+
expectedRunId: string;
|
|
2322
|
+
}): Promise<boolean>;
|
|
2323
|
+
readClaim(input: IdempotencyLookupInput): Promise<IdempotencyClaimResult | null>;
|
|
2324
|
+
lookupIdempotency(input: IdempotencyLookupInput): Promise<string | null>;
|
|
2325
|
+
resetIdempotency(input: IdempotencyLookupInput): Promise<{
|
|
2326
|
+
clearedRunId: string | null;
|
|
2327
|
+
}>;
|
|
2328
|
+
ack(runId: string): Promise<void>;
|
|
2329
|
+
requeue(runId: string): Promise<void>;
|
|
2330
|
+
fail(runId: string, error: {
|
|
2331
|
+
code: string;
|
|
2332
|
+
message: string;
|
|
2333
|
+
}): Promise<boolean>;
|
|
2334
|
+
getDrainingCount(): Promise<number>;
|
|
2335
|
+
listStaleDraining(olderThanMs: number, limit: number): Promise<string[]>;
|
|
2336
|
+
getEntryTtlSeconds(runId: string): Promise<number>;
|
|
2337
|
+
evaluateTrip(envId: string, options: {
|
|
2338
|
+
windowMs: number;
|
|
2339
|
+
threshold: number;
|
|
2340
|
+
holdMs: number;
|
|
2341
|
+
}): Promise<{
|
|
2342
|
+
tripped: boolean;
|
|
2343
|
+
count: number;
|
|
2344
|
+
}>;
|
|
2345
|
+
close(): Promise<void>;
|
|
2346
|
+
}
|
|
2347
|
+
declare module "@internal/redis" {
|
|
2348
|
+
interface RedisCommander<Context> {
|
|
2349
|
+
acceptMollifierEntry(entryKey: string, queueKey: string, orgsKey: string, runId: string, envId: string, orgId: string, payload: string, createdAt: string, createdAtMicros: string, orgEnvsPrefix: string, idempotencyLookupKey: string, entryPrefix: string, callback?: Callback<number | string>): Result<number | string, Context>;
|
|
2350
|
+
popAndMarkDraining(queueKey: string, orgsKey: string, drainingSetKey: string, entryPrefix: string, envId: string, orgEnvsPrefix: string, callback?: Callback<string | null>): Result<string | null, Context>;
|
|
2351
|
+
requeueMollifierEntry(entryKey: string, orgsKey: string, drainingSetKey: string, queuePrefix: string, runId: string, orgEnvsPrefix: string, callback?: Callback<number>): Result<number, Context>;
|
|
2352
|
+
mutateMollifierSnapshot(entryKey: string, patchJson: string, callback?: Callback<string>): Result<string, Context>;
|
|
2353
|
+
casSetMollifierMetadata(entryKey: string, expectedVersion: string, newMetadata: string, newMetadataType: string, callback?: Callback<string>): Result<string, Context>;
|
|
2354
|
+
resetMollifierIdempotency(lookupKey: string, entryPrefix: string, claimKey: string, callback?: Callback<string>): Result<string, Context>;
|
|
2355
|
+
claimMollifierIdempotency(claimKey: string, pendingMarker: string, pendingPrefix: string, ttlSeconds: string, callback?: Callback<string>): Result<string, Context>;
|
|
2356
|
+
publishMollifierClaim(claimKey: string, ownerMarker: string, runId: string, ttlSeconds: string, callback?: Callback<number>): Result<number, Context>;
|
|
2357
|
+
releaseMollifierClaim(claimKey: string, ownerMarker: string, callback?: Callback<number>): Result<number, Context>;
|
|
2358
|
+
ackMollifierEntry(entryKey: string, drainingSetKey: string, graceTtlSeconds: string, runId: string, callback?: Callback<number>): Result<number, Context>;
|
|
2359
|
+
failMollifierEntry(entryKey: string, drainingSetKey: string, errorPayload: string, runId: string, callback?: Callback<number>): Result<number, Context>;
|
|
2360
|
+
delMollifierKeyIfEquals(key: string, expected: string, callback?: Callback<number>): Result<number, Context>;
|
|
2361
|
+
mollifierEvaluateTrip(rateKey: string, trippedKey: string, windowMs: string, threshold: string, holdMs: string, callback?: Callback<[number, number]>): Result<[number, number], Context>;
|
|
2362
|
+
}
|
|
2363
|
+
}
|
|
2364
|
+
//#endregion
|
|
2365
|
+
//#region src/mollifier/drainer.d.ts
|
|
2366
|
+
type MollifierDrainerHandler<TPayload> = (input: {
|
|
2367
|
+
runId: string;
|
|
2368
|
+
envId: string;
|
|
2369
|
+
orgId: string;
|
|
2370
|
+
payload: TPayload;
|
|
2371
|
+
attempts: number;
|
|
2372
|
+
createdAt: Date;
|
|
2358
2373
|
}) => Promise<void>;
|
|
2359
2374
|
type MollifierDrainerTerminalFailureCause = "non-retryable" | "max-attempts-exhausted";
|
|
2360
2375
|
type MollifierDrainerTerminalFailureHandler<TPayload> = (input: {
|
|
2361
|
-
|
|
2362
|
-
|
|
2363
|
-
|
|
2364
|
-
|
|
2365
|
-
|
|
2366
|
-
|
|
2367
|
-
|
|
2368
|
-
|
|
2369
|
-
|
|
2370
|
-
|
|
2371
|
-
|
|
2376
|
+
runId: string;
|
|
2377
|
+
envId: string;
|
|
2378
|
+
orgId: string;
|
|
2379
|
+
payload: TPayload;
|
|
2380
|
+
attempts: number;
|
|
2381
|
+
createdAt: Date;
|
|
2382
|
+
error: {
|
|
2383
|
+
code: string;
|
|
2384
|
+
message: string;
|
|
2385
|
+
};
|
|
2386
|
+
cause: MollifierDrainerTerminalFailureCause;
|
|
2372
2387
|
}) => Promise<void>;
|
|
2373
2388
|
type MollifierDrainerOptions<TPayload> = {
|
|
2374
|
-
|
|
2375
|
-
|
|
2376
|
-
|
|
2377
|
-
|
|
2378
|
-
|
|
2379
|
-
|
|
2380
|
-
|
|
2381
|
-
|
|
2382
|
-
|
|
2383
|
-
|
|
2384
|
-
|
|
2385
|
-
|
|
2389
|
+
buffer: MollifierBuffer;
|
|
2390
|
+
handler: MollifierDrainerHandler<TPayload>;
|
|
2391
|
+
onTerminalFailure?: MollifierDrainerTerminalFailureHandler<TPayload>;
|
|
2392
|
+
concurrency: number;
|
|
2393
|
+
maxAttempts: number;
|
|
2394
|
+
isRetryable: (err: unknown) => boolean;
|
|
2395
|
+
pollIntervalMs?: number;
|
|
2396
|
+
maxOrgsPerTick?: number;
|
|
2397
|
+
drainBatchSize?: number;
|
|
2398
|
+
maxBackoffMs?: number;
|
|
2399
|
+
backoffFloorMs?: number;
|
|
2400
|
+
logger?: Logger;
|
|
2386
2401
|
};
|
|
2387
2402
|
type DrainResult = {
|
|
2388
|
-
|
|
2389
|
-
|
|
2403
|
+
drained: number;
|
|
2404
|
+
failed: number;
|
|
2390
2405
|
};
|
|
2391
2406
|
declare class MollifierDrainer<TPayload = unknown> {
|
|
2392
|
-
|
|
2393
|
-
|
|
2394
|
-
|
|
2395
|
-
|
|
2396
|
-
|
|
2397
|
-
|
|
2398
|
-
|
|
2399
|
-
|
|
2400
|
-
|
|
2401
|
-
|
|
2402
|
-
|
|
2403
|
-
|
|
2404
|
-
|
|
2405
|
-
|
|
2406
|
-
|
|
2407
|
-
|
|
2408
|
-
|
|
2409
|
-
|
|
2410
|
-
|
|
2411
|
-
|
|
2412
|
-
|
|
2413
|
-
|
|
2414
|
-
|
|
2415
|
-
|
|
2416
|
-
|
|
2417
|
-
|
|
2418
|
-
|
|
2419
|
-
|
|
2420
|
-
|
|
2421
|
-
}
|
|
2422
|
-
|
|
2423
|
-
export { type AcceptResult,
|
|
2407
|
+
private readonly buffer;
|
|
2408
|
+
private readonly handler;
|
|
2409
|
+
private readonly onTerminalFailure?;
|
|
2410
|
+
private readonly maxAttempts;
|
|
2411
|
+
private readonly isRetryable;
|
|
2412
|
+
private readonly pollIntervalMs;
|
|
2413
|
+
private readonly maxOrgsPerTick;
|
|
2414
|
+
private readonly drainBatchSize;
|
|
2415
|
+
private readonly concurrency;
|
|
2416
|
+
private readonly maxBackoffMs;
|
|
2417
|
+
private readonly backoffFloorMs;
|
|
2418
|
+
private readonly logger;
|
|
2419
|
+
private orgCursor;
|
|
2420
|
+
private perOrgEnvCursors;
|
|
2421
|
+
private isRunning;
|
|
2422
|
+
private stopping;
|
|
2423
|
+
private loopPromise;
|
|
2424
|
+
constructor(options: MollifierDrainerOptions<TPayload>);
|
|
2425
|
+
runOnce(): Promise<DrainResult>;
|
|
2426
|
+
start(): void;
|
|
2427
|
+
stop(options?: {
|
|
2428
|
+
timeoutMs?: number;
|
|
2429
|
+
}): Promise<void>;
|
|
2430
|
+
private loop;
|
|
2431
|
+
private backoffMs;
|
|
2432
|
+
private delay;
|
|
2433
|
+
private takeOrgSlice;
|
|
2434
|
+
private pickEnvForOrg;
|
|
2435
|
+
private processEntry;
|
|
2436
|
+
}
|
|
2437
|
+
//#endregion
|
|
2438
|
+
export { type AcceptResult, AnyMessageCatalog, AnyQueueItem, BaseScheduler, BatchConfig, BatchedSpanManager, BatchedSpanManagerOptions, type BufferEntry, BufferEntryError, BufferEntrySchema, BufferEntryStatus, CallbackFairQueueKeyProducer, type CasSetMetadataResult, ClaimResult, ConcurrencyCheckResult, ConcurrencyGroupConfig, ConcurrencyManager, ConcurrencyManagerOptions, ConcurrencyState, ConsumerLoopState, CooloffOptions, CronSchema, CustomRetry, DRRScheduler, DRRSchedulerConfig, DeadLetterMessage, DefaultFairQueueKeyProducer, DispatchSchedulerContext, type DrainResult, EnqueueBatchOptions, EnqueueOptions, ExponentialBackoffRetry, FairQueue, FairQueueAttributes, FairQueueKeyProducer, FairQueueMetrics, FairQueueOptions, FairQueueTelemetry, FairScheduler, FixedDelayRetry, GlobalRateLimiter, type IdempotencyClaimResult, type IdempotencyLookupInput, ImmediateRetry, InFlightMessage, JobHandler, JobHandlerParams, LinearBackoffRetry, MasterQueue, MasterQueueOptions, MessageCatalogKey, MessageCatalogSchema, MessageCatalogValue, MessageHandler, MessageHandlerContext, MessagingAttributes, MollifierBuffer, type MollifierBufferOptions, MollifierDrainer, type MollifierDrainerHandler, type MollifierDrainerOptions, type MollifierDrainerTerminalFailureCause, type MollifierDrainerTerminalFailureHandler, type MutateSnapshotResult, NoRetry, NoopScheduler, QueueCooloffState, QueueDescriptor, QueueItem, QueueMessage, QueueWithScore, ReclaimedMessageInfo, RetryOptions, RetryStrategy, RoundRobinScheduler, SchedulerContext, SimpleQueue, type SnapshotPatch, StoredMessage, TelemetryOptions, TenantDispatch, TenantDispatchOptions, TenantQueues, TenantWithScore, VisibilityManager, VisibilityManagerOptions, WeightedScheduler, WeightedSchedulerBiases, WeightedSchedulerConfig, Worker, WorkerCatalog, WorkerConcurrencyOptions, WorkerQueueManager, WorkerQueueManagerOptions, WorkerQueueOptions, createDefaultRetryStrategy, defaultRetryOptions, deserialiseSnapshot, idempotencyLookupKeyFor, isAbortError, makeIdempotencyClaimKey, noopTelemetry, serialiseSnapshot };
|