@trigger.dev/redis-worker 4.5.0 → 4.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +231 -231
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +350 -350
- package/dist/index.d.ts +350 -350
- package/dist/index.js +231 -231
- package/dist/index.js.map +1 -1
- package/package.json +2 -2
package/dist/index.d.ts
CHANGED
|
@@ -910,6 +910,101 @@ interface WeightedSchedulerConfig {
|
|
|
910
910
|
tracer?: Tracer;
|
|
911
911
|
}
|
|
912
912
|
|
|
913
|
+
interface ConcurrencyManagerOptions {
|
|
914
|
+
redis: RedisOptions;
|
|
915
|
+
keys: FairQueueKeyProducer;
|
|
916
|
+
groups: ConcurrencyGroupConfig[];
|
|
917
|
+
}
|
|
918
|
+
/**
|
|
919
|
+
* ConcurrencyManager handles multi-level concurrency tracking and limiting.
|
|
920
|
+
*
|
|
921
|
+
* Features:
|
|
922
|
+
* - Multiple concurrent concurrency groups (tenant, org, project, etc.)
|
|
923
|
+
* - Atomic reserve/release operations using Lua scripts
|
|
924
|
+
* - Efficient batch checking of all groups
|
|
925
|
+
*/
|
|
926
|
+
declare class ConcurrencyManager {
|
|
927
|
+
#private;
|
|
928
|
+
private options;
|
|
929
|
+
private redis;
|
|
930
|
+
private keys;
|
|
931
|
+
private groups;
|
|
932
|
+
private groupsByName;
|
|
933
|
+
constructor(options: ConcurrencyManagerOptions);
|
|
934
|
+
/**
|
|
935
|
+
* Check if a message can be processed given all concurrency constraints.
|
|
936
|
+
* Checks all configured groups and returns the first one at capacity.
|
|
937
|
+
*/
|
|
938
|
+
canProcess(queue: QueueDescriptor): Promise<ConcurrencyCheckResult>;
|
|
939
|
+
/**
|
|
940
|
+
* Reserve concurrency slots for a message across all groups.
|
|
941
|
+
* Atomic - either all groups are reserved or none.
|
|
942
|
+
*
|
|
943
|
+
* @returns true if reservation successful, false if any group is at capacity
|
|
944
|
+
*/
|
|
945
|
+
reserve(queue: QueueDescriptor, messageId: string): Promise<boolean>;
|
|
946
|
+
/**
|
|
947
|
+
* Release concurrency slots for a message across all groups.
|
|
948
|
+
*/
|
|
949
|
+
release(queue: QueueDescriptor, messageId: string): Promise<void>;
|
|
950
|
+
/**
|
|
951
|
+
* Release concurrency slots for multiple messages in a single pipeline.
|
|
952
|
+
* More efficient than calling release() multiple times.
|
|
953
|
+
*/
|
|
954
|
+
releaseBatch(messages: Array<{
|
|
955
|
+
queue: QueueDescriptor;
|
|
956
|
+
messageId: string;
|
|
957
|
+
}>): Promise<void>;
|
|
958
|
+
/**
|
|
959
|
+
* Get current concurrency for a specific group.
|
|
960
|
+
*/
|
|
961
|
+
getCurrentConcurrency(groupName: string, groupId: string): Promise<number>;
|
|
962
|
+
/**
|
|
963
|
+
* Get available capacity for a queue across all concurrency groups.
|
|
964
|
+
* Returns the minimum available capacity across all groups.
|
|
965
|
+
*/
|
|
966
|
+
getAvailableCapacity(queue: QueueDescriptor): Promise<number>;
|
|
967
|
+
/**
|
|
968
|
+
* Get concurrency limit for a specific group.
|
|
969
|
+
*/
|
|
970
|
+
getConcurrencyLimit(groupName: string, groupId: string): Promise<number>;
|
|
971
|
+
/**
|
|
972
|
+
* Check if a group is at capacity.
|
|
973
|
+
*/
|
|
974
|
+
isAtCapacity(groupName: string, groupId: string): Promise<boolean>;
|
|
975
|
+
/**
|
|
976
|
+
* Get full state for a group.
|
|
977
|
+
*/
|
|
978
|
+
getState(groupName: string, groupId: string): Promise<ConcurrencyState>;
|
|
979
|
+
/**
|
|
980
|
+
* Get all active message IDs for a group.
|
|
981
|
+
*/
|
|
982
|
+
getActiveMessages(groupName: string, groupId: string): Promise<string[]>;
|
|
983
|
+
/**
|
|
984
|
+
* Force-clear concurrency for a group (use with caution).
|
|
985
|
+
* Useful for cleanup after crashes.
|
|
986
|
+
*/
|
|
987
|
+
clearGroup(groupName: string, groupId: string): Promise<void>;
|
|
988
|
+
/**
|
|
989
|
+
* Remove a specific message from concurrency tracking.
|
|
990
|
+
* Useful for cleanup.
|
|
991
|
+
*/
|
|
992
|
+
removeMessage(messageId: string, queue: QueueDescriptor): Promise<void>;
|
|
993
|
+
/**
|
|
994
|
+
* Get configured group names.
|
|
995
|
+
*/
|
|
996
|
+
getGroupNames(): string[];
|
|
997
|
+
/**
|
|
998
|
+
* Close the Redis connection.
|
|
999
|
+
*/
|
|
1000
|
+
close(): Promise<void>;
|
|
1001
|
+
}
|
|
1002
|
+
declare module "@internal/redis" {
|
|
1003
|
+
interface RedisCommander<Context> {
|
|
1004
|
+
reserveConcurrency(numKeys: number, keys: string[], messageId: string, ...limits: string[]): Promise<number>;
|
|
1005
|
+
}
|
|
1006
|
+
}
|
|
1007
|
+
|
|
913
1008
|
/**
|
|
914
1009
|
* Default key producer for the fair queue system.
|
|
915
1010
|
* Uses a configurable prefix and standard key structure.
|
|
@@ -1059,359 +1154,12 @@ declare class MasterQueue {
|
|
|
1059
1154
|
/**
|
|
1060
1155
|
* Close the Redis connection.
|
|
1061
1156
|
*/
|
|
1062
|
-
close(): Promise<void>;
|
|
1063
|
-
}
|
|
1064
|
-
declare module "@internal/redis" {
|
|
1065
|
-
interface RedisCommander<Context> {
|
|
1066
|
-
addQueueIfNotEmpty(masterKey: string, queueKey: string, queueId: string): Promise<number>;
|
|
1067
|
-
removeQueueIfEmpty(masterKey: string, queueKey: string, queueId: string): Promise<number>;
|
|
1068
|
-
}
|
|
1069
|
-
}
|
|
1070
|
-
|
|
1071
|
-
interface ConcurrencyManagerOptions {
|
|
1072
|
-
redis: RedisOptions;
|
|
1073
|
-
keys: FairQueueKeyProducer;
|
|
1074
|
-
groups: ConcurrencyGroupConfig[];
|
|
1075
|
-
}
|
|
1076
|
-
/**
|
|
1077
|
-
* ConcurrencyManager handles multi-level concurrency tracking and limiting.
|
|
1078
|
-
*
|
|
1079
|
-
* Features:
|
|
1080
|
-
* - Multiple concurrent concurrency groups (tenant, org, project, etc.)
|
|
1081
|
-
* - Atomic reserve/release operations using Lua scripts
|
|
1082
|
-
* - Efficient batch checking of all groups
|
|
1083
|
-
*/
|
|
1084
|
-
declare class ConcurrencyManager {
|
|
1085
|
-
#private;
|
|
1086
|
-
private options;
|
|
1087
|
-
private redis;
|
|
1088
|
-
private keys;
|
|
1089
|
-
private groups;
|
|
1090
|
-
private groupsByName;
|
|
1091
|
-
constructor(options: ConcurrencyManagerOptions);
|
|
1092
|
-
/**
|
|
1093
|
-
* Check if a message can be processed given all concurrency constraints.
|
|
1094
|
-
* Checks all configured groups and returns the first one at capacity.
|
|
1095
|
-
*/
|
|
1096
|
-
canProcess(queue: QueueDescriptor): Promise<ConcurrencyCheckResult>;
|
|
1097
|
-
/**
|
|
1098
|
-
* Reserve concurrency slots for a message across all groups.
|
|
1099
|
-
* Atomic - either all groups are reserved or none.
|
|
1100
|
-
*
|
|
1101
|
-
* @returns true if reservation successful, false if any group is at capacity
|
|
1102
|
-
*/
|
|
1103
|
-
reserve(queue: QueueDescriptor, messageId: string): Promise<boolean>;
|
|
1104
|
-
/**
|
|
1105
|
-
* Release concurrency slots for a message across all groups.
|
|
1106
|
-
*/
|
|
1107
|
-
release(queue: QueueDescriptor, messageId: string): Promise<void>;
|
|
1108
|
-
/**
|
|
1109
|
-
* Release concurrency slots for multiple messages in a single pipeline.
|
|
1110
|
-
* More efficient than calling release() multiple times.
|
|
1111
|
-
*/
|
|
1112
|
-
releaseBatch(messages: Array<{
|
|
1113
|
-
queue: QueueDescriptor;
|
|
1114
|
-
messageId: string;
|
|
1115
|
-
}>): Promise<void>;
|
|
1116
|
-
/**
|
|
1117
|
-
* Get current concurrency for a specific group.
|
|
1118
|
-
*/
|
|
1119
|
-
getCurrentConcurrency(groupName: string, groupId: string): Promise<number>;
|
|
1120
|
-
/**
|
|
1121
|
-
* Get available capacity for a queue across all concurrency groups.
|
|
1122
|
-
* Returns the minimum available capacity across all groups.
|
|
1123
|
-
*/
|
|
1124
|
-
getAvailableCapacity(queue: QueueDescriptor): Promise<number>;
|
|
1125
|
-
/**
|
|
1126
|
-
* Get concurrency limit for a specific group.
|
|
1127
|
-
*/
|
|
1128
|
-
getConcurrencyLimit(groupName: string, groupId: string): Promise<number>;
|
|
1129
|
-
/**
|
|
1130
|
-
* Check if a group is at capacity.
|
|
1131
|
-
*/
|
|
1132
|
-
isAtCapacity(groupName: string, groupId: string): Promise<boolean>;
|
|
1133
|
-
/**
|
|
1134
|
-
* Get full state for a group.
|
|
1135
|
-
*/
|
|
1136
|
-
getState(groupName: string, groupId: string): Promise<ConcurrencyState>;
|
|
1137
|
-
/**
|
|
1138
|
-
* Get all active message IDs for a group.
|
|
1139
|
-
*/
|
|
1140
|
-
getActiveMessages(groupName: string, groupId: string): Promise<string[]>;
|
|
1141
|
-
/**
|
|
1142
|
-
* Force-clear concurrency for a group (use with caution).
|
|
1143
|
-
* Useful for cleanup after crashes.
|
|
1144
|
-
*/
|
|
1145
|
-
clearGroup(groupName: string, groupId: string): Promise<void>;
|
|
1146
|
-
/**
|
|
1147
|
-
* Remove a specific message from concurrency tracking.
|
|
1148
|
-
* Useful for cleanup.
|
|
1149
|
-
*/
|
|
1150
|
-
removeMessage(messageId: string, queue: QueueDescriptor): Promise<void>;
|
|
1151
|
-
/**
|
|
1152
|
-
* Get configured group names.
|
|
1153
|
-
*/
|
|
1154
|
-
getGroupNames(): string[];
|
|
1155
|
-
/**
|
|
1156
|
-
* Close the Redis connection.
|
|
1157
|
-
*/
|
|
1158
|
-
close(): Promise<void>;
|
|
1159
|
-
}
|
|
1160
|
-
declare module "@internal/redis" {
|
|
1161
|
-
interface RedisCommander<Context> {
|
|
1162
|
-
reserveConcurrency(numKeys: number, keys: string[], messageId: string, ...limits: string[]): Promise<number>;
|
|
1163
|
-
}
|
|
1164
|
-
}
|
|
1165
|
-
|
|
1166
|
-
interface VisibilityManagerOptions {
|
|
1167
|
-
redis: RedisOptions;
|
|
1168
|
-
keys: FairQueueKeyProducer;
|
|
1169
|
-
shardCount: number;
|
|
1170
|
-
defaultTimeoutMs: number;
|
|
1171
|
-
logger?: {
|
|
1172
|
-
debug: (message: string, context?: Record<string, unknown>) => void;
|
|
1173
|
-
error: (message: string, context?: Record<string, unknown>) => void;
|
|
1174
|
-
};
|
|
1175
|
-
}
|
|
1176
|
-
/**
|
|
1177
|
-
* VisibilityManager handles message visibility timeouts for safe message processing.
|
|
1178
|
-
*
|
|
1179
|
-
* Features:
|
|
1180
|
-
* - Claim messages with visibility timeout
|
|
1181
|
-
* - Heartbeat to extend timeout
|
|
1182
|
-
* - Automatic reclaim of timed-out messages
|
|
1183
|
-
* - Per-shard in-flight tracking
|
|
1184
|
-
*
|
|
1185
|
-
* Data structures:
|
|
1186
|
-
* - In-flight sorted set: score = deadline timestamp, member = "{messageId}:{queueId}"
|
|
1187
|
-
* - In-flight data hash: field = messageId, value = JSON message data
|
|
1188
|
-
*/
|
|
1189
|
-
declare class VisibilityManager {
|
|
1190
|
-
#private;
|
|
1191
|
-
private options;
|
|
1192
|
-
private redis;
|
|
1193
|
-
private keys;
|
|
1194
|
-
private shardCount;
|
|
1195
|
-
private defaultTimeoutMs;
|
|
1196
|
-
private logger;
|
|
1197
|
-
constructor(options: VisibilityManagerOptions);
|
|
1198
|
-
/**
|
|
1199
|
-
* Claim a message for processing.
|
|
1200
|
-
* Moves the message from its queue to the in-flight set with a visibility timeout.
|
|
1201
|
-
*
|
|
1202
|
-
* @param queueId - The queue to claim from
|
|
1203
|
-
* @param queueKey - The Redis key for the queue sorted set
|
|
1204
|
-
* @param queueItemsKey - The Redis key for the queue items hash
|
|
1205
|
-
* @param consumerId - ID of the consumer claiming the message
|
|
1206
|
-
* @param timeoutMs - Visibility timeout in milliseconds
|
|
1207
|
-
* @returns Claim result with the message if successful
|
|
1208
|
-
*/
|
|
1209
|
-
claim<TPayload = unknown>(queueId: string, queueKey: string, queueItemsKey: string, consumerId: string, timeoutMs?: number): Promise<ClaimResult<TPayload>>;
|
|
1210
|
-
/**
|
|
1211
|
-
* Claim multiple messages for processing (batch claim).
|
|
1212
|
-
* Moves up to maxCount messages from the queue to the in-flight set.
|
|
1213
|
-
*
|
|
1214
|
-
* @param queueId - The queue to claim from
|
|
1215
|
-
* @param queueKey - The Redis key for the queue sorted set
|
|
1216
|
-
* @param queueItemsKey - The Redis key for the queue items hash
|
|
1217
|
-
* @param consumerId - ID of the consumer claiming the messages
|
|
1218
|
-
* @param maxCount - Maximum number of messages to claim
|
|
1219
|
-
* @param timeoutMs - Visibility timeout in milliseconds
|
|
1220
|
-
* @returns Array of claimed messages
|
|
1221
|
-
*/
|
|
1222
|
-
claimBatch<TPayload = unknown>(queueId: string, queueKey: string, queueItemsKey: string, consumerId: string, maxCount: number, timeoutMs?: number): Promise<Array<InFlightMessage<TPayload>>>;
|
|
1223
|
-
/**
|
|
1224
|
-
* Extend the visibility timeout for a message (heartbeat).
|
|
1225
|
-
*
|
|
1226
|
-
* @param messageId - The message ID
|
|
1227
|
-
* @param queueId - The queue ID
|
|
1228
|
-
* @param extendMs - Additional milliseconds to add to the deadline
|
|
1229
|
-
* @returns true if the heartbeat was successful
|
|
1230
|
-
*/
|
|
1231
|
-
heartbeat(messageId: string, queueId: string, extendMs: number): Promise<boolean>;
|
|
1232
|
-
/**
|
|
1233
|
-
* Mark a message as successfully processed.
|
|
1234
|
-
* Removes the message from in-flight tracking.
|
|
1235
|
-
*
|
|
1236
|
-
* @param messageId - The message ID
|
|
1237
|
-
* @param queueId - The queue ID
|
|
1238
|
-
*/
|
|
1239
|
-
complete(messageId: string, queueId: string): Promise<void>;
|
|
1240
|
-
/**
|
|
1241
|
-
* Release a message back to its queue.
|
|
1242
|
-
* Used when processing fails or consumer wants to retry later.
|
|
1243
|
-
*
|
|
1244
|
-
* @param messageId - The message ID
|
|
1245
|
-
* @param queueId - The queue ID
|
|
1246
|
-
* @param queueKey - The Redis key for the queue
|
|
1247
|
-
* @param queueItemsKey - The Redis key for the queue items hash
|
|
1248
|
-
* @param tenantQueueIndexKey - The Redis key for the tenant queue index (Level 2)
|
|
1249
|
-
* @param dispatchKey - The Redis key for the dispatch index (Level 1)
|
|
1250
|
-
* @param tenantId - The tenant ID
|
|
1251
|
-
* @param score - Optional score for the message (defaults to now)
|
|
1252
|
-
*/
|
|
1253
|
-
release<TPayload = unknown>(messageId: string, queueId: string, queueKey: string, queueItemsKey: string, tenantQueueIndexKey: string, dispatchKey: string, tenantId: string, score?: number, updatedData?: string): Promise<void>;
|
|
1254
|
-
/**
|
|
1255
|
-
* Release multiple messages back to their queue in a single operation.
|
|
1256
|
-
* Used when processing fails or consumer wants to retry later.
|
|
1257
|
-
* All messages must belong to the same queue.
|
|
1258
|
-
*
|
|
1259
|
-
* @param messages - Array of messages to release (must all have same queueId)
|
|
1260
|
-
* @param queueId - The queue ID
|
|
1261
|
-
* @param queueKey - The Redis key for the queue
|
|
1262
|
-
* @param queueItemsKey - The Redis key for the queue items hash
|
|
1263
|
-
* @param tenantQueueIndexKey - The Redis key for the tenant queue index (Level 2)
|
|
1264
|
-
* @param dispatchKey - The Redis key for the dispatch index (Level 1)
|
|
1265
|
-
* @param tenantId - The tenant ID
|
|
1266
|
-
* @param score - Optional score for the messages (defaults to now)
|
|
1267
|
-
*/
|
|
1268
|
-
releaseBatch(messages: Array<{
|
|
1269
|
-
messageId: string;
|
|
1270
|
-
}>, queueId: string, queueKey: string, queueItemsKey: string, tenantQueueIndexKey: string, dispatchKey: string, tenantId: string, score?: number): Promise<void>;
|
|
1271
|
-
/**
|
|
1272
|
-
* Reclaim timed-out messages from a shard.
|
|
1273
|
-
* Returns messages to their original queues.
|
|
1274
|
-
*
|
|
1275
|
-
* @param shardId - The shard to check
|
|
1276
|
-
* @param getQueueKeys - Function to get queue keys for a queue ID
|
|
1277
|
-
* @returns Array of reclaimed message info for concurrency release
|
|
1278
|
-
*/
|
|
1279
|
-
reclaimTimedOut(shardId: number, getQueueKeys: (queueId: string) => {
|
|
1280
|
-
queueKey: string;
|
|
1281
|
-
queueItemsKey: string;
|
|
1282
|
-
tenantQueueIndexKey: string;
|
|
1283
|
-
dispatchKey: string;
|
|
1284
|
-
tenantId: string;
|
|
1285
|
-
}): Promise<ReclaimedMessageInfo[]>;
|
|
1286
|
-
/**
|
|
1287
|
-
* Get all in-flight messages for a shard.
|
|
1288
|
-
*/
|
|
1289
|
-
getInflightMessages(shardId: number): Promise<Array<{
|
|
1290
|
-
messageId: string;
|
|
1291
|
-
queueId: string;
|
|
1292
|
-
deadline: number;
|
|
1293
|
-
}>>;
|
|
1294
|
-
/**
|
|
1295
|
-
* Get count of in-flight messages for a shard.
|
|
1296
|
-
*/
|
|
1297
|
-
getInflightCount(shardId: number): Promise<number>;
|
|
1298
|
-
/**
|
|
1299
|
-
* Get total in-flight count across all shards.
|
|
1300
|
-
*/
|
|
1301
|
-
getTotalInflightCount(): Promise<number>;
|
|
1302
|
-
/**
|
|
1303
|
-
* Close the Redis connection.
|
|
1304
|
-
*/
|
|
1305
|
-
close(): Promise<void>;
|
|
1306
|
-
}
|
|
1307
|
-
declare module "@internal/redis" {
|
|
1308
|
-
interface RedisCommander<Context> {
|
|
1309
|
-
claimMessage(queueKey: string, queueItemsKey: string, inflightKey: string, inflightDataKey: string, queueId: string, consumerId: string, deadline: string): Promise<[string, string] | null>;
|
|
1310
|
-
claimMessageBatch(queueKey: string, queueItemsKey: string, inflightKey: string, inflightDataKey: string, queueId: string, deadline: string, maxCount: string): Promise<string[]>;
|
|
1311
|
-
releaseMessage(inflightKey: string, inflightDataKey: string, queueKey: string, queueItemsKey: string, tenantQueueIndexKey: string, dispatchKey: string, member: string, messageId: string, score: string, queueId: string, updatedData: string, tenantId: string): Promise<number>;
|
|
1312
|
-
releaseMessageBatch(inflightKey: string, inflightDataKey: string, queueKey: string, queueItemsKey: string, tenantQueueIndexKey: string, dispatchKey: string, score: string, queueId: string, tenantId: string, ...membersAndMessageIds: string[]): Promise<number>;
|
|
1313
|
-
heartbeatMessage(inflightKey: string, member: string, newDeadline: string): Promise<number>;
|
|
1314
|
-
}
|
|
1315
|
-
}
|
|
1316
|
-
|
|
1317
|
-
interface WorkerQueueManagerOptions {
|
|
1318
|
-
redis: RedisOptions;
|
|
1319
|
-
keys: FairQueueKeyProducer;
|
|
1320
|
-
logger?: {
|
|
1321
|
-
debug: (message: string, context?: Record<string, unknown>) => void;
|
|
1322
|
-
error: (message: string, context?: Record<string, unknown>) => void;
|
|
1323
|
-
};
|
|
1324
|
-
}
|
|
1325
|
-
/**
|
|
1326
|
-
* WorkerQueueManager handles the intermediate worker queue layer.
|
|
1327
|
-
*
|
|
1328
|
-
* This provides:
|
|
1329
|
-
* - Low-latency message delivery via blocking pop (BLPOP)
|
|
1330
|
-
* - Routing of messages to specific workers/consumers
|
|
1331
|
-
* - Efficient waiting without polling
|
|
1332
|
-
*
|
|
1333
|
-
* Flow:
|
|
1334
|
-
* 1. Master queue consumer claims message from message queue
|
|
1335
|
-
* 2. Message key is pushed to worker queue
|
|
1336
|
-
* 3. Worker queue consumer does blocking pop to receive message
|
|
1337
|
-
*/
|
|
1338
|
-
declare class WorkerQueueManager {
|
|
1339
|
-
#private;
|
|
1340
|
-
private options;
|
|
1341
|
-
private redis;
|
|
1342
|
-
private keys;
|
|
1343
|
-
private logger;
|
|
1344
|
-
constructor(options: WorkerQueueManagerOptions);
|
|
1345
|
-
/**
|
|
1346
|
-
* Push a message key to a worker queue.
|
|
1347
|
-
* Called after claiming a message from the message queue.
|
|
1348
|
-
*
|
|
1349
|
-
* @param workerQueueId - The worker queue identifier
|
|
1350
|
-
* @param messageKey - The message key to push (typically "messageId:queueId")
|
|
1351
|
-
*/
|
|
1352
|
-
push(workerQueueId: string, messageKey: string): Promise<void>;
|
|
1353
|
-
/**
|
|
1354
|
-
* Push multiple message keys to a worker queue.
|
|
1355
|
-
*
|
|
1356
|
-
* @param workerQueueId - The worker queue identifier
|
|
1357
|
-
* @param messageKeys - The message keys to push
|
|
1358
|
-
*/
|
|
1359
|
-
pushBatch(workerQueueId: string, messageKeys: string[]): Promise<void>;
|
|
1360
|
-
/**
|
|
1361
|
-
* Blocking pop from a worker queue.
|
|
1362
|
-
* Waits until a message is available or timeout expires.
|
|
1363
|
-
*
|
|
1364
|
-
* @param workerQueueId - The worker queue identifier
|
|
1365
|
-
* @param timeoutSeconds - Maximum time to wait (0 = wait forever)
|
|
1366
|
-
* @param signal - Optional abort signal to cancel waiting
|
|
1367
|
-
* @returns The message key, or null if timeout
|
|
1368
|
-
*/
|
|
1369
|
-
blockingPop(workerQueueId: string, timeoutSeconds: number, signal?: AbortSignal): Promise<string | null>;
|
|
1370
|
-
/**
|
|
1371
|
-
* Non-blocking pop from a worker queue.
|
|
1372
|
-
*
|
|
1373
|
-
* @param workerQueueId - The worker queue identifier
|
|
1374
|
-
* @returns The message key and queue length, or null if empty
|
|
1375
|
-
*/
|
|
1376
|
-
pop(workerQueueId: string): Promise<{
|
|
1377
|
-
messageKey: string;
|
|
1378
|
-
queueLength: number;
|
|
1379
|
-
} | null>;
|
|
1380
|
-
/**
|
|
1381
|
-
* Get the current length of a worker queue.
|
|
1382
|
-
*/
|
|
1383
|
-
getLength(workerQueueId: string): Promise<number>;
|
|
1384
|
-
/**
|
|
1385
|
-
* Peek at all messages in a worker queue without removing them.
|
|
1386
|
-
* Useful for debugging and tests.
|
|
1387
|
-
*/
|
|
1388
|
-
peek(workerQueueId: string): Promise<string[]>;
|
|
1389
|
-
/**
|
|
1390
|
-
* Remove a specific message from the worker queue.
|
|
1391
|
-
* Used when a message needs to be removed without processing.
|
|
1392
|
-
*
|
|
1393
|
-
* @param workerQueueId - The worker queue identifier
|
|
1394
|
-
* @param messageKey - The message key to remove
|
|
1395
|
-
* @returns Number of removed items
|
|
1396
|
-
*/
|
|
1397
|
-
remove(workerQueueId: string, messageKey: string): Promise<number>;
|
|
1398
|
-
/**
|
|
1399
|
-
* Clear all messages from a worker queue.
|
|
1400
|
-
*/
|
|
1401
|
-
clear(workerQueueId: string): Promise<void>;
|
|
1402
|
-
/**
|
|
1403
|
-
* Close the Redis connection.
|
|
1404
|
-
*/
|
|
1405
|
-
close(): Promise<void>;
|
|
1406
|
-
/**
|
|
1407
|
-
* Register custom commands on an external Redis client.
|
|
1408
|
-
* Use this when initializing FairQueue with worker queues.
|
|
1409
|
-
*/
|
|
1410
|
-
registerCommands(redis: Redis): void;
|
|
1157
|
+
close(): Promise<void>;
|
|
1411
1158
|
}
|
|
1412
1159
|
declare module "@internal/redis" {
|
|
1413
1160
|
interface RedisCommander<Context> {
|
|
1414
|
-
|
|
1161
|
+
addQueueIfNotEmpty(masterKey: string, queueKey: string, queueId: string): Promise<number>;
|
|
1162
|
+
removeQueueIfEmpty(masterKey: string, queueKey: string, queueId: string): Promise<number>;
|
|
1415
1163
|
}
|
|
1416
1164
|
}
|
|
1417
1165
|
|
|
@@ -1942,6 +1690,258 @@ declare class TenantDispatch {
|
|
|
1942
1690
|
close(): Promise<void>;
|
|
1943
1691
|
}
|
|
1944
1692
|
|
|
1693
|
+
interface VisibilityManagerOptions {
|
|
1694
|
+
redis: RedisOptions;
|
|
1695
|
+
keys: FairQueueKeyProducer;
|
|
1696
|
+
shardCount: number;
|
|
1697
|
+
defaultTimeoutMs: number;
|
|
1698
|
+
logger?: {
|
|
1699
|
+
debug: (message: string, context?: Record<string, unknown>) => void;
|
|
1700
|
+
error: (message: string, context?: Record<string, unknown>) => void;
|
|
1701
|
+
};
|
|
1702
|
+
}
|
|
1703
|
+
/**
|
|
1704
|
+
* VisibilityManager handles message visibility timeouts for safe message processing.
|
|
1705
|
+
*
|
|
1706
|
+
* Features:
|
|
1707
|
+
* - Claim messages with visibility timeout
|
|
1708
|
+
* - Heartbeat to extend timeout
|
|
1709
|
+
* - Automatic reclaim of timed-out messages
|
|
1710
|
+
* - Per-shard in-flight tracking
|
|
1711
|
+
*
|
|
1712
|
+
* Data structures:
|
|
1713
|
+
* - In-flight sorted set: score = deadline timestamp, member = "{messageId}:{queueId}"
|
|
1714
|
+
* - In-flight data hash: field = messageId, value = JSON message data
|
|
1715
|
+
*/
|
|
1716
|
+
declare class VisibilityManager {
|
|
1717
|
+
#private;
|
|
1718
|
+
private options;
|
|
1719
|
+
private redis;
|
|
1720
|
+
private keys;
|
|
1721
|
+
private shardCount;
|
|
1722
|
+
private defaultTimeoutMs;
|
|
1723
|
+
private logger;
|
|
1724
|
+
constructor(options: VisibilityManagerOptions);
|
|
1725
|
+
/**
|
|
1726
|
+
* Claim a message for processing.
|
|
1727
|
+
* Moves the message from its queue to the in-flight set with a visibility timeout.
|
|
1728
|
+
*
|
|
1729
|
+
* @param queueId - The queue to claim from
|
|
1730
|
+
* @param queueKey - The Redis key for the queue sorted set
|
|
1731
|
+
* @param queueItemsKey - The Redis key for the queue items hash
|
|
1732
|
+
* @param consumerId - ID of the consumer claiming the message
|
|
1733
|
+
* @param timeoutMs - Visibility timeout in milliseconds
|
|
1734
|
+
* @returns Claim result with the message if successful
|
|
1735
|
+
*/
|
|
1736
|
+
claim<TPayload = unknown>(queueId: string, queueKey: string, queueItemsKey: string, consumerId: string, timeoutMs?: number): Promise<ClaimResult<TPayload>>;
|
|
1737
|
+
/**
|
|
1738
|
+
* Claim multiple messages for processing (batch claim).
|
|
1739
|
+
* Moves up to maxCount messages from the queue to the in-flight set.
|
|
1740
|
+
*
|
|
1741
|
+
* @param queueId - The queue to claim from
|
|
1742
|
+
* @param queueKey - The Redis key for the queue sorted set
|
|
1743
|
+
* @param queueItemsKey - The Redis key for the queue items hash
|
|
1744
|
+
* @param consumerId - ID of the consumer claiming the messages
|
|
1745
|
+
* @param maxCount - Maximum number of messages to claim
|
|
1746
|
+
* @param timeoutMs - Visibility timeout in milliseconds
|
|
1747
|
+
* @returns Array of claimed messages
|
|
1748
|
+
*/
|
|
1749
|
+
claimBatch<TPayload = unknown>(queueId: string, queueKey: string, queueItemsKey: string, consumerId: string, maxCount: number, timeoutMs?: number): Promise<Array<InFlightMessage<TPayload>>>;
|
|
1750
|
+
/**
|
|
1751
|
+
* Extend the visibility timeout for a message (heartbeat).
|
|
1752
|
+
*
|
|
1753
|
+
* @param messageId - The message ID
|
|
1754
|
+
* @param queueId - The queue ID
|
|
1755
|
+
* @param extendMs - Additional milliseconds to add to the deadline
|
|
1756
|
+
* @returns true if the heartbeat was successful
|
|
1757
|
+
*/
|
|
1758
|
+
heartbeat(messageId: string, queueId: string, extendMs: number): Promise<boolean>;
|
|
1759
|
+
/**
|
|
1760
|
+
* Mark a message as successfully processed.
|
|
1761
|
+
* Removes the message from in-flight tracking.
|
|
1762
|
+
*
|
|
1763
|
+
* @param messageId - The message ID
|
|
1764
|
+
* @param queueId - The queue ID
|
|
1765
|
+
*/
|
|
1766
|
+
complete(messageId: string, queueId: string): Promise<void>;
|
|
1767
|
+
/**
|
|
1768
|
+
* Release a message back to its queue.
|
|
1769
|
+
* Used when processing fails or consumer wants to retry later.
|
|
1770
|
+
*
|
|
1771
|
+
* @param messageId - The message ID
|
|
1772
|
+
* @param queueId - The queue ID
|
|
1773
|
+
* @param queueKey - The Redis key for the queue
|
|
1774
|
+
* @param queueItemsKey - The Redis key for the queue items hash
|
|
1775
|
+
* @param tenantQueueIndexKey - The Redis key for the tenant queue index (Level 2)
|
|
1776
|
+
* @param dispatchKey - The Redis key for the dispatch index (Level 1)
|
|
1777
|
+
* @param tenantId - The tenant ID
|
|
1778
|
+
* @param score - Optional score for the message (defaults to now)
|
|
1779
|
+
*/
|
|
1780
|
+
release<_TPayload = unknown>(messageId: string, queueId: string, queueKey: string, queueItemsKey: string, tenantQueueIndexKey: string, dispatchKey: string, tenantId: string, score?: number, updatedData?: string): Promise<void>;
|
|
1781
|
+
/**
|
|
1782
|
+
* Release multiple messages back to their queue in a single operation.
|
|
1783
|
+
* Used when processing fails or consumer wants to retry later.
|
|
1784
|
+
* All messages must belong to the same queue.
|
|
1785
|
+
*
|
|
1786
|
+
* @param messages - Array of messages to release (must all have same queueId)
|
|
1787
|
+
* @param queueId - The queue ID
|
|
1788
|
+
* @param queueKey - The Redis key for the queue
|
|
1789
|
+
* @param queueItemsKey - The Redis key for the queue items hash
|
|
1790
|
+
* @param tenantQueueIndexKey - The Redis key for the tenant queue index (Level 2)
|
|
1791
|
+
* @param dispatchKey - The Redis key for the dispatch index (Level 1)
|
|
1792
|
+
* @param tenantId - The tenant ID
|
|
1793
|
+
* @param score - Optional score for the messages (defaults to now)
|
|
1794
|
+
*/
|
|
1795
|
+
releaseBatch(messages: Array<{
|
|
1796
|
+
messageId: string;
|
|
1797
|
+
}>, queueId: string, queueKey: string, queueItemsKey: string, tenantQueueIndexKey: string, dispatchKey: string, tenantId: string, score?: number): Promise<void>;
|
|
1798
|
+
/**
|
|
1799
|
+
* Reclaim timed-out messages from a shard.
|
|
1800
|
+
* Returns messages to their original queues.
|
|
1801
|
+
*
|
|
1802
|
+
* @param shardId - The shard to check
|
|
1803
|
+
* @param getQueueKeys - Function to get queue keys for a queue ID
|
|
1804
|
+
* @returns Array of reclaimed message info for concurrency release
|
|
1805
|
+
*/
|
|
1806
|
+
reclaimTimedOut(shardId: number, getQueueKeys: (queueId: string) => {
|
|
1807
|
+
queueKey: string;
|
|
1808
|
+
queueItemsKey: string;
|
|
1809
|
+
tenantQueueIndexKey: string;
|
|
1810
|
+
dispatchKey: string;
|
|
1811
|
+
tenantId: string;
|
|
1812
|
+
}): Promise<ReclaimedMessageInfo[]>;
|
|
1813
|
+
/**
|
|
1814
|
+
* Get all in-flight messages for a shard.
|
|
1815
|
+
*/
|
|
1816
|
+
getInflightMessages(shardId: number): Promise<Array<{
|
|
1817
|
+
messageId: string;
|
|
1818
|
+
queueId: string;
|
|
1819
|
+
deadline: number;
|
|
1820
|
+
}>>;
|
|
1821
|
+
/**
|
|
1822
|
+
* Get count of in-flight messages for a shard.
|
|
1823
|
+
*/
|
|
1824
|
+
getInflightCount(shardId: number): Promise<number>;
|
|
1825
|
+
/**
|
|
1826
|
+
* Get total in-flight count across all shards.
|
|
1827
|
+
*/
|
|
1828
|
+
getTotalInflightCount(): Promise<number>;
|
|
1829
|
+
/**
|
|
1830
|
+
* Close the Redis connection.
|
|
1831
|
+
*/
|
|
1832
|
+
close(): Promise<void>;
|
|
1833
|
+
}
|
|
1834
|
+
declare module "@internal/redis" {
|
|
1835
|
+
interface RedisCommander<Context> {
|
|
1836
|
+
claimMessage(queueKey: string, queueItemsKey: string, inflightKey: string, inflightDataKey: string, queueId: string, consumerId: string, deadline: string): Promise<[string, string] | null>;
|
|
1837
|
+
claimMessageBatch(queueKey: string, queueItemsKey: string, inflightKey: string, inflightDataKey: string, queueId: string, deadline: string, maxCount: string): Promise<string[]>;
|
|
1838
|
+
releaseMessage(inflightKey: string, inflightDataKey: string, queueKey: string, queueItemsKey: string, tenantQueueIndexKey: string, dispatchKey: string, member: string, messageId: string, score: string, queueId: string, updatedData: string, tenantId: string): Promise<number>;
|
|
1839
|
+
releaseMessageBatch(inflightKey: string, inflightDataKey: string, queueKey: string, queueItemsKey: string, tenantQueueIndexKey: string, dispatchKey: string, score: string, queueId: string, tenantId: string, ...membersAndMessageIds: string[]): Promise<number>;
|
|
1840
|
+
heartbeatMessage(inflightKey: string, member: string, newDeadline: string): Promise<number>;
|
|
1841
|
+
}
|
|
1842
|
+
}
|
|
1843
|
+
|
|
1844
|
+
interface WorkerQueueManagerOptions {
|
|
1845
|
+
redis: RedisOptions;
|
|
1846
|
+
keys: FairQueueKeyProducer;
|
|
1847
|
+
logger?: {
|
|
1848
|
+
debug: (message: string, context?: Record<string, unknown>) => void;
|
|
1849
|
+
error: (message: string, context?: Record<string, unknown>) => void;
|
|
1850
|
+
};
|
|
1851
|
+
}
|
|
1852
|
+
/**
|
|
1853
|
+
* WorkerQueueManager handles the intermediate worker queue layer.
|
|
1854
|
+
*
|
|
1855
|
+
* This provides:
|
|
1856
|
+
* - Low-latency message delivery via blocking pop (BLPOP)
|
|
1857
|
+
* - Routing of messages to specific workers/consumers
|
|
1858
|
+
* - Efficient waiting without polling
|
|
1859
|
+
*
|
|
1860
|
+
* Flow:
|
|
1861
|
+
* 1. Master queue consumer claims message from message queue
|
|
1862
|
+
* 2. Message key is pushed to worker queue
|
|
1863
|
+
* 3. Worker queue consumer does blocking pop to receive message
|
|
1864
|
+
*/
|
|
1865
|
+
declare class WorkerQueueManager {
|
|
1866
|
+
#private;
|
|
1867
|
+
private options;
|
|
1868
|
+
private redis;
|
|
1869
|
+
private keys;
|
|
1870
|
+
private logger;
|
|
1871
|
+
constructor(options: WorkerQueueManagerOptions);
|
|
1872
|
+
/**
|
|
1873
|
+
* Push a message key to a worker queue.
|
|
1874
|
+
* Called after claiming a message from the message queue.
|
|
1875
|
+
*
|
|
1876
|
+
* @param workerQueueId - The worker queue identifier
|
|
1877
|
+
* @param messageKey - The message key to push (typically "messageId:queueId")
|
|
1878
|
+
*/
|
|
1879
|
+
push(workerQueueId: string, messageKey: string): Promise<void>;
|
|
1880
|
+
/**
|
|
1881
|
+
* Push multiple message keys to a worker queue.
|
|
1882
|
+
*
|
|
1883
|
+
* @param workerQueueId - The worker queue identifier
|
|
1884
|
+
* @param messageKeys - The message keys to push
|
|
1885
|
+
*/
|
|
1886
|
+
pushBatch(workerQueueId: string, messageKeys: string[]): Promise<void>;
|
|
1887
|
+
/**
|
|
1888
|
+
* Blocking pop from a worker queue.
|
|
1889
|
+
* Waits until a message is available or timeout expires.
|
|
1890
|
+
*
|
|
1891
|
+
* @param workerQueueId - The worker queue identifier
|
|
1892
|
+
* @param timeoutSeconds - Maximum time to wait (0 = wait forever)
|
|
1893
|
+
* @param signal - Optional abort signal to cancel waiting
|
|
1894
|
+
* @returns The message key, or null if timeout
|
|
1895
|
+
*/
|
|
1896
|
+
blockingPop(workerQueueId: string, timeoutSeconds: number, signal?: AbortSignal): Promise<string | null>;
|
|
1897
|
+
/**
|
|
1898
|
+
* Non-blocking pop from a worker queue.
|
|
1899
|
+
*
|
|
1900
|
+
* @param workerQueueId - The worker queue identifier
|
|
1901
|
+
* @returns The message key and queue length, or null if empty
|
|
1902
|
+
*/
|
|
1903
|
+
pop(workerQueueId: string): Promise<{
|
|
1904
|
+
messageKey: string;
|
|
1905
|
+
queueLength: number;
|
|
1906
|
+
} | null>;
|
|
1907
|
+
/**
|
|
1908
|
+
* Get the current length of a worker queue.
|
|
1909
|
+
*/
|
|
1910
|
+
getLength(workerQueueId: string): Promise<number>;
|
|
1911
|
+
/**
|
|
1912
|
+
* Peek at all messages in a worker queue without removing them.
|
|
1913
|
+
* Useful for debugging and tests.
|
|
1914
|
+
*/
|
|
1915
|
+
peek(workerQueueId: string): Promise<string[]>;
|
|
1916
|
+
/**
|
|
1917
|
+
* Remove a specific message from the worker queue.
|
|
1918
|
+
* Used when a message needs to be removed without processing.
|
|
1919
|
+
*
|
|
1920
|
+
* @param workerQueueId - The worker queue identifier
|
|
1921
|
+
* @param messageKey - The message key to remove
|
|
1922
|
+
* @returns Number of removed items
|
|
1923
|
+
*/
|
|
1924
|
+
remove(workerQueueId: string, messageKey: string): Promise<number>;
|
|
1925
|
+
/**
|
|
1926
|
+
* Clear all messages from a worker queue.
|
|
1927
|
+
*/
|
|
1928
|
+
clear(workerQueueId: string): Promise<void>;
|
|
1929
|
+
/**
|
|
1930
|
+
* Close the Redis connection.
|
|
1931
|
+
*/
|
|
1932
|
+
close(): Promise<void>;
|
|
1933
|
+
/**
|
|
1934
|
+
* Register custom commands on an external Redis client.
|
|
1935
|
+
* Use this when initializing FairQueue with worker queues.
|
|
1936
|
+
*/
|
|
1937
|
+
registerCommands(redis: Redis): void;
|
|
1938
|
+
}
|
|
1939
|
+
declare module "@internal/redis" {
|
|
1940
|
+
interface RedisCommander<Context> {
|
|
1941
|
+
popWithLength(workerQueueKey: string): Promise<[string, string] | null>;
|
|
1942
|
+
}
|
|
1943
|
+
}
|
|
1944
|
+
|
|
1945
1945
|
/**
|
|
1946
1946
|
* FairQueue is the main orchestrator for fair queue message routing.
|
|
1947
1947
|
*
|