@trigger.dev/redis-worker 0.0.0-prerelease-20251205122901 → 0.0.0-prerelease-20251209103333

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.cjs CHANGED
@@ -4,18 +4,21 @@ var process2 = require('process');
4
4
  var os = require('os');
5
5
  var tty = require('tty');
6
6
  var logger$1 = require('@trigger.dev/core/logger');
7
- var crypto = require('crypto');
7
+ var crypto$1 = require('crypto');
8
8
  require('@trigger.dev/core/v3/utils/flattenAttributes');
9
9
  var v3 = require('@trigger.dev/core/v3');
10
10
  var serverOnly = require('@trigger.dev/core/v3/serverOnly');
11
11
  var zod = require('zod');
12
12
  var cronParser = require('cron-parser');
13
+ var promises = require('timers/promises');
14
+ var seedrandom = require('seedrandom');
13
15
 
14
16
  function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; }
15
17
 
16
18
  var process2__default = /*#__PURE__*/_interopDefault(process2);
17
19
  var os__default = /*#__PURE__*/_interopDefault(os);
18
20
  var tty__default = /*#__PURE__*/_interopDefault(tty);
21
+ var seedrandom__default = /*#__PURE__*/_interopDefault(seedrandom);
19
22
 
20
23
  var __create = Object.create;
21
24
  var __defProp = Object.defineProperty;
@@ -9419,10 +9422,10 @@ var poolOffset;
9419
9422
  function fillPool(bytes) {
9420
9423
  if (!pool || pool.length < bytes) {
9421
9424
  pool = Buffer.allocUnsafe(bytes * POOL_SIZE_MULTIPLIER);
9422
- crypto.webcrypto.getRandomValues(pool);
9425
+ crypto$1.webcrypto.getRandomValues(pool);
9423
9426
  poolOffset = 0;
9424
9427
  } else if (poolOffset + bytes > pool.length) {
9425
- crypto.webcrypto.getRandomValues(pool);
9428
+ crypto$1.webcrypto.getRandomValues(pool);
9426
9429
  poolOffset = 0;
9427
9430
  }
9428
9431
  poolOffset += bytes;
@@ -11727,8 +11730,3316 @@ var Worker = class _Worker {
11727
11730
  }
11728
11731
  };
11729
11732
 
11733
+ // src/fair-queue/concurrency.ts
11734
+ var ConcurrencyManager = class {
11735
+ constructor(options) {
11736
+ this.options = options;
11737
+ this.redis = createRedisClient(options.redis);
11738
+ this.keys = options.keys;
11739
+ this.groups = options.groups;
11740
+ this.groupsByName = new Map(options.groups.map((g) => [g.name, g]));
11741
+ this.#registerCommands();
11742
+ }
11743
+ redis;
11744
+ keys;
11745
+ groups;
11746
+ groupsByName;
11747
+ // ============================================================================
11748
+ // Public Methods
11749
+ // ============================================================================
11750
+ /**
11751
+ * Check if a message can be processed given all concurrency constraints.
11752
+ * Checks all configured groups and returns the first one at capacity.
11753
+ */
11754
+ async canProcess(queue) {
11755
+ for (const group of this.groups) {
11756
+ const groupId = group.extractGroupId(queue);
11757
+ const isAtCapacity = await this.isAtCapacity(group.name, groupId);
11758
+ if (isAtCapacity) {
11759
+ const state = await this.getState(group.name, groupId);
11760
+ return {
11761
+ allowed: false,
11762
+ blockedBy: state
11763
+ };
11764
+ }
11765
+ }
11766
+ return { allowed: true };
11767
+ }
11768
+ /**
11769
+ * Reserve concurrency slots for a message across all groups.
11770
+ * Atomic - either all groups are reserved or none.
11771
+ *
11772
+ * @returns true if reservation successful, false if any group is at capacity
11773
+ */
11774
+ async reserve(queue, messageId) {
11775
+ const groupData = await Promise.all(
11776
+ this.groups.map(async (group) => {
11777
+ const groupId = group.extractGroupId(queue);
11778
+ const limit = await group.getLimit(groupId);
11779
+ return {
11780
+ key: this.keys.concurrencyKey(group.name, groupId),
11781
+ limit: limit || group.defaultLimit
11782
+ };
11783
+ })
11784
+ );
11785
+ const keys = groupData.map((g) => g.key);
11786
+ const limits = groupData.map((g) => g.limit.toString());
11787
+ const result = await this.redis.reserveConcurrency(
11788
+ keys.length.toString(),
11789
+ messageId,
11790
+ ...keys,
11791
+ ...limits
11792
+ );
11793
+ return result === 1;
11794
+ }
11795
+ /**
11796
+ * Release concurrency slots for a message across all groups.
11797
+ */
11798
+ async release(queue, messageId) {
11799
+ const pipeline = this.redis.pipeline();
11800
+ for (const group of this.groups) {
11801
+ const groupId = group.extractGroupId(queue);
11802
+ const key = this.keys.concurrencyKey(group.name, groupId);
11803
+ pipeline.srem(key, messageId);
11804
+ }
11805
+ await pipeline.exec();
11806
+ }
11807
+ /**
11808
+ * Get current concurrency for a specific group.
11809
+ */
11810
+ async getCurrentConcurrency(groupName, groupId) {
11811
+ const key = this.keys.concurrencyKey(groupName, groupId);
11812
+ return await this.redis.scard(key);
11813
+ }
11814
+ /**
11815
+ * Get concurrency limit for a specific group.
11816
+ */
11817
+ async getConcurrencyLimit(groupName, groupId) {
11818
+ const group = this.groupsByName.get(groupName);
11819
+ if (!group) {
11820
+ throw new Error(`Unknown concurrency group: ${groupName}`);
11821
+ }
11822
+ return await group.getLimit(groupId) || group.defaultLimit;
11823
+ }
11824
+ /**
11825
+ * Check if a group is at capacity.
11826
+ */
11827
+ async isAtCapacity(groupName, groupId) {
11828
+ const [current, limit] = await Promise.all([
11829
+ this.getCurrentConcurrency(groupName, groupId),
11830
+ this.getConcurrencyLimit(groupName, groupId)
11831
+ ]);
11832
+ return current >= limit;
11833
+ }
11834
+ /**
11835
+ * Get full state for a group.
11836
+ */
11837
+ async getState(groupName, groupId) {
11838
+ const [current, limit] = await Promise.all([
11839
+ this.getCurrentConcurrency(groupName, groupId),
11840
+ this.getConcurrencyLimit(groupName, groupId)
11841
+ ]);
11842
+ return {
11843
+ groupName,
11844
+ groupId,
11845
+ current,
11846
+ limit
11847
+ };
11848
+ }
11849
+ /**
11850
+ * Get all active message IDs for a group.
11851
+ */
11852
+ async getActiveMessages(groupName, groupId) {
11853
+ const key = this.keys.concurrencyKey(groupName, groupId);
11854
+ return await this.redis.smembers(key);
11855
+ }
11856
+ /**
11857
+ * Force-clear concurrency for a group (use with caution).
11858
+ * Useful for cleanup after crashes.
11859
+ */
11860
+ async clearGroup(groupName, groupId) {
11861
+ const key = this.keys.concurrencyKey(groupName, groupId);
11862
+ await this.redis.del(key);
11863
+ }
11864
+ /**
11865
+ * Remove a specific message from concurrency tracking.
11866
+ * Useful for cleanup.
11867
+ */
11868
+ async removeMessage(messageId, queue) {
11869
+ await this.release(queue, messageId);
11870
+ }
11871
+ /**
11872
+ * Get configured group names.
11873
+ */
11874
+ getGroupNames() {
11875
+ return this.groups.map((g) => g.name);
11876
+ }
11877
+ /**
11878
+ * Close the Redis connection.
11879
+ */
11880
+ async close() {
11881
+ await this.redis.quit();
11882
+ }
11883
+ // ============================================================================
11884
+ // Private Methods
11885
+ // ============================================================================
11886
+ #registerCommands() {
11887
+ this.redis.defineCommand("reserveConcurrency", {
11888
+ numberOfKeys: 0,
11889
+ // Will pass number of keys in ARGV
11890
+ lua: `
11891
+ local numGroups = tonumber(ARGV[1])
11892
+ local messageId = ARGV[2]
11893
+
11894
+ -- Check all groups first
11895
+ for i = 1, numGroups do
11896
+ local key = ARGV[2 + i] -- Keys start at ARGV[3]
11897
+ local limit = tonumber(ARGV[2 + numGroups + i]) -- Limits come after keys
11898
+ local current = redis.call('SCARD', key)
11899
+
11900
+ if current >= limit then
11901
+ return 0 -- At capacity
11902
+ end
11903
+ end
11904
+
11905
+ -- All groups have capacity, add message to all
11906
+ for i = 1, numGroups do
11907
+ local key = ARGV[2 + i]
11908
+ redis.call('SADD', key, messageId)
11909
+ end
11910
+
11911
+ return 1
11912
+ `
11913
+ });
11914
+ }
11915
+ };
11916
+ var MasterQueue = class {
11917
+ constructor(options) {
11918
+ this.options = options;
11919
+ this.redis = createRedisClient(options.redis);
11920
+ this.keys = options.keys;
11921
+ this.shardCount = Math.max(1, options.shardCount);
11922
+ this.#registerCommands();
11923
+ }
11924
+ redis;
11925
+ keys;
11926
+ shardCount;
11927
+ // ============================================================================
11928
+ // Public Methods
11929
+ // ============================================================================
11930
+ /**
11931
+ * Get the shard ID for a queue.
11932
+ * Uses consistent hashing based on queue ID.
11933
+ */
11934
+ getShardForQueue(queueId) {
11935
+ return this.#hashToShard(queueId);
11936
+ }
11937
+ /**
11938
+ * Add a queue to its master queue shard.
11939
+ * Updates the score to the oldest message timestamp.
11940
+ *
11941
+ * @param queueId - The queue identifier
11942
+ * @param oldestMessageTimestamp - Timestamp of the oldest message in the queue
11943
+ */
11944
+ async addQueue(queueId, oldestMessageTimestamp) {
11945
+ const shardId = this.getShardForQueue(queueId);
11946
+ const masterKey = this.keys.masterQueueKey(shardId);
11947
+ await this.redis.zadd(masterKey, oldestMessageTimestamp, queueId);
11948
+ }
11949
+ /**
11950
+ * Update a queue's score in the master queue.
11951
+ * This is typically called after dequeuing to update to the new oldest message.
11952
+ *
11953
+ * @param queueId - The queue identifier
11954
+ * @param newOldestTimestamp - New timestamp of the oldest message
11955
+ */
11956
+ async updateQueueScore(queueId, newOldestTimestamp) {
11957
+ const shardId = this.getShardForQueue(queueId);
11958
+ const masterKey = this.keys.masterQueueKey(shardId);
11959
+ await this.redis.zadd(masterKey, newOldestTimestamp, queueId);
11960
+ }
11961
+ /**
11962
+ * Remove a queue from its master queue shard.
11963
+ * Called when a queue becomes empty.
11964
+ *
11965
+ * @param queueId - The queue identifier
11966
+ */
11967
+ async removeQueue(queueId) {
11968
+ const shardId = this.getShardForQueue(queueId);
11969
+ const masterKey = this.keys.masterQueueKey(shardId);
11970
+ await this.redis.zrem(masterKey, queueId);
11971
+ }
11972
+ /**
11973
+ * Get queues from a shard, ordered by oldest message (lowest score first).
11974
+ *
11975
+ * @param shardId - The shard to query
11976
+ * @param limit - Maximum number of queues to return (default: 1000)
11977
+ * @param maxScore - Maximum score (timestamp) to include (default: now)
11978
+ */
11979
+ async getQueuesFromShard(shardId, limit = 1e3, maxScore) {
11980
+ const masterKey = this.keys.masterQueueKey(shardId);
11981
+ const score = maxScore ?? Date.now();
11982
+ const results = await this.redis.zrangebyscore(
11983
+ masterKey,
11984
+ "-inf",
11985
+ score,
11986
+ "WITHSCORES",
11987
+ "LIMIT",
11988
+ 0,
11989
+ limit
11990
+ );
11991
+ const queues = [];
11992
+ for (let i = 0; i < results.length; i += 2) {
11993
+ const queueId = results[i];
11994
+ const scoreStr = results[i + 1];
11995
+ if (queueId && scoreStr) {
11996
+ queues.push({
11997
+ queueId,
11998
+ score: parseFloat(scoreStr),
11999
+ tenantId: this.keys.extractTenantId(queueId)
12000
+ });
12001
+ }
12002
+ }
12003
+ return queues;
12004
+ }
12005
+ /**
12006
+ * Get the number of queues in a shard.
12007
+ */
12008
+ async getShardQueueCount(shardId) {
12009
+ const masterKey = this.keys.masterQueueKey(shardId);
12010
+ return await this.redis.zcard(masterKey);
12011
+ }
12012
+ /**
12013
+ * Get total queue count across all shards.
12014
+ */
12015
+ async getTotalQueueCount() {
12016
+ const counts = await Promise.all(
12017
+ Array.from({ length: this.shardCount }, (_, i) => this.getShardQueueCount(i))
12018
+ );
12019
+ return counts.reduce((sum, count) => sum + count, 0);
12020
+ }
12021
+ /**
12022
+ * Atomically add a queue to master queue only if queue has messages.
12023
+ * Uses Lua script for atomicity.
12024
+ *
12025
+ * @param queueId - The queue identifier
12026
+ * @param queueKey - The actual queue sorted set key
12027
+ * @returns Whether the queue was added to the master queue
12028
+ */
12029
+ async addQueueIfNotEmpty(queueId, queueKey) {
12030
+ const shardId = this.getShardForQueue(queueId);
12031
+ const masterKey = this.keys.masterQueueKey(shardId);
12032
+ const result = await this.redis.addQueueIfNotEmpty(masterKey, queueKey, queueId);
12033
+ return result === 1;
12034
+ }
12035
+ /**
12036
+ * Atomically remove a queue from master queue only if queue is empty.
12037
+ * Uses Lua script for atomicity.
12038
+ *
12039
+ * @param queueId - The queue identifier
12040
+ * @param queueKey - The actual queue sorted set key
12041
+ * @returns Whether the queue was removed from the master queue
12042
+ */
12043
+ async removeQueueIfEmpty(queueId, queueKey) {
12044
+ const shardId = this.getShardForQueue(queueId);
12045
+ const masterKey = this.keys.masterQueueKey(shardId);
12046
+ const result = await this.redis.removeQueueIfEmpty(masterKey, queueKey, queueId);
12047
+ return result === 1;
12048
+ }
12049
+ /**
12050
+ * Close the Redis connection.
12051
+ */
12052
+ async close() {
12053
+ await this.redis.quit();
12054
+ }
12055
+ // ============================================================================
12056
+ // Private Methods
12057
+ // ============================================================================
12058
+ /**
12059
+ * Map queue ID to shard using Jump Consistent Hash.
12060
+ * Provides better distribution than djb2 and minimal reshuffling when shard count changes.
12061
+ */
12062
+ #hashToShard(queueId) {
12063
+ return serverOnly.jumpHash(queueId, this.shardCount);
12064
+ }
12065
+ #registerCommands() {
12066
+ this.redis.defineCommand("addQueueIfNotEmpty", {
12067
+ numberOfKeys: 2,
12068
+ lua: `
12069
+ local masterKey = KEYS[1]
12070
+ local queueKey = KEYS[2]
12071
+ local queueId = ARGV[1]
12072
+
12073
+ -- Check if queue has any messages
12074
+ local count = redis.call('ZCARD', queueKey)
12075
+ if count == 0 then
12076
+ return 0
12077
+ end
12078
+
12079
+ -- Get the oldest message timestamp (lowest score)
12080
+ local oldest = redis.call('ZRANGE', queueKey, 0, 0, 'WITHSCORES')
12081
+ if #oldest == 0 then
12082
+ return 0
12083
+ end
12084
+
12085
+ local score = oldest[2]
12086
+
12087
+ -- Add to master queue with the oldest message score
12088
+ redis.call('ZADD', masterKey, score, queueId)
12089
+ return 1
12090
+ `
12091
+ });
12092
+ this.redis.defineCommand("removeQueueIfEmpty", {
12093
+ numberOfKeys: 2,
12094
+ lua: `
12095
+ local masterKey = KEYS[1]
12096
+ local queueKey = KEYS[2]
12097
+ local queueId = ARGV[1]
12098
+
12099
+ -- Check if queue is empty
12100
+ local count = redis.call('ZCARD', queueKey)
12101
+ if count > 0 then
12102
+ return 0
12103
+ end
12104
+
12105
+ -- Remove from master queue
12106
+ redis.call('ZREM', masterKey, queueId)
12107
+ return 1
12108
+ `
12109
+ });
12110
+ }
12111
+ };
12112
+
12113
+ // src/fair-queue/telemetry.ts
12114
+ var FairQueueAttributes = {
12115
+ QUEUE_ID: "fairqueue.queue_id",
12116
+ TENANT_ID: "fairqueue.tenant_id",
12117
+ MESSAGE_ID: "fairqueue.message_id",
12118
+ SHARD_ID: "fairqueue.shard_id",
12119
+ WORKER_QUEUE: "fairqueue.worker_queue",
12120
+ CONSUMER_ID: "fairqueue.consumer_id",
12121
+ ATTEMPT: "fairqueue.attempt",
12122
+ CONCURRENCY_GROUP: "fairqueue.concurrency_group",
12123
+ MESSAGE_COUNT: "fairqueue.message_count",
12124
+ RESULT: "fairqueue.result"
12125
+ };
12126
+ var MessagingAttributes = {
12127
+ SYSTEM: "messaging.system",
12128
+ OPERATION: "messaging.operation",
12129
+ MESSAGE_ID: "messaging.message_id",
12130
+ DESTINATION_NAME: "messaging.destination.name"
12131
+ };
12132
+ var FairQueueTelemetry = class {
12133
+ tracer;
12134
+ meter;
12135
+ metrics;
12136
+ name;
12137
+ constructor(options) {
12138
+ this.tracer = options.tracer;
12139
+ this.meter = options.meter;
12140
+ this.name = options.name ?? "fairqueue";
12141
+ if (this.meter) {
12142
+ this.#initializeMetrics();
12143
+ }
12144
+ }
12145
+ // ============================================================================
12146
+ // Tracing
12147
+ // ============================================================================
12148
+ /**
12149
+ * Create a traced span for an operation.
12150
+ * Returns the result of the function, or throws any error after recording it.
12151
+ */
12152
+ async trace(name, fn, options) {
12153
+ if (!this.tracer) {
12154
+ return fn(noopSpan);
12155
+ }
12156
+ const spanOptions = {
12157
+ kind: options?.kind,
12158
+ attributes: {
12159
+ [MessagingAttributes.SYSTEM]: this.name,
12160
+ ...options?.attributes
12161
+ }
12162
+ };
12163
+ return this.tracer.startActiveSpan(`${this.name}.${name}`, spanOptions, async (span) => {
12164
+ try {
12165
+ const result = await fn(span);
12166
+ return result;
12167
+ } catch (error) {
12168
+ if (error instanceof Error) {
12169
+ span.recordException(error);
12170
+ } else {
12171
+ span.recordException(new Error(String(error)));
12172
+ }
12173
+ throw error;
12174
+ } finally {
12175
+ span.end();
12176
+ }
12177
+ });
12178
+ }
12179
+ /**
12180
+ * Synchronous version of trace.
12181
+ */
12182
+ traceSync(name, fn, options) {
12183
+ if (!this.tracer) {
12184
+ return fn(noopSpan);
12185
+ }
12186
+ const spanOptions = {
12187
+ kind: options?.kind,
12188
+ attributes: {
12189
+ [MessagingAttributes.SYSTEM]: this.name,
12190
+ ...options?.attributes
12191
+ }
12192
+ };
12193
+ return this.tracer.startActiveSpan(`${this.name}.${name}`, spanOptions, (span) => {
12194
+ try {
12195
+ return fn(span);
12196
+ } catch (error) {
12197
+ if (error instanceof Error) {
12198
+ span.recordException(error);
12199
+ } else {
12200
+ span.recordException(new Error(String(error)));
12201
+ }
12202
+ throw error;
12203
+ } finally {
12204
+ span.end();
12205
+ }
12206
+ });
12207
+ }
12208
+ // ============================================================================
12209
+ // Metrics
12210
+ // ============================================================================
12211
+ /**
12212
+ * Record a message enqueued.
12213
+ */
12214
+ recordEnqueue(attributes) {
12215
+ this.metrics?.messagesEnqueued.add(1, attributes);
12216
+ }
12217
+ /**
12218
+ * Record a batch of messages enqueued.
12219
+ */
12220
+ recordEnqueueBatch(count, attributes) {
12221
+ this.metrics?.messagesEnqueued.add(count, attributes);
12222
+ }
12223
+ /**
12224
+ * Record a message completed successfully.
12225
+ */
12226
+ recordComplete(attributes) {
12227
+ this.metrics?.messagesCompleted.add(1, attributes);
12228
+ }
12229
+ /**
12230
+ * Record a message processing failure.
12231
+ */
12232
+ recordFailure(attributes) {
12233
+ this.metrics?.messagesFailed.add(1, attributes);
12234
+ }
12235
+ /**
12236
+ * Record a message retry.
12237
+ */
12238
+ recordRetry(attributes) {
12239
+ this.metrics?.messagesRetried.add(1, attributes);
12240
+ }
12241
+ /**
12242
+ * Record a message sent to DLQ.
12243
+ */
12244
+ recordDLQ(attributes) {
12245
+ this.metrics?.messagesToDLQ.add(1, attributes);
12246
+ }
12247
+ /**
12248
+ * Record message processing time.
12249
+ *
12250
+ * @param durationMs - Processing duration in milliseconds
12251
+ */
12252
+ recordProcessingTime(durationMs, attributes) {
12253
+ this.metrics?.processingTime.record(durationMs, attributes);
12254
+ }
12255
+ /**
12256
+ * Record time a message spent waiting in queue.
12257
+ *
12258
+ * @param durationMs - Queue wait time in milliseconds
12259
+ */
12260
+ recordQueueTime(durationMs, attributes) {
12261
+ this.metrics?.queueTime.record(durationMs, attributes);
12262
+ }
12263
+ /**
12264
+ * Register observable gauge callbacks.
12265
+ * Call this after FairQueue is initialized to register the gauge callbacks.
12266
+ */
12267
+ registerGaugeCallbacks(callbacks) {
12268
+ if (!this.metrics) return;
12269
+ if (callbacks.getQueueLength && callbacks.observedQueues) {
12270
+ const getQueueLength = callbacks.getQueueLength;
12271
+ const queues = callbacks.observedQueues;
12272
+ this.metrics.queueLength.addCallback(async (observableResult) => {
12273
+ for (const queueId of queues) {
12274
+ const length = await getQueueLength(queueId);
12275
+ observableResult.observe(length, {
12276
+ [FairQueueAttributes.QUEUE_ID]: queueId
12277
+ });
12278
+ }
12279
+ });
12280
+ }
12281
+ if (callbacks.getMasterQueueLength && callbacks.shardCount) {
12282
+ const getMasterQueueLength = callbacks.getMasterQueueLength;
12283
+ const shardCount = callbacks.shardCount;
12284
+ this.metrics.masterQueueLength.addCallback(async (observableResult) => {
12285
+ for (let shardId = 0; shardId < shardCount; shardId++) {
12286
+ const length = await getMasterQueueLength(shardId);
12287
+ observableResult.observe(length, {
12288
+ [FairQueueAttributes.SHARD_ID]: shardId.toString()
12289
+ });
12290
+ }
12291
+ });
12292
+ }
12293
+ if (callbacks.getInflightCount && callbacks.shardCount) {
12294
+ const getInflightCount = callbacks.getInflightCount;
12295
+ const shardCount = callbacks.shardCount;
12296
+ this.metrics.inflightCount.addCallback(async (observableResult) => {
12297
+ for (let shardId = 0; shardId < shardCount; shardId++) {
12298
+ const count = await getInflightCount(shardId);
12299
+ observableResult.observe(count, {
12300
+ [FairQueueAttributes.SHARD_ID]: shardId.toString()
12301
+ });
12302
+ }
12303
+ });
12304
+ }
12305
+ if (callbacks.getDLQLength && callbacks.observedTenants) {
12306
+ const getDLQLength = callbacks.getDLQLength;
12307
+ const tenants = callbacks.observedTenants;
12308
+ this.metrics.dlqLength.addCallback(async (observableResult) => {
12309
+ for (const tenantId of tenants) {
12310
+ const length = await getDLQLength(tenantId);
12311
+ observableResult.observe(length, {
12312
+ [FairQueueAttributes.TENANT_ID]: tenantId
12313
+ });
12314
+ }
12315
+ });
12316
+ }
12317
+ }
12318
+ // ============================================================================
12319
+ // Helper Methods
12320
+ // ============================================================================
12321
+ /**
12322
+ * Create standard attributes for a message operation.
12323
+ */
12324
+ messageAttributes(params) {
12325
+ const attrs = {};
12326
+ if (params.queueId) attrs[FairQueueAttributes.QUEUE_ID] = params.queueId;
12327
+ if (params.tenantId) attrs[FairQueueAttributes.TENANT_ID] = params.tenantId;
12328
+ if (params.messageId) attrs[FairQueueAttributes.MESSAGE_ID] = params.messageId;
12329
+ if (params.attempt !== void 0) attrs[FairQueueAttributes.ATTEMPT] = params.attempt;
12330
+ if (params.workerQueue) attrs[FairQueueAttributes.WORKER_QUEUE] = params.workerQueue;
12331
+ if (params.consumerId) attrs[FairQueueAttributes.CONSUMER_ID] = params.consumerId;
12332
+ return attrs;
12333
+ }
12334
+ /**
12335
+ * Check if telemetry is enabled.
12336
+ */
12337
+ get isEnabled() {
12338
+ return !!this.tracer || !!this.meter;
12339
+ }
12340
+ /**
12341
+ * Check if tracing is enabled.
12342
+ */
12343
+ get hasTracer() {
12344
+ return !!this.tracer;
12345
+ }
12346
+ /**
12347
+ * Check if metrics are enabled.
12348
+ */
12349
+ get hasMetrics() {
12350
+ return !!this.meter;
12351
+ }
12352
+ // ============================================================================
12353
+ // Private Methods
12354
+ // ============================================================================
12355
+ #initializeMetrics() {
12356
+ if (!this.meter) return;
12357
+ this.metrics = {
12358
+ // Counters
12359
+ messagesEnqueued: this.meter.createCounter(`${this.name}.messages.enqueued`, {
12360
+ description: "Number of messages enqueued",
12361
+ unit: "messages"
12362
+ }),
12363
+ messagesCompleted: this.meter.createCounter(`${this.name}.messages.completed`, {
12364
+ description: "Number of messages completed successfully",
12365
+ unit: "messages"
12366
+ }),
12367
+ messagesFailed: this.meter.createCounter(`${this.name}.messages.failed`, {
12368
+ description: "Number of messages that failed processing",
12369
+ unit: "messages"
12370
+ }),
12371
+ messagesRetried: this.meter.createCounter(`${this.name}.messages.retried`, {
12372
+ description: "Number of message retries",
12373
+ unit: "messages"
12374
+ }),
12375
+ messagesToDLQ: this.meter.createCounter(`${this.name}.messages.dlq`, {
12376
+ description: "Number of messages sent to dead letter queue",
12377
+ unit: "messages"
12378
+ }),
12379
+ // Histograms
12380
+ processingTime: this.meter.createHistogram(`${this.name}.message.processing_time`, {
12381
+ description: "Message processing time",
12382
+ unit: "ms"
12383
+ }),
12384
+ queueTime: this.meter.createHistogram(`${this.name}.message.queue_time`, {
12385
+ description: "Time message spent waiting in queue",
12386
+ unit: "ms"
12387
+ }),
12388
+ // Observable gauges
12389
+ queueLength: this.meter.createObservableGauge(`${this.name}.queue.length`, {
12390
+ description: "Number of messages in a queue",
12391
+ unit: "messages"
12392
+ }),
12393
+ masterQueueLength: this.meter.createObservableGauge(`${this.name}.master_queue.length`, {
12394
+ description: "Number of queues in master queue shard",
12395
+ unit: "queues"
12396
+ }),
12397
+ inflightCount: this.meter.createObservableGauge(`${this.name}.inflight.count`, {
12398
+ description: "Number of messages currently being processed",
12399
+ unit: "messages"
12400
+ }),
12401
+ dlqLength: this.meter.createObservableGauge(`${this.name}.dlq.length`, {
12402
+ description: "Number of messages in dead letter queue",
12403
+ unit: "messages"
12404
+ })
12405
+ };
12406
+ }
12407
+ };
12408
+ var noopSpan = {
12409
+ spanContext: () => ({
12410
+ traceId: "",
12411
+ spanId: "",
12412
+ traceFlags: 0
12413
+ }),
12414
+ setAttribute: () => noopSpan,
12415
+ setAttributes: () => noopSpan,
12416
+ addEvent: () => noopSpan,
12417
+ addLink: () => noopSpan,
12418
+ addLinks: () => noopSpan,
12419
+ setStatus: () => noopSpan,
12420
+ updateName: () => noopSpan,
12421
+ end: () => {
12422
+ },
12423
+ isRecording: () => false,
12424
+ recordException: () => {
12425
+ }
12426
+ };
12427
+ var noopTelemetry = new FairQueueTelemetry({});
12428
+ var VisibilityManager = class {
12429
+ constructor(options) {
12430
+ this.options = options;
12431
+ this.redis = createRedisClient(options.redis);
12432
+ this.keys = options.keys;
12433
+ this.shardCount = options.shardCount;
12434
+ this.defaultTimeoutMs = options.defaultTimeoutMs;
12435
+ this.logger = options.logger ?? {
12436
+ debug: () => {
12437
+ },
12438
+ error: () => {
12439
+ }
12440
+ };
12441
+ this.#registerCommands();
12442
+ }
12443
+ redis;
12444
+ keys;
12445
+ shardCount;
12446
+ defaultTimeoutMs;
12447
+ logger;
12448
+ // ============================================================================
12449
+ // Public Methods
12450
+ // ============================================================================
12451
+ /**
12452
+ * Claim a message for processing.
12453
+ * Moves the message from its queue to the in-flight set with a visibility timeout.
12454
+ *
12455
+ * @param queueId - The queue to claim from
12456
+ * @param queueKey - The Redis key for the queue sorted set
12457
+ * @param queueItemsKey - The Redis key for the queue items hash
12458
+ * @param consumerId - ID of the consumer claiming the message
12459
+ * @param timeoutMs - Visibility timeout in milliseconds
12460
+ * @returns Claim result with the message if successful
12461
+ */
12462
+ async claim(queueId, queueKey, queueItemsKey, consumerId, timeoutMs) {
12463
+ const timeout = timeoutMs ?? this.defaultTimeoutMs;
12464
+ const deadline = Date.now() + timeout;
12465
+ const shardId = this.#getShardForQueue(queueId);
12466
+ const inflightKey = this.keys.inflightKey(shardId);
12467
+ const inflightDataKey = this.keys.inflightDataKey(shardId);
12468
+ const result = await this.redis.claimMessage(
12469
+ queueKey,
12470
+ queueItemsKey,
12471
+ inflightKey,
12472
+ inflightDataKey,
12473
+ queueId,
12474
+ consumerId,
12475
+ deadline.toString()
12476
+ );
12477
+ if (!result) {
12478
+ return { claimed: false };
12479
+ }
12480
+ const [messageId, payloadJson] = result;
12481
+ try {
12482
+ const payload = JSON.parse(payloadJson);
12483
+ const message = {
12484
+ messageId,
12485
+ queueId,
12486
+ payload,
12487
+ deadline,
12488
+ consumerId
12489
+ };
12490
+ this.logger.debug("Message claimed", {
12491
+ messageId,
12492
+ queueId,
12493
+ consumerId,
12494
+ deadline
12495
+ });
12496
+ return { claimed: true, message };
12497
+ } catch (error) {
12498
+ this.logger.error("Failed to parse claimed message", {
12499
+ messageId,
12500
+ queueId,
12501
+ error: error instanceof Error ? error.message : String(error)
12502
+ });
12503
+ await this.#removeFromInflight(shardId, messageId, queueId);
12504
+ return { claimed: false };
12505
+ }
12506
+ }
12507
+ /**
12508
+ * Extend the visibility timeout for a message (heartbeat).
12509
+ *
12510
+ * @param messageId - The message ID
12511
+ * @param queueId - The queue ID
12512
+ * @param extendMs - Additional milliseconds to add to the deadline
12513
+ * @returns true if the heartbeat was successful
12514
+ */
12515
+ async heartbeat(messageId, queueId, extendMs) {
12516
+ const shardId = this.#getShardForQueue(queueId);
12517
+ const inflightKey = this.keys.inflightKey(shardId);
12518
+ const member = this.#makeMember(messageId, queueId);
12519
+ const newDeadline = Date.now() + extendMs;
12520
+ const result = await this.redis.zadd(inflightKey, "XX", newDeadline, member);
12521
+ const success = result !== 0;
12522
+ if (success) {
12523
+ this.logger.debug("Heartbeat successful", {
12524
+ messageId,
12525
+ queueId,
12526
+ newDeadline
12527
+ });
12528
+ }
12529
+ return success;
12530
+ }
12531
+ /**
12532
+ * Mark a message as successfully processed.
12533
+ * Removes the message from in-flight tracking.
12534
+ *
12535
+ * @param messageId - The message ID
12536
+ * @param queueId - The queue ID
12537
+ */
12538
+ async complete(messageId, queueId) {
12539
+ const shardId = this.#getShardForQueue(queueId);
12540
+ await this.#removeFromInflight(shardId, messageId, queueId);
12541
+ this.logger.debug("Message completed", {
12542
+ messageId,
12543
+ queueId
12544
+ });
12545
+ }
12546
+ /**
12547
+ * Release a message back to its queue.
12548
+ * Used when processing fails or consumer wants to retry later.
12549
+ *
12550
+ * @param messageId - The message ID
12551
+ * @param queueId - The queue ID
12552
+ * @param queueKey - The Redis key for the queue
12553
+ * @param queueItemsKey - The Redis key for the queue items hash
12554
+ * @param score - Optional score for the message (defaults to now)
12555
+ */
12556
+ async release(messageId, queueId, queueKey, queueItemsKey, score) {
12557
+ const shardId = this.#getShardForQueue(queueId);
12558
+ const inflightKey = this.keys.inflightKey(shardId);
12559
+ const inflightDataKey = this.keys.inflightDataKey(shardId);
12560
+ const member = this.#makeMember(messageId, queueId);
12561
+ const messageScore = score ?? Date.now();
12562
+ await this.redis.releaseMessage(
12563
+ inflightKey,
12564
+ inflightDataKey,
12565
+ queueKey,
12566
+ queueItemsKey,
12567
+ member,
12568
+ messageId,
12569
+ messageScore.toString()
12570
+ );
12571
+ this.logger.debug("Message released", {
12572
+ messageId,
12573
+ queueId,
12574
+ score: messageScore
12575
+ });
12576
+ }
12577
+ /**
12578
+ * Reclaim timed-out messages from a shard.
12579
+ * Returns messages to their original queues.
12580
+ *
12581
+ * @param shardId - The shard to check
12582
+ * @param getQueueKeys - Function to get queue keys for a queue ID
12583
+ * @returns Number of messages reclaimed
12584
+ */
12585
+ async reclaimTimedOut(shardId, getQueueKeys) {
12586
+ const inflightKey = this.keys.inflightKey(shardId);
12587
+ const inflightDataKey = this.keys.inflightDataKey(shardId);
12588
+ const now = Date.now();
12589
+ const timedOut = await this.redis.zrangebyscore(
12590
+ inflightKey,
12591
+ "-inf",
12592
+ now,
12593
+ "WITHSCORES",
12594
+ "LIMIT",
12595
+ 0,
12596
+ 100
12597
+ // Process in batches
12598
+ );
12599
+ let reclaimed = 0;
12600
+ for (let i = 0; i < timedOut.length; i += 2) {
12601
+ const member = timedOut[i];
12602
+ const originalScore = timedOut[i + 1];
12603
+ if (!member || !originalScore) {
12604
+ continue;
12605
+ }
12606
+ const { messageId, queueId } = this.#parseMember(member);
12607
+ const { queueKey, queueItemsKey } = getQueueKeys(queueId);
12608
+ try {
12609
+ const score = parseFloat(originalScore) || now;
12610
+ await this.redis.releaseMessage(
12611
+ inflightKey,
12612
+ inflightDataKey,
12613
+ queueKey,
12614
+ queueItemsKey,
12615
+ member,
12616
+ messageId,
12617
+ score.toString()
12618
+ );
12619
+ reclaimed++;
12620
+ this.logger.debug("Reclaimed timed-out message", {
12621
+ messageId,
12622
+ queueId,
12623
+ originalScore
12624
+ });
12625
+ } catch (error) {
12626
+ this.logger.error("Failed to reclaim message", {
12627
+ messageId,
12628
+ queueId,
12629
+ error: error instanceof Error ? error.message : String(error)
12630
+ });
12631
+ }
12632
+ }
12633
+ return reclaimed;
12634
+ }
12635
+ /**
12636
+ * Get all in-flight messages for a shard.
12637
+ */
12638
+ async getInflightMessages(shardId) {
12639
+ const inflightKey = this.keys.inflightKey(shardId);
12640
+ const results = await this.redis.zrange(inflightKey, 0, -1, "WITHSCORES");
12641
+ const messages = [];
12642
+ for (let i = 0; i < results.length; i += 2) {
12643
+ const member = results[i];
12644
+ const deadlineStr = results[i + 1];
12645
+ if (!member || !deadlineStr) {
12646
+ continue;
12647
+ }
12648
+ const deadline = parseFloat(deadlineStr);
12649
+ const { messageId, queueId } = this.#parseMember(member);
12650
+ messages.push({ messageId, queueId, deadline });
12651
+ }
12652
+ return messages;
12653
+ }
12654
+ /**
12655
+ * Get count of in-flight messages for a shard.
12656
+ */
12657
+ async getInflightCount(shardId) {
12658
+ const inflightKey = this.keys.inflightKey(shardId);
12659
+ return await this.redis.zcard(inflightKey);
12660
+ }
12661
+ /**
12662
+ * Get total in-flight count across all shards.
12663
+ */
12664
+ async getTotalInflightCount() {
12665
+ const counts = await Promise.all(
12666
+ Array.from({ length: this.shardCount }, (_, i) => this.getInflightCount(i))
12667
+ );
12668
+ return counts.reduce((sum, count) => sum + count, 0);
12669
+ }
12670
+ /**
12671
+ * Close the Redis connection.
12672
+ */
12673
+ async close() {
12674
+ await this.redis.quit();
12675
+ }
12676
+ // ============================================================================
12677
+ // Private Methods
12678
+ // ============================================================================
12679
+ /**
12680
+ * Map queue ID to shard using Jump Consistent Hash.
12681
+ * Must use same algorithm as MasterQueue for consistency.
12682
+ */
12683
+ #getShardForQueue(queueId) {
12684
+ return serverOnly.jumpHash(queueId, this.shardCount);
12685
+ }
12686
+ #makeMember(messageId, queueId) {
12687
+ return `${messageId}:${queueId}`;
12688
+ }
12689
+ #parseMember(member) {
12690
+ const colonIndex = member.indexOf(":");
12691
+ if (colonIndex === -1) {
12692
+ return { messageId: member, queueId: "" };
12693
+ }
12694
+ return {
12695
+ messageId: member.substring(0, colonIndex),
12696
+ queueId: member.substring(colonIndex + 1)
12697
+ };
12698
+ }
12699
+ async #removeFromInflight(shardId, messageId, queueId) {
12700
+ const inflightKey = this.keys.inflightKey(shardId);
12701
+ const inflightDataKey = this.keys.inflightDataKey(shardId);
12702
+ const member = this.#makeMember(messageId, queueId);
12703
+ const pipeline = this.redis.pipeline();
12704
+ pipeline.zrem(inflightKey, member);
12705
+ pipeline.hdel(inflightDataKey, messageId);
12706
+ await pipeline.exec();
12707
+ }
12708
+ #registerCommands() {
12709
+ this.redis.defineCommand("claimMessage", {
12710
+ numberOfKeys: 4,
12711
+ lua: `
12712
+ local queueKey = KEYS[1]
12713
+ local queueItemsKey = KEYS[2]
12714
+ local inflightKey = KEYS[3]
12715
+ local inflightDataKey = KEYS[4]
12716
+
12717
+ local queueId = ARGV[1]
12718
+ local consumerId = ARGV[2]
12719
+ local deadline = tonumber(ARGV[3])
12720
+
12721
+ -- Get oldest message from queue
12722
+ local items = redis.call('ZRANGE', queueKey, 0, 0)
12723
+ if #items == 0 then
12724
+ return nil
12725
+ end
12726
+
12727
+ local messageId = items[1]
12728
+
12729
+ -- Get message data
12730
+ local payload = redis.call('HGET', queueItemsKey, messageId)
12731
+ if not payload then
12732
+ -- Message data missing, remove from queue and return nil
12733
+ redis.call('ZREM', queueKey, messageId)
12734
+ return nil
12735
+ end
12736
+
12737
+ -- Remove from queue
12738
+ redis.call('ZREM', queueKey, messageId)
12739
+ redis.call('HDEL', queueItemsKey, messageId)
12740
+
12741
+ -- Add to in-flight set with deadline
12742
+ local member = messageId .. ':' .. queueId
12743
+ redis.call('ZADD', inflightKey, deadline, member)
12744
+
12745
+ -- Store message data for potential release
12746
+ redis.call('HSET', inflightDataKey, messageId, payload)
12747
+
12748
+ return {messageId, payload}
12749
+ `
12750
+ });
12751
+ this.redis.defineCommand("releaseMessage", {
12752
+ numberOfKeys: 4,
12753
+ lua: `
12754
+ local inflightKey = KEYS[1]
12755
+ local inflightDataKey = KEYS[2]
12756
+ local queueKey = KEYS[3]
12757
+ local queueItemsKey = KEYS[4]
12758
+
12759
+ local member = ARGV[1]
12760
+ local messageId = ARGV[2]
12761
+ local score = tonumber(ARGV[3])
12762
+
12763
+ -- Get message data from in-flight
12764
+ local payload = redis.call('HGET', inflightDataKey, messageId)
12765
+ if not payload then
12766
+ -- Message not in in-flight or already released
12767
+ return 0
12768
+ end
12769
+
12770
+ -- Remove from in-flight
12771
+ redis.call('ZREM', inflightKey, member)
12772
+ redis.call('HDEL', inflightDataKey, messageId)
12773
+
12774
+ -- Add back to queue
12775
+ redis.call('ZADD', queueKey, score, messageId)
12776
+ redis.call('HSET', queueItemsKey, messageId, payload)
12777
+
12778
+ return 1
12779
+ `
12780
+ });
12781
+ }
12782
+ };
12783
+
12784
+ // src/fair-queue/workerQueue.ts
12785
+ var WorkerQueueManager = class {
12786
+ constructor(options) {
12787
+ this.options = options;
12788
+ this.redis = createRedisClient(options.redis);
12789
+ this.keys = options.keys;
12790
+ this.logger = options.logger ?? {
12791
+ debug: () => {
12792
+ },
12793
+ error: () => {
12794
+ }
12795
+ };
12796
+ this.#registerCommands();
12797
+ }
12798
+ redis;
12799
+ keys;
12800
+ logger;
12801
+ // ============================================================================
12802
+ // Public Methods
12803
+ // ============================================================================
12804
+ /**
12805
+ * Push a message key to a worker queue.
12806
+ * Called after claiming a message from the message queue.
12807
+ *
12808
+ * @param workerQueueId - The worker queue identifier
12809
+ * @param messageKey - The message key to push (typically "messageId:queueId")
12810
+ */
12811
+ async push(workerQueueId, messageKey) {
12812
+ const workerQueueKey = this.keys.workerQueueKey(workerQueueId);
12813
+ await this.redis.rpush(workerQueueKey, messageKey);
12814
+ this.logger.debug("Pushed to worker queue", {
12815
+ workerQueueId,
12816
+ workerQueueKey,
12817
+ messageKey
12818
+ });
12819
+ }
12820
+ /**
12821
+ * Push multiple message keys to a worker queue.
12822
+ *
12823
+ * @param workerQueueId - The worker queue identifier
12824
+ * @param messageKeys - The message keys to push
12825
+ */
12826
+ async pushBatch(workerQueueId, messageKeys) {
12827
+ if (messageKeys.length === 0) {
12828
+ return;
12829
+ }
12830
+ const workerQueueKey = this.keys.workerQueueKey(workerQueueId);
12831
+ await this.redis.rpush(workerQueueKey, ...messageKeys);
12832
+ this.logger.debug("Pushed batch to worker queue", {
12833
+ workerQueueId,
12834
+ workerQueueKey,
12835
+ count: messageKeys.length
12836
+ });
12837
+ }
12838
+ /**
12839
+ * Blocking pop from a worker queue.
12840
+ * Waits until a message is available or timeout expires.
12841
+ *
12842
+ * @param workerQueueId - The worker queue identifier
12843
+ * @param timeoutSeconds - Maximum time to wait (0 = wait forever)
12844
+ * @param signal - Optional abort signal to cancel waiting
12845
+ * @returns The message key, or null if timeout
12846
+ */
12847
+ async blockingPop(workerQueueId, timeoutSeconds, signal) {
12848
+ const workerQueueKey = this.keys.workerQueueKey(workerQueueId);
12849
+ const blockingClient = this.redis.duplicate();
12850
+ try {
12851
+ if (signal) {
12852
+ const cleanup = () => {
12853
+ blockingClient.disconnect();
12854
+ };
12855
+ signal.addEventListener("abort", cleanup, { once: true });
12856
+ if (signal.aborted) {
12857
+ return null;
12858
+ }
12859
+ }
12860
+ const result = await blockingClient.blpop(workerQueueKey, timeoutSeconds);
12861
+ if (!result) {
12862
+ return null;
12863
+ }
12864
+ const [, messageKey] = result;
12865
+ this.logger.debug("Blocking pop received message", {
12866
+ workerQueueId,
12867
+ workerQueueKey,
12868
+ messageKey
12869
+ });
12870
+ return messageKey;
12871
+ } catch (error) {
12872
+ if (signal?.aborted) {
12873
+ return null;
12874
+ }
12875
+ this.logger.error("Blocking pop error", {
12876
+ workerQueueId,
12877
+ error: error instanceof Error ? error.message : String(error)
12878
+ });
12879
+ throw error;
12880
+ } finally {
12881
+ await blockingClient.quit().catch(() => {
12882
+ });
12883
+ }
12884
+ }
12885
+ /**
12886
+ * Non-blocking pop from a worker queue.
12887
+ *
12888
+ * @param workerQueueId - The worker queue identifier
12889
+ * @returns The message key and queue length, or null if empty
12890
+ */
12891
+ async pop(workerQueueId) {
12892
+ const workerQueueKey = this.keys.workerQueueKey(workerQueueId);
12893
+ const result = await this.redis.popWithLength(workerQueueKey);
12894
+ if (!result) {
12895
+ return null;
12896
+ }
12897
+ const [messageKey, queueLength] = result;
12898
+ this.logger.debug("Non-blocking pop received message", {
12899
+ workerQueueId,
12900
+ workerQueueKey,
12901
+ messageKey,
12902
+ queueLength
12903
+ });
12904
+ return { messageKey, queueLength: Number(queueLength) };
12905
+ }
12906
+ /**
12907
+ * Get the current length of a worker queue.
12908
+ */
12909
+ async getLength(workerQueueId) {
12910
+ const workerQueueKey = this.keys.workerQueueKey(workerQueueId);
12911
+ return await this.redis.llen(workerQueueKey);
12912
+ }
12913
+ /**
12914
+ * Peek at all messages in a worker queue without removing them.
12915
+ * Useful for debugging and tests.
12916
+ */
12917
+ async peek(workerQueueId) {
12918
+ const workerQueueKey = this.keys.workerQueueKey(workerQueueId);
12919
+ return await this.redis.lrange(workerQueueKey, 0, -1);
12920
+ }
12921
+ /**
12922
+ * Remove a specific message from the worker queue.
12923
+ * Used when a message needs to be removed without processing.
12924
+ *
12925
+ * @param workerQueueId - The worker queue identifier
12926
+ * @param messageKey - The message key to remove
12927
+ * @returns Number of removed items
12928
+ */
12929
+ async remove(workerQueueId, messageKey) {
12930
+ const workerQueueKey = this.keys.workerQueueKey(workerQueueId);
12931
+ return await this.redis.lrem(workerQueueKey, 0, messageKey);
12932
+ }
12933
+ /**
12934
+ * Clear all messages from a worker queue.
12935
+ */
12936
+ async clear(workerQueueId) {
12937
+ const workerQueueKey = this.keys.workerQueueKey(workerQueueId);
12938
+ await this.redis.del(workerQueueKey);
12939
+ }
12940
+ /**
12941
+ * Close the Redis connection.
12942
+ */
12943
+ async close() {
12944
+ await this.redis.quit();
12945
+ }
12946
+ // ============================================================================
12947
+ // Private - Register Commands
12948
+ // ============================================================================
12949
+ /**
12950
+ * Initialize custom Redis commands.
12951
+ */
12952
+ #registerCommands() {
12953
+ this.redis.defineCommand("popWithLength", {
12954
+ numberOfKeys: 1,
12955
+ lua: `
12956
+ local workerQueueKey = KEYS[1]
12957
+
12958
+ -- Pop the first message
12959
+ local messageKey = redis.call('LPOP', workerQueueKey)
12960
+ if not messageKey then
12961
+ return nil
12962
+ end
12963
+
12964
+ -- Get remaining queue length
12965
+ local queueLength = redis.call('LLEN', workerQueueKey)
12966
+
12967
+ return {messageKey, queueLength}
12968
+ `
12969
+ });
12970
+ }
12971
+ /**
12972
+ * Register custom commands on an external Redis client.
12973
+ * Use this when initializing FairQueue with worker queues.
12974
+ */
12975
+ registerCommands(redis) {
12976
+ redis.defineCommand("popWithLength", {
12977
+ numberOfKeys: 1,
12978
+ lua: `
12979
+ local workerQueueKey = KEYS[1]
12980
+
12981
+ -- Pop the first message
12982
+ local messageKey = redis.call('LPOP', workerQueueKey)
12983
+ if not messageKey then
12984
+ return nil
12985
+ end
12986
+
12987
+ -- Get remaining queue length
12988
+ local queueLength = redis.call('LLEN', workerQueueKey)
12989
+
12990
+ return {messageKey, queueLength}
12991
+ `
12992
+ });
12993
+ }
12994
+ };
12995
+
12996
+ // src/fair-queue/keyProducer.ts
12997
+ var DefaultFairQueueKeyProducer = class {
12998
+ prefix;
12999
+ separator;
13000
+ constructor(options = {}) {
13001
+ this.prefix = options.prefix ?? "fq";
13002
+ this.separator = options.separator ?? ":";
13003
+ }
13004
+ // ============================================================================
13005
+ // Master Queue Keys
13006
+ // ============================================================================
13007
+ masterQueueKey(shardId) {
13008
+ return this.#buildKey("master", shardId.toString());
13009
+ }
13010
+ // ============================================================================
13011
+ // Queue Keys
13012
+ // ============================================================================
13013
+ queueKey(queueId) {
13014
+ return this.#buildKey("queue", queueId);
13015
+ }
13016
+ queueItemsKey(queueId) {
13017
+ return this.#buildKey("queue", queueId, "items");
13018
+ }
13019
+ // ============================================================================
13020
+ // Concurrency Keys
13021
+ // ============================================================================
13022
+ concurrencyKey(groupName, groupId) {
13023
+ return this.#buildKey("concurrency", groupName, groupId);
13024
+ }
13025
+ // ============================================================================
13026
+ // In-Flight Keys
13027
+ // ============================================================================
13028
+ inflightKey(shardId) {
13029
+ return this.#buildKey("inflight", shardId.toString());
13030
+ }
13031
+ inflightDataKey(shardId) {
13032
+ return this.#buildKey("inflight", shardId.toString(), "data");
13033
+ }
13034
+ // ============================================================================
13035
+ // Worker Queue Keys
13036
+ // ============================================================================
13037
+ workerQueueKey(consumerId) {
13038
+ return this.#buildKey("worker", consumerId);
13039
+ }
13040
+ // ============================================================================
13041
+ // Dead Letter Queue Keys
13042
+ // ============================================================================
13043
+ deadLetterQueueKey(tenantId) {
13044
+ return this.#buildKey("dlq", tenantId);
13045
+ }
13046
+ deadLetterQueueDataKey(tenantId) {
13047
+ return this.#buildKey("dlq", tenantId, "data");
13048
+ }
13049
+ // ============================================================================
13050
+ // Extraction Methods
13051
+ // ============================================================================
13052
+ /**
13053
+ * Extract tenant ID from a queue ID.
13054
+ * Default implementation assumes queue IDs are formatted as: tenant:{tenantId}:...
13055
+ * Override this method for custom queue ID formats.
13056
+ */
13057
+ extractTenantId(queueId) {
13058
+ const parts = queueId.split(this.separator);
13059
+ if (parts.length >= 2 && parts[0] === "tenant" && parts[1]) {
13060
+ return parts[1];
13061
+ }
13062
+ return parts[0] ?? "";
13063
+ }
13064
+ /**
13065
+ * Extract a group ID from a queue ID.
13066
+ * Default implementation looks for pattern: {groupName}:{groupId}:...
13067
+ * Override this method for custom queue ID formats.
13068
+ */
13069
+ extractGroupId(groupName, queueId) {
13070
+ const parts = queueId.split(this.separator);
13071
+ for (let i = 0; i < parts.length - 1; i++) {
13072
+ if (parts[i] === groupName) {
13073
+ const nextPart = parts[i + 1];
13074
+ if (nextPart) {
13075
+ return nextPart;
13076
+ }
13077
+ }
13078
+ }
13079
+ return "";
13080
+ }
13081
+ // ============================================================================
13082
+ // Helper Methods
13083
+ // ============================================================================
13084
+ #buildKey(...parts) {
13085
+ return [this.prefix, ...parts].join(this.separator);
13086
+ }
13087
+ };
13088
+ var CallbackFairQueueKeyProducer = class extends DefaultFairQueueKeyProducer {
13089
+ tenantExtractor;
13090
+ groupExtractor;
13091
+ constructor(options) {
13092
+ super({ prefix: options.prefix, separator: options.separator });
13093
+ this.tenantExtractor = options.extractTenantId;
13094
+ this.groupExtractor = options.extractGroupId;
13095
+ }
13096
+ extractTenantId(queueId) {
13097
+ return this.tenantExtractor(queueId);
13098
+ }
13099
+ extractGroupId(groupName, queueId) {
13100
+ return this.groupExtractor(groupName, queueId);
13101
+ }
13102
+ };
13103
+
13104
+ // src/fair-queue/scheduler.ts
13105
+ var BaseScheduler = class {
13106
+ /**
13107
+ * Called after processing a message to update scheduler state.
13108
+ * Default implementation does nothing.
13109
+ */
13110
+ async recordProcessed(_tenantId, _queueId) {
13111
+ }
13112
+ /**
13113
+ * Initialize the scheduler.
13114
+ * Default implementation does nothing.
13115
+ */
13116
+ async initialize() {
13117
+ }
13118
+ /**
13119
+ * Cleanup scheduler resources.
13120
+ * Default implementation does nothing.
13121
+ */
13122
+ async close() {
13123
+ }
13124
+ /**
13125
+ * Helper to group queues by tenant.
13126
+ */
13127
+ groupQueuesByTenant(queues) {
13128
+ const grouped = /* @__PURE__ */ new Map();
13129
+ for (const { queueId, tenantId } of queues) {
13130
+ const existing = grouped.get(tenantId) ?? [];
13131
+ existing.push(queueId);
13132
+ grouped.set(tenantId, existing);
13133
+ }
13134
+ return grouped;
13135
+ }
13136
+ /**
13137
+ * Helper to convert grouped queues to TenantQueues array.
13138
+ */
13139
+ toTenantQueuesArray(grouped) {
13140
+ return Array.from(grouped.entries()).map(([tenantId, queues]) => ({
13141
+ tenantId,
13142
+ queues
13143
+ }));
13144
+ }
13145
+ /**
13146
+ * Helper to filter out tenants at capacity.
13147
+ */
13148
+ async filterAtCapacity(tenants, context2, groupName = "tenant") {
13149
+ const filtered = [];
13150
+ for (const tenant of tenants) {
13151
+ const isAtCapacity = await context2.isAtCapacity(groupName, tenant.tenantId);
13152
+ if (!isAtCapacity) {
13153
+ filtered.push(tenant);
13154
+ }
13155
+ }
13156
+ return filtered;
13157
+ }
13158
+ };
13159
+ var NoopScheduler = class extends BaseScheduler {
13160
+ async selectQueues(_masterQueueShard, _consumerId, _context) {
13161
+ return [];
13162
+ }
13163
+ };
13164
+
13165
+ // src/fair-queue/schedulers/drr.ts
13166
+ var DRRScheduler = class extends BaseScheduler {
13167
+ constructor(config) {
13168
+ super();
13169
+ this.config = config;
13170
+ this.redis = createRedisClient(config.redis);
13171
+ this.keys = config.keys;
13172
+ this.quantum = config.quantum;
13173
+ this.maxDeficit = config.maxDeficit;
13174
+ this.logger = config.logger ?? {
13175
+ debug: () => {
13176
+ },
13177
+ error: () => {
13178
+ }
13179
+ };
13180
+ this.#registerCommands();
13181
+ }
13182
+ redis;
13183
+ keys;
13184
+ quantum;
13185
+ maxDeficit;
13186
+ logger;
13187
+ // ============================================================================
13188
+ // FairScheduler Implementation
13189
+ // ============================================================================
13190
+ /**
13191
+ * Select queues for processing using DRR algorithm.
13192
+ *
13193
+ * Algorithm:
13194
+ * 1. Get all queues from the master shard
13195
+ * 2. Group by tenant
13196
+ * 3. Filter out tenants at concurrency capacity
13197
+ * 4. Add quantum to each tenant's deficit (atomically)
13198
+ * 5. Select queues from tenants with deficit >= 1
13199
+ * 6. Order tenants by deficit (highest first for fairness)
13200
+ */
13201
+ async selectQueues(masterQueueShard, consumerId, context2) {
13202
+ const queues = await this.#getQueuesFromShard(masterQueueShard);
13203
+ if (queues.length === 0) {
13204
+ return [];
13205
+ }
13206
+ const queuesByTenant = this.groupQueuesByTenant(
13207
+ queues.map((q) => ({ queueId: q.queueId, tenantId: q.tenantId }))
13208
+ );
13209
+ const tenantIds = Array.from(queuesByTenant.keys());
13210
+ const deficits = await this.#addQuantumToTenants(tenantIds);
13211
+ const tenantData = await Promise.all(
13212
+ tenantIds.map(async (tenantId, index) => {
13213
+ const isAtCapacity = await context2.isAtCapacity("tenant", tenantId);
13214
+ return {
13215
+ tenantId,
13216
+ deficit: deficits[index] ?? 0,
13217
+ queues: queuesByTenant.get(tenantId) ?? [],
13218
+ isAtCapacity
13219
+ };
13220
+ })
13221
+ );
13222
+ const eligibleTenants = tenantData.filter(
13223
+ (t) => !t.isAtCapacity && t.deficit >= 1
13224
+ );
13225
+ eligibleTenants.sort((a, b) => b.deficit - a.deficit);
13226
+ return eligibleTenants.map((t) => ({
13227
+ tenantId: t.tenantId,
13228
+ queues: t.queues
13229
+ }));
13230
+ }
13231
+ /**
13232
+ * Record that a message was processed from a tenant.
13233
+ * Decrements the tenant's deficit.
13234
+ */
13235
+ async recordProcessed(tenantId, _queueId) {
13236
+ await this.#decrementDeficit(tenantId);
13237
+ }
13238
+ async close() {
13239
+ await this.redis.quit();
13240
+ }
13241
+ // ============================================================================
13242
+ // Public Methods for Deficit Management
13243
+ // ============================================================================
13244
+ /**
13245
+ * Get the current deficit for a tenant.
13246
+ */
13247
+ async getDeficit(tenantId) {
13248
+ const key = this.#deficitKey();
13249
+ const value = await this.redis.hget(key, tenantId);
13250
+ return value ? parseFloat(value) : 0;
13251
+ }
13252
+ /**
13253
+ * Reset deficit for a tenant.
13254
+ * Used when a tenant has no more active queues.
13255
+ */
13256
+ async resetDeficit(tenantId) {
13257
+ const key = this.#deficitKey();
13258
+ await this.redis.hdel(key, tenantId);
13259
+ }
13260
+ /**
13261
+ * Get all tenant deficits.
13262
+ */
13263
+ async getAllDeficits() {
13264
+ const key = this.#deficitKey();
13265
+ const data = await this.redis.hgetall(key);
13266
+ const result = /* @__PURE__ */ new Map();
13267
+ for (const [tenantId, value] of Object.entries(data)) {
13268
+ result.set(tenantId, parseFloat(value));
13269
+ }
13270
+ return result;
13271
+ }
13272
+ // ============================================================================
13273
+ // Private Methods
13274
+ // ============================================================================
13275
+ #deficitKey() {
13276
+ return `${this.keys.masterQueueKey(0).split(":")[0]}:drr:deficit`;
13277
+ }
13278
+ async #getQueuesFromShard(shardKey) {
13279
+ const now = Date.now();
13280
+ const results = await this.redis.zrangebyscore(
13281
+ shardKey,
13282
+ "-inf",
13283
+ now,
13284
+ "WITHSCORES",
13285
+ "LIMIT",
13286
+ 0,
13287
+ 1e3
13288
+ // Limit for performance
13289
+ );
13290
+ const queues = [];
13291
+ for (let i = 0; i < results.length; i += 2) {
13292
+ const queueId = results[i];
13293
+ const scoreStr = results[i + 1];
13294
+ if (queueId && scoreStr) {
13295
+ queues.push({
13296
+ queueId,
13297
+ score: parseFloat(scoreStr),
13298
+ tenantId: this.keys.extractTenantId(queueId)
13299
+ });
13300
+ }
13301
+ }
13302
+ return queues;
13303
+ }
13304
+ /**
13305
+ * Add quantum to multiple tenants atomically.
13306
+ * Returns the new deficit values.
13307
+ */
13308
+ async #addQuantumToTenants(tenantIds) {
13309
+ if (tenantIds.length === 0) {
13310
+ return [];
13311
+ }
13312
+ const key = this.#deficitKey();
13313
+ const results = await this.redis.drrAddQuantum(
13314
+ key,
13315
+ this.quantum.toString(),
13316
+ this.maxDeficit.toString(),
13317
+ ...tenantIds
13318
+ );
13319
+ return results.map((r) => parseFloat(r));
13320
+ }
13321
+ /**
13322
+ * Decrement deficit for a tenant atomically.
13323
+ */
13324
+ async #decrementDeficit(tenantId) {
13325
+ const key = this.#deficitKey();
13326
+ const result = await this.redis.drrDecrementDeficit(key, tenantId);
13327
+ return parseFloat(result);
13328
+ }
13329
+ #registerCommands() {
13330
+ this.redis.defineCommand("drrAddQuantum", {
13331
+ numberOfKeys: 1,
13332
+ lua: `
13333
+ local deficitKey = KEYS[1]
13334
+ local quantum = tonumber(ARGV[1])
13335
+ local maxDeficit = tonumber(ARGV[2])
13336
+ local results = {}
13337
+
13338
+ for i = 3, #ARGV do
13339
+ local tenantId = ARGV[i]
13340
+
13341
+ -- Add quantum to deficit
13342
+ local newDeficit = redis.call('HINCRBYFLOAT', deficitKey, tenantId, quantum)
13343
+ newDeficit = tonumber(newDeficit)
13344
+
13345
+ -- Cap at maxDeficit
13346
+ if newDeficit > maxDeficit then
13347
+ redis.call('HSET', deficitKey, tenantId, maxDeficit)
13348
+ newDeficit = maxDeficit
13349
+ end
13350
+
13351
+ table.insert(results, tostring(newDeficit))
13352
+ end
13353
+
13354
+ return results
13355
+ `
13356
+ });
13357
+ this.redis.defineCommand("drrDecrementDeficit", {
13358
+ numberOfKeys: 1,
13359
+ lua: `
13360
+ local deficitKey = KEYS[1]
13361
+ local tenantId = ARGV[1]
13362
+
13363
+ local newDeficit = redis.call('HINCRBYFLOAT', deficitKey, tenantId, -1)
13364
+ newDeficit = tonumber(newDeficit)
13365
+
13366
+ -- Floor at 0
13367
+ if newDeficit < 0 then
13368
+ redis.call('HSET', deficitKey, tenantId, 0)
13369
+ newDeficit = 0
13370
+ end
13371
+
13372
+ return tostring(newDeficit)
13373
+ `
13374
+ });
13375
+ }
13376
+ };
13377
+ var defaultBiases = {
13378
+ concurrencyLimitBias: 0,
13379
+ availableCapacityBias: 0,
13380
+ queueAgeRandomization: 0
13381
+ };
13382
+ var WeightedScheduler = class extends BaseScheduler {
13383
+ constructor(config) {
13384
+ super();
13385
+ this.config = config;
13386
+ this.redis = createRedisClient(config.redis);
13387
+ this.keys = config.keys;
13388
+ this.rng = seedrandom__default.default(config.seed);
13389
+ this.biases = config.biases ?? defaultBiases;
13390
+ this.defaultTenantLimit = config.defaultTenantConcurrencyLimit ?? 100;
13391
+ this.masterQueueLimit = config.masterQueueLimit ?? 100;
13392
+ this.reuseSnapshotCount = config.reuseSnapshotCount ?? 0;
13393
+ this.maximumTenantCount = config.maximumTenantCount ?? 0;
13394
+ }
13395
+ redis;
13396
+ keys;
13397
+ rng;
13398
+ biases;
13399
+ defaultTenantLimit;
13400
+ masterQueueLimit;
13401
+ reuseSnapshotCount;
13402
+ maximumTenantCount;
13403
+ // Snapshot cache
13404
+ snapshotCache = /* @__PURE__ */ new Map();
13405
+ // ============================================================================
13406
+ // FairScheduler Implementation
13407
+ // ============================================================================
13408
+ async selectQueues(masterQueueShard, consumerId, context2) {
13409
+ const snapshot = await this.#getOrCreateSnapshot(
13410
+ masterQueueShard,
13411
+ consumerId,
13412
+ context2
13413
+ );
13414
+ if (snapshot.queues.length === 0) {
13415
+ return [];
13416
+ }
13417
+ const shuffledTenants = this.#shuffleTenantsByWeight(snapshot);
13418
+ return shuffledTenants.map((tenantId) => ({
13419
+ tenantId,
13420
+ queues: this.#orderQueuesForTenant(snapshot, tenantId)
13421
+ }));
13422
+ }
13423
+ async close() {
13424
+ this.snapshotCache.clear();
13425
+ await this.redis.quit();
13426
+ }
13427
+ // ============================================================================
13428
+ // Private Methods
13429
+ // ============================================================================
13430
+ async #getOrCreateSnapshot(masterQueueShard, consumerId, context2) {
13431
+ const cacheKey = `${masterQueueShard}:${consumerId}`;
13432
+ if (this.reuseSnapshotCount > 0) {
13433
+ const cached = this.snapshotCache.get(cacheKey);
13434
+ if (cached && cached.reuseCount < this.reuseSnapshotCount) {
13435
+ this.snapshotCache.set(cacheKey, {
13436
+ snapshot: cached.snapshot,
13437
+ reuseCount: cached.reuseCount + 1
13438
+ });
13439
+ return cached.snapshot;
13440
+ }
13441
+ }
13442
+ const snapshot = await this.#createSnapshot(masterQueueShard, context2);
13443
+ if (this.reuseSnapshotCount > 0) {
13444
+ this.snapshotCache.set(cacheKey, { snapshot, reuseCount: 0 });
13445
+ }
13446
+ return snapshot;
13447
+ }
13448
+ async #createSnapshot(masterQueueShard, context2) {
13449
+ const now = Date.now();
13450
+ let rawQueues = await this.#getQueuesFromShard(masterQueueShard, now);
13451
+ if (rawQueues.length === 0) {
13452
+ return { id: crypto.randomUUID(), tenants: /* @__PURE__ */ new Map(), queues: [] };
13453
+ }
13454
+ if (this.maximumTenantCount > 0) {
13455
+ rawQueues = this.#selectTopTenantQueues(rawQueues);
13456
+ }
13457
+ const tenantIds = /* @__PURE__ */ new Set();
13458
+ const queuesByTenant = /* @__PURE__ */ new Map();
13459
+ for (const queue of rawQueues) {
13460
+ tenantIds.add(queue.tenantId);
13461
+ const tenantQueues = queuesByTenant.get(queue.tenantId) ?? [];
13462
+ tenantQueues.push({
13463
+ queueId: queue.queueId,
13464
+ age: now - queue.score
13465
+ });
13466
+ queuesByTenant.set(queue.tenantId, tenantQueues);
13467
+ }
13468
+ const tenants = /* @__PURE__ */ new Map();
13469
+ for (const tenantId of tenantIds) {
13470
+ const [current, limit] = await Promise.all([
13471
+ context2.getCurrentConcurrency("tenant", tenantId),
13472
+ context2.getConcurrencyLimit("tenant", tenantId)
13473
+ ]);
13474
+ if (current >= limit) {
13475
+ continue;
13476
+ }
13477
+ tenants.set(tenantId, {
13478
+ tenantId,
13479
+ concurrency: { current, limit },
13480
+ queues: queuesByTenant.get(tenantId) ?? []
13481
+ });
13482
+ }
13483
+ const queues = rawQueues.filter((q) => tenants.has(q.tenantId)).map((q) => ({
13484
+ queueId: q.queueId,
13485
+ tenantId: q.tenantId,
13486
+ age: now - q.score
13487
+ }));
13488
+ return {
13489
+ id: crypto.randomUUID(),
13490
+ tenants,
13491
+ queues
13492
+ };
13493
+ }
13494
+ async #getQueuesFromShard(shardKey, maxScore) {
13495
+ const results = await this.redis.zrangebyscore(
13496
+ shardKey,
13497
+ "-inf",
13498
+ maxScore,
13499
+ "WITHSCORES",
13500
+ "LIMIT",
13501
+ 0,
13502
+ this.masterQueueLimit
13503
+ );
13504
+ const queues = [];
13505
+ for (let i = 0; i < results.length; i += 2) {
13506
+ const queueId = results[i];
13507
+ const scoreStr = results[i + 1];
13508
+ if (queueId && scoreStr) {
13509
+ queues.push({
13510
+ queueId,
13511
+ score: parseFloat(scoreStr),
13512
+ tenantId: this.keys.extractTenantId(queueId)
13513
+ });
13514
+ }
13515
+ }
13516
+ return queues;
13517
+ }
13518
+ #selectTopTenantQueues(queues) {
13519
+ const queuesByTenant = /* @__PURE__ */ new Map();
13520
+ for (const queue of queues) {
13521
+ const tenantQueues = queuesByTenant.get(queue.tenantId) ?? [];
13522
+ tenantQueues.push(queue);
13523
+ queuesByTenant.set(queue.tenantId, tenantQueues);
13524
+ }
13525
+ const tenantAges = Array.from(queuesByTenant.entries()).map(([tenantId, tQueues]) => {
13526
+ const avgAge = tQueues.reduce((sum, q) => sum + q.score, 0) / tQueues.length;
13527
+ return { tenantId, avgAge };
13528
+ });
13529
+ const maxAge = Math.max(...tenantAges.map((t) => t.avgAge));
13530
+ const weightedTenants = tenantAges.map((t) => ({
13531
+ tenantId: t.tenantId,
13532
+ weight: t.avgAge / maxAge
13533
+ }));
13534
+ const selectedTenants = /* @__PURE__ */ new Set();
13535
+ let remaining = [...weightedTenants];
13536
+ let totalWeight = remaining.reduce((sum, t) => sum + t.weight, 0);
13537
+ while (selectedTenants.size < this.maximumTenantCount && remaining.length > 0) {
13538
+ let random = this.rng() * totalWeight;
13539
+ let index = 0;
13540
+ while (random > 0 && index < remaining.length) {
13541
+ const item = remaining[index];
13542
+ if (item) {
13543
+ random -= item.weight;
13544
+ }
13545
+ index++;
13546
+ }
13547
+ index = Math.max(0, index - 1);
13548
+ const selected = remaining[index];
13549
+ if (selected) {
13550
+ selectedTenants.add(selected.tenantId);
13551
+ totalWeight -= selected.weight;
13552
+ remaining.splice(index, 1);
13553
+ }
13554
+ }
13555
+ return queues.filter((q) => selectedTenants.has(q.tenantId));
13556
+ }
13557
+ #shuffleTenantsByWeight(snapshot) {
13558
+ const tenantIds = Array.from(snapshot.tenants.keys());
13559
+ if (tenantIds.length === 0) {
13560
+ return [];
13561
+ }
13562
+ const { concurrencyLimitBias, availableCapacityBias } = this.biases;
13563
+ if (concurrencyLimitBias === 0 && availableCapacityBias === 0) {
13564
+ return this.#shuffle(tenantIds);
13565
+ }
13566
+ const maxLimit = Math.max(
13567
+ ...tenantIds.map((id) => snapshot.tenants.get(id).concurrency.limit)
13568
+ );
13569
+ const weightedTenants = tenantIds.map((tenantId) => {
13570
+ const tenant = snapshot.tenants.get(tenantId);
13571
+ let weight = 1;
13572
+ if (concurrencyLimitBias > 0) {
13573
+ const normalizedLimit = tenant.concurrency.limit / maxLimit;
13574
+ weight *= 1 + Math.pow(normalizedLimit * concurrencyLimitBias, 2);
13575
+ }
13576
+ if (availableCapacityBias > 0) {
13577
+ const usedPercentage = tenant.concurrency.current / tenant.concurrency.limit;
13578
+ const availableBonus = 1 - usedPercentage;
13579
+ weight *= 1 + Math.pow(availableBonus * availableCapacityBias, 2);
13580
+ }
13581
+ return { tenantId, weight };
13582
+ });
13583
+ return this.#weightedShuffle(weightedTenants);
13584
+ }
13585
+ #orderQueuesForTenant(snapshot, tenantId) {
13586
+ const tenant = snapshot.tenants.get(tenantId);
13587
+ if (!tenant || tenant.queues.length === 0) {
13588
+ return [];
13589
+ }
13590
+ const queues = [...tenant.queues];
13591
+ const { queueAgeRandomization } = this.biases;
13592
+ if (queueAgeRandomization === 0) {
13593
+ return queues.sort((a, b) => b.age - a.age).map((q) => q.queueId);
13594
+ }
13595
+ const maxAge = Math.max(...queues.map((q) => q.age));
13596
+ const weightedQueues = queues.map((q) => ({
13597
+ queue: q,
13598
+ weight: 1 + q.age / maxAge * queueAgeRandomization
13599
+ }));
13600
+ const result = [];
13601
+ let remaining = [...weightedQueues];
13602
+ let totalWeight = remaining.reduce((sum, q) => sum + q.weight, 0);
13603
+ while (remaining.length > 0) {
13604
+ let random = this.rng() * totalWeight;
13605
+ let index = 0;
13606
+ while (random > 0 && index < remaining.length) {
13607
+ const item = remaining[index];
13608
+ if (item) {
13609
+ random -= item.weight;
13610
+ }
13611
+ index++;
13612
+ }
13613
+ index = Math.max(0, index - 1);
13614
+ const selected = remaining[index];
13615
+ if (selected) {
13616
+ result.push(selected.queue.queueId);
13617
+ totalWeight -= selected.weight;
13618
+ remaining.splice(index, 1);
13619
+ }
13620
+ }
13621
+ return result;
13622
+ }
13623
+ #shuffle(array) {
13624
+ const result = [...array];
13625
+ for (let i = result.length - 1; i > 0; i--) {
13626
+ const j = Math.floor(this.rng() * (i + 1));
13627
+ const temp = result[i];
13628
+ const swapValue = result[j];
13629
+ if (temp !== void 0 && swapValue !== void 0) {
13630
+ result[i] = swapValue;
13631
+ result[j] = temp;
13632
+ }
13633
+ }
13634
+ return result;
13635
+ }
13636
+ #weightedShuffle(items) {
13637
+ const result = [];
13638
+ let remaining = [...items];
13639
+ let totalWeight = remaining.reduce((sum, item) => sum + item.weight, 0);
13640
+ while (remaining.length > 0) {
13641
+ let random = this.rng() * totalWeight;
13642
+ let index = 0;
13643
+ while (random > 0 && index < remaining.length) {
13644
+ const item = remaining[index];
13645
+ if (item) {
13646
+ random -= item.weight;
13647
+ }
13648
+ index++;
13649
+ }
13650
+ index = Math.max(0, index - 1);
13651
+ const selected = remaining[index];
13652
+ if (selected) {
13653
+ result.push(selected.tenantId);
13654
+ totalWeight -= selected.weight;
13655
+ remaining.splice(index, 1);
13656
+ }
13657
+ }
13658
+ return result;
13659
+ }
13660
+ };
13661
+
13662
+ // src/fair-queue/schedulers/roundRobin.ts
13663
+ var RoundRobinScheduler = class extends BaseScheduler {
13664
+ constructor(config) {
13665
+ super();
13666
+ this.config = config;
13667
+ this.redis = createRedisClient(config.redis);
13668
+ this.keys = config.keys;
13669
+ this.masterQueueLimit = config.masterQueueLimit ?? 1e3;
13670
+ }
13671
+ redis;
13672
+ keys;
13673
+ masterQueueLimit;
13674
+ // ============================================================================
13675
+ // FairScheduler Implementation
13676
+ // ============================================================================
13677
+ async selectQueues(masterQueueShard, consumerId, context2) {
13678
+ const now = Date.now();
13679
+ const queues = await this.#getQueuesFromShard(masterQueueShard, now);
13680
+ if (queues.length === 0) {
13681
+ return [];
13682
+ }
13683
+ const queuesByTenant = /* @__PURE__ */ new Map();
13684
+ const tenantOrder = [];
13685
+ for (const queue of queues) {
13686
+ if (!queuesByTenant.has(queue.tenantId)) {
13687
+ queuesByTenant.set(queue.tenantId, []);
13688
+ tenantOrder.push(queue.tenantId);
13689
+ }
13690
+ queuesByTenant.get(queue.tenantId).push(queue.queueId);
13691
+ }
13692
+ const lastServedIndex = await this.#getLastServedIndex(masterQueueShard);
13693
+ const rotatedTenants = this.#rotateArray(tenantOrder, lastServedIndex);
13694
+ const eligibleTenants = [];
13695
+ for (const tenantId of rotatedTenants) {
13696
+ const isAtCapacity = await context2.isAtCapacity("tenant", tenantId);
13697
+ if (!isAtCapacity) {
13698
+ const tenantQueues = queuesByTenant.get(tenantId) ?? [];
13699
+ eligibleTenants.push({
13700
+ tenantId,
13701
+ queues: tenantQueues
13702
+ });
13703
+ }
13704
+ }
13705
+ const firstEligible = eligibleTenants[0];
13706
+ if (firstEligible) {
13707
+ const firstTenantIndex = tenantOrder.indexOf(firstEligible.tenantId);
13708
+ await this.#setLastServedIndex(masterQueueShard, firstTenantIndex + 1);
13709
+ }
13710
+ return eligibleTenants;
13711
+ }
13712
+ async close() {
13713
+ await this.redis.quit();
13714
+ }
13715
+ // ============================================================================
13716
+ // Private Methods
13717
+ // ============================================================================
13718
+ async #getQueuesFromShard(shardKey, maxScore) {
13719
+ const results = await this.redis.zrangebyscore(
13720
+ shardKey,
13721
+ "-inf",
13722
+ maxScore,
13723
+ "WITHSCORES",
13724
+ "LIMIT",
13725
+ 0,
13726
+ this.masterQueueLimit
13727
+ );
13728
+ const queues = [];
13729
+ for (let i = 0; i < results.length; i += 2) {
13730
+ const queueId = results[i];
13731
+ const scoreStr = results[i + 1];
13732
+ if (queueId && scoreStr) {
13733
+ queues.push({
13734
+ queueId,
13735
+ score: parseFloat(scoreStr),
13736
+ tenantId: this.keys.extractTenantId(queueId)
13737
+ });
13738
+ }
13739
+ }
13740
+ return queues;
13741
+ }
13742
+ #lastServedKey(shardKey) {
13743
+ return `${shardKey}:rr:lastServed`;
13744
+ }
13745
+ async #getLastServedIndex(shardKey) {
13746
+ const key = this.#lastServedKey(shardKey);
13747
+ const value = await this.redis.get(key);
13748
+ return value ? parseInt(value, 10) : 0;
13749
+ }
13750
+ async #setLastServedIndex(shardKey, index) {
13751
+ const key = this.#lastServedKey(shardKey);
13752
+ await this.redis.set(key, index.toString());
13753
+ }
13754
+ #rotateArray(array, startIndex) {
13755
+ if (array.length === 0) return [];
13756
+ const normalizedIndex = startIndex % array.length;
13757
+ return [...array.slice(normalizedIndex), ...array.slice(0, normalizedIndex)];
13758
+ }
13759
+ };
13760
+ var ExponentialBackoffRetry = class {
13761
+ maxAttempts;
13762
+ options;
13763
+ constructor(options) {
13764
+ this.options = {
13765
+ maxAttempts: options?.maxAttempts ?? 12,
13766
+ factor: options?.factor ?? 2,
13767
+ minTimeoutInMs: options?.minTimeoutInMs ?? 1e3,
13768
+ maxTimeoutInMs: options?.maxTimeoutInMs ?? 36e5,
13769
+ // 1 hour
13770
+ randomize: options?.randomize ?? true
13771
+ };
13772
+ this.maxAttempts = this.options.maxAttempts ?? 12;
13773
+ }
13774
+ getNextDelay(attempt, _error) {
13775
+ if (attempt >= this.maxAttempts) {
13776
+ return null;
13777
+ }
13778
+ const delay = v3.calculateNextRetryDelay(this.options, attempt);
13779
+ return delay ?? null;
13780
+ }
13781
+ };
13782
+ var FixedDelayRetry = class {
13783
+ maxAttempts;
13784
+ delayMs;
13785
+ constructor(options) {
13786
+ this.maxAttempts = options.maxAttempts;
13787
+ this.delayMs = options.delayMs;
13788
+ }
13789
+ getNextDelay(attempt, _error) {
13790
+ if (attempt >= this.maxAttempts) {
13791
+ return null;
13792
+ }
13793
+ return this.delayMs;
13794
+ }
13795
+ };
13796
+ var LinearBackoffRetry = class {
13797
+ maxAttempts;
13798
+ baseDelayMs;
13799
+ maxDelayMs;
13800
+ constructor(options) {
13801
+ this.maxAttempts = options.maxAttempts;
13802
+ this.baseDelayMs = options.baseDelayMs;
13803
+ this.maxDelayMs = options.maxDelayMs ?? options.baseDelayMs * options.maxAttempts;
13804
+ }
13805
+ getNextDelay(attempt, _error) {
13806
+ if (attempt >= this.maxAttempts) {
13807
+ return null;
13808
+ }
13809
+ const delay = this.baseDelayMs * attempt;
13810
+ return Math.min(delay, this.maxDelayMs);
13811
+ }
13812
+ };
13813
+ var NoRetry = class {
13814
+ maxAttempts = 1;
13815
+ getNextDelay(_attempt, _error) {
13816
+ return null;
13817
+ }
13818
+ };
13819
+ var ImmediateRetry = class {
13820
+ maxAttempts;
13821
+ constructor(maxAttempts) {
13822
+ this.maxAttempts = maxAttempts;
13823
+ }
13824
+ getNextDelay(attempt, _error) {
13825
+ if (attempt >= this.maxAttempts) {
13826
+ return null;
13827
+ }
13828
+ return 0;
13829
+ }
13830
+ };
13831
+ var CustomRetry = class {
13832
+ maxAttempts;
13833
+ calculateDelay;
13834
+ constructor(options) {
13835
+ this.maxAttempts = options.maxAttempts;
13836
+ this.calculateDelay = options.calculateDelay;
13837
+ }
13838
+ getNextDelay(attempt, error) {
13839
+ if (attempt >= this.maxAttempts) {
13840
+ return null;
13841
+ }
13842
+ return this.calculateDelay(attempt, error);
13843
+ }
13844
+ };
13845
+ var defaultRetryOptions = {
13846
+ maxAttempts: 12,
13847
+ factor: 2,
13848
+ minTimeoutInMs: 1e3,
13849
+ maxTimeoutInMs: 36e5,
13850
+ randomize: true
13851
+ };
13852
+ function createDefaultRetryStrategy() {
13853
+ return new ExponentialBackoffRetry(defaultRetryOptions);
13854
+ }
13855
+
13856
+ // src/fair-queue/index.ts
13857
+ var FairQueue = class {
13858
+ constructor(options) {
13859
+ this.options = options;
13860
+ this.redis = createRedisClient(options.redis);
13861
+ this.keys = options.keys;
13862
+ this.scheduler = options.scheduler;
13863
+ this.logger = options.logger ?? new logger$1.Logger("FairQueue", "info");
13864
+ this.abortController = new AbortController();
13865
+ this.payloadSchema = options.payloadSchema;
13866
+ this.validateOnEnqueue = options.validateOnEnqueue ?? false;
13867
+ this.retryStrategy = options.retry?.strategy;
13868
+ this.deadLetterQueueEnabled = options.retry?.deadLetterQueue ?? true;
13869
+ this.shardCount = options.shardCount ?? 1;
13870
+ this.consumerCount = options.consumerCount ?? 1;
13871
+ this.consumerIntervalMs = options.consumerIntervalMs ?? 100;
13872
+ this.visibilityTimeoutMs = options.visibilityTimeoutMs ?? 3e4;
13873
+ this.heartbeatIntervalMs = options.heartbeatIntervalMs ?? this.visibilityTimeoutMs / 3;
13874
+ this.reclaimIntervalMs = options.reclaimIntervalMs ?? 5e3;
13875
+ this.workerQueueEnabled = options.workerQueue?.enabled ?? false;
13876
+ this.workerQueueBlockingTimeoutSeconds = options.workerQueue?.blockingTimeoutSeconds ?? 10;
13877
+ this.workerQueueResolver = options.workerQueue?.resolveWorkerQueue;
13878
+ this.cooloffEnabled = options.cooloff?.enabled ?? true;
13879
+ this.cooloffThreshold = options.cooloff?.threshold ?? 10;
13880
+ this.cooloffPeriodMs = options.cooloff?.periodMs ?? 1e4;
13881
+ this.telemetry = new FairQueueTelemetry({
13882
+ tracer: options.tracer,
13883
+ meter: options.meter,
13884
+ name: options.name ?? "fairqueue"
13885
+ });
13886
+ this.masterQueue = new MasterQueue({
13887
+ redis: options.redis,
13888
+ keys: options.keys,
13889
+ shardCount: this.shardCount
13890
+ });
13891
+ if (options.concurrencyGroups && options.concurrencyGroups.length > 0) {
13892
+ this.concurrencyManager = new ConcurrencyManager({
13893
+ redis: options.redis,
13894
+ keys: options.keys,
13895
+ groups: options.concurrencyGroups
13896
+ });
13897
+ }
13898
+ this.visibilityManager = new VisibilityManager({
13899
+ redis: options.redis,
13900
+ keys: options.keys,
13901
+ shardCount: this.shardCount,
13902
+ defaultTimeoutMs: this.visibilityTimeoutMs,
13903
+ logger: {
13904
+ debug: (msg, ctx) => this.logger.debug(msg, ctx),
13905
+ error: (msg, ctx) => this.logger.error(msg, ctx)
13906
+ }
13907
+ });
13908
+ if (this.workerQueueEnabled) {
13909
+ this.workerQueueManager = new WorkerQueueManager({
13910
+ redis: options.redis,
13911
+ keys: options.keys,
13912
+ logger: {
13913
+ debug: (msg, ctx) => this.logger.debug(msg, ctx),
13914
+ error: (msg, ctx) => this.logger.error(msg, ctx)
13915
+ }
13916
+ });
13917
+ }
13918
+ this.#registerCommands();
13919
+ if (options.startConsumers !== false) {
13920
+ this.start();
13921
+ }
13922
+ }
13923
+ redis;
13924
+ keys;
13925
+ scheduler;
13926
+ masterQueue;
13927
+ concurrencyManager;
13928
+ visibilityManager;
13929
+ workerQueueManager;
13930
+ telemetry;
13931
+ logger;
13932
+ // Configuration
13933
+ payloadSchema;
13934
+ validateOnEnqueue;
13935
+ retryStrategy;
13936
+ deadLetterQueueEnabled;
13937
+ shardCount;
13938
+ consumerCount;
13939
+ consumerIntervalMs;
13940
+ visibilityTimeoutMs;
13941
+ heartbeatIntervalMs;
13942
+ reclaimIntervalMs;
13943
+ workerQueueEnabled;
13944
+ workerQueueBlockingTimeoutSeconds;
13945
+ workerQueueResolver;
13946
+ // Cooloff state
13947
+ cooloffEnabled;
13948
+ cooloffThreshold;
13949
+ cooloffPeriodMs;
13950
+ queueCooloffStates = /* @__PURE__ */ new Map();
13951
+ // Runtime state
13952
+ messageHandler;
13953
+ isRunning = false;
13954
+ abortController;
13955
+ masterQueueConsumerLoops = [];
13956
+ workerQueueConsumerLoops = [];
13957
+ reclaimLoop;
13958
+ // Queue descriptor cache for message processing
13959
+ queueDescriptorCache = /* @__PURE__ */ new Map();
13960
+ // ============================================================================
13961
+ // Public API - Message Handler
13962
+ // ============================================================================
13963
+ /**
13964
+ * Set the message handler for processing dequeued messages.
13965
+ */
13966
+ onMessage(handler) {
13967
+ this.messageHandler = handler;
13968
+ }
13969
+ // ============================================================================
13970
+ // Public API - Enqueueing
13971
+ // ============================================================================
13972
+ /**
13973
+ * Enqueue a single message to a queue.
13974
+ */
13975
+ async enqueue(options) {
13976
+ return this.telemetry.trace(
13977
+ "enqueue",
13978
+ async (span) => {
13979
+ const messageId = options.messageId ?? nanoid();
13980
+ const timestamp = options.timestamp ?? Date.now();
13981
+ const queueKey = this.keys.queueKey(options.queueId);
13982
+ const queueItemsKey = this.keys.queueItemsKey(options.queueId);
13983
+ const shardId = this.masterQueue.getShardForQueue(options.queueId);
13984
+ const masterQueueKey = this.keys.masterQueueKey(shardId);
13985
+ if (this.validateOnEnqueue && this.payloadSchema) {
13986
+ const result = this.payloadSchema.safeParse(options.payload);
13987
+ if (!result.success) {
13988
+ throw new Error(`Payload validation failed: ${result.error.message}`);
13989
+ }
13990
+ }
13991
+ const descriptor = {
13992
+ id: options.queueId,
13993
+ tenantId: options.tenantId,
13994
+ metadata: options.metadata ?? {}
13995
+ };
13996
+ this.queueDescriptorCache.set(options.queueId, descriptor);
13997
+ const storedMessage = {
13998
+ id: messageId,
13999
+ queueId: options.queueId,
14000
+ tenantId: options.tenantId,
14001
+ payload: options.payload,
14002
+ timestamp,
14003
+ attempt: 1,
14004
+ workerQueue: this.workerQueueResolver ? this.workerQueueResolver({
14005
+ id: messageId,
14006
+ queueId: options.queueId,
14007
+ tenantId: options.tenantId,
14008
+ payload: options.payload,
14009
+ timestamp,
14010
+ attempt: 1,
14011
+ metadata: options.metadata
14012
+ }) : options.queueId,
14013
+ metadata: options.metadata
14014
+ };
14015
+ await this.redis.enqueueMessageAtomic(
14016
+ queueKey,
14017
+ queueItemsKey,
14018
+ masterQueueKey,
14019
+ options.queueId,
14020
+ messageId,
14021
+ timestamp.toString(),
14022
+ JSON.stringify(storedMessage)
14023
+ );
14024
+ span.setAttributes({
14025
+ [FairQueueAttributes.QUEUE_ID]: options.queueId,
14026
+ [FairQueueAttributes.TENANT_ID]: options.tenantId,
14027
+ [FairQueueAttributes.MESSAGE_ID]: messageId,
14028
+ [FairQueueAttributes.SHARD_ID]: shardId.toString()
14029
+ });
14030
+ this.telemetry.recordEnqueue(
14031
+ this.telemetry.messageAttributes({
14032
+ queueId: options.queueId,
14033
+ tenantId: options.tenantId,
14034
+ messageId
14035
+ })
14036
+ );
14037
+ this.logger.debug("Message enqueued", {
14038
+ queueId: options.queueId,
14039
+ messageId,
14040
+ timestamp
14041
+ });
14042
+ return messageId;
14043
+ },
14044
+ {
14045
+ kind: SpanKind.PRODUCER,
14046
+ attributes: {
14047
+ [MessagingAttributes.OPERATION]: "publish"
14048
+ }
14049
+ }
14050
+ );
14051
+ }
14052
+ /**
14053
+ * Enqueue multiple messages to a queue.
14054
+ */
14055
+ async enqueueBatch(options) {
14056
+ return this.telemetry.trace(
14057
+ "enqueueBatch",
14058
+ async (span) => {
14059
+ const queueKey = this.keys.queueKey(options.queueId);
14060
+ const queueItemsKey = this.keys.queueItemsKey(options.queueId);
14061
+ const shardId = this.masterQueue.getShardForQueue(options.queueId);
14062
+ const masterQueueKey = this.keys.masterQueueKey(shardId);
14063
+ const now = Date.now();
14064
+ const descriptor = {
14065
+ id: options.queueId,
14066
+ tenantId: options.tenantId,
14067
+ metadata: options.metadata ?? {}
14068
+ };
14069
+ this.queueDescriptorCache.set(options.queueId, descriptor);
14070
+ const messageIds = [];
14071
+ const args = [];
14072
+ for (const message of options.messages) {
14073
+ const messageId = message.messageId ?? nanoid();
14074
+ const timestamp = message.timestamp ?? now;
14075
+ if (this.validateOnEnqueue && this.payloadSchema) {
14076
+ const result = this.payloadSchema.safeParse(message.payload);
14077
+ if (!result.success) {
14078
+ throw new Error(
14079
+ `Payload validation failed for message ${messageId}: ${result.error.message}`
14080
+ );
14081
+ }
14082
+ }
14083
+ const storedMessage = {
14084
+ id: messageId,
14085
+ queueId: options.queueId,
14086
+ tenantId: options.tenantId,
14087
+ payload: message.payload,
14088
+ timestamp,
14089
+ attempt: 1,
14090
+ workerQueue: this.workerQueueResolver ? this.workerQueueResolver({
14091
+ id: messageId,
14092
+ queueId: options.queueId,
14093
+ tenantId: options.tenantId,
14094
+ payload: message.payload,
14095
+ timestamp,
14096
+ attempt: 1,
14097
+ metadata: options.metadata
14098
+ }) : options.queueId,
14099
+ metadata: options.metadata
14100
+ };
14101
+ messageIds.push(messageId);
14102
+ args.push(messageId, timestamp.toString(), JSON.stringify(storedMessage));
14103
+ }
14104
+ await this.redis.enqueueBatchAtomic(
14105
+ queueKey,
14106
+ queueItemsKey,
14107
+ masterQueueKey,
14108
+ options.queueId,
14109
+ ...args
14110
+ );
14111
+ span.setAttributes({
14112
+ [FairQueueAttributes.QUEUE_ID]: options.queueId,
14113
+ [FairQueueAttributes.TENANT_ID]: options.tenantId,
14114
+ [FairQueueAttributes.MESSAGE_COUNT]: messageIds.length,
14115
+ [FairQueueAttributes.SHARD_ID]: shardId.toString()
14116
+ });
14117
+ this.telemetry.recordEnqueueBatch(
14118
+ messageIds.length,
14119
+ this.telemetry.messageAttributes({
14120
+ queueId: options.queueId,
14121
+ tenantId: options.tenantId
14122
+ })
14123
+ );
14124
+ this.logger.debug("Batch enqueued", {
14125
+ queueId: options.queueId,
14126
+ messageCount: messageIds.length
14127
+ });
14128
+ return messageIds;
14129
+ },
14130
+ {
14131
+ kind: SpanKind.PRODUCER,
14132
+ attributes: {
14133
+ [MessagingAttributes.OPERATION]: "publish"
14134
+ }
14135
+ }
14136
+ );
14137
+ }
14138
+ // ============================================================================
14139
+ // Public API - Dead Letter Queue
14140
+ // ============================================================================
14141
+ /**
14142
+ * Get messages from the dead letter queue for a tenant.
14143
+ */
14144
+ async getDeadLetterMessages(tenantId, limit = 100) {
14145
+ if (!this.deadLetterQueueEnabled) {
14146
+ return [];
14147
+ }
14148
+ const dlqKey = this.keys.deadLetterQueueKey(tenantId);
14149
+ const dlqDataKey = this.keys.deadLetterQueueDataKey(tenantId);
14150
+ const results = await this.redis.zrange(dlqKey, 0, limit - 1, "WITHSCORES");
14151
+ const messages = [];
14152
+ for (let i = 0; i < results.length; i += 2) {
14153
+ const messageId = results[i];
14154
+ const deadLetteredAtStr = results[i + 1];
14155
+ if (!messageId || !deadLetteredAtStr) continue;
14156
+ const dataJson = await this.redis.hget(dlqDataKey, messageId);
14157
+ if (!dataJson) continue;
14158
+ try {
14159
+ const data = JSON.parse(dataJson);
14160
+ data.deadLetteredAt = parseFloat(deadLetteredAtStr);
14161
+ messages.push(data);
14162
+ } catch {
14163
+ this.logger.error("Failed to parse DLQ message", { messageId, tenantId });
14164
+ }
14165
+ }
14166
+ return messages;
14167
+ }
14168
+ /**
14169
+ * Redrive a message from DLQ back to its original queue.
14170
+ */
14171
+ async redriveMessage(tenantId, messageId) {
14172
+ if (!this.deadLetterQueueEnabled) {
14173
+ return false;
14174
+ }
14175
+ return this.telemetry.trace(
14176
+ "redriveMessage",
14177
+ async (span) => {
14178
+ const dlqKey = this.keys.deadLetterQueueKey(tenantId);
14179
+ const dlqDataKey = this.keys.deadLetterQueueDataKey(tenantId);
14180
+ const dataJson = await this.redis.hget(dlqDataKey, messageId);
14181
+ if (!dataJson) {
14182
+ return false;
14183
+ }
14184
+ const dlqMessage = JSON.parse(dataJson);
14185
+ await this.enqueue({
14186
+ queueId: dlqMessage.queueId,
14187
+ tenantId: dlqMessage.tenantId,
14188
+ payload: dlqMessage.payload,
14189
+ messageId: dlqMessage.id,
14190
+ timestamp: Date.now()
14191
+ });
14192
+ const pipeline = this.redis.pipeline();
14193
+ pipeline.zrem(dlqKey, messageId);
14194
+ pipeline.hdel(dlqDataKey, messageId);
14195
+ await pipeline.exec();
14196
+ span.setAttributes({
14197
+ [FairQueueAttributes.TENANT_ID]: tenantId,
14198
+ [FairQueueAttributes.MESSAGE_ID]: messageId
14199
+ });
14200
+ this.logger.info("Redrived message from DLQ", { tenantId, messageId });
14201
+ return true;
14202
+ },
14203
+ {
14204
+ kind: SpanKind.PRODUCER,
14205
+ attributes: {
14206
+ [MessagingAttributes.OPERATION]: "redrive"
14207
+ }
14208
+ }
14209
+ );
14210
+ }
14211
+ /**
14212
+ * Redrive all messages from DLQ back to their original queues.
14213
+ */
14214
+ async redriveAll(tenantId) {
14215
+ const messages = await this.getDeadLetterMessages(tenantId, 1e3);
14216
+ let count = 0;
14217
+ for (const message of messages) {
14218
+ const success = await this.redriveMessage(tenantId, message.id);
14219
+ if (success) count++;
14220
+ }
14221
+ return count;
14222
+ }
14223
+ /**
14224
+ * Purge all messages from a tenant's DLQ.
14225
+ */
14226
+ async purgeDeadLetterQueue(tenantId) {
14227
+ if (!this.deadLetterQueueEnabled) {
14228
+ return 0;
14229
+ }
14230
+ const dlqKey = this.keys.deadLetterQueueKey(tenantId);
14231
+ const dlqDataKey = this.keys.deadLetterQueueDataKey(tenantId);
14232
+ const count = await this.redis.zcard(dlqKey);
14233
+ const pipeline = this.redis.pipeline();
14234
+ pipeline.del(dlqKey);
14235
+ pipeline.del(dlqDataKey);
14236
+ await pipeline.exec();
14237
+ this.logger.info("Purged DLQ", { tenantId, count });
14238
+ return count;
14239
+ }
14240
+ /**
14241
+ * Get the number of messages in a tenant's DLQ.
14242
+ */
14243
+ async getDeadLetterQueueLength(tenantId) {
14244
+ if (!this.deadLetterQueueEnabled) {
14245
+ return 0;
14246
+ }
14247
+ const dlqKey = this.keys.deadLetterQueueKey(tenantId);
14248
+ return await this.redis.zcard(dlqKey);
14249
+ }
14250
+ // ============================================================================
14251
+ // Public API - Lifecycle
14252
+ // ============================================================================
14253
+ /**
14254
+ * Start the consumer loops and reclaim loop.
14255
+ */
14256
+ start() {
14257
+ if (this.isRunning) {
14258
+ return;
14259
+ }
14260
+ this.isRunning = true;
14261
+ this.abortController = new AbortController();
14262
+ if (this.workerQueueEnabled && this.workerQueueManager) {
14263
+ for (let shardId = 0; shardId < this.shardCount; shardId++) {
14264
+ const loop = this.#runMasterQueueConsumerLoop(shardId);
14265
+ this.masterQueueConsumerLoops.push(loop);
14266
+ }
14267
+ for (let consumerId = 0; consumerId < this.consumerCount; consumerId++) {
14268
+ const loop = this.#runWorkerQueueConsumerLoop(consumerId);
14269
+ this.workerQueueConsumerLoops.push(loop);
14270
+ }
14271
+ } else {
14272
+ for (let consumerId = 0; consumerId < this.consumerCount; consumerId++) {
14273
+ for (let shardId = 0; shardId < this.shardCount; shardId++) {
14274
+ const loop = this.#runDirectConsumerLoop(consumerId, shardId);
14275
+ this.masterQueueConsumerLoops.push(loop);
14276
+ }
14277
+ }
14278
+ }
14279
+ this.reclaimLoop = this.#runReclaimLoop();
14280
+ this.logger.info("FairQueue started", {
14281
+ consumerCount: this.consumerCount,
14282
+ shardCount: this.shardCount,
14283
+ workerQueueEnabled: this.workerQueueEnabled,
14284
+ consumerIntervalMs: this.consumerIntervalMs
14285
+ });
14286
+ }
14287
+ /**
14288
+ * Stop the consumer loops gracefully.
14289
+ */
14290
+ async stop() {
14291
+ if (!this.isRunning) {
14292
+ return;
14293
+ }
14294
+ this.isRunning = false;
14295
+ this.abortController.abort();
14296
+ await Promise.allSettled([
14297
+ ...this.masterQueueConsumerLoops,
14298
+ ...this.workerQueueConsumerLoops,
14299
+ this.reclaimLoop
14300
+ ]);
14301
+ this.masterQueueConsumerLoops = [];
14302
+ this.workerQueueConsumerLoops = [];
14303
+ this.reclaimLoop = void 0;
14304
+ this.logger.info("FairQueue stopped");
14305
+ }
14306
+ /**
14307
+ * Close all resources.
14308
+ */
14309
+ async close() {
14310
+ await this.stop();
14311
+ await Promise.all([
14312
+ this.masterQueue.close(),
14313
+ this.concurrencyManager?.close(),
14314
+ this.visibilityManager.close(),
14315
+ this.workerQueueManager?.close(),
14316
+ this.scheduler.close?.(),
14317
+ this.redis.quit()
14318
+ ]);
14319
+ }
14320
+ // ============================================================================
14321
+ // Public API - Inspection
14322
+ // ============================================================================
14323
+ /**
14324
+ * Get the number of messages in a queue.
14325
+ */
14326
+ async getQueueLength(queueId) {
14327
+ const queueKey = this.keys.queueKey(queueId);
14328
+ return await this.redis.zcard(queueKey);
14329
+ }
14330
+ /**
14331
+ * Get total queue count across all shards.
14332
+ */
14333
+ async getTotalQueueCount() {
14334
+ return await this.masterQueue.getTotalQueueCount();
14335
+ }
14336
+ /**
14337
+ * Get total in-flight message count.
14338
+ */
14339
+ async getTotalInflightCount() {
14340
+ return await this.visibilityManager.getTotalInflightCount();
14341
+ }
14342
+ /**
14343
+ * Get the shard ID for a queue.
14344
+ */
14345
+ getShardForQueue(queueId) {
14346
+ return this.masterQueue.getShardForQueue(queueId);
14347
+ }
14348
+ // ============================================================================
14349
+ // Private - Master Queue Consumer Loop (Two-Stage)
14350
+ // ============================================================================
14351
+ async #runMasterQueueConsumerLoop(shardId) {
14352
+ const loopId = `master-shard-${shardId}`;
14353
+ try {
14354
+ for await (const _ of promises.setInterval(this.consumerIntervalMs, null, {
14355
+ signal: this.abortController.signal
14356
+ })) {
14357
+ try {
14358
+ await this.#processMasterQueueShard(loopId, shardId);
14359
+ } catch (error) {
14360
+ this.logger.error("Master queue consumer error", {
14361
+ loopId,
14362
+ shardId,
14363
+ error: error instanceof Error ? error.message : String(error)
14364
+ });
14365
+ }
14366
+ }
14367
+ } catch (error) {
14368
+ if (error instanceof Error && error.name === "AbortError") {
14369
+ this.logger.debug("Master queue consumer aborted", { loopId });
14370
+ return;
14371
+ }
14372
+ throw error;
14373
+ }
14374
+ }
14375
+ async #processMasterQueueShard(loopId, shardId) {
14376
+ const masterQueueKey = this.keys.masterQueueKey(shardId);
14377
+ const context2 = this.#createSchedulerContext();
14378
+ const tenantQueues = await this.scheduler.selectQueues(masterQueueKey, loopId, context2);
14379
+ if (tenantQueues.length === 0) {
14380
+ return;
14381
+ }
14382
+ for (const { tenantId, queues } of tenantQueues) {
14383
+ for (const queueId of queues) {
14384
+ if (this.cooloffEnabled && this.#isInCooloff(queueId)) {
14385
+ continue;
14386
+ }
14387
+ const processed = await this.#claimAndPushToWorkerQueue(loopId, queueId, tenantId, shardId);
14388
+ if (processed) {
14389
+ await this.scheduler.recordProcessed?.(tenantId, queueId);
14390
+ this.#resetCooloff(queueId);
14391
+ } else {
14392
+ this.#incrementCooloff(queueId);
14393
+ }
14394
+ }
14395
+ }
14396
+ }
14397
+ async #claimAndPushToWorkerQueue(loopId, queueId, tenantId, shardId) {
14398
+ const queueKey = this.keys.queueKey(queueId);
14399
+ const queueItemsKey = this.keys.queueItemsKey(queueId);
14400
+ const masterQueueKey = this.keys.masterQueueKey(shardId);
14401
+ const descriptor = this.queueDescriptorCache.get(queueId) ?? {
14402
+ id: queueId,
14403
+ tenantId,
14404
+ metadata: {}
14405
+ };
14406
+ if (this.concurrencyManager) {
14407
+ const check = await this.concurrencyManager.canProcess(descriptor);
14408
+ if (!check.allowed) {
14409
+ return false;
14410
+ }
14411
+ }
14412
+ const claimResult = await this.visibilityManager.claim(
14413
+ queueId,
14414
+ queueKey,
14415
+ queueItemsKey,
14416
+ loopId,
14417
+ this.visibilityTimeoutMs
14418
+ );
14419
+ if (!claimResult.claimed || !claimResult.message) {
14420
+ await this.redis.updateMasterQueueIfEmpty(masterQueueKey, queueKey, queueId);
14421
+ return false;
14422
+ }
14423
+ const { message } = claimResult;
14424
+ if (this.concurrencyManager) {
14425
+ const reserved = await this.concurrencyManager.reserve(descriptor, message.messageId);
14426
+ if (!reserved) {
14427
+ await this.visibilityManager.release(message.messageId, queueId, queueKey, queueItemsKey);
14428
+ return false;
14429
+ }
14430
+ }
14431
+ const workerQueueId = message.payload.workerQueue ?? queueId;
14432
+ const messageKey = `${message.messageId}:${queueId}`;
14433
+ await this.workerQueueManager.push(workerQueueId, messageKey);
14434
+ return true;
14435
+ }
14436
+ // ============================================================================
14437
+ // Private - Worker Queue Consumer Loop (Two-Stage)
14438
+ // ============================================================================
14439
+ async #runWorkerQueueConsumerLoop(consumerId) {
14440
+ const loopId = `worker-${consumerId}`;
14441
+ const workerQueueId = loopId;
14442
+ try {
14443
+ while (this.isRunning) {
14444
+ if (!this.messageHandler) {
14445
+ await new Promise((resolve) => setTimeout(resolve, this.consumerIntervalMs));
14446
+ continue;
14447
+ }
14448
+ try {
14449
+ const messageKey = await this.workerQueueManager.blockingPop(
14450
+ workerQueueId,
14451
+ this.workerQueueBlockingTimeoutSeconds,
14452
+ this.abortController.signal
14453
+ );
14454
+ if (!messageKey) {
14455
+ continue;
14456
+ }
14457
+ const colonIndex = messageKey.indexOf(":");
14458
+ if (colonIndex === -1) {
14459
+ this.logger.error("Invalid message key format", { messageKey });
14460
+ continue;
14461
+ }
14462
+ const messageId = messageKey.substring(0, colonIndex);
14463
+ const queueId = messageKey.substring(colonIndex + 1);
14464
+ await this.#processMessageFromWorkerQueue(loopId, messageId, queueId);
14465
+ } catch (error) {
14466
+ if (this.abortController.signal.aborted) {
14467
+ break;
14468
+ }
14469
+ this.logger.error("Worker queue consumer error", {
14470
+ loopId,
14471
+ error: error instanceof Error ? error.message : String(error)
14472
+ });
14473
+ }
14474
+ }
14475
+ } catch (error) {
14476
+ if (error instanceof Error && error.name === "AbortError") {
14477
+ this.logger.debug("Worker queue consumer aborted", { loopId });
14478
+ return;
14479
+ }
14480
+ throw error;
14481
+ }
14482
+ }
14483
+ async #processMessageFromWorkerQueue(loopId, messageId, queueId) {
14484
+ const shardId = this.masterQueue.getShardForQueue(queueId);
14485
+ const inflightDataKey = this.keys.inflightDataKey(shardId);
14486
+ const dataJson = await this.redis.hget(inflightDataKey, messageId);
14487
+ if (!dataJson) {
14488
+ this.logger.error("Message not found in in-flight data", { messageId, queueId });
14489
+ return;
14490
+ }
14491
+ let storedMessage;
14492
+ try {
14493
+ storedMessage = JSON.parse(dataJson);
14494
+ } catch {
14495
+ this.logger.error("Failed to parse message data", { messageId, queueId });
14496
+ return;
14497
+ }
14498
+ await this.#processMessage(loopId, storedMessage, queueId);
14499
+ }
14500
+ // ============================================================================
14501
+ // Private - Direct Consumer Loop (No Worker Queue)
14502
+ // ============================================================================
14503
+ async #runDirectConsumerLoop(consumerId, shardId) {
14504
+ const loopId = `consumer-${consumerId}-shard-${shardId}`;
14505
+ try {
14506
+ for await (const _ of promises.setInterval(this.consumerIntervalMs, null, {
14507
+ signal: this.abortController.signal
14508
+ })) {
14509
+ if (!this.messageHandler) {
14510
+ continue;
14511
+ }
14512
+ try {
14513
+ await this.#processDirectIteration(loopId, shardId);
14514
+ } catch (error) {
14515
+ this.logger.error("Direct consumer iteration error", {
14516
+ loopId,
14517
+ error: error instanceof Error ? error.message : String(error)
14518
+ });
14519
+ }
14520
+ }
14521
+ } catch (error) {
14522
+ if (error instanceof Error && error.name === "AbortError") {
14523
+ this.logger.debug("Direct consumer loop aborted", { loopId });
14524
+ return;
14525
+ }
14526
+ throw error;
14527
+ }
14528
+ }
14529
+ async #processDirectIteration(loopId, shardId) {
14530
+ const masterQueueKey = this.keys.masterQueueKey(shardId);
14531
+ const context2 = this.#createSchedulerContext();
14532
+ const tenantQueues = await this.scheduler.selectQueues(masterQueueKey, loopId, context2);
14533
+ if (tenantQueues.length === 0) {
14534
+ return;
14535
+ }
14536
+ for (const { tenantId, queues } of tenantQueues) {
14537
+ for (const queueId of queues) {
14538
+ if (this.cooloffEnabled && this.#isInCooloff(queueId)) {
14539
+ continue;
14540
+ }
14541
+ const processed = await this.#processOneMessage(loopId, queueId, tenantId, shardId);
14542
+ if (processed) {
14543
+ await this.scheduler.recordProcessed?.(tenantId, queueId);
14544
+ this.#resetCooloff(queueId);
14545
+ } else {
14546
+ this.#incrementCooloff(queueId);
14547
+ }
14548
+ break;
14549
+ }
14550
+ }
14551
+ }
14552
+ async #processOneMessage(loopId, queueId, tenantId, shardId) {
14553
+ const queueKey = this.keys.queueKey(queueId);
14554
+ const queueItemsKey = this.keys.queueItemsKey(queueId);
14555
+ const masterQueueKey = this.keys.masterQueueKey(shardId);
14556
+ const descriptor = this.queueDescriptorCache.get(queueId) ?? {
14557
+ id: queueId,
14558
+ tenantId,
14559
+ metadata: {}
14560
+ };
14561
+ if (this.concurrencyManager) {
14562
+ const check = await this.concurrencyManager.canProcess(descriptor);
14563
+ if (!check.allowed) {
14564
+ return false;
14565
+ }
14566
+ }
14567
+ const claimResult = await this.visibilityManager.claim(
14568
+ queueId,
14569
+ queueKey,
14570
+ queueItemsKey,
14571
+ loopId,
14572
+ this.visibilityTimeoutMs
14573
+ );
14574
+ if (!claimResult.claimed || !claimResult.message) {
14575
+ await this.redis.updateMasterQueueIfEmpty(masterQueueKey, queueKey, queueId);
14576
+ return false;
14577
+ }
14578
+ const { message } = claimResult;
14579
+ if (this.concurrencyManager) {
14580
+ const reserved = await this.concurrencyManager.reserve(descriptor, message.messageId);
14581
+ if (!reserved) {
14582
+ await this.visibilityManager.release(message.messageId, queueId, queueKey, queueItemsKey);
14583
+ return false;
14584
+ }
14585
+ }
14586
+ await this.#processMessage(loopId, message.payload, queueId);
14587
+ return true;
14588
+ }
14589
+ // ============================================================================
14590
+ // Private - Message Processing
14591
+ // ============================================================================
14592
+ async #processMessage(loopId, storedMessage, queueId) {
14593
+ const startTime = Date.now();
14594
+ const queueKey = this.keys.queueKey(queueId);
14595
+ const queueItemsKey = this.keys.queueItemsKey(queueId);
14596
+ const shardId = this.masterQueue.getShardForQueue(queueId);
14597
+ const masterQueueKey = this.keys.masterQueueKey(shardId);
14598
+ const descriptor = this.queueDescriptorCache.get(queueId) ?? {
14599
+ id: queueId,
14600
+ tenantId: storedMessage.tenantId,
14601
+ metadata: storedMessage.metadata ?? {}
14602
+ };
14603
+ let payload;
14604
+ if (this.payloadSchema) {
14605
+ const result = this.payloadSchema.safeParse(storedMessage.payload);
14606
+ if (!result.success) {
14607
+ this.logger.error("Payload validation failed on dequeue", {
14608
+ messageId: storedMessage.id,
14609
+ queueId,
14610
+ error: result.error.message
14611
+ });
14612
+ await this.#moveToDeadLetterQueue(storedMessage, "Payload validation failed");
14613
+ return;
14614
+ }
14615
+ payload = result.data;
14616
+ } else {
14617
+ payload = storedMessage.payload;
14618
+ }
14619
+ const queueMessage = {
14620
+ id: storedMessage.id,
14621
+ queueId,
14622
+ payload,
14623
+ timestamp: storedMessage.timestamp,
14624
+ attempt: storedMessage.attempt,
14625
+ metadata: storedMessage.metadata
14626
+ };
14627
+ const queueTime = startTime - storedMessage.timestamp;
14628
+ this.telemetry.recordQueueTime(
14629
+ queueTime,
14630
+ this.telemetry.messageAttributes({
14631
+ queueId,
14632
+ tenantId: storedMessage.tenantId,
14633
+ messageId: storedMessage.id
14634
+ })
14635
+ );
14636
+ const handlerContext = {
14637
+ message: queueMessage,
14638
+ queue: descriptor,
14639
+ consumerId: loopId,
14640
+ heartbeat: async () => {
14641
+ return this.visibilityManager.heartbeat(
14642
+ storedMessage.id,
14643
+ queueId,
14644
+ this.heartbeatIntervalMs
14645
+ );
14646
+ },
14647
+ complete: async () => {
14648
+ await this.#completeMessage(storedMessage, queueId, queueKey, masterQueueKey, descriptor);
14649
+ this.telemetry.recordComplete(
14650
+ this.telemetry.messageAttributes({
14651
+ queueId,
14652
+ tenantId: storedMessage.tenantId,
14653
+ messageId: storedMessage.id
14654
+ })
14655
+ );
14656
+ this.telemetry.recordProcessingTime(
14657
+ Date.now() - startTime,
14658
+ this.telemetry.messageAttributes({
14659
+ queueId,
14660
+ tenantId: storedMessage.tenantId,
14661
+ messageId: storedMessage.id
14662
+ })
14663
+ );
14664
+ },
14665
+ release: async () => {
14666
+ await this.#releaseMessage(storedMessage, queueId, queueKey, queueItemsKey, descriptor);
14667
+ },
14668
+ fail: async (error) => {
14669
+ await this.#handleMessageFailure(
14670
+ storedMessage,
14671
+ queueId,
14672
+ queueKey,
14673
+ queueItemsKey,
14674
+ masterQueueKey,
14675
+ descriptor,
14676
+ error
14677
+ );
14678
+ }
14679
+ };
14680
+ try {
14681
+ await this.telemetry.trace(
14682
+ "processMessage",
14683
+ async (span) => {
14684
+ span.setAttributes({
14685
+ [FairQueueAttributes.QUEUE_ID]: queueId,
14686
+ [FairQueueAttributes.TENANT_ID]: storedMessage.tenantId,
14687
+ [FairQueueAttributes.MESSAGE_ID]: storedMessage.id,
14688
+ [FairQueueAttributes.ATTEMPT]: storedMessage.attempt,
14689
+ [FairQueueAttributes.CONSUMER_ID]: loopId
14690
+ });
14691
+ await this.messageHandler(handlerContext);
14692
+ },
14693
+ {
14694
+ kind: SpanKind.CONSUMER,
14695
+ attributes: {
14696
+ [MessagingAttributes.OPERATION]: "process"
14697
+ }
14698
+ }
14699
+ );
14700
+ } catch (error) {
14701
+ this.logger.error("Message handler error", {
14702
+ messageId: storedMessage.id,
14703
+ queueId,
14704
+ error: error instanceof Error ? error.message : String(error)
14705
+ });
14706
+ await handlerContext.fail(error instanceof Error ? error : new Error(String(error)));
14707
+ }
14708
+ }
14709
+ async #completeMessage(storedMessage, queueId, queueKey, masterQueueKey, descriptor) {
14710
+ this.masterQueue.getShardForQueue(queueId);
14711
+ await this.visibilityManager.complete(storedMessage.id, queueId);
14712
+ if (this.concurrencyManager) {
14713
+ await this.concurrencyManager.release(descriptor, storedMessage.id);
14714
+ }
14715
+ await this.redis.updateMasterQueueIfEmpty(masterQueueKey, queueKey, queueId);
14716
+ this.logger.debug("Message completed", {
14717
+ messageId: storedMessage.id,
14718
+ queueId
14719
+ });
14720
+ }
14721
+ async #releaseMessage(storedMessage, queueId, queueKey, queueItemsKey, descriptor) {
14722
+ await this.visibilityManager.release(
14723
+ storedMessage.id,
14724
+ queueId,
14725
+ queueKey,
14726
+ queueItemsKey,
14727
+ Date.now()
14728
+ // Put at back of queue
14729
+ );
14730
+ if (this.concurrencyManager) {
14731
+ await this.concurrencyManager.release(descriptor, storedMessage.id);
14732
+ }
14733
+ this.logger.debug("Message released", {
14734
+ messageId: storedMessage.id,
14735
+ queueId
14736
+ });
14737
+ }
14738
+ async #handleMessageFailure(storedMessage, queueId, queueKey, queueItemsKey, masterQueueKey, descriptor, error) {
14739
+ this.telemetry.recordFailure(
14740
+ this.telemetry.messageAttributes({
14741
+ queueId,
14742
+ tenantId: storedMessage.tenantId,
14743
+ messageId: storedMessage.id,
14744
+ attempt: storedMessage.attempt
14745
+ })
14746
+ );
14747
+ if (this.retryStrategy) {
14748
+ const nextDelay = this.retryStrategy.getNextDelay(storedMessage.attempt, error);
14749
+ if (nextDelay !== null) {
14750
+ const updatedMessage = {
14751
+ ...storedMessage,
14752
+ attempt: storedMessage.attempt + 1
14753
+ };
14754
+ await this.visibilityManager.release(
14755
+ storedMessage.id,
14756
+ queueId,
14757
+ queueKey,
14758
+ queueItemsKey,
14759
+ Date.now() + nextDelay
14760
+ );
14761
+ await this.redis.hset(queueItemsKey, storedMessage.id, JSON.stringify(updatedMessage));
14762
+ if (this.concurrencyManager) {
14763
+ await this.concurrencyManager.release(descriptor, storedMessage.id);
14764
+ }
14765
+ this.telemetry.recordRetry(
14766
+ this.telemetry.messageAttributes({
14767
+ queueId,
14768
+ tenantId: storedMessage.tenantId,
14769
+ messageId: storedMessage.id,
14770
+ attempt: storedMessage.attempt + 1
14771
+ })
14772
+ );
14773
+ this.logger.debug("Message scheduled for retry", {
14774
+ messageId: storedMessage.id,
14775
+ queueId,
14776
+ attempt: storedMessage.attempt + 1,
14777
+ delayMs: nextDelay
14778
+ });
14779
+ return;
14780
+ }
14781
+ }
14782
+ await this.#moveToDeadLetterQueue(storedMessage, error?.message);
14783
+ if (this.concurrencyManager) {
14784
+ await this.concurrencyManager.release(descriptor, storedMessage.id);
14785
+ }
14786
+ }
14787
+ async #moveToDeadLetterQueue(storedMessage, errorMessage) {
14788
+ if (!this.deadLetterQueueEnabled) {
14789
+ this.masterQueue.getShardForQueue(storedMessage.queueId);
14790
+ await this.visibilityManager.complete(storedMessage.id, storedMessage.queueId);
14791
+ return;
14792
+ }
14793
+ const dlqKey = this.keys.deadLetterQueueKey(storedMessage.tenantId);
14794
+ const dlqDataKey = this.keys.deadLetterQueueDataKey(storedMessage.tenantId);
14795
+ this.masterQueue.getShardForQueue(storedMessage.queueId);
14796
+ const dlqMessage = {
14797
+ id: storedMessage.id,
14798
+ queueId: storedMessage.queueId,
14799
+ tenantId: storedMessage.tenantId,
14800
+ payload: storedMessage.payload,
14801
+ deadLetteredAt: Date.now(),
14802
+ attempts: storedMessage.attempt,
14803
+ lastError: errorMessage,
14804
+ originalTimestamp: storedMessage.timestamp
14805
+ };
14806
+ await this.visibilityManager.complete(storedMessage.id, storedMessage.queueId);
14807
+ const pipeline = this.redis.pipeline();
14808
+ pipeline.zadd(dlqKey, dlqMessage.deadLetteredAt, storedMessage.id);
14809
+ pipeline.hset(dlqDataKey, storedMessage.id, JSON.stringify(dlqMessage));
14810
+ await pipeline.exec();
14811
+ this.telemetry.recordDLQ(
14812
+ this.telemetry.messageAttributes({
14813
+ queueId: storedMessage.queueId,
14814
+ tenantId: storedMessage.tenantId,
14815
+ messageId: storedMessage.id,
14816
+ attempt: storedMessage.attempt
14817
+ })
14818
+ );
14819
+ this.logger.info("Message moved to DLQ", {
14820
+ messageId: storedMessage.id,
14821
+ queueId: storedMessage.queueId,
14822
+ tenantId: storedMessage.tenantId,
14823
+ attempts: storedMessage.attempt,
14824
+ error: errorMessage
14825
+ });
14826
+ }
14827
+ // ============================================================================
14828
+ // Private - Reclaim Loop
14829
+ // ============================================================================
14830
+ async #runReclaimLoop() {
14831
+ try {
14832
+ for await (const _ of promises.setInterval(this.reclaimIntervalMs, null, {
14833
+ signal: this.abortController.signal
14834
+ })) {
14835
+ try {
14836
+ await this.#reclaimTimedOutMessages();
14837
+ } catch (error) {
14838
+ this.logger.error("Reclaim loop error", {
14839
+ error: error instanceof Error ? error.message : String(error)
14840
+ });
14841
+ }
14842
+ }
14843
+ } catch (error) {
14844
+ if (error instanceof Error && error.name === "AbortError") {
14845
+ this.logger.debug("Reclaim loop aborted");
14846
+ return;
14847
+ }
14848
+ throw error;
14849
+ }
14850
+ }
14851
+ async #reclaimTimedOutMessages() {
14852
+ let totalReclaimed = 0;
14853
+ for (let shardId = 0; shardId < this.shardCount; shardId++) {
14854
+ const reclaimed = await this.visibilityManager.reclaimTimedOut(shardId, (queueId) => ({
14855
+ queueKey: this.keys.queueKey(queueId),
14856
+ queueItemsKey: this.keys.queueItemsKey(queueId)
14857
+ }));
14858
+ totalReclaimed += reclaimed;
14859
+ }
14860
+ if (totalReclaimed > 0) {
14861
+ this.logger.info("Reclaimed timed-out messages", { count: totalReclaimed });
14862
+ }
14863
+ }
14864
+ // ============================================================================
14865
+ // Private - Cooloff State
14866
+ // ============================================================================
14867
+ #isInCooloff(queueId) {
14868
+ const state = this.queueCooloffStates.get(queueId);
14869
+ if (!state) return false;
14870
+ if (state.tag === "cooloff") {
14871
+ if (Date.now() >= state.expiresAt) {
14872
+ this.queueCooloffStates.delete(queueId);
14873
+ return false;
14874
+ }
14875
+ return true;
14876
+ }
14877
+ return false;
14878
+ }
14879
+ #incrementCooloff(queueId) {
14880
+ const state = this.queueCooloffStates.get(queueId) ?? {
14881
+ tag: "normal",
14882
+ consecutiveFailures: 0
14883
+ };
14884
+ if (state.tag === "normal") {
14885
+ const newFailures = state.consecutiveFailures + 1;
14886
+ if (newFailures >= this.cooloffThreshold) {
14887
+ this.queueCooloffStates.set(queueId, {
14888
+ tag: "cooloff",
14889
+ expiresAt: Date.now() + this.cooloffPeriodMs
14890
+ });
14891
+ } else {
14892
+ this.queueCooloffStates.set(queueId, {
14893
+ tag: "normal",
14894
+ consecutiveFailures: newFailures
14895
+ });
14896
+ }
14897
+ }
14898
+ }
14899
+ #resetCooloff(queueId) {
14900
+ this.queueCooloffStates.delete(queueId);
14901
+ }
14902
+ // ============================================================================
14903
+ // Private - Helpers
14904
+ // ============================================================================
14905
+ #createSchedulerContext() {
14906
+ return {
14907
+ getCurrentConcurrency: async (groupName, groupId) => {
14908
+ if (!this.concurrencyManager) return 0;
14909
+ return this.concurrencyManager.getCurrentConcurrency(groupName, groupId);
14910
+ },
14911
+ getConcurrencyLimit: async (groupName, groupId) => {
14912
+ if (!this.concurrencyManager) return Infinity;
14913
+ return this.concurrencyManager.getConcurrencyLimit(groupName, groupId);
14914
+ },
14915
+ isAtCapacity: async (groupName, groupId) => {
14916
+ if (!this.concurrencyManager) return false;
14917
+ return this.concurrencyManager.isAtCapacity(groupName, groupId);
14918
+ },
14919
+ getQueueDescriptor: (queueId) => {
14920
+ return this.queueDescriptorCache.get(queueId) ?? {
14921
+ id: queueId,
14922
+ tenantId: this.keys.extractTenantId(queueId),
14923
+ metadata: {}
14924
+ };
14925
+ }
14926
+ };
14927
+ }
14928
+ // ============================================================================
14929
+ // Private - Redis Commands
14930
+ // ============================================================================
14931
+ #registerCommands() {
14932
+ this.redis.defineCommand("enqueueMessageAtomic", {
14933
+ numberOfKeys: 3,
14934
+ lua: `
14935
+ local queueKey = KEYS[1]
14936
+ local queueItemsKey = KEYS[2]
14937
+ local masterQueueKey = KEYS[3]
14938
+
14939
+ local queueId = ARGV[1]
14940
+ local messageId = ARGV[2]
14941
+ local timestamp = tonumber(ARGV[3])
14942
+ local payload = ARGV[4]
14943
+
14944
+ -- Add to sorted set (score = timestamp)
14945
+ redis.call('ZADD', queueKey, timestamp, messageId)
14946
+
14947
+ -- Store payload in hash
14948
+ redis.call('HSET', queueItemsKey, messageId, payload)
14949
+
14950
+ -- Update master queue with oldest message timestamp
14951
+ local oldest = redis.call('ZRANGE', queueKey, 0, 0, 'WITHSCORES')
14952
+ if #oldest >= 2 then
14953
+ redis.call('ZADD', masterQueueKey, oldest[2], queueId)
14954
+ end
14955
+
14956
+ return 1
14957
+ `
14958
+ });
14959
+ this.redis.defineCommand("enqueueBatchAtomic", {
14960
+ numberOfKeys: 3,
14961
+ lua: `
14962
+ local queueKey = KEYS[1]
14963
+ local queueItemsKey = KEYS[2]
14964
+ local masterQueueKey = KEYS[3]
14965
+
14966
+ local queueId = ARGV[1]
14967
+
14968
+ -- Args after queueId are triples: [messageId, timestamp, payload, ...]
14969
+ for i = 2, #ARGV, 3 do
14970
+ local messageId = ARGV[i]
14971
+ local timestamp = tonumber(ARGV[i + 1])
14972
+ local payload = ARGV[i + 2]
14973
+
14974
+ -- Add to sorted set
14975
+ redis.call('ZADD', queueKey, timestamp, messageId)
14976
+
14977
+ -- Store payload in hash
14978
+ redis.call('HSET', queueItemsKey, messageId, payload)
14979
+ end
14980
+
14981
+ -- Update master queue with oldest message timestamp
14982
+ local oldest = redis.call('ZRANGE', queueKey, 0, 0, 'WITHSCORES')
14983
+ if #oldest >= 2 then
14984
+ redis.call('ZADD', masterQueueKey, oldest[2], queueId)
14985
+ end
14986
+
14987
+ return (#ARGV - 1) / 3
14988
+ `
14989
+ });
14990
+ this.redis.defineCommand("updateMasterQueueIfEmpty", {
14991
+ numberOfKeys: 2,
14992
+ lua: `
14993
+ local masterQueueKey = KEYS[1]
14994
+ local queueKey = KEYS[2]
14995
+ local queueId = ARGV[1]
14996
+
14997
+ local count = redis.call('ZCARD', queueKey)
14998
+ if count == 0 then
14999
+ redis.call('ZREM', masterQueueKey, queueId)
15000
+ return 1
15001
+ else
15002
+ -- Update with oldest message timestamp
15003
+ local oldest = redis.call('ZRANGE', queueKey, 0, 0, 'WITHSCORES')
15004
+ if #oldest >= 2 then
15005
+ redis.call('ZADD', masterQueueKey, oldest[2], queueId)
15006
+ end
15007
+ return 0
15008
+ end
15009
+ `
15010
+ });
15011
+ if (this.workerQueueManager) {
15012
+ this.workerQueueManager.registerCommands(this.redis);
15013
+ }
15014
+ }
15015
+ };
15016
+
15017
+ exports.BaseScheduler = BaseScheduler;
15018
+ exports.CallbackFairQueueKeyProducer = CallbackFairQueueKeyProducer;
15019
+ exports.ConcurrencyManager = ConcurrencyManager;
11730
15020
  exports.CronSchema = CronSchema;
15021
+ exports.CustomRetry = CustomRetry;
15022
+ exports.DRRScheduler = DRRScheduler;
15023
+ exports.DefaultFairQueueKeyProducer = DefaultFairQueueKeyProducer;
15024
+ exports.ExponentialBackoffRetry = ExponentialBackoffRetry;
15025
+ exports.FairQueue = FairQueue;
15026
+ exports.FairQueueAttributes = FairQueueAttributes;
15027
+ exports.FairQueueTelemetry = FairQueueTelemetry;
15028
+ exports.FixedDelayRetry = FixedDelayRetry;
15029
+ exports.ImmediateRetry = ImmediateRetry;
15030
+ exports.LinearBackoffRetry = LinearBackoffRetry;
15031
+ exports.MasterQueue = MasterQueue;
15032
+ exports.MessagingAttributes = MessagingAttributes;
15033
+ exports.NoRetry = NoRetry;
15034
+ exports.NoopScheduler = NoopScheduler;
15035
+ exports.RoundRobinScheduler = RoundRobinScheduler;
11731
15036
  exports.SimpleQueue = SimpleQueue;
15037
+ exports.VisibilityManager = VisibilityManager;
15038
+ exports.WeightedScheduler = WeightedScheduler;
11732
15039
  exports.Worker = Worker;
15040
+ exports.WorkerQueueManager = WorkerQueueManager;
15041
+ exports.createDefaultRetryStrategy = createDefaultRetryStrategy;
15042
+ exports.defaultRetryOptions = defaultRetryOptions;
15043
+ exports.noopTelemetry = noopTelemetry;
11733
15044
  //# sourceMappingURL=index.cjs.map
11734
15045
  //# sourceMappingURL=index.cjs.map