@trigger.dev/redis-worker 4.5.0-rc.7 → 4.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +368 -337
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +460 -443
- package/dist/index.d.ts +460 -443
- package/dist/index.js +368 -337
- package/dist/index.js.map +1 -1
- package/package.json +2 -2
package/dist/index.cjs
CHANGED
|
@@ -9060,7 +9060,7 @@ var require_Redis = __commonJS({
|
|
|
9060
9060
|
var lodash_1 = require_lodash3();
|
|
9061
9061
|
var Deque = require_denque();
|
|
9062
9062
|
var debug = (0, utils_1.Debug)("redis");
|
|
9063
|
-
var
|
|
9063
|
+
var Redis3 = class _Redis extends Commander_1.default {
|
|
9064
9064
|
constructor(arg1, arg2, arg3) {
|
|
9065
9065
|
super();
|
|
9066
9066
|
this.status = "wait";
|
|
@@ -9639,12 +9639,12 @@ var require_Redis = __commonJS({
|
|
|
9639
9639
|
}).catch(lodash_1.noop);
|
|
9640
9640
|
}
|
|
9641
9641
|
};
|
|
9642
|
-
|
|
9643
|
-
|
|
9644
|
-
|
|
9645
|
-
(0, applyMixin_1.default)(
|
|
9646
|
-
(0, transaction_1.addTransactionSupport)(
|
|
9647
|
-
exports$1.default =
|
|
9642
|
+
Redis3.Cluster = cluster_1.default;
|
|
9643
|
+
Redis3.Command = Command_1.default;
|
|
9644
|
+
Redis3.defaultOptions = RedisOptions_1.DEFAULT_REDIS_OPTIONS;
|
|
9645
|
+
(0, applyMixin_1.default)(Redis3, events_1.EventEmitter);
|
|
9646
|
+
(0, transaction_1.addTransactionSupport)(Redis3.prototype);
|
|
9647
|
+
exports$1.default = Redis3;
|
|
9648
9648
|
}
|
|
9649
9649
|
});
|
|
9650
9650
|
|
|
@@ -9992,6 +9992,37 @@ var SimpleQueue = class {
|
|
|
9992
9992
|
throw e;
|
|
9993
9993
|
}
|
|
9994
9994
|
}
|
|
9995
|
+
/**
|
|
9996
|
+
* Age (in ms) of the oldest *overdue* message — the oldest item whose scheduled
|
|
9997
|
+
* time has already passed (score <= now). Returns 0 when the queue is empty or
|
|
9998
|
+
* only holds future/delayed or in-flight (future-scored) items.
|
|
9999
|
+
*
|
|
10000
|
+
* Resolves the candidate against the `items` hash so orphaned `queue` entries
|
|
10001
|
+
* (a member whose payload is missing — the same stale state `dequeueItems`
|
|
10002
|
+
* cleans up) don't report a phantom stall for work that can't be dequeued. The
|
|
10003
|
+
* Lua scans due items oldest-first and returns the first score whose payload
|
|
10004
|
+
* still exists.
|
|
10005
|
+
*
|
|
10006
|
+
* This is the generic stall signal: it stays at 0 while a queue drains healthily
|
|
10007
|
+
* and rises only when due work sits undrained (poison block, dead consumer,
|
|
10008
|
+
* backpressure).
|
|
10009
|
+
*/
|
|
10010
|
+
async oldestMessageAge() {
|
|
10011
|
+
try {
|
|
10012
|
+
const now = Date.now();
|
|
10013
|
+
const score = Number(await this.redis.getOldestDueScore(`queue`, `items`, now));
|
|
10014
|
+
if (!Number.isFinite(score) || score < 0) {
|
|
10015
|
+
return 0;
|
|
10016
|
+
}
|
|
10017
|
+
return Math.max(0, now - score);
|
|
10018
|
+
} catch (e) {
|
|
10019
|
+
this.logger.error(`SimpleQueue ${this.name}.oldestMessageAge(): error getting oldest age`, {
|
|
10020
|
+
queue: this.name,
|
|
10021
|
+
error: e
|
|
10022
|
+
});
|
|
10023
|
+
return 0;
|
|
10024
|
+
}
|
|
10025
|
+
}
|
|
9995
10026
|
async getJob(id) {
|
|
9996
10027
|
const result = await this.redis.getJob(`queue`, `items`, id);
|
|
9997
10028
|
if (!result) {
|
|
@@ -10158,6 +10189,29 @@ var SimpleQueue = class {
|
|
|
10158
10189
|
return dequeued
|
|
10159
10190
|
`
|
|
10160
10191
|
});
|
|
10192
|
+
this.redis.defineCommand("getOldestDueScore", {
|
|
10193
|
+
numberOfKeys: 2,
|
|
10194
|
+
lua: `
|
|
10195
|
+
local queue = KEYS[1]
|
|
10196
|
+
local items = KEYS[2]
|
|
10197
|
+
local now = tonumber(ARGV[1])
|
|
10198
|
+
|
|
10199
|
+
-- Oldest-first scan of due items, bounded so a long prefix of orphans can't
|
|
10200
|
+
-- make this O(n). Orphans are rare (dequeueItems removes them), so in the
|
|
10201
|
+
-- common case this returns on the first iteration. Read-only: unlike
|
|
10202
|
+
-- dequeueItems we don't ZREM orphans here \u2014 a metric probe must not mutate.
|
|
10203
|
+
local result = redis.call('ZRANGEBYSCORE', queue, '-inf', now, 'WITHSCORES', 'LIMIT', 0, 100)
|
|
10204
|
+
|
|
10205
|
+
for i = 1, #result, 2 do
|
|
10206
|
+
local id = result[i]
|
|
10207
|
+
if redis.call('HEXISTS', items, id) == 1 then
|
|
10208
|
+
return result[i + 1]
|
|
10209
|
+
end
|
|
10210
|
+
end
|
|
10211
|
+
|
|
10212
|
+
return -1
|
|
10213
|
+
`
|
|
10214
|
+
});
|
|
10161
10215
|
this.redis.defineCommand("getJob", {
|
|
10162
10216
|
numberOfKeys: 2,
|
|
10163
10217
|
lua: `
|
|
@@ -11402,6 +11456,15 @@ var Worker = class _Worker {
|
|
|
11402
11456
|
concurrencyLimitPendingObservableGauge.addCallback(
|
|
11403
11457
|
this.#updateConcurrencyLimitPendingMetric.bind(this)
|
|
11404
11458
|
);
|
|
11459
|
+
const oldestMessageAgeObservableGauge = this.meter.createObservableGauge(
|
|
11460
|
+
"redis_worker.queue.oldest_message_age",
|
|
11461
|
+
{
|
|
11462
|
+
description: "Age of the oldest overdue message in the queue",
|
|
11463
|
+
unit: "ms",
|
|
11464
|
+
valueType: ValueType.INT
|
|
11465
|
+
}
|
|
11466
|
+
);
|
|
11467
|
+
oldestMessageAgeObservableGauge.addCallback(this.#updateOldestMessageAgeMetric.bind(this));
|
|
11405
11468
|
}
|
|
11406
11469
|
subscriber;
|
|
11407
11470
|
tracer;
|
|
@@ -11430,6 +11493,12 @@ var Worker = class _Worker {
|
|
|
11430
11493
|
worker_name: this.options.name
|
|
11431
11494
|
});
|
|
11432
11495
|
}
|
|
11496
|
+
async #updateOldestMessageAgeMetric(observableResult) {
|
|
11497
|
+
const oldestMessageAge = await this.queue.oldestMessageAge();
|
|
11498
|
+
observableResult.observe(oldestMessageAge, {
|
|
11499
|
+
worker_name: this.options.name
|
|
11500
|
+
});
|
|
11501
|
+
}
|
|
11433
11502
|
async #updateConcurrencyLimitActiveMetric(observableResult) {
|
|
11434
11503
|
for (const [workerId, limiter] of Object.entries(this.limiters)) {
|
|
11435
11504
|
observableResult.observe(limiter.activeCount, {
|
|
@@ -11723,15 +11792,15 @@ var Worker = class _Worker {
|
|
|
11723
11792
|
await this.flushBatch(queueItem.job, workerId, limiter);
|
|
11724
11793
|
}
|
|
11725
11794
|
} else {
|
|
11726
|
-
limiter(
|
|
11727
|
-
() =>
|
|
11728
|
-
|
|
11729
|
-
|
|
11730
|
-
|
|
11731
|
-
|
|
11732
|
-
|
|
11733
|
-
}
|
|
11734
|
-
|
|
11795
|
+
limiter(() => this.processItem(queueItem, items.length, workerId, limiter)).catch(
|
|
11796
|
+
(err) => {
|
|
11797
|
+
this.logger.error("Unhandled error in processItem:", {
|
|
11798
|
+
error: err,
|
|
11799
|
+
workerId,
|
|
11800
|
+
item
|
|
11801
|
+
});
|
|
11802
|
+
}
|
|
11803
|
+
);
|
|
11735
11804
|
}
|
|
11736
11805
|
}
|
|
11737
11806
|
} catch (error) {
|
|
@@ -11881,13 +11950,25 @@ var Worker = class _Worker {
|
|
|
11881
11950
|
attempt: newAttempt
|
|
11882
11951
|
};
|
|
11883
11952
|
if (!shouldLogError) {
|
|
11884
|
-
this.logger.info(
|
|
11953
|
+
this.logger.info(
|
|
11954
|
+
`Worker batch item reached max attempts. Moving to DLQ.`,
|
|
11955
|
+
dlqLogAttributes
|
|
11956
|
+
);
|
|
11885
11957
|
} else if (errorLogLevel === "warn") {
|
|
11886
|
-
this.logger.warn(
|
|
11958
|
+
this.logger.warn(
|
|
11959
|
+
`Worker batch item reached max attempts. Moving to DLQ.`,
|
|
11960
|
+
dlqLogAttributes
|
|
11961
|
+
);
|
|
11887
11962
|
} else if (errorLogLevel === "info") {
|
|
11888
|
-
this.logger.info(
|
|
11963
|
+
this.logger.info(
|
|
11964
|
+
`Worker batch item reached max attempts. Moving to DLQ.`,
|
|
11965
|
+
dlqLogAttributes
|
|
11966
|
+
);
|
|
11889
11967
|
} else {
|
|
11890
|
-
this.logger.error(
|
|
11968
|
+
this.logger.error(
|
|
11969
|
+
`Worker batch item reached max attempts. Moving to DLQ.`,
|
|
11970
|
+
dlqLogAttributes
|
|
11971
|
+
);
|
|
11891
11972
|
}
|
|
11892
11973
|
await this.queue.moveToDeadLetterQueue(item.id, errorMessage);
|
|
11893
11974
|
continue;
|
|
@@ -12644,135 +12725,6 @@ return 1
|
|
|
12644
12725
|
});
|
|
12645
12726
|
}
|
|
12646
12727
|
};
|
|
12647
|
-
var TenantDispatch = class {
|
|
12648
|
-
constructor(options) {
|
|
12649
|
-
this.options = options;
|
|
12650
|
-
this.redis = createRedisClient(options.redis);
|
|
12651
|
-
this.keys = options.keys;
|
|
12652
|
-
this.shardCount = Math.max(1, options.shardCount);
|
|
12653
|
-
}
|
|
12654
|
-
redis;
|
|
12655
|
-
keys;
|
|
12656
|
-
shardCount;
|
|
12657
|
-
/**
|
|
12658
|
-
* Get the dispatch shard ID for a tenant.
|
|
12659
|
-
* Uses jump consistent hash on the tenant ID so each tenant
|
|
12660
|
-
* always maps to exactly one dispatch shard.
|
|
12661
|
-
*/
|
|
12662
|
-
getShardForTenant(tenantId) {
|
|
12663
|
-
return serverOnly.jumpHash(tenantId, this.shardCount);
|
|
12664
|
-
}
|
|
12665
|
-
/**
|
|
12666
|
-
* Get eligible tenants from a dispatch shard (Level 1).
|
|
12667
|
-
* Returns tenants ordered by oldest message (lowest score first).
|
|
12668
|
-
*/
|
|
12669
|
-
async getTenantsFromShard(shardId, limit = 1e3, maxScore) {
|
|
12670
|
-
const dispatchKey = this.keys.dispatchKey(shardId);
|
|
12671
|
-
const score = maxScore ?? Date.now();
|
|
12672
|
-
const results = await this.redis.zrangebyscore(
|
|
12673
|
-
dispatchKey,
|
|
12674
|
-
"-inf",
|
|
12675
|
-
score,
|
|
12676
|
-
"WITHSCORES",
|
|
12677
|
-
"LIMIT",
|
|
12678
|
-
0,
|
|
12679
|
-
limit
|
|
12680
|
-
);
|
|
12681
|
-
const tenants = [];
|
|
12682
|
-
for (let i = 0; i < results.length; i += 2) {
|
|
12683
|
-
const tenantId = results[i];
|
|
12684
|
-
const scoreStr = results[i + 1];
|
|
12685
|
-
if (tenantId && scoreStr) {
|
|
12686
|
-
tenants.push({
|
|
12687
|
-
tenantId,
|
|
12688
|
-
score: parseFloat(scoreStr)
|
|
12689
|
-
});
|
|
12690
|
-
}
|
|
12691
|
-
}
|
|
12692
|
-
return tenants;
|
|
12693
|
-
}
|
|
12694
|
-
/**
|
|
12695
|
-
* Get queues for a specific tenant (Level 2).
|
|
12696
|
-
* Returns queues ordered by oldest message (lowest score first).
|
|
12697
|
-
*/
|
|
12698
|
-
async getQueuesForTenant(tenantId, limit = 1e3, maxScore) {
|
|
12699
|
-
const tenantQueueKey = this.keys.tenantQueueIndexKey(tenantId);
|
|
12700
|
-
const score = maxScore ?? Date.now();
|
|
12701
|
-
const results = await this.redis.zrangebyscore(
|
|
12702
|
-
tenantQueueKey,
|
|
12703
|
-
"-inf",
|
|
12704
|
-
score,
|
|
12705
|
-
"WITHSCORES",
|
|
12706
|
-
"LIMIT",
|
|
12707
|
-
0,
|
|
12708
|
-
limit
|
|
12709
|
-
);
|
|
12710
|
-
const queues = [];
|
|
12711
|
-
for (let i = 0; i < results.length; i += 2) {
|
|
12712
|
-
const queueId = results[i];
|
|
12713
|
-
const scoreStr = results[i + 1];
|
|
12714
|
-
if (queueId && scoreStr) {
|
|
12715
|
-
queues.push({
|
|
12716
|
-
queueId,
|
|
12717
|
-
score: parseFloat(scoreStr),
|
|
12718
|
-
tenantId
|
|
12719
|
-
});
|
|
12720
|
-
}
|
|
12721
|
-
}
|
|
12722
|
-
return queues;
|
|
12723
|
-
}
|
|
12724
|
-
/**
|
|
12725
|
-
* Get the number of tenants in a dispatch shard.
|
|
12726
|
-
*/
|
|
12727
|
-
async getShardTenantCount(shardId) {
|
|
12728
|
-
const dispatchKey = this.keys.dispatchKey(shardId);
|
|
12729
|
-
return await this.redis.zcard(dispatchKey);
|
|
12730
|
-
}
|
|
12731
|
-
/**
|
|
12732
|
-
* Get total tenant count across all dispatch shards.
|
|
12733
|
-
* Note: tenants may appear in multiple shards, so this may overcount.
|
|
12734
|
-
*/
|
|
12735
|
-
async getTotalTenantCount() {
|
|
12736
|
-
const counts = await Promise.all(
|
|
12737
|
-
Array.from({ length: this.shardCount }, (_, i) => this.getShardTenantCount(i))
|
|
12738
|
-
);
|
|
12739
|
-
return counts.reduce((sum, count) => sum + count, 0);
|
|
12740
|
-
}
|
|
12741
|
-
/**
|
|
12742
|
-
* Get the number of queues for a tenant.
|
|
12743
|
-
*/
|
|
12744
|
-
async getTenantQueueCount(tenantId) {
|
|
12745
|
-
const tenantQueueKey = this.keys.tenantQueueIndexKey(tenantId);
|
|
12746
|
-
return await this.redis.zcard(tenantQueueKey);
|
|
12747
|
-
}
|
|
12748
|
-
/**
|
|
12749
|
-
* Remove a tenant from a specific dispatch shard.
|
|
12750
|
-
*/
|
|
12751
|
-
async removeTenantFromShard(shardId, tenantId) {
|
|
12752
|
-
const dispatchKey = this.keys.dispatchKey(shardId);
|
|
12753
|
-
await this.redis.zrem(dispatchKey, tenantId);
|
|
12754
|
-
}
|
|
12755
|
-
/**
|
|
12756
|
-
* Add a tenant to a dispatch shard with the given score.
|
|
12757
|
-
*/
|
|
12758
|
-
async addTenantToShard(shardId, tenantId, score) {
|
|
12759
|
-
const dispatchKey = this.keys.dispatchKey(shardId);
|
|
12760
|
-
await this.redis.zadd(dispatchKey, score, tenantId);
|
|
12761
|
-
}
|
|
12762
|
-
/**
|
|
12763
|
-
* Remove a queue from a tenant's queue index.
|
|
12764
|
-
*/
|
|
12765
|
-
async removeQueueFromTenant(tenantId, queueId) {
|
|
12766
|
-
const tenantQueueKey = this.keys.tenantQueueIndexKey(tenantId);
|
|
12767
|
-
await this.redis.zrem(tenantQueueKey, queueId);
|
|
12768
|
-
}
|
|
12769
|
-
/**
|
|
12770
|
-
* Close the Redis connection.
|
|
12771
|
-
*/
|
|
12772
|
-
async close() {
|
|
12773
|
-
await this.redis.quit();
|
|
12774
|
-
}
|
|
12775
|
-
};
|
|
12776
12728
|
|
|
12777
12729
|
// src/fair-queue/telemetry.ts
|
|
12778
12730
|
var FairQueueAttributes = {
|
|
@@ -13252,41 +13204,170 @@ var BatchedSpanManager = class {
|
|
|
13252
13204
|
}
|
|
13253
13205
|
}
|
|
13254
13206
|
/**
|
|
13255
|
-
* Clean up state for a loop when it's stopped.
|
|
13207
|
+
* Clean up state for a loop when it's stopped.
|
|
13208
|
+
*/
|
|
13209
|
+
cleanup(loopId) {
|
|
13210
|
+
this.endCurrentSpan(loopId);
|
|
13211
|
+
this.loopStates.delete(loopId);
|
|
13212
|
+
}
|
|
13213
|
+
/**
|
|
13214
|
+
* Clean up all loop states.
|
|
13215
|
+
*/
|
|
13216
|
+
cleanupAll() {
|
|
13217
|
+
for (const loopId of this.loopStates.keys()) {
|
|
13218
|
+
this.cleanup(loopId);
|
|
13219
|
+
}
|
|
13220
|
+
}
|
|
13221
|
+
};
|
|
13222
|
+
var noopSpan = {
|
|
13223
|
+
spanContext: () => ({
|
|
13224
|
+
traceId: "",
|
|
13225
|
+
spanId: "",
|
|
13226
|
+
traceFlags: 0
|
|
13227
|
+
}),
|
|
13228
|
+
setAttribute: () => noopSpan,
|
|
13229
|
+
setAttributes: () => noopSpan,
|
|
13230
|
+
addEvent: () => noopSpan,
|
|
13231
|
+
addLink: () => noopSpan,
|
|
13232
|
+
addLinks: () => noopSpan,
|
|
13233
|
+
setStatus: () => noopSpan,
|
|
13234
|
+
updateName: () => noopSpan,
|
|
13235
|
+
end: () => {
|
|
13236
|
+
},
|
|
13237
|
+
isRecording: () => false,
|
|
13238
|
+
recordException: () => {
|
|
13239
|
+
}
|
|
13240
|
+
};
|
|
13241
|
+
var noopTelemetry = new FairQueueTelemetry({});
|
|
13242
|
+
var TenantDispatch = class {
|
|
13243
|
+
constructor(options) {
|
|
13244
|
+
this.options = options;
|
|
13245
|
+
this.redis = createRedisClient(options.redis);
|
|
13246
|
+
this.keys = options.keys;
|
|
13247
|
+
this.shardCount = Math.max(1, options.shardCount);
|
|
13248
|
+
}
|
|
13249
|
+
redis;
|
|
13250
|
+
keys;
|
|
13251
|
+
shardCount;
|
|
13252
|
+
/**
|
|
13253
|
+
* Get the dispatch shard ID for a tenant.
|
|
13254
|
+
* Uses jump consistent hash on the tenant ID so each tenant
|
|
13255
|
+
* always maps to exactly one dispatch shard.
|
|
13256
|
+
*/
|
|
13257
|
+
getShardForTenant(tenantId) {
|
|
13258
|
+
return serverOnly.jumpHash(tenantId, this.shardCount);
|
|
13259
|
+
}
|
|
13260
|
+
/**
|
|
13261
|
+
* Get eligible tenants from a dispatch shard (Level 1).
|
|
13262
|
+
* Returns tenants ordered by oldest message (lowest score first).
|
|
13263
|
+
*/
|
|
13264
|
+
async getTenantsFromShard(shardId, limit = 1e3, maxScore) {
|
|
13265
|
+
const dispatchKey = this.keys.dispatchKey(shardId);
|
|
13266
|
+
const score = maxScore ?? Date.now();
|
|
13267
|
+
const results = await this.redis.zrangebyscore(
|
|
13268
|
+
dispatchKey,
|
|
13269
|
+
"-inf",
|
|
13270
|
+
score,
|
|
13271
|
+
"WITHSCORES",
|
|
13272
|
+
"LIMIT",
|
|
13273
|
+
0,
|
|
13274
|
+
limit
|
|
13275
|
+
);
|
|
13276
|
+
const tenants = [];
|
|
13277
|
+
for (let i = 0; i < results.length; i += 2) {
|
|
13278
|
+
const tenantId = results[i];
|
|
13279
|
+
const scoreStr = results[i + 1];
|
|
13280
|
+
if (tenantId && scoreStr) {
|
|
13281
|
+
tenants.push({
|
|
13282
|
+
tenantId,
|
|
13283
|
+
score: parseFloat(scoreStr)
|
|
13284
|
+
});
|
|
13285
|
+
}
|
|
13286
|
+
}
|
|
13287
|
+
return tenants;
|
|
13288
|
+
}
|
|
13289
|
+
/**
|
|
13290
|
+
* Get queues for a specific tenant (Level 2).
|
|
13291
|
+
* Returns queues ordered by oldest message (lowest score first).
|
|
13292
|
+
*/
|
|
13293
|
+
async getQueuesForTenant(tenantId, limit = 1e3, maxScore) {
|
|
13294
|
+
const tenantQueueKey = this.keys.tenantQueueIndexKey(tenantId);
|
|
13295
|
+
const score = maxScore ?? Date.now();
|
|
13296
|
+
const results = await this.redis.zrangebyscore(
|
|
13297
|
+
tenantQueueKey,
|
|
13298
|
+
"-inf",
|
|
13299
|
+
score,
|
|
13300
|
+
"WITHSCORES",
|
|
13301
|
+
"LIMIT",
|
|
13302
|
+
0,
|
|
13303
|
+
limit
|
|
13304
|
+
);
|
|
13305
|
+
const queues = [];
|
|
13306
|
+
for (let i = 0; i < results.length; i += 2) {
|
|
13307
|
+
const queueId = results[i];
|
|
13308
|
+
const scoreStr = results[i + 1];
|
|
13309
|
+
if (queueId && scoreStr) {
|
|
13310
|
+
queues.push({
|
|
13311
|
+
queueId,
|
|
13312
|
+
score: parseFloat(scoreStr),
|
|
13313
|
+
tenantId
|
|
13314
|
+
});
|
|
13315
|
+
}
|
|
13316
|
+
}
|
|
13317
|
+
return queues;
|
|
13318
|
+
}
|
|
13319
|
+
/**
|
|
13320
|
+
* Get the number of tenants in a dispatch shard.
|
|
13321
|
+
*/
|
|
13322
|
+
async getShardTenantCount(shardId) {
|
|
13323
|
+
const dispatchKey = this.keys.dispatchKey(shardId);
|
|
13324
|
+
return await this.redis.zcard(dispatchKey);
|
|
13325
|
+
}
|
|
13326
|
+
/**
|
|
13327
|
+
* Get total tenant count across all dispatch shards.
|
|
13328
|
+
* Note: tenants may appear in multiple shards, so this may overcount.
|
|
13329
|
+
*/
|
|
13330
|
+
async getTotalTenantCount() {
|
|
13331
|
+
const counts = await Promise.all(
|
|
13332
|
+
Array.from({ length: this.shardCount }, (_, i) => this.getShardTenantCount(i))
|
|
13333
|
+
);
|
|
13334
|
+
return counts.reduce((sum, count) => sum + count, 0);
|
|
13335
|
+
}
|
|
13336
|
+
/**
|
|
13337
|
+
* Get the number of queues for a tenant.
|
|
13338
|
+
*/
|
|
13339
|
+
async getTenantQueueCount(tenantId) {
|
|
13340
|
+
const tenantQueueKey = this.keys.tenantQueueIndexKey(tenantId);
|
|
13341
|
+
return await this.redis.zcard(tenantQueueKey);
|
|
13342
|
+
}
|
|
13343
|
+
/**
|
|
13344
|
+
* Remove a tenant from a specific dispatch shard.
|
|
13345
|
+
*/
|
|
13346
|
+
async removeTenantFromShard(shardId, tenantId) {
|
|
13347
|
+
const dispatchKey = this.keys.dispatchKey(shardId);
|
|
13348
|
+
await this.redis.zrem(dispatchKey, tenantId);
|
|
13349
|
+
}
|
|
13350
|
+
/**
|
|
13351
|
+
* Add a tenant to a dispatch shard with the given score.
|
|
13256
13352
|
*/
|
|
13257
|
-
|
|
13258
|
-
this.
|
|
13259
|
-
this.
|
|
13353
|
+
async addTenantToShard(shardId, tenantId, score) {
|
|
13354
|
+
const dispatchKey = this.keys.dispatchKey(shardId);
|
|
13355
|
+
await this.redis.zadd(dispatchKey, score, tenantId);
|
|
13260
13356
|
}
|
|
13261
13357
|
/**
|
|
13262
|
-
*
|
|
13358
|
+
* Remove a queue from a tenant's queue index.
|
|
13263
13359
|
*/
|
|
13264
|
-
|
|
13265
|
-
|
|
13266
|
-
|
|
13267
|
-
}
|
|
13360
|
+
async removeQueueFromTenant(tenantId, queueId) {
|
|
13361
|
+
const tenantQueueKey = this.keys.tenantQueueIndexKey(tenantId);
|
|
13362
|
+
await this.redis.zrem(tenantQueueKey, queueId);
|
|
13268
13363
|
}
|
|
13269
|
-
|
|
13270
|
-
|
|
13271
|
-
|
|
13272
|
-
|
|
13273
|
-
|
|
13274
|
-
traceFlags: 0
|
|
13275
|
-
}),
|
|
13276
|
-
setAttribute: () => noopSpan,
|
|
13277
|
-
setAttributes: () => noopSpan,
|
|
13278
|
-
addEvent: () => noopSpan,
|
|
13279
|
-
addLink: () => noopSpan,
|
|
13280
|
-
addLinks: () => noopSpan,
|
|
13281
|
-
setStatus: () => noopSpan,
|
|
13282
|
-
updateName: () => noopSpan,
|
|
13283
|
-
end: () => {
|
|
13284
|
-
},
|
|
13285
|
-
isRecording: () => false,
|
|
13286
|
-
recordException: () => {
|
|
13364
|
+
/**
|
|
13365
|
+
* Close the Redis connection.
|
|
13366
|
+
*/
|
|
13367
|
+
async close() {
|
|
13368
|
+
await this.redis.quit();
|
|
13287
13369
|
}
|
|
13288
13370
|
};
|
|
13289
|
-
var noopTelemetry = new FairQueueTelemetry({});
|
|
13290
13371
|
var VisibilityManager = class {
|
|
13291
13372
|
constructor(options) {
|
|
13292
13373
|
this.options = options;
|
|
@@ -14268,6 +14349,101 @@ var CallbackFairQueueKeyProducer = class extends DefaultFairQueueKeyProducer {
|
|
|
14268
14349
|
return this.groupExtractor(groupName, queueId);
|
|
14269
14350
|
}
|
|
14270
14351
|
};
|
|
14352
|
+
var ExponentialBackoffRetry = class {
|
|
14353
|
+
maxAttempts;
|
|
14354
|
+
options;
|
|
14355
|
+
constructor(options) {
|
|
14356
|
+
this.options = {
|
|
14357
|
+
maxAttempts: options?.maxAttempts ?? 12,
|
|
14358
|
+
factor: options?.factor ?? 2,
|
|
14359
|
+
minTimeoutInMs: options?.minTimeoutInMs ?? 1e3,
|
|
14360
|
+
maxTimeoutInMs: options?.maxTimeoutInMs ?? 36e5,
|
|
14361
|
+
// 1 hour
|
|
14362
|
+
randomize: options?.randomize ?? true
|
|
14363
|
+
};
|
|
14364
|
+
this.maxAttempts = this.options.maxAttempts ?? 12;
|
|
14365
|
+
}
|
|
14366
|
+
getNextDelay(attempt, _error) {
|
|
14367
|
+
if (attempt >= this.maxAttempts) {
|
|
14368
|
+
return null;
|
|
14369
|
+
}
|
|
14370
|
+
const delay = v3.calculateNextRetryDelay(this.options, attempt);
|
|
14371
|
+
return delay ?? null;
|
|
14372
|
+
}
|
|
14373
|
+
};
|
|
14374
|
+
var FixedDelayRetry = class {
|
|
14375
|
+
maxAttempts;
|
|
14376
|
+
delayMs;
|
|
14377
|
+
constructor(options) {
|
|
14378
|
+
this.maxAttempts = options.maxAttempts;
|
|
14379
|
+
this.delayMs = options.delayMs;
|
|
14380
|
+
}
|
|
14381
|
+
getNextDelay(attempt, _error) {
|
|
14382
|
+
if (attempt >= this.maxAttempts) {
|
|
14383
|
+
return null;
|
|
14384
|
+
}
|
|
14385
|
+
return this.delayMs;
|
|
14386
|
+
}
|
|
14387
|
+
};
|
|
14388
|
+
var LinearBackoffRetry = class {
|
|
14389
|
+
maxAttempts;
|
|
14390
|
+
baseDelayMs;
|
|
14391
|
+
maxDelayMs;
|
|
14392
|
+
constructor(options) {
|
|
14393
|
+
this.maxAttempts = options.maxAttempts;
|
|
14394
|
+
this.baseDelayMs = options.baseDelayMs;
|
|
14395
|
+
this.maxDelayMs = options.maxDelayMs ?? options.baseDelayMs * options.maxAttempts;
|
|
14396
|
+
}
|
|
14397
|
+
getNextDelay(attempt, _error) {
|
|
14398
|
+
if (attempt >= this.maxAttempts) {
|
|
14399
|
+
return null;
|
|
14400
|
+
}
|
|
14401
|
+
const delay = this.baseDelayMs * attempt;
|
|
14402
|
+
return Math.min(delay, this.maxDelayMs);
|
|
14403
|
+
}
|
|
14404
|
+
};
|
|
14405
|
+
var NoRetry = class {
|
|
14406
|
+
maxAttempts = 1;
|
|
14407
|
+
getNextDelay(_attempt, _error) {
|
|
14408
|
+
return null;
|
|
14409
|
+
}
|
|
14410
|
+
};
|
|
14411
|
+
var ImmediateRetry = class {
|
|
14412
|
+
maxAttempts;
|
|
14413
|
+
constructor(maxAttempts) {
|
|
14414
|
+
this.maxAttempts = maxAttempts;
|
|
14415
|
+
}
|
|
14416
|
+
getNextDelay(attempt, _error) {
|
|
14417
|
+
if (attempt >= this.maxAttempts) {
|
|
14418
|
+
return null;
|
|
14419
|
+
}
|
|
14420
|
+
return 0;
|
|
14421
|
+
}
|
|
14422
|
+
};
|
|
14423
|
+
var CustomRetry = class {
|
|
14424
|
+
maxAttempts;
|
|
14425
|
+
calculateDelay;
|
|
14426
|
+
constructor(options) {
|
|
14427
|
+
this.maxAttempts = options.maxAttempts;
|
|
14428
|
+
this.calculateDelay = options.calculateDelay;
|
|
14429
|
+
}
|
|
14430
|
+
getNextDelay(attempt, error) {
|
|
14431
|
+
if (attempt >= this.maxAttempts) {
|
|
14432
|
+
return null;
|
|
14433
|
+
}
|
|
14434
|
+
return this.calculateDelay(attempt, error);
|
|
14435
|
+
}
|
|
14436
|
+
};
|
|
14437
|
+
var defaultRetryOptions = {
|
|
14438
|
+
maxAttempts: 12,
|
|
14439
|
+
factor: 2,
|
|
14440
|
+
minTimeoutInMs: 1e3,
|
|
14441
|
+
maxTimeoutInMs: 36e5,
|
|
14442
|
+
randomize: true
|
|
14443
|
+
};
|
|
14444
|
+
function createDefaultRetryStrategy() {
|
|
14445
|
+
return new ExponentialBackoffRetry(defaultRetryOptions);
|
|
14446
|
+
}
|
|
14271
14447
|
|
|
14272
14448
|
// src/fair-queue/scheduler.ts
|
|
14273
14449
|
var BaseScheduler = class {
|
|
@@ -14396,9 +14572,7 @@ var DRRScheduler = class extends BaseScheduler {
|
|
|
14396
14572
|
};
|
|
14397
14573
|
})
|
|
14398
14574
|
);
|
|
14399
|
-
const eligibleTenants = tenantData.filter(
|
|
14400
|
-
(t) => !t.isAtCapacity && t.deficit >= 1
|
|
14401
|
-
);
|
|
14575
|
+
const eligibleTenants = tenantData.filter((t) => !t.isAtCapacity && t.deficit >= 1);
|
|
14402
14576
|
const blockedTenants = tenantData.filter((t) => t.isAtCapacity);
|
|
14403
14577
|
if (blockedTenants.length > 0) {
|
|
14404
14578
|
this.logger.debug("DRR: tenants blocked by concurrency", {
|
|
@@ -14691,11 +14865,7 @@ var WeightedScheduler = class extends BaseScheduler {
|
|
|
14691
14865
|
// FairScheduler Implementation
|
|
14692
14866
|
// ============================================================================
|
|
14693
14867
|
async selectQueues(masterQueueShard, consumerId, context2) {
|
|
14694
|
-
const snapshot = await this.#getOrCreateSnapshot(
|
|
14695
|
-
masterQueueShard,
|
|
14696
|
-
consumerId,
|
|
14697
|
-
context2
|
|
14698
|
-
);
|
|
14868
|
+
const snapshot = await this.#getOrCreateSnapshot(masterQueueShard, consumerId, context2);
|
|
14699
14869
|
if (snapshot.queues.length === 0) {
|
|
14700
14870
|
return [];
|
|
14701
14871
|
}
|
|
@@ -15046,101 +15216,6 @@ var RoundRobinScheduler = class extends BaseScheduler {
|
|
|
15046
15216
|
return [...array.slice(normalizedIndex), ...array.slice(0, normalizedIndex)];
|
|
15047
15217
|
}
|
|
15048
15218
|
};
|
|
15049
|
-
var ExponentialBackoffRetry = class {
|
|
15050
|
-
maxAttempts;
|
|
15051
|
-
options;
|
|
15052
|
-
constructor(options) {
|
|
15053
|
-
this.options = {
|
|
15054
|
-
maxAttempts: options?.maxAttempts ?? 12,
|
|
15055
|
-
factor: options?.factor ?? 2,
|
|
15056
|
-
minTimeoutInMs: options?.minTimeoutInMs ?? 1e3,
|
|
15057
|
-
maxTimeoutInMs: options?.maxTimeoutInMs ?? 36e5,
|
|
15058
|
-
// 1 hour
|
|
15059
|
-
randomize: options?.randomize ?? true
|
|
15060
|
-
};
|
|
15061
|
-
this.maxAttempts = this.options.maxAttempts ?? 12;
|
|
15062
|
-
}
|
|
15063
|
-
getNextDelay(attempt, _error) {
|
|
15064
|
-
if (attempt >= this.maxAttempts) {
|
|
15065
|
-
return null;
|
|
15066
|
-
}
|
|
15067
|
-
const delay = v3.calculateNextRetryDelay(this.options, attempt);
|
|
15068
|
-
return delay ?? null;
|
|
15069
|
-
}
|
|
15070
|
-
};
|
|
15071
|
-
var FixedDelayRetry = class {
|
|
15072
|
-
maxAttempts;
|
|
15073
|
-
delayMs;
|
|
15074
|
-
constructor(options) {
|
|
15075
|
-
this.maxAttempts = options.maxAttempts;
|
|
15076
|
-
this.delayMs = options.delayMs;
|
|
15077
|
-
}
|
|
15078
|
-
getNextDelay(attempt, _error) {
|
|
15079
|
-
if (attempt >= this.maxAttempts) {
|
|
15080
|
-
return null;
|
|
15081
|
-
}
|
|
15082
|
-
return this.delayMs;
|
|
15083
|
-
}
|
|
15084
|
-
};
|
|
15085
|
-
var LinearBackoffRetry = class {
|
|
15086
|
-
maxAttempts;
|
|
15087
|
-
baseDelayMs;
|
|
15088
|
-
maxDelayMs;
|
|
15089
|
-
constructor(options) {
|
|
15090
|
-
this.maxAttempts = options.maxAttempts;
|
|
15091
|
-
this.baseDelayMs = options.baseDelayMs;
|
|
15092
|
-
this.maxDelayMs = options.maxDelayMs ?? options.baseDelayMs * options.maxAttempts;
|
|
15093
|
-
}
|
|
15094
|
-
getNextDelay(attempt, _error) {
|
|
15095
|
-
if (attempt >= this.maxAttempts) {
|
|
15096
|
-
return null;
|
|
15097
|
-
}
|
|
15098
|
-
const delay = this.baseDelayMs * attempt;
|
|
15099
|
-
return Math.min(delay, this.maxDelayMs);
|
|
15100
|
-
}
|
|
15101
|
-
};
|
|
15102
|
-
var NoRetry = class {
|
|
15103
|
-
maxAttempts = 1;
|
|
15104
|
-
getNextDelay(_attempt, _error) {
|
|
15105
|
-
return null;
|
|
15106
|
-
}
|
|
15107
|
-
};
|
|
15108
|
-
var ImmediateRetry = class {
|
|
15109
|
-
maxAttempts;
|
|
15110
|
-
constructor(maxAttempts) {
|
|
15111
|
-
this.maxAttempts = maxAttempts;
|
|
15112
|
-
}
|
|
15113
|
-
getNextDelay(attempt, _error) {
|
|
15114
|
-
if (attempt >= this.maxAttempts) {
|
|
15115
|
-
return null;
|
|
15116
|
-
}
|
|
15117
|
-
return 0;
|
|
15118
|
-
}
|
|
15119
|
-
};
|
|
15120
|
-
var CustomRetry = class {
|
|
15121
|
-
maxAttempts;
|
|
15122
|
-
calculateDelay;
|
|
15123
|
-
constructor(options) {
|
|
15124
|
-
this.maxAttempts = options.maxAttempts;
|
|
15125
|
-
this.calculateDelay = options.calculateDelay;
|
|
15126
|
-
}
|
|
15127
|
-
getNextDelay(attempt, error) {
|
|
15128
|
-
if (attempt >= this.maxAttempts) {
|
|
15129
|
-
return null;
|
|
15130
|
-
}
|
|
15131
|
-
return this.calculateDelay(attempt, error);
|
|
15132
|
-
}
|
|
15133
|
-
};
|
|
15134
|
-
var defaultRetryOptions = {
|
|
15135
|
-
maxAttempts: 12,
|
|
15136
|
-
factor: 2,
|
|
15137
|
-
minTimeoutInMs: 1e3,
|
|
15138
|
-
maxTimeoutInMs: 36e5,
|
|
15139
|
-
randomize: true
|
|
15140
|
-
};
|
|
15141
|
-
function createDefaultRetryStrategy() {
|
|
15142
|
-
return new ExponentialBackoffRetry(defaultRetryOptions);
|
|
15143
|
-
}
|
|
15144
15219
|
|
|
15145
15220
|
// src/fair-queue/index.ts
|
|
15146
15221
|
var FairQueue = class {
|
|
@@ -16068,10 +16143,7 @@ var FairQueue = class {
|
|
|
16068
16143
|
if (this.concurrencyManager && storedMessage) {
|
|
16069
16144
|
await this.concurrencyManager.release(descriptor, messageId);
|
|
16070
16145
|
}
|
|
16071
|
-
const { queueEmpty } = await this.#updateAllIndexesAfterDequeue(
|
|
16072
|
-
queueId,
|
|
16073
|
-
descriptor.tenantId
|
|
16074
|
-
);
|
|
16146
|
+
const { queueEmpty } = await this.#updateAllIndexesAfterDequeue(queueId, descriptor.tenantId);
|
|
16075
16147
|
if (queueEmpty) {
|
|
16076
16148
|
this.queueDescriptorCache.delete(queueId);
|
|
16077
16149
|
this.queueCooloffStates.delete(queueId);
|
|
@@ -16328,38 +16400,6 @@ var FairQueue = class {
|
|
|
16328
16400
|
}
|
|
16329
16401
|
return false;
|
|
16330
16402
|
}
|
|
16331
|
-
#incrementCooloff(queueId) {
|
|
16332
|
-
if (this.queueCooloffStates.size >= this.maxCooloffStatesSize) {
|
|
16333
|
-
this.logger.warn("Cooloff states cache hit size cap, clearing all entries", {
|
|
16334
|
-
size: this.queueCooloffStates.size,
|
|
16335
|
-
cap: this.maxCooloffStatesSize
|
|
16336
|
-
});
|
|
16337
|
-
this.queueCooloffStates.clear();
|
|
16338
|
-
}
|
|
16339
|
-
const state = this.queueCooloffStates.get(queueId) ?? {
|
|
16340
|
-
tag: "normal",
|
|
16341
|
-
consecutiveFailures: 0
|
|
16342
|
-
};
|
|
16343
|
-
if (state.tag === "normal") {
|
|
16344
|
-
const newFailures = state.consecutiveFailures + 1;
|
|
16345
|
-
if (newFailures >= this.cooloffThreshold) {
|
|
16346
|
-
this.queueCooloffStates.set(queueId, {
|
|
16347
|
-
tag: "cooloff",
|
|
16348
|
-
expiresAt: Date.now() + this.cooloffPeriodMs
|
|
16349
|
-
});
|
|
16350
|
-
this.logger.debug("Queue entered cooloff", {
|
|
16351
|
-
queueId,
|
|
16352
|
-
cooloffPeriodMs: this.cooloffPeriodMs,
|
|
16353
|
-
consecutiveFailures: newFailures
|
|
16354
|
-
});
|
|
16355
|
-
} else {
|
|
16356
|
-
this.queueCooloffStates.set(queueId, {
|
|
16357
|
-
tag: "normal",
|
|
16358
|
-
consecutiveFailures: newFailures
|
|
16359
|
-
});
|
|
16360
|
-
}
|
|
16361
|
-
}
|
|
16362
|
-
}
|
|
16363
16403
|
#resetCooloff(queueId) {
|
|
16364
16404
|
this.queueCooloffStates.delete(queueId);
|
|
16365
16405
|
}
|
|
@@ -16627,7 +16667,10 @@ var stringToError = zod.z.string().transform((v, ctx) => {
|
|
|
16627
16667
|
try {
|
|
16628
16668
|
return BufferEntryError.parse(JSON.parse(v));
|
|
16629
16669
|
} catch {
|
|
16630
|
-
ctx.addIssue({
|
|
16670
|
+
ctx.addIssue({
|
|
16671
|
+
code: zod.z.ZodIssueCode.custom,
|
|
16672
|
+
message: "expected JSON-encoded BufferEntryError"
|
|
16673
|
+
});
|
|
16631
16674
|
return zod.z.NEVER;
|
|
16632
16675
|
}
|
|
16633
16676
|
});
|
|
@@ -16825,11 +16868,7 @@ var MollifierBuffer = class {
|
|
|
16825
16868
|
// not an error.
|
|
16826
16869
|
async listEntriesForEnv(envId, maxCount) {
|
|
16827
16870
|
if (maxCount <= 0) return [];
|
|
16828
|
-
const runIds = await this.redis.lrange(
|
|
16829
|
-
`mollifier:queue:${envId}`,
|
|
16830
|
-
0,
|
|
16831
|
-
maxCount - 1
|
|
16832
|
-
);
|
|
16871
|
+
const runIds = await this.redis.lrange(`mollifier:queue:${envId}`, 0, maxCount - 1);
|
|
16833
16872
|
if (runIds.length === 0) return [];
|
|
16834
16873
|
const pipeline = this.redis.pipeline();
|
|
16835
16874
|
for (const runId of runIds) {
|
|
@@ -16966,10 +17005,7 @@ var MollifierBuffer = class {
|
|
|
16966
17005
|
// never wiped by a slow predecessor.
|
|
16967
17006
|
async releaseClaim(input) {
|
|
16968
17007
|
const claimKey = makeIdempotencyClaimKey(input);
|
|
16969
|
-
await this.redis.releaseMollifierClaim(
|
|
16970
|
-
claimKey,
|
|
16971
|
-
`${PENDING_PREFIX}${input.token}`
|
|
16972
|
-
);
|
|
17008
|
+
await this.redis.releaseMollifierClaim(claimKey, `${PENDING_PREFIX}${input.token}`);
|
|
16973
17009
|
}
|
|
16974
17010
|
// Read the current claim value, used by the wait/poll loop on losers
|
|
16975
17011
|
// to detect "pending" → "resolved" transitions and timeouts.
|
|
@@ -17654,9 +17690,7 @@ var MollifierDrainer = class {
|
|
|
17654
17690
|
const orgs = await this.buffer.listOrgs();
|
|
17655
17691
|
if (orgs.length === 0) return { drained: 0, failed: 0 };
|
|
17656
17692
|
const orgSlice = this.takeOrgSlice(orgs);
|
|
17657
|
-
const envsByOrg = await Promise.all(
|
|
17658
|
-
orgSlice.map((orgId) => this.buffer.listEnvsForOrg(orgId))
|
|
17659
|
-
);
|
|
17693
|
+
const envsByOrg = await Promise.all(orgSlice.map((orgId) => this.buffer.listEnvsForOrg(orgId)));
|
|
17660
17694
|
const targets = [];
|
|
17661
17695
|
for (let i = 0; i < orgSlice.length; i++) {
|
|
17662
17696
|
const orgId = orgSlice[i];
|
|
@@ -17869,14 +17903,11 @@ var MollifierDrainer = class {
|
|
|
17869
17903
|
} catch (writeErr) {
|
|
17870
17904
|
if (this.isRetryable(writeErr)) {
|
|
17871
17905
|
await this.buffer.requeue(entry.runId);
|
|
17872
|
-
this.logger.warn(
|
|
17873
|
-
|
|
17874
|
-
|
|
17875
|
-
|
|
17876
|
-
|
|
17877
|
-
writeErr
|
|
17878
|
-
}
|
|
17879
|
-
);
|
|
17906
|
+
this.logger.warn("MollifierDrainer: terminal-failure callback retryable; requeued", {
|
|
17907
|
+
runId: entry.runId,
|
|
17908
|
+
attempts: nextAttempts,
|
|
17909
|
+
writeErr
|
|
17910
|
+
});
|
|
17880
17911
|
return "failed";
|
|
17881
17912
|
}
|
|
17882
17913
|
this.logger.error("MollifierDrainer: terminal-failure callback failed", {
|