@trigger.dev/redis-worker 4.7.3 → 4.7.5

package/dist/index.cjs

@@ -9644,6 +9644,39 @@ function createRedisClient(options, handlers) {
 	});
 	return client;
 }
+/**
+* Cluster-mode client. `defaultOptions` go on the INNER per-node options, so a role swap gets the
+* same reconnect-and-retry treatment a single-node client already gets.
+*/
+function createRedisClusterClient(options, handlers) {
+	const client = new import_built.Redis.Cluster(options.nodes, {
+		...options.failFast && { enableOfflineQueue: false },
+		...options.clusterOptions,
+		redisOptions: {
+			...defaultOptions,
+			...options.failFast && { enableOfflineQueue: false },
+			...options.redisOptions
+		}
+	});
+	if (process.env.VITEST) {
+		client.on("error", () => {});
+		return client;
+	}
+	client.on("error", (error) => {
+		if (handlers?.onError) handlers.onError(error);
+		else logger.error(`Redis cluster client error:`, {
+			error,
+			keyPrefix: options.redisOptions?.keyPrefix
+		});
+	});
+	client.on("node error", (error, node) => {
+		logger.warn(`Redis cluster node error:`, {
+			node,
+			error: error.message
+		});
+	});
+	return client;
+}
 
 //#endregion
 //#region ../../node_modules/.pnpm/nanoid@5.1.16/node_modules/nanoid/url-alphabet/index.js
@@ -9729,6 +9762,23 @@ var SimpleQueue = class {
 			throw e;
 		}
 	}
+	/**
+	* Re-enqueues an item for a retry, unless since it was dequeued a newer enqueue replaced it (its
+	* stored deduplication key no longer matches) or it was removed by an ack or cancel. Returns
+	* false when it skipped, so a failed attempt never overwrites the newer item or revives a removed
+	* one. A newer enqueue starts its own retry budget: an item a handler re-enqueues before failing
+	* restarts at attempt 0 rather than moving towards the DLQ.
+	*/
+	async requeue({ id, job, item, attempt, availableAt, visibilityTimeoutMs, deduplicationKey }) {
+		const serializedItem = JSON.stringify({
+			job,
+			item,
+			visibilityTimeoutMs,
+			attempt,
+			deduplicationKey: nanoid()
+		});
+		return await this.redis.requeueItem(`queue`, `items`, id, availableAt.getTime(), serializedItem, deduplicationKey ?? "") === 1;
+	}
 	async enqueueOnce({ id, job, item, attempt, availableAt, visibilityTimeoutMs }) {
 		if (!id) throw new Error("enqueueOnce requires an id");
 		try {
@@ -9896,14 +9946,21 @@ var SimpleQueue = class {
 			deduplicationKey: item.deduplicationKey ?? void 0
 		};
 	}
-	async moveToDeadLetterQueue(id, errorMessage) {
+	/**
+	* Moves an item to the dead letter queue. With a deduplication key, an item a newer enqueue has
+	* replaced is left in the queue and false is returned.
+	*/
+	async moveToDeadLetterQueue(id, errorMessage, deduplicationKey) {
 		try {
 			this.logger.debug(`SimpleQueue ${this.name}.moveToDeadLetterQueue(): moving item to DLQ`, {
 				queue: this.name,
 				id,
 				errorMessage
 			});
-			if (await this.redis.moveToDeadLetterQueue(`queue`, `items`, `dlq`, `dlq:items`, id, errorMessage) !== 1) throw new Error("Move to Dead Letter Queue operation failed");
+			const result = await this.redis.moveToDeadLetterQueue(`queue`, `items`, `dlq`, `dlq:items`, id, errorMessage, deduplicationKey ?? "");
+			if (result === 2) return false;
+			if (result !== 1) throw new Error("Move to Dead Letter Queue operation failed");
+			return true;
 		} catch (e) {
 			this.logger.error(`SimpleQueue ${this.name}.moveToDeadLetterQueue(): error moving item to DLQ`, {
 				queue: this.name,
@@ -10089,6 +10146,35 @@ var SimpleQueue = class {
         -- Remove from sorted set and hash
         redis.call('ZREM', queueKey, id)
         redis.call('HDEL', itemsKey, id)
+        return 1
+      `
+		});
+		this.redis.defineCommand("requeueItem", {
+			numberOfKeys: 2,
+			lua: `
+        local queue = KEYS[1]
+        local items = KEYS[2]
+        local id = ARGV[1]
+        local score = ARGV[2]
+        local serializedItem = ARGV[3]
+        local deduplicationKey = ARGV[4]
+
+        -- Leave the item alone if, while this attempt ran, it was removed (acked or cancelled)
+        -- or a newer enqueue replaced it.
+        if deduplicationKey ~= "" then
+          local current = redis.call('HGET', items, id)
+          if not current then
+            return 2
+          end
+          local success, parsed = pcall(cjson.decode, current)
+          if success and parsed.deduplicationKey and parsed.deduplicationKey ~= deduplicationKey then
+            return 2
+          end
+        end
+
+        redis.call('ZADD', queue, score, id)
+        redis.call('HSET', items, id, serializedItem)
+
         return 1
       `
 		});
@@ -10101,6 +10187,7 @@ var SimpleQueue = class {
         local dlqItems = KEYS[4]
         local id = ARGV[1]
         local errorMessage = ARGV[2]
+        local deduplicationKey = ARGV[3]
 
         local item = redis.call('HGET', items, id)
         if not item then
@@ -10108,6 +10195,9 @@ var SimpleQueue = class {
         end
 
         local parsedItem = cjson.decode(item)
+        if deduplicationKey ~= "" and parsedItem.deduplicationKey and parsedItem.deduplicationKey ~= deduplicationKey then
+          return 2
+        end
         parsedItem.errorMessage = errorMessage
 
         local time = redis.call('TIME')
@@ -11831,7 +11921,7 @@ var Worker = class Worker {
 					else if (errorLogLevel === "warn") this.logger.warn(`Worker batch item reached max attempts. Moving to DLQ.`, dlqLogAttributes);
 					else if (errorLogLevel === "info") this.logger.info(`Worker batch item reached max attempts. Moving to DLQ.`, dlqLogAttributes);
 					else this.logger.error(`Worker batch item reached max attempts. Moving to DLQ.`, dlqLogAttributes);
-					await this.queue.moveToDeadLetterQueue(item.id, errorMessage);
+					if (!await this.queue.moveToDeadLetterQueue(item.id, errorMessage, item.deduplicationKey)) this.#logSuperseded("dead-letter", item.id, jobType);
 					continue;
 				}
 				const retryDate = new Date(Date.now() + retryDelay);
@@ -11843,14 +11933,15 @@ var Worker = class Worker {
 					retryDelay,
 					attempt: newAttempt
 				});
-				await this.queue.enqueue({
+				if (!await this.queue.requeue({
 					id: item.id,
 					job: item.job,
 					item: item.item,
 					availableAt: retryDate,
 					attempt: newAttempt,
-					visibilityTimeoutMs: item.visibilityTimeoutMs
-				});
+					visibilityTimeoutMs: item.visibilityTimeoutMs,
+					deduplicationKey: item.deduplicationKey
+				})) this.#logSuperseded("retry", item.id, jobType);
 			} catch (requeueError) {
 				this.logger.error(`Worker failed to requeue batch item. It will be retried after the visibility timeout.`, {
 					name: this.options.name,
@@ -11865,6 +11956,13 @@ var Worker = class Worker {
 	/**
 	* Processes a single item.
 	*/
+	#logSuperseded(action, id, job) {
+		this.logger.info(`Worker skipped ${action} for an item replaced or removed while it ran`, {
+			name: this.options.name,
+			id,
+			job
+		});
+	}
 	async processItem({ id, job, item, visibilityTimeoutMs, attempt, timestamp, deduplicationKey }, batchSize, workerId, limiter) {
 		const catalogItem = this.options.catalog[job];
 		const handler = this.jobs[job];
@@ -11941,7 +12039,7 @@ var Worker = class Worker {
 						...logAttributes,
 						attempt: newAttempt
 					});
-					await this.queue.moveToDeadLetterQueue(id, errorMessage);
+					if (!await this.queue.moveToDeadLetterQueue(id, errorMessage, deduplicationKey)) this.#logSuperseded("dead-letter", id, job);
 					if (catalogItem.cron) await this.rescheduleCronJob(job, catalogItem, item);
 					return;
 				}
@@ -11955,14 +12053,15 @@ var Worker = class Worker {
 					visibilityTimeoutMs,
 					attempt: newAttempt
 				});
-				await this.queue.enqueue({
+				if (!await this.queue.requeue({
 					id,
 					job,
 					item,
 					availableAt: retryDate,
 					attempt: newAttempt,
-					visibilityTimeoutMs
-				});
+					visibilityTimeoutMs,
+					deduplicationKey
+				})) this.#logSuperseded("retry", id, job);
 			} catch (requeueError) {
 				this.logger.error(`Worker failed to requeue item. It will be retried after the visibility timeout.`, {
 					name: this.options.name,
@@ -12121,6 +12220,33 @@ function isAbortError(error) {
 	return error instanceof Error && (error.name === "AbortError" || error.message === "AbortError");
 }
 
+//#endregion
+//#region src/fair-queue/connection.ts
+function isClusterConnection(options) {
+	return "cluster" in options && options.cluster !== void 0;
+}
+function createFairQueueRedisClient(options) {
+	return isClusterConnection(options) ? createRedisClusterClient(options.cluster) : createRedisClient(options);
+}
+function keyPrefixOf(options) {
+	return isClusterConnection(options) ? options.cluster.clusterOptions?.keyPrefix ?? "" : options.keyPrefix ?? "";
+}
+/**
+* Every key matching `pattern` (which must include the key prefix), page by page. A cluster doesn't
+* fan SCAN out to its nodes, so this scans each primary in turn.
+*/
+async function* scanKeys(client, pattern, count = 1e3) {
+	const nodes = client instanceof import_built.Cluster ? client.nodes("master") : [client];
+	for (const node of nodes) {
+		let cursor = "0";
+		do {
+			const [next, keys] = await node.scan(cursor, "MATCH", pattern, "COUNT", count);
+			cursor = next;
+			if (keys.length > 0) yield keys;
+		} while (cursor !== "0");
+	}
+}
+
 //#endregion
 //#region src/fair-queue/concurrency.ts
 /**
@@ -12147,7 +12273,7 @@ var ConcurrencyManager = class {
 	logger;
 	constructor(options) {
 		this.options = options;
-		this.redis = createRedisClient(options.redis);
+		this.redis = createFairQueueRedisClient(options.redis);
 		this.keys = options.keys;
 		this.groups = options.groups;
 		this.groupsByName = new Map(options.groups.map((g) => [g.name, g]));
@@ -12235,7 +12361,10 @@ var ConcurrencyManager = class {
 	* record can only be a leak; if the message is about to be re-claimed, reserve simply
 	* re-adds the member.
 	*
-	* @param inflightDataKeys - The in-flight data hash keys for every shard. The sweep
+	* With a tenant-sharded key producer, each shard's sets are checked against that shard's
+	* in-flight data only, so the check stays in one cluster slot.
+	*
+	* @param inflightDataKeys - The in-flight data hash keys for every shard, in shard order. The sweep
 	*   refuses to run when this is empty, since with nowhere to look for running messages
 	*   every member would look orphaned and all concurrency accounting would be erased.
 	* @returns The message ids that were removed, and how many sets were checked
@@ -12248,46 +12377,49 @@ var ConcurrencyManager = class {
 				removed: []
 			};
 		}
-		const keyPrefix = this.options.redis.keyPrefix ?? "";
+		const keyPrefix = keyPrefixOf(this.options.redis);
 		let scannedSets = 0;
 		const removed = [];
-		for (const group of this.groups) {
-			const pattern = `${keyPrefix}${this.keys.concurrencyKey(group.name, "*")}`;
-			let cursor = "0";
-			do {
-				const [nextCursor, foundKeys] = await this.redis.scan(cursor, "MATCH", pattern, "COUNT", 1e3);
-				cursor = nextCursor;
-				for (const fullKey of foundKeys) {
-					const key = keyPrefix && fullKey.startsWith(keyPrefix) ? fullKey.slice(keyPrefix.length) : fullKey;
-					try {
-						let sawMembers = false;
-						let memberCursor = "0";
-						do {
-							const [nextMemberCursor, page] = await this.redis.sscan(key, memberCursor, "COUNT", SWEEP_MEMBER_CHUNK_SIZE);
-							memberCursor = nextMemberCursor;
-							if (page.length === 0) continue;
-							sawMembers = true;
-							for (let i = 0; i < page.length; i += SWEEP_MEMBER_CHUNK_SIZE) {
-								const chunk = page.slice(i, i + SWEEP_MEMBER_CHUNK_SIZE);
-								const removedIds = await this.redis.removeOrphanedConcurrencySlots(1 + inflightDataKeys.length, [key, ...inflightDataKeys], ...chunk);
-								removed.push(...removedIds);
-							}
-						} while (memberCursor !== "0");
-						if (sawMembers) scannedSets++;
-					} catch (error) {
-						this.logger.error("Failed to sweep concurrency set, skipping it", {
-							key,
-							error: error instanceof Error ? error.message : String(error)
-						});
+		for (const scope of this.#sweepScopes(inflightDataKeys)) for await (const foundKeys of scanKeys(this.redis, `${keyPrefix}${scope.pattern}`)) for (const fullKey of foundKeys) {
+			const key = keyPrefix && fullKey.startsWith(keyPrefix) ? fullKey.slice(keyPrefix.length) : fullKey;
+			try {
+				let sawMembers = false;
+				let memberCursor = "0";
+				do {
+					const [nextMemberCursor, page] = await this.redis.sscan(key, memberCursor, "COUNT", SWEEP_MEMBER_CHUNK_SIZE);
+					memberCursor = nextMemberCursor;
+					if (page.length === 0) continue;
+					sawMembers = true;
+					for (let i = 0; i < page.length; i += SWEEP_MEMBER_CHUNK_SIZE) {
+						const chunk = page.slice(i, i + SWEEP_MEMBER_CHUNK_SIZE);
+						const removedIds = await this.redis.removeOrphanedConcurrencySlots(1 + scope.inflightDataKeys.length, [key, ...scope.inflightDataKeys], ...chunk);
+						removed.push(...removedIds);
 					}
-				}
-			} while (cursor !== "0");
+				} while (memberCursor !== "0");
+				if (sawMembers) scannedSets++;
+			} catch (error) {
+				this.logger.error("Failed to sweep concurrency set, skipping it", {
+					key,
+					error: error instanceof Error ? error.message : String(error)
+				});
+			}
 		}
 		return {
 			scannedSets,
 			removed
 		};
 	}
+	#sweepScopes(inflightDataKeys) {
+		const keys = this.keys;
+		if (keys.tenantShard && keys.concurrencyKeyPattern) return this.groups.flatMap((group) => inflightDataKeys.map((inflightDataKey, shardId) => ({
+			pattern: keys.concurrencyKeyPattern(group.name, shardId),
+			inflightDataKeys: [inflightDataKey]
+		})));
+		return this.groups.map((group) => ({
+			pattern: keys.concurrencyKey(group.name, "*"),
+			inflightDataKeys
+		}));
+	}
 	/**
 	* Get current concurrency for a specific group.
 	*/
@@ -12448,7 +12580,7 @@ var MasterQueue = class {
 	shardCount;
 	constructor(options) {
 		this.options = options;
-		this.redis = createRedisClient(options.redis);
+		this.redis = createFairQueueRedisClient(options.redis);
 		this.keys = options.keys;
 		this.shardCount = Math.max(1, options.shardCount);
 		this.#registerCommands();
@@ -12568,6 +12700,8 @@ var MasterQueue = class {
 	* Provides better distribution than djb2 and minimal reshuffling when shard count changes.
 	*/
 	#hashToShard(queueId) {
+		const keys = this.options.keys;
+		if (keys.tenantShard) return keys.tenantShard(keys.extractTenantId(queueId));
 		return (0, _trigger_dev_core_v3_serverOnly.jumpHash)(queueId, this.shardCount);
 	}
 	#registerCommands() {
@@ -12810,6 +12944,16 @@ var FairQueueTelemetry = class {
 				}
 			});
 		}
+		if (callbacks.getOldestMessageAge && callbacks.shardCount) {
+			const getOldestMessageAge = callbacks.getOldestMessageAge;
+			const shardCount = callbacks.shardCount;
+			this.metrics.oldestMessageAge.addCallback(async (observableResult) => {
+				for (let shardId = 0; shardId < shardCount; shardId++) {
+					const age = await getOldestMessageAge(shardId);
+					observableResult.observe(age, { [FairQueueAttributes.SHARD_ID]: shardId.toString() });
+				}
+			});
+		}
 		if (callbacks.getDLQLength && callbacks.observedTenants) {
 			const getDLQLength = callbacks.getDLQLength;
 			const tenants = callbacks.observedTenants;
@@ -12903,6 +13047,10 @@ var FairQueueTelemetry = class {
 			dlqLength: this.meter.createObservableGauge(`${this.name}.dlq.length`, {
 				description: "Number of messages in dead letter queue",
 				unit: "messages"
+			}),
+			oldestMessageAge: this.meter.createObservableGauge(`${this.name}.oldest_message_age`, {
+				description: "Age of the oldest due message not yet completed in a shard, including claimed messages until their queue next completes or empties",
+				unit: "ms"
 			})
 		};
 	}
@@ -13120,7 +13268,7 @@ var TenantDispatch = class {
 	shardCount;
 	constructor(options) {
 		this.options = options;
-		this.redis = createRedisClient(options.redis);
+		this.redis = createFairQueueRedisClient(options.redis);
 		this.keys = options.keys;
 		this.shardCount = Math.max(1, options.shardCount);
 	}
@@ -13130,6 +13278,7 @@ var TenantDispatch = class {
 	* always maps to exactly one dispatch shard.
 	*/
 	getShardForTenant(tenantId) {
+		if (this.options.keys.tenantShard) return this.options.keys.tenantShard(tenantId);
 		return (0, _trigger_dev_core_v3_serverOnly.jumpHash)(tenantId, this.shardCount);
 	}
 	/**
@@ -13245,7 +13394,7 @@ var VisibilityManager = class {
 	logger;
 	constructor(options) {
 		this.options = options;
-		this.redis = createRedisClient(options.redis);
+		this.redis = createFairQueueRedisClient(options.redis);
 		this.keys = options.keys;
 		this.shardCount = options.shardCount;
 		this.defaultTimeoutMs = options.defaultTimeoutMs;
@@ -13558,6 +13707,7 @@ var VisibilityManager = class {
 	* Must use same algorithm as MasterQueue for consistency.
 	*/
 	#getShardForQueue(queueId) {
+		if (this.keys.tenantShard) return this.keys.tenantShard(this.keys.extractTenantId(queueId));
 		return (0, _trigger_dev_core_v3_serverOnly.jumpHash)(queueId, this.shardCount);
 	}
 	#makeMember(messageId, queueId) {
@@ -13836,7 +13986,7 @@ var WorkerQueueManager = class {
 	logger;
 	constructor(options) {
 		this.options = options;
-		this.redis = createRedisClient(options.redis);
+		this.redis = createFairQueueRedisClient(options.redis);
 		this.keys = options.keys;
 		this.logger = options.logger ?? {
 			debug: () => {},
@@ -13887,6 +14037,9 @@ var WorkerQueueManager = class {
 	*/
 	async blockingPop(workerQueueId, timeoutSeconds, signal) {
 		const workerQueueKey = this.keys.workerQueueKey(workerQueueId);
+		if (signal?.aborted) return null;
+		const readyMessageKey = await this.redis.lpop(workerQueueKey);
+		if (readyMessageKey) return readyMessageKey;
 		const blockingClient = this.redis.duplicate();
 		const cleanup = signal ? () => {
 			blockingClient.disconnect();
@@ -14132,6 +14285,100 @@ var CallbackFairQueueKeyProducer = class extends DefaultFairQueueKeyProducer {
 		return this.groupExtractor(groupName, queueId);
 	}
 };
+/**
+* Key producer for running a fair queue on a Redis cluster. Each tenant belongs to one shard,
+* `jumpHash(tenantId, shardCount)`, and every key of a shard carries the hash tag
+* `{prefix:shard}`, so a shard's queues, items, payloads, tenant indexes, dispatch index,
+* in-flight set and concurrency sets share a slot and every script stays single-slot.
+*
+* Concurrency groups must be keyed by tenant id. The shard count can't change without moving keys.
+*
+* Key structure (S = the tenant's shard):
+* - Dispatch index: {prefix:S}:dispatch
+* - Tenant queue index: {prefix:S}:tenantq:{tenantId}
+* - Queue: {prefix:S}:queue:{queueId}, items {prefix:S}:queue:{queueId}:items
+* - Payload: {prefix:S}:payload:{queueId}:{messageId}
+* - Payload bytes per tenant: {prefix:S}:payloadbytes:{tenantId}
+* - Concurrency: {prefix:S}:concurrency:{groupName}:{tenantId}
+* - In-flight: {prefix:S}:inflight, data {prefix:S}:inflight:data
+* - Dead letter queue: {prefix:S}:dlq:{tenantId}, data {prefix:S}:dlq:{tenantId}:data
+* - Legacy master queue: {prefix:S}:master
+* - Worker queue: {prefix}:worker:{consumerId} (its own slot)
+* - DRR deficits: {prefix}:drr:deficit (its own slot)
+*/
+var TenantShardedFairQueueKeyProducer = class {
+	shardCount;
+	prefix;
+	tenantExtractor;
+	groupExtractor;
+	constructor(options) {
+		if (!Number.isInteger(options.shardCount) || options.shardCount < 1) throw new Error(`shardCount must be a positive integer, got ${options.shardCount}`);
+		this.prefix = options.prefix;
+		this.shardCount = options.shardCount;
+		this.tenantExtractor = options.extractTenantId;
+		this.groupExtractor = options.extractGroupId;
+	}
+	tenantShard(tenantId) {
+		return (0, _trigger_dev_core_v3_serverOnly.jumpHash)(tenantId, this.shardCount);
+	}
+	masterQueueKey(shardId) {
+		return `${this.#tag(shardId)}:master`;
+	}
+	queueKey(queueId) {
+		return `${this.#queueTag(queueId)}:queue:${queueId}`;
+	}
+	queueItemsKey(queueId) {
+		return `${this.#queueTag(queueId)}:queue:${queueId}:items`;
+	}
+	payloadKey(queueId, messageId) {
+		return `${this.#queueTag(queueId)}:payload:${queueId}:${messageId}`;
+	}
+	tenantPayloadBytesKey(tenantId) {
+		return `${this.#tag(this.tenantShard(tenantId))}:payloadbytes:${tenantId}`;
+	}
+	concurrencyKey(groupName, groupId) {
+		return `${this.#tag(this.tenantShard(groupId))}:concurrency:${groupName}:${groupId}`;
+	}
+	concurrencyKeyPattern(groupName, shardId) {
+		return `${this.#tag(shardId)}:concurrency:${groupName}:*`;
+	}
+	inflightKey(shardId) {
+		return `${this.#tag(shardId)}:inflight`;
+	}
+	inflightDataKey(shardId) {
+		return `${this.#tag(shardId)}:inflight:data`;
+	}
+	workerQueueKey(consumerId) {
+		return `${this.prefix}:worker:${consumerId}`;
+	}
+	deadLetterQueueKey(tenantId) {
+		return `${this.#tag(this.tenantShard(tenantId))}:dlq:${tenantId}`;
+	}
+	deadLetterQueueDataKey(tenantId) {
+		return `${this.#tag(this.tenantShard(tenantId))}:dlq:${tenantId}:data`;
+	}
+	dispatchKey(shardId) {
+		return `${this.#tag(shardId)}:dispatch`;
+	}
+	tenantQueueIndexKey(tenantId) {
+		return `${this.#tag(this.tenantShard(tenantId))}:tenantq:${tenantId}`;
+	}
+	drrDeficitKey() {
+		return `${this.prefix}:drr:deficit`;
+	}
+	extractTenantId(queueId) {
+		return this.tenantExtractor(queueId);
+	}
+	extractGroupId(groupName, queueId) {
+		return this.groupExtractor(groupName, queueId);
+	}
+	#queueTag(queueId) {
+		return this.#tag(this.tenantShard(this.tenantExtractor(queueId)));
+	}
+	#tag(shardId) {
+		return `{${this.prefix}:${shardId}}`;
+	}
+};
 
 //#endregion
 //#region src/fair-queue/retry.ts
@@ -14348,7 +14595,7 @@ var DRRScheduler = class extends BaseScheduler {
 	constructor(config) {
 		super();
 		this.config = config;
-		this.redis = createRedisClient(config.redis);
+		this.redis = createFairQueueRedisClient(config.redis);
 		this.keys = config.keys;
 		this.quantum = config.quantum;
 		this.maxDeficit = config.maxDeficit;
@@ -14491,6 +14738,7 @@ var DRRScheduler = class extends BaseScheduler {
 		return result;
 	}
 	#deficitKey() {
+		if (this.keys.drrDeficitKey) return this.keys.drrDeficitKey();
 		return `${this.keys.masterQueueKey(0).split(":")[0]}:drr:deficit`;
 	}
 	async #getTenantsFromDispatch(dispatchKey) {
@@ -14649,7 +14897,7 @@ var WeightedScheduler = class extends BaseScheduler {
 	constructor(config) {
 		super();
 		this.config = config;
-		this.redis = createRedisClient(config.redis);
+		this.redis = createFairQueueRedisClient(config.redis);
 		this.keys = config.keys;
 		this.rng = (0, seedrandom.default)(config.seed);
 		this.biases = config.biases ?? defaultBiases;
@@ -14904,7 +15152,7 @@ var RoundRobinScheduler = class extends BaseScheduler {
 	constructor(config) {
 		super();
 		this.config = config;
-		this.redis = createRedisClient(config.redis);
+		this.redis = createFairQueueRedisClient(config.redis);
 		this.keys = config.keys;
 		this.masterQueueLimit = config.masterQueueLimit ?? 1e3;
 	}
@@ -14976,6 +15224,17 @@ var RoundRobinScheduler = class extends BaseScheduler {
 
 //#endregion
 //#region src/fair-queue/index.ts
+/** An enqueue that would take a tenant past `tenantPayloadBytesLimit`. */
+var TenantQueueFullError = class extends Error {
+	tenantId;
+	limitBytes;
+	constructor(tenantId, limitBytes) {
+		super(`Tenant ${tenantId} is at its limit of ${limitBytes} queued payload bytes`);
+		this.tenantId = tenantId;
+		this.limitBytes = limitBytes;
+		this.name = "TenantQueueFullError";
+	}
+};
 /**
 * FairQueue is the main orchestrator for fair queue message routing.
 *
@@ -14994,7 +15253,7 @@ var RoundRobinScheduler = class extends BaseScheduler {
 *
 * @typeParam TPayloadSchema - Zod schema for message payload validation
 */
-var FairQueue = class {
+var FairQueue = class FairQueue {
 	options;
 	redis;
 	keys;
@@ -15018,6 +15277,7 @@ var FairQueue = class {
 	reconcileIntervalMs;
 	workerQueueResolver;
 	batchClaimSize;
+	separatePayloads;
 	cooloffEnabled;
 	cooloffThreshold;
 	cooloffPeriodMs;
@@ -15037,8 +15297,10 @@ var FairQueue = class {
 	tenantDispatch;
 	constructor(options) {
 		this.options = options;
-		this.redis = createRedisClient(options.redis);
+		FairQueue.#validateLayout(options);
+		this.redis = createFairQueueRedisClient(options.redis);
 		this.keys = options.keys;
+		this.separatePayloads = options.separatePayloads ?? false;
 		this.scheduler = options.scheduler;
 		this.logger = options.logger ?? new _trigger_dev_core_logger.Logger("FairQueue", "info");
 		this.abortController = new AbortController();
@@ -15046,7 +15308,7 @@ var FairQueue = class {
 		this.validateOnEnqueue = options.validateOnEnqueue ?? false;
 		this.retryStrategy = options.retry?.strategy;
 		this.deadLetterQueueEnabled = options.retry?.deadLetterQueue ?? true;
-		this.shardCount = options.shardCount ?? 1;
+		this.shardCount = options.keys.shardCount ?? options.shardCount ?? 1;
 		this.consumerCount = options.consumerCount ?? 1;
 		this.consumerIntervalMs = options.consumerIntervalMs ?? 100;
 		this.visibilityTimeoutMs = options.visibilityTimeoutMs ?? 3e4;
@@ -15138,6 +15400,9 @@ var FairQueue = class {
 			getDLQLength: async (tenantId) => {
 				return await this.getDeadLetterQueueLength(tenantId);
 			},
+			getOldestMessageAge: async (shardId) => {
+				return await this.getOldestMessageAge(shardId);
+			},
 			shardCount: this.shardCount,
 			observedTenants: options?.observedTenants
 		});
@@ -15182,7 +15447,13 @@ var FairQueue = class {
 				}) : void 0,
 				metadata: options.metadata
 			};
-			await this.redis.enqueueMessageAtomicV2(queueKey, queueItemsKey, tenantQueueIndexKey, dispatchKey, options.queueId, messageId, timestamp.toString(), JSON.stringify(storedMessage), options.tenantId);
+			if (this.separatePayloads) {
+				const { payload, ...envelope } = storedMessage;
+				const limit = options.ignorePayloadBytesLimit ? 0 : await this.options.tenantPayloadBytesLimit?.(options.tenantId) ?? 0;
+				const result = await this.redis.enqueueMessageSeparatePayload(queueKey, queueItemsKey, tenantQueueIndexKey, dispatchKey, this.keys.payloadKey(options.queueId, messageId), this.keys.inflightKey(this.masterQueue.getShardForQueue(options.queueId)), this.keys.tenantPayloadBytesKey(options.tenantId), options.queueId, messageId, timestamp.toString(), JSON.stringify(envelope), options.tenantId, JSON.stringify(payload), `${messageId}:${options.queueId}`, String(limit));
+				if (result === -1) throw new TenantQueueFullError(options.tenantId, limit);
+				span.setAttribute("enqueued", result === 1);
+			} else await this.redis.enqueueMessageAtomicV2(queueKey, queueItemsKey, tenantQueueIndexKey, dispatchKey, options.queueId, messageId, timestamp.toString(), JSON.stringify(storedMessage), options.tenantId);
 			span.setAttributes({
 				[FairQueueAttributes.QUEUE_ID]: options.queueId,
 				[FairQueueAttributes.TENANT_ID]: options.tenantId,
@@ -15206,6 +15477,7 @@ var FairQueue = class {
 	*/
 	async enqueueBatch(options) {
 		return this.telemetry.trace("enqueueBatch", async (span) => {
+			if (this.separatePayloads) throw new Error("enqueueBatch isn't supported with separatePayloads");
 			const queueKey = this.keys.queueKey(options.queueId);
 			const queueItemsKey = this.keys.queueItemsKey(options.queueId);
 			const dispatchShardId = this.tenantDispatch.getShardForTenant(options.tenantId);
@@ -15460,6 +15732,17 @@ var FairQueue = class {
 		return dispatchCount + legacyCount;
 	}
 	/**
+	* Get the age in ms of the oldest due message not yet completed in a shard, or 0 if none.
+	* Claims don't refresh the index scores, so claimed messages count until their queue next completes or empties.
+	*/
+	async getOldestMessageAge(shardId) {
+		const now = Date.now();
+		const [[oldestTenant], [oldestLegacyQueue]] = await Promise.all([this.tenantDispatch.getTenantsFromShard(shardId, 1, now), this.masterQueue.getQueuesFromShard(shardId, 1, now)]);
+		const oldestScore = Math.min(oldestTenant?.score ?? Infinity, oldestLegacyQueue?.score ?? Infinity);
+		if (!Number.isFinite(oldestScore)) return 0;
+		return Math.max(0, now - oldestScore);
+	}
+	/**
 	* Get total in-flight message count.
 	*/
 	async getTotalInflightCount() {
@@ -15722,10 +16005,22 @@ var FairQueue = class {
 	async getMessageData(messageId, queueId) {
 		const shardId = this.masterQueue.getShardForQueue(queueId);
 		const inflightDataKey = this.keys.inflightDataKey(shardId);
-		const dataJson = await this.redis.hget(inflightDataKey, messageId);
+		const [dataJson, payloadJson] = await Promise.all([this.redis.hget(inflightDataKey, messageId), this.separatePayloads ? this.redis.get(this.keys.payloadKey(queueId, messageId)) : Promise.resolve(null)]);
 		if (!dataJson) return null;
 		try {
-			return JSON.parse(dataJson);
+			const message = JSON.parse(dataJson);
+			if (!this.separatePayloads) return message;
+			if (payloadJson === null) {
+				this.logger.error("In-flight message has no payload", {
+					messageId,
+					queueId
+				});
+				return null;
+			}
+			return {
+				...message,
+				payload: JSON.parse(payloadJson)
+			};
 		} catch {
 			this.logger.error("Failed to parse message data", {
 				messageId,
@@ -15766,7 +16061,7 @@ var FairQueue = class {
 			metadata: storedMessage?.metadata ?? {}
 		};
 		await this.#releaseConcurrencySafely(descriptor, messageId, "completeMessage");
-		await this.visibilityManager.complete(messageId, queueId);
+		await this.#completeInVisibility(messageId, queueId, descriptor.tenantId);
 		const { queueEmpty } = await this.#updateAllIndexesAfterDequeue(queueId, descriptor.tenantId);
 		if (queueEmpty) {
 			this.queueDescriptorCache.delete(queueId);
@@ -15916,9 +16211,16 @@ var FairQueue = class {
 	}
 	async #moveToDeadLetterQueue(storedMessage, errorMessage) {
 		if (!this.deadLetterQueueEnabled) {
-			await this.visibilityManager.complete(storedMessage.id, storedMessage.queueId);
+			await this.#completeInVisibility(storedMessage.id, storedMessage.queueId, storedMessage.tenantId);
 			return;
 		}
+		if (this.separatePayloads) {
+			const payloadJson = await this.redis.get(this.keys.payloadKey(storedMessage.queueId, storedMessage.id));
+			storedMessage = {
+				...storedMessage,
+				payload: payloadJson === null ? void 0 : JSON.parse(payloadJson)
+			};
+		}
 		const dlqKey = this.keys.deadLetterQueueKey(storedMessage.tenantId);
 		const dlqDataKey = this.keys.deadLetterQueueDataKey(storedMessage.tenantId);
 		this.masterQueue.getShardForQueue(storedMessage.queueId);
@@ -15932,7 +16234,7 @@ var FairQueue = class {
 			lastError: errorMessage,
 			originalTimestamp: storedMessage.timestamp
 		};
-		await this.visibilityManager.complete(storedMessage.id, storedMessage.queueId);
+		await this.#completeInVisibility(storedMessage.id, storedMessage.queueId, storedMessage.tenantId);
 		const pipeline = this.redis.pipeline();
 		pipeline.zadd(dlqKey, dlqMessage.deadLetteredAt, storedMessage.id);
 		pipeline.hset(dlqDataKey, storedMessage.id, JSON.stringify(dlqMessage));
@@ -16078,6 +16380,40 @@ var FairQueue = class {
 		const removedFromDispatch = await this.redis.updateDispatchIndexes(queueKey, tenantQueueIndexKey, dispatchKey, queueId, tenantId);
 		return { queueEmpty: removedFromMaster === 1 || removedFromDispatch === 1 };
 	}
+	/**
+	* Remove a settled message from in-flight. With separate payloads this also deletes the payload
+	* and takes its bytes off the tenant's count, in one script, so the count never drifts.
+	*/
+	async #completeInVisibility(messageId, queueId, tenantId) {
+		if (!this.separatePayloads) {
+			await this.visibilityManager.complete(messageId, queueId);
+			return;
+		}
+		const shardId = this.masterQueue.getShardForQueue(queueId);
+		await this.redis.completeSeparatePayload(this.keys.inflightKey(shardId), this.keys.inflightDataKey(shardId), this.keys.payloadKey(queueId, messageId), this.keys.tenantPayloadBytesKey(tenantId), `${messageId}:${queueId}`, messageId);
+	}
+	/** A tenant's queued and in-flight payload bytes, with `separatePayloads`. */
+	async getTenantPayloadBytes(tenantId) {
+		if (!this.separatePayloads) return 0;
+		return Number(await this.redis.get(this.keys.tenantPayloadBytesKey(tenantId)) ?? 0);
+	}
+	/**
+	* A cluster needs every script's keys in one slot, which only a tenant-sharded key producer
+	* gives, and the concurrency reservation touches every group's set at once, so it allows one
+	* group, keyed by tenant.
+	*/
+	static #validateLayout(options) {
+		const keys = options.keys;
+		if (options.separatePayloads && (!keys.payloadKey || !keys.tenantPayloadBytesKey)) throw new Error("separatePayloads needs a key producer with payloadKey and tenantPayloadBytesKey");
+		if (keys.tenantShard) {
+			if (!keys.shardCount) throw new Error("A key producer with tenantShard must set shardCount");
+			if (options.shardCount !== void 0 && options.shardCount !== keys.shardCount) throw new Error(`shardCount ${options.shardCount} doesn't match the key producer's ${keys.shardCount}`);
+		}
+		if (isClusterConnection(options.redis)) {
+			if (!keys.tenantShard) throw new Error("A cluster connection needs a tenant-sharded key producer");
+			if ((options.concurrencyGroups?.length ?? 0) > 1) throw new Error("A cluster connection supports at most one concurrency group");
+		}
+	}
 	#createSchedulerContext() {
 		return {
 			getCurrentConcurrency: async (groupName, groupId) => {
@@ -16236,6 +16572,77 @@ if #tenantOldest >= 2 then
 end
 
 return (#ARGV - 2) / 3
+      `
+		});
+		this.redis.defineCommand("enqueueMessageSeparatePayload", {
+			numberOfKeys: 7,
+			lua: `#!lua
+local queueKey = KEYS[1]
+local queueItemsKey = KEYS[2]
+local tenantQueueIndexKey = KEYS[3]
+local dispatchKey = KEYS[4]
+local payloadKey = KEYS[5]
+local inflightKey = KEYS[6]
+local bytesKey = KEYS[7]
+
+local queueId = ARGV[1]
+local messageId = ARGV[2]
+local timestamp = tonumber(ARGV[3])
+local envelope = ARGV[4]
+local tenantId = ARGV[5]
+local payload = ARGV[6]
+local inflightMember = ARGV[7]
+local limit = tonumber(ARGV[8])
+
+if redis.call('HEXISTS', queueItemsKey, messageId) == 1 then
+  return 0
+end
+if redis.call('ZSCORE', inflightKey, inflightMember) then
+  return 0
+end
+
+local size = string.len(payload)
+if limit > 0 and tonumber(redis.call('GET', bytesKey) or '0') + size > limit then
+  return -1
+end
+
+redis.call('SET', payloadKey, payload)
+redis.call('INCRBY', bytesKey, size)
+redis.call('ZADD', queueKey, timestamp, messageId)
+redis.call('HSET', queueItemsKey, messageId, envelope)
+
+local oldest = redis.call('ZRANGE', queueKey, 0, 0, 'WITHSCORES')
+if #oldest >= 2 then
+  redis.call('ZADD', tenantQueueIndexKey, oldest[2], queueId)
+end
+
+local tenantOldest = redis.call('ZRANGE', tenantQueueIndexKey, 0, 0, 'WITHSCORES')
+if #tenantOldest >= 2 then
+  redis.call('ZADD', dispatchKey, tenantOldest[2], tenantId)
+end
+
+return 1
+      `
+		});
+		this.redis.defineCommand("completeSeparatePayload", {
+			numberOfKeys: 4,
+			lua: `
+local inflightKey = KEYS[1]
+local inflightDataKey = KEYS[2]
+local payloadKey = KEYS[3]
+local bytesKey = KEYS[4]
+
+redis.call('ZREM', inflightKey, ARGV[1])
+redis.call('HDEL', inflightDataKey, ARGV[2])
+
+local size = redis.call('STRLEN', payloadKey)
+if size > 0 then
+  redis.call('DEL', payloadKey)
+  if redis.call('DECRBY', bytesKey, size) <= 0 then
+    redis.call('DEL', bytesKey)
+  end
+end
+return size
       `
 		});
 		this.redis.defineCommand("updateDispatchIndexes", {
@@ -17385,16 +17792,22 @@ exports.NoopScheduler = NoopScheduler;
 exports.RoundRobinScheduler = RoundRobinScheduler;
 exports.SimpleQueue = SimpleQueue;
 exports.TenantDispatch = TenantDispatch;
+exports.TenantQueueFullError = TenantQueueFullError;
+exports.TenantShardedFairQueueKeyProducer = TenantShardedFairQueueKeyProducer;
 exports.VisibilityManager = VisibilityManager;
 exports.WeightedScheduler = WeightedScheduler;
 exports.Worker = Worker;
 exports.WorkerQueueManager = WorkerQueueManager;
 exports.createDefaultRetryStrategy = createDefaultRetryStrategy;
+exports.createFairQueueRedisClient = createFairQueueRedisClient;
 exports.defaultRetryOptions = defaultRetryOptions;
 exports.deserialiseSnapshot = deserialiseSnapshot;
 exports.idempotencyLookupKeyFor = idempotencyLookupKeyFor;
 exports.isAbortError = isAbortError;
+exports.isClusterConnection = isClusterConnection;
+exports.keyPrefixOf = keyPrefixOf;
 exports.makeIdempotencyClaimKey = makeIdempotencyClaimKey;
 exports.noopTelemetry = noopTelemetry;
+exports.scanKeys = scanKeys;
 exports.serialiseSnapshot = serialiseSnapshot;
 //# sourceMappingURL=index.cjs.map