@getstrata/core 2.2.0 → 2.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +7 -0
- package/dist/core/contracts/serviceTokens.d.ts +3 -2
- package/dist/core/http/requestMetaContext.d.ts +2 -0
- package/dist/core/queue/createAppQueue.d.ts +2 -1
- package/dist/core/queue/failedJobService.d.ts +3 -1
- package/dist/core/queue/index.d.ts +10 -2
- package/dist/core/queue/jobRunner.d.ts +7 -1
- package/dist/core/queue/publicQueue.d.ts +2 -1
- package/dist/core/queue/queueConfig.d.ts +3 -0
- package/dist/core/queue/redisQueue.d.ts +5 -7
- package/dist/core/queue/redisQueueKeys.d.ts +8 -0
- package/dist/core/queue/redisStreams.d.ts +73 -0
- package/dist/core/queue/types.d.ts +1 -0
- package/dist/core/tenant/publicTenantHost.d.ts +11 -0
- package/dist/core/tenant/tenantMiddleware.d.ts +4 -0
- package/dist/entries/auth/basicAuthGuard.js +1 -0
- package/dist/entries/auth/jwtGuard.js +1 -0
- package/dist/entries/auth/sessionGuard.js +1 -0
- package/dist/entries/contracts/serviceTokens.js +2 -0
- package/dist/entries/queue/createAppQueue.js +681 -66
- package/dist/entries/queue/failedJobRepository.js +1 -1
- package/dist/entries/queue/failedJobService.js +5 -1
- package/dist/entries/queue/jobRunner.js +67 -2
- package/dist/entries/queue/publicQueue.js +679 -65
- package/dist/entries/queue/queueMetrics.js +694 -72
- package/dist/entries/queue/redisQueue.js +765 -148
- package/dist/framework/public-api.d.ts +2 -2
- package/dist/index.js +711 -67
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -1136,6 +1136,7 @@ class ConfigStore {
|
|
|
1136
1136
|
|
|
1137
1137
|
// ../../src/core/contracts/serviceTokens.ts
|
|
1138
1138
|
var CORE_THROTTLE_QUOTA_POLICY_TOKEN = createServiceToken("core.throttleQuotaPolicy");
|
|
1139
|
+
var CORE_PUBLIC_TENANCY_TOKEN = createServiceToken("core.publicTenancy");
|
|
1139
1140
|
var CORE_TENANT_RESOLVER_TOKEN = createServiceToken("core.tenantResolver");
|
|
1140
1141
|
var CORE_HTTP_CLEANUP_TOKEN = createServiceToken("core.httpCleanup");
|
|
1141
1142
|
var CORE_QUEUE_TOKEN = "core.queue";
|
|
@@ -12490,7 +12491,7 @@ function createQueue(driver) {
|
|
|
12490
12491
|
var failedJobTable = defineTable({
|
|
12491
12492
|
name: "failed_job",
|
|
12492
12493
|
primaryKey: "id",
|
|
12493
|
-
columns: ["id", "job_name", "payload", "exception", "failed_at"],
|
|
12494
|
+
columns: ["id", "job_name", "job_id", "payload", "exception", "failed_at"],
|
|
12494
12495
|
defaultOrderBy: { column: "failed_at", direction: "DESC" }
|
|
12495
12496
|
});
|
|
12496
12497
|
|
|
@@ -12511,6 +12512,7 @@ class FailedJobService {
|
|
|
12511
12512
|
async recordFailure(input) {
|
|
12512
12513
|
return await this.repository.create({
|
|
12513
12514
|
job_name: input.jobName,
|
|
12515
|
+
job_id: input.jobId ?? null,
|
|
12514
12516
|
payload: input.payload,
|
|
12515
12517
|
exception: input.exception,
|
|
12516
12518
|
failed_at: new Date
|
|
@@ -12522,8 +12524,11 @@ class FailedJobService {
|
|
|
12522
12524
|
orderBy: { column: "failed_at", direction: "DESC" }
|
|
12523
12525
|
});
|
|
12524
12526
|
}
|
|
12525
|
-
async retry(id) {
|
|
12527
|
+
async retry(id, enqueue) {
|
|
12528
|
+
if (typeof enqueue !== "function")
|
|
12529
|
+
throw new Error("Failed-job retry requires an admission callback.");
|
|
12526
12530
|
const failedJob = await this.repository.findByIdOrThrow(id, (jobId) => new Error(`Failed job ${jobId} not found.`));
|
|
12531
|
+
await enqueue(failedJob);
|
|
12527
12532
|
await this.repository.deleteById(id);
|
|
12528
12533
|
return failedJob;
|
|
12529
12534
|
}
|
|
@@ -12546,6 +12551,9 @@ class FailedJobService {
|
|
|
12546
12551
|
}
|
|
12547
12552
|
var failedJobService_default = FailedJobService;
|
|
12548
12553
|
|
|
12554
|
+
// ../../src/core/queue/jobRunner.ts
|
|
12555
|
+
import { setTimeout as delay } from "timers/promises";
|
|
12556
|
+
|
|
12549
12557
|
// ../../src/core/queue/queueConfig.ts
|
|
12550
12558
|
function resolveQueueConfig() {
|
|
12551
12559
|
const driver = process.env.QUEUE_DRIVER;
|
|
@@ -12568,22 +12576,82 @@ var queueConfig = {
|
|
|
12568
12576
|
return resolveQueueConfig().backoffMs;
|
|
12569
12577
|
}
|
|
12570
12578
|
};
|
|
12579
|
+
function resolveRedisQueueTransport() {
|
|
12580
|
+
const transport = process.env.QUEUE_REDIS_TRANSPORT ?? "lists";
|
|
12581
|
+
if (transport !== "lists" && transport !== "streams")
|
|
12582
|
+
throw new Error('QUEUE_REDIS_TRANSPORT must be "lists" or "streams".');
|
|
12583
|
+
return transport;
|
|
12584
|
+
}
|
|
12585
|
+
function readQueueVisibilityMs() {
|
|
12586
|
+
const value = Number(process.env.QUEUE_VISIBILITY_MS ?? "60000");
|
|
12587
|
+
if (!Number.isSafeInteger(value) || value < 30)
|
|
12588
|
+
throw new Error("QUEUE_VISIBILITY_MS must be an integer of at least 30 milliseconds.");
|
|
12589
|
+
return value;
|
|
12590
|
+
}
|
|
12571
12591
|
|
|
12572
12592
|
// ../../src/core/queue/jobRunner.ts
|
|
12593
|
+
function parseQueueJobEnvelope(rawPayload) {
|
|
12594
|
+
let parsed;
|
|
12595
|
+
try {
|
|
12596
|
+
parsed = JSON.parse(rawPayload);
|
|
12597
|
+
} catch {
|
|
12598
|
+
console.error("[QueueWorker] Ignoring malformed queue payload");
|
|
12599
|
+
return null;
|
|
12600
|
+
}
|
|
12601
|
+
if (!parsed || typeof parsed !== "object") {
|
|
12602
|
+
console.error("[QueueWorker] Ignoring non-object queue payload");
|
|
12603
|
+
return null;
|
|
12604
|
+
}
|
|
12605
|
+
const envelope = parsed;
|
|
12606
|
+
if (typeof envelope.name !== "string" || envelope.name.length === 0) {
|
|
12607
|
+
console.error("[QueueWorker] Ignoring queue payload without job name");
|
|
12608
|
+
return null;
|
|
12609
|
+
}
|
|
12610
|
+
if (!jobRegistry.create(envelope.name)) {
|
|
12611
|
+
console.error(`[QueueWorker] Ignoring unknown job name: ${envelope.name}`);
|
|
12612
|
+
return null;
|
|
12613
|
+
}
|
|
12614
|
+
if (envelope.payload !== undefined && (typeof envelope.payload !== "object" || envelope.payload === null)) {
|
|
12615
|
+
console.error("[QueueWorker] Ignoring queue payload with invalid payload object");
|
|
12616
|
+
return null;
|
|
12617
|
+
}
|
|
12618
|
+
if (envelope.jobId !== undefined && (typeof envelope.jobId !== "string" || !envelope.jobId.length))
|
|
12619
|
+
return null;
|
|
12620
|
+
if (envelope.attempts !== undefined && (!Number.isSafeInteger(envelope.attempts) || envelope.attempts < 0))
|
|
12621
|
+
return null;
|
|
12622
|
+
if (envelope.deadlineAtMs !== undefined && (!Number.isSafeInteger(envelope.deadlineAtMs) || envelope.deadlineAtMs <= 0))
|
|
12623
|
+
return null;
|
|
12624
|
+
if (envelope.cancellable !== undefined && (typeof envelope.cancellable !== "boolean" || envelope.cancellable && !envelope.jobId))
|
|
12625
|
+
return null;
|
|
12626
|
+
return {
|
|
12627
|
+
...envelope.cancellable ? { cancellable: true } : {},
|
|
12628
|
+
...envelope.deadlineAtMs === undefined ? {} : { deadlineAtMs: envelope.deadlineAtMs },
|
|
12629
|
+
jobId: envelope.jobId,
|
|
12630
|
+
name: envelope.name,
|
|
12631
|
+
payload: envelope.payload ?? {},
|
|
12632
|
+
attempts: typeof envelope.attempts === "number" ? envelope.attempts : 0
|
|
12633
|
+
};
|
|
12634
|
+
}
|
|
12573
12635
|
async function runQueueJob(envelope, failedJobs, options = {}) {
|
|
12636
|
+
options.context?.signal.throwIfAborted();
|
|
12574
12637
|
const job = jobRegistry.create(envelope.name);
|
|
12575
12638
|
if (!job) {
|
|
12576
12639
|
throw new Error(`Unknown job "${envelope.name}".`);
|
|
12577
12640
|
}
|
|
12578
12641
|
const attempts = envelope.attempts ?? 0;
|
|
12579
12642
|
try {
|
|
12580
|
-
await job.handle(envelope.payload);
|
|
12643
|
+
await job.handle(envelope.payload, options.context);
|
|
12581
12644
|
} catch (error) {
|
|
12645
|
+
options.context?.signal.throwIfAborted();
|
|
12582
12646
|
const nextAttempt = attempts + 1;
|
|
12583
12647
|
const maxAttempts = job.maxAttempts ?? queueConfig.maxAttempts;
|
|
12584
12648
|
if (nextAttempt < maxAttempts) {
|
|
12585
12649
|
const backoffMs = job.backoffMs ?? queueConfig.backoffMs;
|
|
12586
|
-
|
|
12650
|
+
if (options.deferRetry) {
|
|
12651
|
+
await options.deferRetry({ ...envelope, attempts: nextAttempt }, backoffMs * nextAttempt);
|
|
12652
|
+
return;
|
|
12653
|
+
}
|
|
12654
|
+
await delay(backoffMs * nextAttempt, undefined, { signal: options.context?.signal });
|
|
12587
12655
|
await runQueueJob({
|
|
12588
12656
|
...envelope,
|
|
12589
12657
|
attempts: nextAttempt
|
|
@@ -12592,6 +12660,7 @@ async function runQueueJob(envelope, failedJobs, options = {}) {
|
|
|
12592
12660
|
}
|
|
12593
12661
|
await failedJobs.recordFailure({
|
|
12594
12662
|
jobName: envelope.name,
|
|
12663
|
+
jobId: options.context?.jobId ?? envelope.jobId,
|
|
12595
12664
|
payload: envelope.payload,
|
|
12596
12665
|
exception: error instanceof Error ? error.stack ?? error.message : String(error)
|
|
12597
12666
|
});
|
|
@@ -12601,7 +12670,9 @@ async function runQueueJob(envelope, failedJobs, options = {}) {
|
|
|
12601
12670
|
}
|
|
12602
12671
|
|
|
12603
12672
|
// ../../src/core/queue/redisQueue.ts
|
|
12604
|
-
var {RedisClient:
|
|
12673
|
+
var {RedisClient: RedisClient4 } = globalThis.Bun;
|
|
12674
|
+
|
|
12675
|
+
// ../../src/core/queue/redisQueueKeys.ts
|
|
12605
12676
|
function queueListKey() {
|
|
12606
12677
|
return namespacedRedisKey("queue:default");
|
|
12607
12678
|
}
|
|
@@ -12614,21 +12685,10 @@ function queueLowKey() {
|
|
|
12614
12685
|
var QUEUE_LIST_KEY = queueListKey();
|
|
12615
12686
|
var QUEUE_HIGH_KEY = queueHighKey();
|
|
12616
12687
|
var QUEUE_LOW_KEY = queueLowKey();
|
|
12617
|
-
|
|
12618
|
-
|
|
12619
|
-
function queueProcessingKey(queueKey) {
|
|
12620
|
-
return `${queueKey}:processing`;
|
|
12621
|
-
}
|
|
12622
|
-
function queueProcessingLeaseKey(queueKey) {
|
|
12623
|
-
return `${queueKey}:processing:leases`;
|
|
12624
|
-
}
|
|
12625
|
-
function readVisibilityMs() {
|
|
12626
|
-
const parsed = Number(process.env.QUEUE_VISIBILITY_MS ?? String(DEFAULT_VISIBILITY_MS));
|
|
12627
|
-
if (!Number.isSafeInteger(parsed) || parsed < 30) {
|
|
12628
|
-
throw new Error("QUEUE_VISIBILITY_MS must be an integer of at least 30 milliseconds.");
|
|
12629
|
-
}
|
|
12630
|
-
return parsed;
|
|
12688
|
+
function defaultQueueKeys() {
|
|
12689
|
+
return [queueHighKey(), queueListKey(), queueLowKey()];
|
|
12631
12690
|
}
|
|
12691
|
+
var QUEUE_KEYS = defaultQueueKeys();
|
|
12632
12692
|
function queueKeyForPriority(priority = "default") {
|
|
12633
12693
|
switch (priority) {
|
|
12634
12694
|
case "high":
|
|
@@ -12639,36 +12699,557 @@ function queueKeyForPriority(priority = "default") {
|
|
|
12639
12699
|
return queueListKey();
|
|
12640
12700
|
}
|
|
12641
12701
|
}
|
|
12642
|
-
|
|
12643
|
-
|
|
12702
|
+
|
|
12703
|
+
// ../../src/core/queue/redisStreams.ts
|
|
12704
|
+
var {RedisClient: RedisClient3 } = globalThis.Bun;
|
|
12705
|
+
var STREAM_GROUP = "strata-workers-v1";
|
|
12706
|
+
function streamQueueKey(queueKey) {
|
|
12707
|
+
return `${queueKey}:stream`;
|
|
12708
|
+
}
|
|
12709
|
+
function streamDeadLetterKey(queueKey) {
|
|
12710
|
+
return `${streamQueueKey(queueKey)}:invalid`;
|
|
12711
|
+
}
|
|
12712
|
+
function streamRetryKey(queueKey) {
|
|
12713
|
+
return `${streamQueueKey(queueKey)}:retries`;
|
|
12714
|
+
}
|
|
12715
|
+
function streamControlKey(key, jobId) {
|
|
12716
|
+
const prefix = `${streamQueueKey(key)}/`;
|
|
12717
|
+
if (!jobId.startsWith(prefix) || !/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/.test(jobId.slice(prefix.length)))
|
|
12718
|
+
throw new Error("Invalid cancellable Streams job ID");
|
|
12719
|
+
return `${streamQueueKey(key)}:control:${jobId.slice(prefix.length)}`;
|
|
12720
|
+
}
|
|
12721
|
+
var ENQUEUE_CONTROLLED = `
|
|
12722
|
+
local kind = redis.call('TYPE', KEYS[1]).ok
|
|
12723
|
+
if kind ~= 'none' and kind ~= 'stream' then return redis.error_reply('Invalid stream admission key type') end
|
|
12724
|
+
if redis.call('EXISTS', KEYS[2]) ~= 0 then return redis.error_reply('Streams control identity already exists') end
|
|
12725
|
+
redis.call('XADD', KEYS[1], '*', 'payload', ARGV[1])
|
|
12726
|
+
redis.call('HSET', KEYS[2], 'state', 'active')
|
|
12727
|
+
return 1
|
|
12728
|
+
`;
|
|
12729
|
+
var CANCEL_STREAM = `
|
|
12730
|
+
local state = redis.call('HGET', KEYS[1], 'state')
|
|
12731
|
+
if not state then return 0 end
|
|
12732
|
+
local kind = redis.call('TYPE', KEYS[2]).ok
|
|
12733
|
+
if kind ~= 'none' and kind ~= 'zset' then return redis.error_reply('Invalid stream retry key type') end
|
|
12734
|
+
local payload = redis.call('HGET', KEYS[1], 'retryPayload')
|
|
12735
|
+
if payload and redis.call('ZSCORE', KEYS[2], payload) then redis.call('ZADD', KEYS[2], 0, payload) end
|
|
12736
|
+
redis.call('HSET', KEYS[1], 'state', 'cancelled')
|
|
12737
|
+
return 1
|
|
12738
|
+
`;
|
|
12739
|
+
async function cancelStreamJob(client, key, jobId) {
|
|
12740
|
+
return Number(await client.send("EVAL", [
|
|
12741
|
+
CANCEL_STREAM,
|
|
12742
|
+
"2",
|
|
12743
|
+
streamControlKey(key, jobId),
|
|
12744
|
+
streamRetryKey(key)
|
|
12745
|
+
])) === 1;
|
|
12746
|
+
}
|
|
12747
|
+
async function readStreamCancellation(client, controlKey) {
|
|
12748
|
+
const state = await client.send("HGET", [controlKey, "state"]);
|
|
12749
|
+
if (state !== "active" && state !== "cancelled")
|
|
12750
|
+
throw new Error("Missing or invalid Streams control state");
|
|
12751
|
+
return state === "cancelled";
|
|
12752
|
+
}
|
|
12753
|
+
var MAX_TIMER_MS = 2147483647;
|
|
12754
|
+
async function readStreamTime(client) {
|
|
12755
|
+
const value = await client.send("TIME", []);
|
|
12756
|
+
if (!Array.isArray(value) || value.length !== 2 || typeof value[0] !== "string" || typeof value[1] !== "string" || !/^\d+$/.test(value[0]) || !/^\d+$/.test(value[1]))
|
|
12757
|
+
throw new Error("Invalid Redis TIME response");
|
|
12758
|
+
const seconds = Number(value[0]);
|
|
12759
|
+
const micros = Number(value[1]);
|
|
12760
|
+
const now = seconds * 1000 + Math.floor(micros / 1000);
|
|
12761
|
+
if (!Number.isSafeInteger(seconds) || seconds < 0 || !Number.isSafeInteger(micros) || micros < 0 || micros >= 1e6 || !Number.isSafeInteger(now))
|
|
12762
|
+
throw new Error("Invalid Redis TIME response");
|
|
12763
|
+
return now;
|
|
12764
|
+
}
|
|
12765
|
+
function watchStreamDeadline(client, deadlineAtMs, remainingMs, expire, fail) {
|
|
12766
|
+
let stopped = false;
|
|
12767
|
+
let timer;
|
|
12768
|
+
const schedule = (remaining) => {
|
|
12769
|
+
timer = setTimeout(() => {
|
|
12770
|
+
(async () => {
|
|
12771
|
+
try {
|
|
12772
|
+
const remaining = deadlineAtMs - await readStreamTime(client);
|
|
12773
|
+
if (stopped)
|
|
12774
|
+
return;
|
|
12775
|
+
if (remaining <= 0)
|
|
12776
|
+
expire();
|
|
12777
|
+
else
|
|
12778
|
+
schedule(remaining);
|
|
12779
|
+
} catch (error) {
|
|
12780
|
+
if (!stopped)
|
|
12781
|
+
fail(error);
|
|
12782
|
+
}
|
|
12783
|
+
})();
|
|
12784
|
+
}, Math.max(1, Math.min(MAX_TIMER_MS, remaining)));
|
|
12785
|
+
};
|
|
12786
|
+
schedule(remainingMs);
|
|
12787
|
+
return () => {
|
|
12788
|
+
stopped = true;
|
|
12789
|
+
clearTimeout(timer);
|
|
12790
|
+
};
|
|
12791
|
+
}
|
|
12792
|
+
var SCHEDULE_RETRY = `
|
|
12793
|
+
local pending = redis.call('XPENDING', KEYS[1], ARGV[1], ARGV[2], ARGV[2], 1)
|
|
12794
|
+
if #pending == 0 or pending[1][2] ~= ARGV[3] or tonumber(pending[1][3]) >= tonumber(ARGV[4]) then return 0 end
|
|
12795
|
+
local kind = redis.call('TYPE', KEYS[2]).ok
|
|
12796
|
+
if kind ~= 'none' and kind ~= 'zset' then return redis.error_reply('Invalid stream retry key type') end
|
|
12797
|
+
if ARGV[8] == '1' then
|
|
12798
|
+
local state = redis.call('HGET', KEYS[3], 'state')
|
|
12799
|
+
if state == 'cancelled' then return redis.error_reply('Streams job cancelled') end
|
|
12800
|
+
if state ~= 'active' then return redis.error_reply('Missing or invalid Streams control state') end
|
|
12801
|
+
end
|
|
12802
|
+
local now = redis.call('TIME')
|
|
12803
|
+
local due = tonumber(now[1])*1000 + math.floor(tonumber(now[2])/1000) + tonumber(ARGV[5])
|
|
12804
|
+
if ARGV[7] ~= '' then due = math.min(due, tonumber(ARGV[7])) end
|
|
12805
|
+
redis.call('ZADD', KEYS[2], due, ARGV[6])
|
|
12806
|
+
if ARGV[8] == '1' then redis.call('HSET', KEYS[3], 'retryPayload', ARGV[6]) end
|
|
12807
|
+
redis.call('XACK', KEYS[1], ARGV[1], ARGV[2])
|
|
12808
|
+
redis.call('XDEL', KEYS[1], ARGV[2])
|
|
12809
|
+
if #redis.call('XPENDING', KEYS[1], ARGV[1], '-', '+', 1, ARGV[3]) == 0 then redis.call('XGROUP', 'DELCONSUMER', KEYS[1], ARGV[1], ARGV[3]) end
|
|
12810
|
+
return 1
|
|
12811
|
+
`;
|
|
12812
|
+
var PROMOTE_RETRIES = `
|
|
12813
|
+
local kinds = {'stream','zset'}
|
|
12814
|
+
for i=1,2 do
|
|
12815
|
+
local kind = redis.call('TYPE', KEYS[i]).ok
|
|
12816
|
+
if kind ~= 'none' and kind ~= kinds[i] then return redis.error_reply('Invalid stream retry promotion key type') end
|
|
12817
|
+
end
|
|
12818
|
+
local now = redis.call('TIME')
|
|
12819
|
+
local due = tonumber(now[1])*1000 + math.floor(tonumber(now[2])/1000)
|
|
12820
|
+
local entries = redis.call('ZRANGE', KEYS[2], '-inf', due, 'BYSCORE', 'LIMIT', 0, 100)
|
|
12821
|
+
for _, payload in ipairs(entries) do
|
|
12822
|
+
redis.call('XADD', KEYS[1], '*', 'payload', payload)
|
|
12823
|
+
redis.call('ZREM', KEYS[2], payload)
|
|
12824
|
+
end
|
|
12825
|
+
return #entries
|
|
12826
|
+
`;
|
|
12827
|
+
async function scheduleStreamRetry(client, key, reservation, envelope, delayMs) {
|
|
12828
|
+
if (!Number.isSafeInteger(delayMs) || delayMs < 0 || delayMs > Number.MAX_SAFE_INTEGER / 2)
|
|
12829
|
+
throw new Error("Invalid Streams retry delay");
|
|
12830
|
+
const scheduled = await client.send("EVAL", [
|
|
12831
|
+
SCHEDULE_RETRY,
|
|
12832
|
+
"3",
|
|
12833
|
+
streamQueueKey(key),
|
|
12834
|
+
streamRetryKey(key),
|
|
12835
|
+
envelope.cancellable ? streamControlKey(key, envelope.jobId ?? "") : streamQueueKey(key),
|
|
12836
|
+
STREAM_GROUP,
|
|
12837
|
+
reservation.id,
|
|
12838
|
+
reservation.owner,
|
|
12839
|
+
String(readQueueVisibilityMs()),
|
|
12840
|
+
String(delayMs),
|
|
12841
|
+
JSON.stringify(envelope),
|
|
12842
|
+
envelope.deadlineAtMs === undefined ? "" : String(envelope.deadlineAtMs),
|
|
12843
|
+
envelope.cancellable ? "1" : "0"
|
|
12844
|
+
]);
|
|
12845
|
+
if (Number(scheduled) !== 1)
|
|
12846
|
+
throw new Error("Queue lease lost while scheduling retry");
|
|
12847
|
+
}
|
|
12848
|
+
async function promoteStreamRetries(client, key) {
|
|
12849
|
+
return Number(await client.send("EVAL", [PROMOTE_RETRIES, "2", streamQueueKey(key), streamRetryKey(key)]));
|
|
12850
|
+
}
|
|
12851
|
+
async function readStreamQueueDepth(client, key) {
|
|
12852
|
+
return Number(await client.send("EVAL", [
|
|
12853
|
+
"return redis.call('XLEN', KEYS[1]) + redis.call('ZCARD', KEYS[2])",
|
|
12854
|
+
"2",
|
|
12855
|
+
streamQueueKey(key),
|
|
12856
|
+
streamRetryKey(key)
|
|
12857
|
+
]));
|
|
12858
|
+
}
|
|
12859
|
+
var ASSERT_DRAINED = `
|
|
12860
|
+
if redis.call('LLEN', KEYS[1]) > 0 or redis.call('LLEN', KEYS[2]) > 0 or redis.call('HLEN', KEYS[3]) > 0 then
|
|
12861
|
+
return redis.error_reply('Legacy queue state requires maintenance conversion before Streams activation')
|
|
12862
|
+
end
|
|
12863
|
+
return 1
|
|
12864
|
+
`;
|
|
12865
|
+
async function assertLegacyDrained(client, key) {
|
|
12866
|
+
await client.send("EVAL", [
|
|
12867
|
+
ASSERT_DRAINED,
|
|
12868
|
+
"3",
|
|
12869
|
+
key,
|
|
12870
|
+
`${key}:processing`,
|
|
12871
|
+
`${key}:processing:leases`
|
|
12872
|
+
]);
|
|
12873
|
+
}
|
|
12874
|
+
var UPDATE_STREAM = `
|
|
12875
|
+
local pending = redis.call('XPENDING', KEYS[1], ARGV[1], ARGV[2], ARGV[2], 1)
|
|
12876
|
+
if #pending == 0 or pending[1][2] ~= ARGV[3] or tonumber(pending[1][3]) >= tonumber(ARGV[4]) then return 0 end
|
|
12877
|
+
if ARGV[7] == '1' then
|
|
12878
|
+
local kind = redis.call('TYPE', KEYS[3]).ok
|
|
12879
|
+
if kind ~= 'none' and kind ~= 'hash' then return redis.error_reply('Invalid stream control key type') end
|
|
12880
|
+
end
|
|
12881
|
+
if ARGV[5] == 'renew' then
|
|
12882
|
+
local claimed = redis.call('XCLAIM', KEYS[1], ARGV[1], ARGV[3], 0, ARGV[2], 'JUSTID')
|
|
12883
|
+
return #claimed
|
|
12884
|
+
end
|
|
12885
|
+
if ARGV[5] == 'quarantine' then
|
|
12886
|
+
local kind = redis.call('TYPE', KEYS[2]).ok
|
|
12887
|
+
if kind ~= 'none' and kind ~= 'stream' then return redis.error_reply('Invalid stream quarantine key type') end
|
|
12888
|
+
redis.call('XADD', KEYS[2], '*', 'sourceId', ARGV[2], 'payload', ARGV[6])
|
|
12889
|
+
end
|
|
12890
|
+
local acknowledged = redis.call('XACK', KEYS[1], ARGV[1], ARGV[2])
|
|
12891
|
+
if acknowledged == 1 then
|
|
12892
|
+
redis.call('XDEL', KEYS[1], ARGV[2])
|
|
12893
|
+
if ARGV[7] == '1' then redis.call('DEL', KEYS[3]) end
|
|
12894
|
+
end
|
|
12895
|
+
if #redis.call('XPENDING', KEYS[1], ARGV[1], '-', '+', 1, ARGV[3]) == 0 then redis.call('XGROUP', 'DELCONSUMER', KEYS[1], ARGV[1], ARGV[3]) end
|
|
12896
|
+
return acknowledged
|
|
12897
|
+
`;
|
|
12898
|
+
var CLAIM_STREAM = `
|
|
12899
|
+
local window = redis.call('XPENDING', KEYS[1], ARGV[1], ARGV[4], '+', 10)
|
|
12900
|
+
local claimed = redis.call('XAUTOCLAIM', KEYS[1], ARGV[1], ARGV[2], ARGV[3], ARGV[4], 'COUNT', 1)
|
|
12901
|
+
for _, entry in ipairs(window) do
|
|
12902
|
+
if entry[2] ~= ARGV[2] and #redis.call('XPENDING', KEYS[1], ARGV[1], '-', '+', 1, entry[2]) == 0 then redis.call('XGROUP', 'DELCONSUMER', KEYS[1], ARGV[1], entry[2]) end
|
|
12903
|
+
end
|
|
12904
|
+
if #redis.call('XPENDING', KEYS[1], ARGV[1], '-', '+', 1, ARGV[2]) == 0 then redis.call('XGROUP', 'DELCONSUMER', KEYS[1], ARGV[1], ARGV[2]) end
|
|
12905
|
+
return claimed
|
|
12906
|
+
`;
|
|
12907
|
+
var READ_STREAM = `
|
|
12908
|
+
local entries = redis.call('XREADGROUP', 'GROUP', ARGV[1], ARGV[2], 'COUNT', 1, 'STREAMS', KEYS[1], '>')
|
|
12909
|
+
if not entries and #redis.call('XPENDING', KEYS[1], ARGV[1], '-', '+', 1, ARGV[2]) == 0 then redis.call('XGROUP', 'DELCONSUMER', KEYS[1], ARGV[1], ARGV[2]) end
|
|
12910
|
+
return entries
|
|
12911
|
+
`;
|
|
12912
|
+
async function ensureStreamGroup(client, key) {
|
|
12913
|
+
await assertLegacyDrained(client, key);
|
|
12644
12914
|
try {
|
|
12645
|
-
|
|
12646
|
-
} catch {
|
|
12647
|
-
|
|
12648
|
-
|
|
12915
|
+
await client.send("XGROUP", ["CREATE", streamQueueKey(key), STREAM_GROUP, "0", "MKSTREAM"]);
|
|
12916
|
+
} catch (error) {
|
|
12917
|
+
if (!(error instanceof Error) || !error.message.includes("BUSYGROUP"))
|
|
12918
|
+
throw error;
|
|
12649
12919
|
}
|
|
12650
|
-
|
|
12651
|
-
|
|
12652
|
-
|
|
12920
|
+
}
|
|
12921
|
+
function readStreamEntry(value, owner) {
|
|
12922
|
+
if (!Array.isArray(value) || typeof value[0] !== "string" || !Array.isArray(value[1]))
|
|
12923
|
+
throw new Error("Invalid Redis stream entry response");
|
|
12924
|
+
const fields = value[1];
|
|
12925
|
+
let payload;
|
|
12926
|
+
for (let index = 0;index < fields.length; index += 2) {
|
|
12927
|
+
if (fields[index] === "payload" && typeof fields[index + 1] === "string")
|
|
12928
|
+
payload = fields[index + 1];
|
|
12653
12929
|
}
|
|
12654
|
-
|
|
12655
|
-
|
|
12656
|
-
|
|
12657
|
-
|
|
12930
|
+
return { id: value[0], owner, payload: payload ?? "" };
|
|
12931
|
+
}
|
|
12932
|
+
async function updateStreamReservation(client, key, reservation, action, controlKey) {
|
|
12933
|
+
return Number(await client.send("EVAL", [
|
|
12934
|
+
UPDATE_STREAM,
|
|
12935
|
+
"3",
|
|
12936
|
+
streamQueueKey(key),
|
|
12937
|
+
streamDeadLetterKey(key),
|
|
12938
|
+
controlKey ?? streamQueueKey(key),
|
|
12939
|
+
STREAM_GROUP,
|
|
12940
|
+
reservation.id,
|
|
12941
|
+
reservation.owner,
|
|
12942
|
+
String(readQueueVisibilityMs()),
|
|
12943
|
+
action,
|
|
12944
|
+
reservation.payload,
|
|
12945
|
+
controlKey ? "1" : "0"
|
|
12946
|
+
])) === 1;
|
|
12947
|
+
}
|
|
12948
|
+
async function reserveStreamJob(client, key, owner, cursor = "0-0") {
|
|
12949
|
+
const claimed = await client.send("EVAL", [
|
|
12950
|
+
CLAIM_STREAM,
|
|
12951
|
+
"1",
|
|
12952
|
+
streamQueueKey(key),
|
|
12953
|
+
STREAM_GROUP,
|
|
12954
|
+
owner,
|
|
12955
|
+
String(readQueueVisibilityMs()),
|
|
12956
|
+
cursor
|
|
12957
|
+
]);
|
|
12958
|
+
if (!Array.isArray(claimed) || typeof claimed[0] !== "string" || !Array.isArray(claimed[1]))
|
|
12959
|
+
throw new Error("Invalid Redis XAUTOCLAIM response");
|
|
12960
|
+
const nextCursor = claimed[0];
|
|
12961
|
+
if (claimed[1].length > 0)
|
|
12962
|
+
return { reservation: readStreamEntry(claimed[1][0], owner), cursor: nextCursor };
|
|
12963
|
+
const fresh = await client.send("EVAL", [
|
|
12964
|
+
READ_STREAM,
|
|
12965
|
+
"1",
|
|
12966
|
+
streamQueueKey(key),
|
|
12967
|
+
STREAM_GROUP,
|
|
12968
|
+
owner
|
|
12969
|
+
]);
|
|
12970
|
+
if (fresh === null)
|
|
12971
|
+
return { reservation: null, cursor: nextCursor };
|
|
12972
|
+
if (!Array.isArray(fresh) || !Array.isArray(fresh[0]) || !Array.isArray(fresh[0][1]))
|
|
12973
|
+
throw new Error("Invalid Redis XREADGROUP response");
|
|
12974
|
+
return { reservation: readStreamEntry(fresh[0][1][0], owner), cursor: nextCursor };
|
|
12975
|
+
}
|
|
12976
|
+
|
|
12977
|
+
class RedisStreamsQueue {
|
|
12978
|
+
client;
|
|
12979
|
+
constructor(redisUrl) {
|
|
12980
|
+
this.client = new RedisClient3(redisUrl);
|
|
12658
12981
|
}
|
|
12659
|
-
|
|
12660
|
-
|
|
12661
|
-
return null;
|
|
12982
|
+
close() {
|
|
12983
|
+
this.client.close();
|
|
12662
12984
|
}
|
|
12663
|
-
|
|
12664
|
-
|
|
12665
|
-
|
|
12985
|
+
async dispatch(job, payload) {
|
|
12986
|
+
await this.enqueue(job, payload);
|
|
12987
|
+
}
|
|
12988
|
+
async enqueue(job, payload, options = {}) {
|
|
12989
|
+
return this.admit(job, payload, options);
|
|
12990
|
+
}
|
|
12991
|
+
async replay(job, payload, jobId) {
|
|
12992
|
+
const prefix = defaultQueueKeys().find((key) => jobId.startsWith(`${streamQueueKey(key)}/`));
|
|
12993
|
+
if (!prefix || !/^(?:[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}|[0-9]+-[0-9]+)$/.test(jobId.slice(streamQueueKey(prefix).length + 1)))
|
|
12994
|
+
throw new Error("Replay identity does not belong to this Streams namespace");
|
|
12995
|
+
await this.admit(job, payload, {}, jobId);
|
|
12996
|
+
}
|
|
12997
|
+
async admit(job, payload, options, replayId) {
|
|
12998
|
+
if (options.cancellable !== undefined && typeof options.cancellable !== "boolean")
|
|
12999
|
+
throw new Error("cancellable must be a boolean");
|
|
13000
|
+
if (options.timeoutMs !== undefined && (!Number.isSafeInteger(options.timeoutMs) || options.timeoutMs <= 0 || options.timeoutMs > MAX_TIMER_MS))
|
|
13001
|
+
throw new Error("Streams timeoutMs must be an integer between 1 and 2147483647");
|
|
13002
|
+
const name = jobRegistry.resolveName(job);
|
|
13003
|
+
if (!name)
|
|
13004
|
+
throw new Error("Job is not registered with the queue worker registry.");
|
|
13005
|
+
const key = queueKeyForPriority(job.priority);
|
|
13006
|
+
await assertLegacyDrained(this.client, key);
|
|
13007
|
+
const jobId = replayId ?? `${streamQueueKey(key)}/${crypto.randomUUID()}`;
|
|
13008
|
+
const deadlineAtMs = options.timeoutMs === undefined ? undefined : await readStreamTime(this.client) + options.timeoutMs;
|
|
13009
|
+
const envelope = JSON.stringify({
|
|
13010
|
+
name,
|
|
13011
|
+
payload,
|
|
13012
|
+
attempts: 0,
|
|
13013
|
+
jobId,
|
|
13014
|
+
deadlineAtMs,
|
|
13015
|
+
...options.cancellable ? { cancellable: true } : {}
|
|
13016
|
+
});
|
|
13017
|
+
if (options.cancellable)
|
|
13018
|
+
await this.client.send("EVAL", [
|
|
13019
|
+
ENQUEUE_CONTROLLED,
|
|
13020
|
+
"2",
|
|
13021
|
+
streamQueueKey(key),
|
|
13022
|
+
streamControlKey(key, jobId),
|
|
13023
|
+
envelope
|
|
13024
|
+
]);
|
|
13025
|
+
else
|
|
13026
|
+
await this.client.send("XADD", [streamQueueKey(key), "*", "payload", envelope]);
|
|
13027
|
+
return jobId;
|
|
13028
|
+
}
|
|
13029
|
+
async cancel(jobId) {
|
|
13030
|
+
const key = defaultQueueKeys().find((key) => jobId.startsWith(`${streamQueueKey(key)}/`));
|
|
13031
|
+
if (!key)
|
|
13032
|
+
throw new Error("Invalid cancellable Streams job ID");
|
|
13033
|
+
return cancelStreamJob(this.client, key, jobId);
|
|
12666
13034
|
}
|
|
12667
|
-
|
|
12668
|
-
|
|
12669
|
-
|
|
12670
|
-
|
|
12671
|
-
|
|
13035
|
+
}
|
|
13036
|
+
|
|
13037
|
+
class RedisStreamsWorker {
|
|
13038
|
+
failedJobs;
|
|
13039
|
+
timeoutSeconds;
|
|
13040
|
+
queueKeys;
|
|
13041
|
+
client;
|
|
13042
|
+
owner = crypto.randomUUID();
|
|
13043
|
+
cursors = new Map;
|
|
13044
|
+
processing = false;
|
|
13045
|
+
initialized = false;
|
|
13046
|
+
running = false;
|
|
13047
|
+
stopping = false;
|
|
13048
|
+
active;
|
|
13049
|
+
constructor(redisUrl, failedJobs, timeoutSeconds = 5, queueKeys = defaultQueueKeys()) {
|
|
13050
|
+
this.failedJobs = failedJobs;
|
|
13051
|
+
this.timeoutSeconds = timeoutSeconds;
|
|
13052
|
+
this.queueKeys = queueKeys;
|
|
13053
|
+
readQueueVisibilityMs();
|
|
13054
|
+
this.client = new RedisClient3(redisUrl);
|
|
13055
|
+
}
|
|
13056
|
+
requestStop() {
|
|
13057
|
+
this.stopping = true;
|
|
13058
|
+
}
|
|
13059
|
+
isRunning() {
|
|
13060
|
+
return this.running;
|
|
13061
|
+
}
|
|
13062
|
+
close() {
|
|
13063
|
+
this.stopping = true;
|
|
13064
|
+
this.active?.abort(new Error("Queue worker closed"));
|
|
13065
|
+
this.client.close();
|
|
13066
|
+
}
|
|
13067
|
+
async initialize() {
|
|
13068
|
+
if (this.initialized)
|
|
13069
|
+
return;
|
|
13070
|
+
for (const key of this.queueKeys)
|
|
13071
|
+
await ensureStreamGroup(this.client, key);
|
|
13072
|
+
this.initialized = true;
|
|
13073
|
+
}
|
|
13074
|
+
async reserve(key) {
|
|
13075
|
+
await promoteStreamRetries(this.client, key);
|
|
13076
|
+
const result = await reserveStreamJob(this.client, key, this.owner, this.cursors.get(key));
|
|
13077
|
+
this.cursors.set(key, result.cursor);
|
|
13078
|
+
return result.reservation;
|
|
13079
|
+
}
|
|
13080
|
+
async processNext() {
|
|
13081
|
+
if (this.processing)
|
|
13082
|
+
throw new Error("Queue worker already processing a job.");
|
|
13083
|
+
this.processing = true;
|
|
13084
|
+
try {
|
|
13085
|
+
return await this.processOne();
|
|
13086
|
+
} finally {
|
|
13087
|
+
this.processing = false;
|
|
13088
|
+
}
|
|
13089
|
+
}
|
|
13090
|
+
async processOne() {
|
|
13091
|
+
if (this.stopping)
|
|
13092
|
+
return false;
|
|
13093
|
+
await this.initialize();
|
|
13094
|
+
const deadline = performance.now() + this.timeoutSeconds * 1000;
|
|
13095
|
+
do {
|
|
13096
|
+
for (const key of this.queueKeys) {
|
|
13097
|
+
if (this.stopping)
|
|
13098
|
+
return false;
|
|
13099
|
+
const reservation = await this.reserve(key);
|
|
13100
|
+
if (!reservation)
|
|
13101
|
+
continue;
|
|
13102
|
+
await this.handle(key, reservation);
|
|
13103
|
+
return true;
|
|
13104
|
+
}
|
|
13105
|
+
if (performance.now() >= deadline || this.stopping)
|
|
13106
|
+
return false;
|
|
13107
|
+
await Bun.sleep(Math.min(100, Math.max(1, deadline - performance.now())));
|
|
13108
|
+
} while (!this.stopping);
|
|
13109
|
+
return false;
|
|
13110
|
+
}
|
|
13111
|
+
async handle(key, reservation) {
|
|
13112
|
+
const envelope = parseQueueJobEnvelope(reservation.payload);
|
|
13113
|
+
if (!envelope) {
|
|
13114
|
+
await updateStreamReservation(this.client, key, reservation, "quarantine");
|
|
13115
|
+
return;
|
|
13116
|
+
}
|
|
13117
|
+
envelope.jobId ??= `${streamQueueKey(key)}/${reservation.id}`;
|
|
13118
|
+
let controlKey;
|
|
13119
|
+
if (envelope.cancellable) {
|
|
13120
|
+
try {
|
|
13121
|
+
controlKey = streamControlKey(key, envelope.jobId);
|
|
13122
|
+
} catch {
|
|
13123
|
+
await updateStreamReservation(this.client, key, reservation, "quarantine");
|
|
13124
|
+
return;
|
|
13125
|
+
}
|
|
13126
|
+
}
|
|
13127
|
+
const cancellationError = new Error("Streams job cancelled");
|
|
13128
|
+
if (controlKey && await readStreamCancellation(this.client, controlKey)) {
|
|
13129
|
+
await this.failedJobs.recordFailure({
|
|
13130
|
+
jobName: envelope.name,
|
|
13131
|
+
jobId: envelope.jobId,
|
|
13132
|
+
payload: envelope.payload,
|
|
13133
|
+
exception: cancellationError.message
|
|
13134
|
+
});
|
|
13135
|
+
await updateStreamReservation(this.client, key, reservation, "ack", controlKey);
|
|
13136
|
+
return;
|
|
13137
|
+
}
|
|
13138
|
+
const remainingMs = envelope.deadlineAtMs === undefined ? undefined : envelope.deadlineAtMs - await readStreamTime(this.client);
|
|
13139
|
+
const deadlineError = new Error("Streams job deadline exceeded");
|
|
13140
|
+
if (remainingMs !== undefined && remainingMs <= 0) {
|
|
13141
|
+
await this.failedJobs.recordFailure({
|
|
13142
|
+
jobName: envelope.name,
|
|
13143
|
+
jobId: envelope.jobId,
|
|
13144
|
+
payload: envelope.payload,
|
|
13145
|
+
exception: deadlineError.message
|
|
13146
|
+
});
|
|
13147
|
+
await updateStreamReservation(this.client, key, reservation, "ack", controlKey);
|
|
13148
|
+
return;
|
|
13149
|
+
}
|
|
13150
|
+
const controller = new AbortController;
|
|
13151
|
+
this.active = controller;
|
|
13152
|
+
let acknowledge = false;
|
|
13153
|
+
let released = false;
|
|
13154
|
+
let ownershipLost = false;
|
|
13155
|
+
let renewal;
|
|
13156
|
+
let deadlineExpired = false;
|
|
13157
|
+
let cancellationRequested = false;
|
|
13158
|
+
let failureRecorded = false;
|
|
13159
|
+
const loseOwnership = (error) => {
|
|
13160
|
+
ownershipLost = true;
|
|
13161
|
+
controller.abort(error);
|
|
13162
|
+
};
|
|
13163
|
+
const stopDeadlineWatch = remainingMs === undefined || envelope.deadlineAtMs === undefined ? undefined : watchStreamDeadline(this.client, envelope.deadlineAtMs, remainingMs, () => {
|
|
13164
|
+
deadlineExpired = true;
|
|
13165
|
+
controller.abort(deadlineError);
|
|
13166
|
+
}, loseOwnership);
|
|
13167
|
+
const heartbeat = setInterval(() => {
|
|
13168
|
+
if (renewal || ownershipLost || released)
|
|
13169
|
+
return;
|
|
13170
|
+
renewal = (async () => {
|
|
13171
|
+
try {
|
|
13172
|
+
if (!await updateStreamReservation(this.client, key, reservation, "renew", controlKey))
|
|
13173
|
+
loseOwnership(new Error("Queue lease lost"));
|
|
13174
|
+
else if (controlKey && await readStreamCancellation(this.client, controlKey)) {
|
|
13175
|
+
cancellationRequested = true;
|
|
13176
|
+
controller.abort(cancellationError);
|
|
13177
|
+
}
|
|
13178
|
+
} catch (error) {
|
|
13179
|
+
loseOwnership(error);
|
|
13180
|
+
console.error("[StreamsWorker] Lease renewal failed:", error);
|
|
13181
|
+
}
|
|
13182
|
+
})().finally(() => {
|
|
13183
|
+
renewal = undefined;
|
|
13184
|
+
});
|
|
13185
|
+
}, Math.max(1, Math.floor(readQueueVisibilityMs() / 3)));
|
|
13186
|
+
try {
|
|
13187
|
+
await runQueueJob(envelope, this.failedJobs, {
|
|
13188
|
+
context: {
|
|
13189
|
+
jobId: envelope.jobId,
|
|
13190
|
+
signal: controller.signal,
|
|
13191
|
+
deadlineAtMs: envelope.deadlineAtMs
|
|
13192
|
+
},
|
|
13193
|
+
deferRetry: async (next, delayMs) => {
|
|
13194
|
+
controller.signal.throwIfAborted();
|
|
13195
|
+
await scheduleStreamRetry(this.client, key, reservation, next, delayMs);
|
|
13196
|
+
released = true;
|
|
13197
|
+
},
|
|
13198
|
+
onFailureRecorded: () => {
|
|
13199
|
+
acknowledge = true;
|
|
13200
|
+
failureRecorded = true;
|
|
13201
|
+
}
|
|
13202
|
+
});
|
|
13203
|
+
acknowledge = true;
|
|
13204
|
+
} catch (error) {
|
|
13205
|
+
console.error("[StreamsWorker] Job failed:", error);
|
|
13206
|
+
} finally {
|
|
13207
|
+
stopDeadlineWatch?.();
|
|
13208
|
+
clearInterval(heartbeat);
|
|
13209
|
+
await renewal;
|
|
13210
|
+
this.active = undefined;
|
|
13211
|
+
}
|
|
13212
|
+
if (!ownershipLost && !released && controlKey && (cancellationRequested || await readStreamCancellation(this.client, controlKey))) {
|
|
13213
|
+
if (!failureRecorded)
|
|
13214
|
+
await this.failedJobs.recordFailure({
|
|
13215
|
+
jobName: envelope.name,
|
|
13216
|
+
jobId: envelope.jobId,
|
|
13217
|
+
payload: envelope.payload,
|
|
13218
|
+
exception: cancellationError.message
|
|
13219
|
+
});
|
|
13220
|
+
failureRecorded = true;
|
|
13221
|
+
acknowledge = true;
|
|
13222
|
+
}
|
|
13223
|
+
if (!ownershipLost && !released && envelope.deadlineAtMs !== undefined && (deadlineExpired || envelope.deadlineAtMs <= await readStreamTime(this.client))) {
|
|
13224
|
+
if (!failureRecorded)
|
|
13225
|
+
await this.failedJobs.recordFailure({
|
|
13226
|
+
jobName: envelope.name,
|
|
13227
|
+
jobId: envelope.jobId,
|
|
13228
|
+
payload: envelope.payload,
|
|
13229
|
+
exception: deadlineError.message
|
|
13230
|
+
});
|
|
13231
|
+
acknowledge = true;
|
|
13232
|
+
}
|
|
13233
|
+
if (acknowledge && !ownershipLost && !released)
|
|
13234
|
+
await updateStreamReservation(this.client, key, reservation, "ack", controlKey);
|
|
13235
|
+
}
|
|
13236
|
+
async run() {
|
|
13237
|
+
this.running = true;
|
|
13238
|
+
try {
|
|
13239
|
+
while (!this.stopping)
|
|
13240
|
+
await this.processNext();
|
|
13241
|
+
} finally {
|
|
13242
|
+
this.running = false;
|
|
13243
|
+
}
|
|
13244
|
+
}
|
|
13245
|
+
}
|
|
13246
|
+
|
|
13247
|
+
// ../../src/core/queue/redisQueue.ts
|
|
13248
|
+
function queueProcessingKey(queueKey) {
|
|
13249
|
+
return `${queueKey}:processing`;
|
|
13250
|
+
}
|
|
13251
|
+
function queueProcessingLeaseKey(queueKey) {
|
|
13252
|
+
return `${queueKey}:processing:leases`;
|
|
12672
13253
|
}
|
|
12673
13254
|
var SCRIPT_HEADER = `
|
|
12674
13255
|
local function checkType(key, expected)
|
|
@@ -12757,7 +13338,7 @@ async function reserveQueueJob(client, queueKey, owner) {
|
|
|
12757
13338
|
queueProcessingLeaseKey(queueKey),
|
|
12758
13339
|
crypto.randomUUID(),
|
|
12759
13340
|
owner,
|
|
12760
|
-
String(
|
|
13341
|
+
String(readQueueVisibilityMs())
|
|
12761
13342
|
]);
|
|
12762
13343
|
if (typeof raw !== "string")
|
|
12763
13344
|
return null;
|
|
@@ -12775,7 +13356,7 @@ async function updateQueueReservation(client, queueKey, reservation, action) {
|
|
|
12775
13356
|
reservation.owner,
|
|
12776
13357
|
reservation.record,
|
|
12777
13358
|
action,
|
|
12778
|
-
String(
|
|
13359
|
+
String(readQueueVisibilityMs()),
|
|
12779
13360
|
reservation.payload
|
|
12780
13361
|
])) === 1;
|
|
12781
13362
|
}
|
|
@@ -12788,7 +13369,7 @@ async function reclaimExpiredQueueReservations(client, queueKeys = QUEUE_KEYS) {
|
|
|
12788
13369
|
key,
|
|
12789
13370
|
queueProcessingKey(key),
|
|
12790
13371
|
queueProcessingLeaseKey(key),
|
|
12791
|
-
String(
|
|
13372
|
+
String(readQueueVisibilityMs())
|
|
12792
13373
|
]));
|
|
12793
13374
|
}
|
|
12794
13375
|
return reclaimed;
|
|
@@ -12797,7 +13378,7 @@ async function reclaimExpiredQueueReservations(client, queueKeys = QUEUE_KEYS) {
|
|
|
12797
13378
|
class RedisQueue {
|
|
12798
13379
|
client;
|
|
12799
13380
|
constructor(redisUrl) {
|
|
12800
|
-
this.client = new
|
|
13381
|
+
this.client = new RedisClient4(redisUrl);
|
|
12801
13382
|
}
|
|
12802
13383
|
close() {
|
|
12803
13384
|
this.client.close();
|
|
@@ -12829,7 +13410,7 @@ class QueueWorker {
|
|
|
12829
13410
|
this.failedJobs = failedJobs;
|
|
12830
13411
|
this.timeoutSeconds = timeoutSeconds;
|
|
12831
13412
|
this.queueKeys = queueKeys;
|
|
12832
|
-
this.client = new
|
|
13413
|
+
this.client = new RedisClient4(redisUrl);
|
|
12833
13414
|
}
|
|
12834
13415
|
requestStop() {
|
|
12835
13416
|
this.stopping = true;
|
|
@@ -12884,7 +13465,7 @@ class QueueWorker {
|
|
|
12884
13465
|
})().finally(() => {
|
|
12885
13466
|
renewal = undefined;
|
|
12886
13467
|
});
|
|
12887
|
-
}, Math.max(1, Math.floor(
|
|
13468
|
+
}, Math.max(1, Math.floor(readQueueVisibilityMs() / 3)));
|
|
12888
13469
|
try {
|
|
12889
13470
|
await runQueueJob(envelope, this.failedJobs, {
|
|
12890
13471
|
onFailureRecorded: () => {
|
|
@@ -12962,22 +13543,30 @@ function createProductionQueue(driver, options = {}) {
|
|
|
12962
13543
|
if (!options.redisUrl) {
|
|
12963
13544
|
throw new Error('QUEUE_DRIVER="redis" requires REDIS_URL to be set.');
|
|
12964
13545
|
}
|
|
12965
|
-
return new RedisQueue(options.redisUrl);
|
|
13546
|
+
return resolveRedisQueueTransport() === "streams" ? new RedisStreamsQueue(options.redisUrl) : new RedisQueue(options.redisUrl);
|
|
12966
13547
|
}
|
|
12967
13548
|
return new ResilientQueue(failedJobs, driver === "async");
|
|
12968
13549
|
}
|
|
12969
13550
|
function createQueueWorker(redisUrl, failedJobs = createFailedJobService()) {
|
|
12970
|
-
return new QueueWorker(redisUrl, failedJobs);
|
|
13551
|
+
return resolveRedisQueueTransport() === "streams" ? new RedisStreamsWorker(redisUrl, failedJobs) : new QueueWorker(redisUrl, failedJobs);
|
|
12971
13552
|
}
|
|
12972
13553
|
// ../../src/core/queue/queueMetrics.ts
|
|
12973
|
-
var {RedisClient:
|
|
13554
|
+
var {RedisClient: RedisClient5 } = globalThis.Bun;
|
|
12974
13555
|
async function readRedisQueueDepth(redisUrl) {
|
|
12975
|
-
const client = new
|
|
12976
|
-
|
|
12977
|
-
|
|
12978
|
-
|
|
12979
|
-
|
|
12980
|
-
|
|
13556
|
+
const client = new RedisClient5(redisUrl);
|
|
13557
|
+
let high;
|
|
13558
|
+
let defaultQueue;
|
|
13559
|
+
let low;
|
|
13560
|
+
try {
|
|
13561
|
+
const readDepth = async (key) => resolveRedisQueueTransport() === "streams" ? readStreamQueueDepth(client, key) : client.llen(key);
|
|
13562
|
+
[high, defaultQueue, low] = await Promise.all([
|
|
13563
|
+
readDepth(defaultQueueKeys()[0]),
|
|
13564
|
+
readDepth(defaultQueueKeys()[1]),
|
|
13565
|
+
readDepth(defaultQueueKeys()[2])
|
|
13566
|
+
]);
|
|
13567
|
+
} finally {
|
|
13568
|
+
client.close();
|
|
13569
|
+
}
|
|
12981
13570
|
return {
|
|
12982
13571
|
high: Number(high ?? 0),
|
|
12983
13572
|
default: Number(defaultQueue ?? 0),
|
|
@@ -13025,7 +13614,7 @@ import { createHash as createHash6 } from "crypto";
|
|
|
13025
13614
|
|
|
13026
13615
|
// ../../src/core/scheduler/leases.ts
|
|
13027
13616
|
import { createHash as createHash5 } from "crypto";
|
|
13028
|
-
var {RedisClient:
|
|
13617
|
+
var {RedisClient: RedisClient6 } = globalThis.Bun;
|
|
13029
13618
|
var LEASE_SCRIPT = `
|
|
13030
13619
|
for _, key in ipairs(KEYS) do
|
|
13031
13620
|
local kind = redis.call('TYPE', key).ok
|
|
@@ -13068,7 +13657,7 @@ class RedisSchedulerLeaseStore {
|
|
|
13068
13657
|
if (options.client)
|
|
13069
13658
|
this.client = options.client;
|
|
13070
13659
|
else {
|
|
13071
|
-
const client = new
|
|
13660
|
+
const client = new RedisClient6(options.redisUrl, { autoReconnect: false, maxRetries: 0 });
|
|
13072
13661
|
this.owned = client;
|
|
13073
13662
|
this.client = client;
|
|
13074
13663
|
}
|
|
@@ -13465,6 +14054,53 @@ init_tenantDatabaseScope();
|
|
|
13465
14054
|
import { createHash as createHash7 } from "crypto";
|
|
13466
14055
|
init_repositoryConnection();
|
|
13467
14056
|
init_databaseTenantContext();
|
|
14057
|
+
|
|
14058
|
+
// ../../src/core/tenant/publicTenantHost.ts
|
|
14059
|
+
import { BlockList, isIP } from "net";
|
|
14060
|
+
function normalizeTenantHostname(authority) {
|
|
14061
|
+
if (!authority || authority !== authority.trim() || /[\s\\/@?#%,]/.test(authority)) {
|
|
14062
|
+
throw new ForbiddenError("Invalid tenant hostname.");
|
|
14063
|
+
}
|
|
14064
|
+
let url;
|
|
14065
|
+
try {
|
|
14066
|
+
url = new URL(`http://${authority}`);
|
|
14067
|
+
} catch {
|
|
14068
|
+
throw new ForbiddenError("Invalid tenant hostname.");
|
|
14069
|
+
}
|
|
14070
|
+
const hostname = url.hostname.toLowerCase().replace(/\.$/, "");
|
|
14071
|
+
const ip = hostname.startsWith("[") ? hostname.slice(1, -1) : hostname;
|
|
14072
|
+
if (!isIP(ip) && (hostname.length > 253 || !hostname.split(".").every((label) => label.length > 0 && label.length <= 63 && /^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?$/.test(label)))) {
|
|
14073
|
+
throw new ForbiddenError("Invalid tenant hostname.");
|
|
14074
|
+
}
|
|
14075
|
+
return hostname;
|
|
14076
|
+
}
|
|
14077
|
+
function createPublicTenantSelector(options) {
|
|
14078
|
+
const addresses = options.trustedProxyAddresses ?? [];
|
|
14079
|
+
if (addresses.length > 128)
|
|
14080
|
+
throw new Error("Public tenancy supports at most 128 trusted proxy addresses.");
|
|
14081
|
+
const proxies = new BlockList;
|
|
14082
|
+
for (const address of addresses) {
|
|
14083
|
+
const version = isIP(address);
|
|
14084
|
+
if (!version)
|
|
14085
|
+
throw new Error("Trusted proxy addresses must be literal IP addresses.");
|
|
14086
|
+
proxies.addAddress(address, version === 6 ? "ipv6" : "ipv4");
|
|
14087
|
+
}
|
|
14088
|
+
return async (request) => {
|
|
14089
|
+
const peer = currentRequestMeta().peerAddress;
|
|
14090
|
+
const version = peer ? isIP(peer) : 0;
|
|
14091
|
+
const trusted = peer && version && proxies.check(peer, version === 6 ? "ipv6" : "ipv4");
|
|
14092
|
+
const forwarded = trusted ? request.headers.get("x-forwarded-host") : null;
|
|
14093
|
+
const authority = forwarded ?? request.headers.get("host") ?? new URL(request.url).host;
|
|
14094
|
+
const id = await options.resolveTenantId(normalizeTenantHostname(authority));
|
|
14095
|
+
if (id === null)
|
|
14096
|
+
throw new ForbiddenError("Tenant host not found.");
|
|
14097
|
+
if (!Number.isSafeInteger(id) || id <= 0)
|
|
14098
|
+
throw new Error("Public tenant resolver returned an invalid identity.");
|
|
14099
|
+
return id;
|
|
14100
|
+
};
|
|
14101
|
+
}
|
|
14102
|
+
|
|
14103
|
+
// ../../src/core/tenant/tenantMiddleware.ts
|
|
13468
14104
|
init_tenancyConfig();
|
|
13469
14105
|
init_tenantContext();
|
|
13470
14106
|
init_tenantDatabaseScope();
|
|
@@ -13489,16 +14125,17 @@ async function resolveUserTenantId(userId) {
|
|
|
13489
14125
|
function auditChecksum(payload) {
|
|
13490
14126
|
return createHash7("sha256").update(JSON.stringify(payload)).digest("hex");
|
|
13491
14127
|
}
|
|
13492
|
-
async function resolveTenantForRequest(request, resolver) {
|
|
14128
|
+
async function resolveTenantForRequest(request, resolver, selectPublicTenant) {
|
|
14129
|
+
const hostTenantId = selectPublicTenant ? await selectPublicTenant(request) : null;
|
|
13493
14130
|
const user = currentAuthUser();
|
|
13494
14131
|
const headerValue = request.headers.get("x-tenant-id")?.trim();
|
|
13495
|
-
const parsedHeader = headerValue !== undefined && headerValue.length > 0 ? Number
|
|
14132
|
+
const parsedHeader = headerValue !== undefined && headerValue.length > 0 ? Number(headerValue) : Number.NaN;
|
|
13496
14133
|
if (user) {
|
|
13497
14134
|
const userId = typeof user.id === "number" ? user.id : Number.parseInt(String(user.id), 10);
|
|
13498
14135
|
if (Number.isInteger(userId) && userId > 0) {
|
|
13499
14136
|
const userTenantId = await resolveUserTenantId(userId);
|
|
13500
14137
|
if (!isGlobalAdmin(user)) {
|
|
13501
|
-
if (Number.
|
|
14138
|
+
if (Number.isSafeInteger(parsedHeader) && parsedHeader > 0 && parsedHeader !== userTenantId) {
|
|
13502
14139
|
throw new ForbiddenError("Tenant header does not match your account.");
|
|
13503
14140
|
}
|
|
13504
14141
|
const memberTenant = await resolver(userTenantId);
|
|
@@ -13507,7 +14144,7 @@ async function resolveTenantForRequest(request, resolver) {
|
|
|
13507
14144
|
}
|
|
13508
14145
|
return DEFAULT_TENANT;
|
|
13509
14146
|
}
|
|
13510
|
-
if (Number.
|
|
14147
|
+
if (Number.isSafeInteger(parsedHeader) && parsedHeader > 0) {
|
|
13511
14148
|
const headerTenant = await resolver(parsedHeader);
|
|
13512
14149
|
if (headerTenant) {
|
|
13513
14150
|
return headerTenant;
|
|
@@ -13521,9 +14158,14 @@ async function resolveTenantForRequest(request, resolver) {
|
|
|
13521
14158
|
return DEFAULT_TENANT;
|
|
13522
14159
|
}
|
|
13523
14160
|
}
|
|
13524
|
-
const headerTenantId = Number.
|
|
13525
|
-
const tenantId =
|
|
14161
|
+
const headerTenantId = Number.isSafeInteger(parsedHeader) && parsedHeader > 0 ? parsedHeader : null;
|
|
14162
|
+
const tenantId = hostTenantId !== null ? hostTenantId : !isProductionEnv() && process.env.TENANT_DEV_HEADERS === "true" && headerTenantId !== null ? headerTenantId : DEFAULT_TENANT.id;
|
|
14163
|
+
if (isProductionEnv() && !selectPublicTenant) {
|
|
14164
|
+
throw new ForbiddenError("Public tenant host resolver is required.");
|
|
14165
|
+
}
|
|
13526
14166
|
const guestTenant = await resolver(tenantId);
|
|
14167
|
+
if (selectPublicTenant && !guestTenant)
|
|
14168
|
+
throw new ForbiddenError("Tenant not found.");
|
|
13527
14169
|
if (guestTenant) {
|
|
13528
14170
|
return guestTenant;
|
|
13529
14171
|
}
|
|
@@ -13531,6 +14173,7 @@ async function resolveTenantForRequest(request, resolver) {
|
|
|
13531
14173
|
}
|
|
13532
14174
|
function createTenantMiddleware(options = {}) {
|
|
13533
14175
|
const resolver = options.resolveTenant ?? resolveTenant;
|
|
14176
|
+
const selectPublicTenant = options.publicTenancy ? createPublicTenantSelector(options.publicTenancy) : undefined;
|
|
13534
14177
|
return async (request, next) => {
|
|
13535
14178
|
const pathname = new URL(request.url).pathname;
|
|
13536
14179
|
if (pathname.startsWith("/scim/")) {
|
|
@@ -13549,7 +14192,7 @@ function createTenantMiddleware(options = {}) {
|
|
|
13549
14192
|
throw new Error("Tenant resolver returned a different tenant identity.");
|
|
13550
14193
|
}
|
|
13551
14194
|
return tenant;
|
|
13552
|
-
});
|
|
14195
|
+
}, selectPublicTenant);
|
|
13553
14196
|
return await runWithTenantDatabase(tenant, async () => {
|
|
13554
14197
|
return await next();
|
|
13555
14198
|
});
|
|
@@ -14424,6 +15067,7 @@ export {
|
|
|
14424
15067
|
namespacedRedisKey,
|
|
14425
15068
|
noContentResponse,
|
|
14426
15069
|
normalizeFieldErrors,
|
|
15070
|
+
normalizeTenantHostname,
|
|
14427
15071
|
notFoundHtmlResponse,
|
|
14428
15072
|
notModifiedResponse,
|
|
14429
15073
|
otelServiceName,
|