@getstrata/core 2.2.48 → 2.2.50
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +11 -0
- package/dist/core/audit/exportStore.d.ts +17 -0
- package/dist/core/cache/redisCacheScript.d.ts +2 -0
- package/dist/core/database/baseRepository.d.ts +0 -1
- package/dist/core/database/model.d.ts +6 -113
- package/dist/core/database/modelAttributes.d.ts +17 -0
- package/dist/core/database/modelQuery.d.ts +139 -0
- package/dist/core/database/modelRegistry.d.ts +34 -0
- package/dist/core/database/relationQueries.d.ts +20 -0
- package/dist/core/http/middleware.d.ts +2 -2
- package/dist/core/http/routeMiddleware.d.ts +1 -1
- package/dist/core/mail/mailer.d.ts +1 -4
- package/dist/core/mail/smtpMessage.d.ts +7 -0
- package/dist/core/queue/redisListScripts.d.ts +5 -0
- package/dist/core/queue/redisStreamsScripts.d.ts +11 -0
- package/dist/core/scheduler/leaseScript.d.ts +2 -0
- package/dist/entries/audit/exportAuditLogs.js +63 -58
- package/dist/entries/auth/scimAuthMiddleware.js +31 -20
- package/dist/entries/cache/createCacheStore.js +2 -1
- package/dist/entries/facades.js +103 -101
- package/dist/entries/http/routeMiddleware.js +3 -3
- package/dist/entries/jobs/exportAuditLogsJob.js +63 -58
- package/dist/entries/mail/mailer.js +103 -101
- package/dist/entries/queue/createAppQueue.js +277 -271
- package/dist/entries/queue/publicQueue.js +277 -271
- package/dist/entries/queue/queueMetrics.js +277 -271
- package/dist/entries/queue/redisQueue.js +277 -271
- package/dist/index.js +1482 -1428
- package/package.json +1 -1
|
@@ -225,6 +225,143 @@ async function runQueueJob(envelope, failedJobs, options = {}) {
|
|
|
225
225
|
// ../../src/core/queue/redisQueue.ts
|
|
226
226
|
var {RedisClient: RedisClient2 } = globalThis.Bun;
|
|
227
227
|
|
|
228
|
+
// ../../src/core/queue/redisListScripts.ts
|
|
229
|
+
var SCRIPT_HEADER = `
|
|
230
|
+
local function checkType(key, expected)
|
|
231
|
+
local actual = redis.call('TYPE', key).ok
|
|
232
|
+
if actual ~= 'none' and actual ~= expected then
|
|
233
|
+
return redis.error_reply('Unexpected queue key type: ' .. key)
|
|
234
|
+
end
|
|
235
|
+
end
|
|
236
|
+
local function nowMs()
|
|
237
|
+
local time = redis.call('TIME')
|
|
238
|
+
return tonumber(time[1]) * 1000 + math.floor(tonumber(time[2]) / 1000)
|
|
239
|
+
end
|
|
240
|
+
`;
|
|
241
|
+
var RESERVE_JOB = `${SCRIPT_HEADER}
|
|
242
|
+
local err = checkType(KEYS[1], 'list') or checkType(KEYS[2], 'list') or checkType(KEYS[3], 'hash')
|
|
243
|
+
if err then return err end
|
|
244
|
+
local payload = redis.call('LINDEX', KEYS[1], -1)
|
|
245
|
+
if not payload then return nil end
|
|
246
|
+
local record = cjson.encode({strataReservation=1, id=ARGV[1], owner=ARGV[2], payload=payload})
|
|
247
|
+
local lease = cjson.encode({owner=ARGV[2], expiresAt=nowMs()+tonumber(ARGV[3])})
|
|
248
|
+
redis.call('LPUSH', KEYS[2], record)
|
|
249
|
+
redis.call('HSET', KEYS[3], ARGV[1], lease)
|
|
250
|
+
redis.call('RPOP', KEYS[1])
|
|
251
|
+
return record
|
|
252
|
+
`;
|
|
253
|
+
var RENEW_OR_ACK = `${SCRIPT_HEADER}
|
|
254
|
+
local err = checkType(KEYS[1], 'list') or checkType(KEYS[2], 'hash') or checkType(KEYS[3], 'list')
|
|
255
|
+
if err then return err end
|
|
256
|
+
local raw = redis.call('HGET', KEYS[2], ARGV[1])
|
|
257
|
+
if not raw then return 0 end
|
|
258
|
+
local ok, lease = pcall(cjson.decode, raw)
|
|
259
|
+
local now = nowMs()
|
|
260
|
+
if not ok or type(lease) ~= 'table' or lease.owner ~= ARGV[2] or not tonumber(lease.expiresAt) or tonumber(lease.expiresAt) <= now then
|
|
261
|
+
return 0
|
|
262
|
+
end
|
|
263
|
+
if ARGV[4] == 'renew' then
|
|
264
|
+
redis.call('HSET', KEYS[2], ARGV[1], cjson.encode({owner=ARGV[2], expiresAt=now+tonumber(ARGV[5])}))
|
|
265
|
+
return 1
|
|
266
|
+
end
|
|
267
|
+
if ARGV[4] == 'quarantine' then redis.call('LPUSH', KEYS[3], ARGV[6]) end
|
|
268
|
+
local removed = redis.call('LREM', KEYS[1], 1, ARGV[3])
|
|
269
|
+
redis.call('HDEL', KEYS[2], ARGV[1])
|
|
270
|
+
return removed
|
|
271
|
+
`;
|
|
272
|
+
var RECLAIM_JOBS = `${SCRIPT_HEADER}
|
|
273
|
+
local err = checkType(KEYS[1], 'list') or checkType(KEYS[2], 'list') or checkType(KEYS[3], 'hash')
|
|
274
|
+
if err then return err end
|
|
275
|
+
local now = nowMs()
|
|
276
|
+
local reclaimed = 0
|
|
277
|
+
for _, record in ipairs(redis.call('LRANGE', KEYS[2], 0, -1)) do
|
|
278
|
+
local payload, field = record, record
|
|
279
|
+
local modern = false
|
|
280
|
+
local ok, reservation = pcall(cjson.decode, record)
|
|
281
|
+
if ok and type(reservation) == 'table' and reservation.strataReservation == 1 then
|
|
282
|
+
payload, field, modern = reservation.payload, reservation.id, true
|
|
283
|
+
end
|
|
284
|
+
local raw = redis.call('HGET', KEYS[3], field)
|
|
285
|
+
local expired = true
|
|
286
|
+
if raw then
|
|
287
|
+
if modern then
|
|
288
|
+
local leaseOk, lease = pcall(cjson.decode, raw)
|
|
289
|
+
expired = not leaseOk or type(lease) ~= 'table' or not tonumber(lease.expiresAt) or tonumber(lease.expiresAt) <= now
|
|
290
|
+
else
|
|
291
|
+
local reservedAt = tonumber(raw)
|
|
292
|
+
expired = not reservedAt or now - reservedAt >= tonumber(ARGV[1])
|
|
293
|
+
end
|
|
294
|
+
end
|
|
295
|
+
if expired then
|
|
296
|
+
redis.call('LPUSH', KEYS[1], payload)
|
|
297
|
+
redis.call('LREM', KEYS[2], 1, record)
|
|
298
|
+
redis.call('HDEL', KEYS[3], field)
|
|
299
|
+
reclaimed = reclaimed + 1
|
|
300
|
+
end
|
|
301
|
+
end
|
|
302
|
+
return reclaimed
|
|
303
|
+
`;
|
|
304
|
+
var POLL_LIST_PRIORITIES = `${SCRIPT_HEADER}
|
|
305
|
+
local owner = ARGV[1]
|
|
306
|
+
local visibility = ARGV[2]
|
|
307
|
+
local reservationId = ARGV[3]
|
|
308
|
+
local reclaim = ARGV[4] == '1'
|
|
309
|
+
local function reclaimQueue(listKey, processingKey, leaseKey)
|
|
310
|
+
local err = checkType(listKey, 'list') or checkType(processingKey, 'list') or checkType(leaseKey, 'hash')
|
|
311
|
+
if err then return err end
|
|
312
|
+
local now = nowMs()
|
|
313
|
+
for _, record in ipairs(redis.call('LRANGE', processingKey, 0, -1)) do
|
|
314
|
+
local payload, field = record, record
|
|
315
|
+
local modern = false
|
|
316
|
+
local ok, reservation = pcall(cjson.decode, record)
|
|
317
|
+
if ok and type(reservation) == 'table' and reservation.strataReservation == 1 then
|
|
318
|
+
payload, field, modern = reservation.payload, reservation.id, true
|
|
319
|
+
end
|
|
320
|
+
local raw = redis.call('HGET', leaseKey, field)
|
|
321
|
+
local expired = true
|
|
322
|
+
if raw then
|
|
323
|
+
if modern then
|
|
324
|
+
local leaseOk, lease = pcall(cjson.decode, raw)
|
|
325
|
+
expired = not leaseOk or type(lease) ~= 'table' or not tonumber(lease.expiresAt) or tonumber(lease.expiresAt) <= now
|
|
326
|
+
else
|
|
327
|
+
local reservedAt = tonumber(raw)
|
|
328
|
+
expired = not reservedAt or now - reservedAt >= tonumber(visibility)
|
|
329
|
+
end
|
|
330
|
+
end
|
|
331
|
+
if expired then
|
|
332
|
+
redis.call('LPUSH', listKey, payload)
|
|
333
|
+
redis.call('LREM', processingKey, 1, record)
|
|
334
|
+
redis.call('HDEL', leaseKey, field)
|
|
335
|
+
end
|
|
336
|
+
end
|
|
337
|
+
end
|
|
338
|
+
local function reserveQueue(listKey, processingKey, leaseKey)
|
|
339
|
+
local err = checkType(listKey, 'list') or checkType(processingKey, 'list') or checkType(leaseKey, 'hash')
|
|
340
|
+
if err then return err end
|
|
341
|
+
local payload = redis.call('LINDEX', listKey, -1)
|
|
342
|
+
if not payload then return nil end
|
|
343
|
+
local record = cjson.encode({strataReservation=1, id=reservationId, owner=owner, payload=payload})
|
|
344
|
+
local lease = cjson.encode({owner=owner, expiresAt=nowMs()+tonumber(visibility)})
|
|
345
|
+
redis.call('LPUSH', processingKey, record)
|
|
346
|
+
redis.call('HSET', leaseKey, reservationId, lease)
|
|
347
|
+
redis.call('RPOP', listKey)
|
|
348
|
+
return record
|
|
349
|
+
end
|
|
350
|
+
local count = #KEYS / 3
|
|
351
|
+
if reclaim then
|
|
352
|
+
for i = 1, count do
|
|
353
|
+
local err = reclaimQueue(KEYS[(i - 1) * 3 + 1], KEYS[(i - 1) * 3 + 2], KEYS[(i - 1) * 3 + 3])
|
|
354
|
+
if err then return err end
|
|
355
|
+
end
|
|
356
|
+
end
|
|
357
|
+
for i = 1, count do
|
|
358
|
+
local record = reserveQueue(KEYS[(i - 1) * 3 + 1], KEYS[(i - 1) * 3 + 2], KEYS[(i - 1) * 3 + 3])
|
|
359
|
+
if type(record) == 'string' then return {'1', tostring(i), record} end
|
|
360
|
+
if record then return record end
|
|
361
|
+
end
|
|
362
|
+
return {'0'}
|
|
363
|
+
`;
|
|
364
|
+
|
|
228
365
|
// ../../src/core/runtime/appEnv.ts
|
|
229
366
|
var NON_PRODUCTION_APP_ENVS = new Set(["local", "development", "dev", "test", "testing", "ci"]);
|
|
230
367
|
function normalizeEnvValue(value) {
|
|
@@ -341,22 +478,8 @@ function queueKeyForPriority(priority = "default") {
|
|
|
341
478
|
|
|
342
479
|
// ../../src/core/queue/redisStreams.ts
|
|
343
480
|
var {RedisClient } = globalThis.Bun;
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
return `${queueKey}:stream`;
|
|
347
|
-
}
|
|
348
|
-
function streamDeadLetterKey(queueKey) {
|
|
349
|
-
return `${streamQueueKey(queueKey)}:invalid`;
|
|
350
|
-
}
|
|
351
|
-
function streamRetryKey(queueKey) {
|
|
352
|
-
return `${streamQueueKey(queueKey)}:retries`;
|
|
353
|
-
}
|
|
354
|
-
function streamControlKey(key, jobId) {
|
|
355
|
-
const prefix = `${streamQueueKey(key)}/`;
|
|
356
|
-
if (!jobId.startsWith(prefix) || !/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/.test(jobId.slice(prefix.length)))
|
|
357
|
-
throw new Error("Invalid cancellable Streams job ID");
|
|
358
|
-
return `${streamQueueKey(key)}:control:${jobId.slice(prefix.length)}`;
|
|
359
|
-
}
|
|
481
|
+
|
|
482
|
+
// ../../src/core/queue/redisStreamsScripts.ts
|
|
360
483
|
var ENQUEUE_CONTROLLED = `
|
|
361
484
|
local kind = redis.call('TYPE', KEYS[1]).ok
|
|
362
485
|
if kind ~= 'none' and kind ~= 'stream' then return redis.error_reply('Invalid stream admission key type') end
|
|
@@ -375,6 +498,144 @@ if payload and redis.call('ZSCORE', KEYS[2], payload) then redis.call('ZADD', KE
|
|
|
375
498
|
redis.call('HSET', KEYS[1], 'state', 'cancelled')
|
|
376
499
|
return 1
|
|
377
500
|
`;
|
|
501
|
+
var SCHEDULE_RETRY = `
|
|
502
|
+
local pending = redis.call('XPENDING', KEYS[1], ARGV[1], ARGV[2], ARGV[2], 1)
|
|
503
|
+
if #pending == 0 or pending[1][2] ~= ARGV[3] or tonumber(pending[1][3]) >= tonumber(ARGV[4]) then return 0 end
|
|
504
|
+
local kind = redis.call('TYPE', KEYS[2]).ok
|
|
505
|
+
if kind ~= 'none' and kind ~= 'zset' then return redis.error_reply('Invalid stream retry key type') end
|
|
506
|
+
if ARGV[8] == '1' then
|
|
507
|
+
local state = redis.call('HGET', KEYS[3], 'state')
|
|
508
|
+
if state == 'cancelled' then return redis.error_reply('Streams job cancelled') end
|
|
509
|
+
if state ~= 'active' then return redis.error_reply('Missing or invalid Streams control state') end
|
|
510
|
+
end
|
|
511
|
+
local now = redis.call('TIME')
|
|
512
|
+
local due = tonumber(now[1])*1000 + math.floor(tonumber(now[2])/1000) + tonumber(ARGV[5])
|
|
513
|
+
if ARGV[7] ~= '' then due = math.min(due, tonumber(ARGV[7])) end
|
|
514
|
+
redis.call('ZADD', KEYS[2], due, ARGV[6])
|
|
515
|
+
if ARGV[8] == '1' then redis.call('HSET', KEYS[3], 'retryPayload', ARGV[6]) end
|
|
516
|
+
redis.call('XACK', KEYS[1], ARGV[1], ARGV[2])
|
|
517
|
+
redis.call('XDEL', KEYS[1], ARGV[2])
|
|
518
|
+
if #redis.call('XPENDING', KEYS[1], ARGV[1], '-', '+', 1, ARGV[3]) == 0 then redis.call('XGROUP', 'DELCONSUMER', KEYS[1], ARGV[1], ARGV[3]) end
|
|
519
|
+
return 1
|
|
520
|
+
`;
|
|
521
|
+
var POLL_STREAM_PRIORITIES = `
|
|
522
|
+
local group = ARGV[1]
|
|
523
|
+
local owner = ARGV[2]
|
|
524
|
+
local visibility = ARGV[3]
|
|
525
|
+
local function payloadOf(fields)
|
|
526
|
+
for index = 1, #fields, 2 do
|
|
527
|
+
if fields[index] == 'payload' and type(fields[index + 1]) == 'string' then return fields[index + 1] end
|
|
528
|
+
end
|
|
529
|
+
return ''
|
|
530
|
+
end
|
|
531
|
+
local cursors = {}
|
|
532
|
+
for i = 1, #KEYS / 2 do cursors[i] = ARGV[3 + i] end
|
|
533
|
+
for i = 1, #KEYS / 2 do
|
|
534
|
+
local stream = KEYS[(i - 1) * 2 + 1]
|
|
535
|
+
local retries = KEYS[(i - 1) * 2 + 2]
|
|
536
|
+
local cursor = ARGV[3 + i]
|
|
537
|
+
local streamKind = redis.call('TYPE', stream).ok
|
|
538
|
+
local retryKind = redis.call('TYPE', retries).ok
|
|
539
|
+
if streamKind ~= 'none' and streamKind ~= 'stream' then return redis.error_reply('Invalid stream retry promotion key type') end
|
|
540
|
+
if retryKind ~= 'none' and retryKind ~= 'zset' then return redis.error_reply('Invalid stream retry promotion key type') end
|
|
541
|
+
if streamKind == 'stream' then
|
|
542
|
+
local now = redis.call('TIME')
|
|
543
|
+
local due = tonumber(now[1]) * 1000 + math.floor(tonumber(now[2]) / 1000)
|
|
544
|
+
local ready = redis.call('ZRANGE', retries, '-inf', due, 'BYSCORE', 'LIMIT', 0, 100)
|
|
545
|
+
for _, payload in ipairs(ready) do
|
|
546
|
+
redis.call('XADD', stream, '*', 'payload', payload)
|
|
547
|
+
redis.call('ZREM', retries, payload)
|
|
548
|
+
end
|
|
549
|
+
local window = redis.call('XPENDING', stream, group, cursor, '+', 10)
|
|
550
|
+
local claimed = redis.call('XAUTOCLAIM', stream, group, owner, visibility, cursor, 'COUNT', 1)
|
|
551
|
+
for _, entry in ipairs(window) do
|
|
552
|
+
if entry[2] ~= owner and #redis.call('XPENDING', stream, group, '-', '+', 1, entry[2]) == 0 then
|
|
553
|
+
redis.call('XGROUP', 'DELCONSUMER', stream, group, entry[2])
|
|
554
|
+
end
|
|
555
|
+
end
|
|
556
|
+
if #redis.call('XPENDING', stream, group, '-', '+', 1, owner) == 0 then
|
|
557
|
+
redis.call('XGROUP', 'DELCONSUMER', stream, group, owner)
|
|
558
|
+
end
|
|
559
|
+
cursors[i] = claimed[1]
|
|
560
|
+
if #claimed[2] > 0 then
|
|
561
|
+
return {'1', tostring(i), claimed[2][1][1], payloadOf(claimed[2][1][2]), unpack(cursors)}
|
|
562
|
+
end
|
|
563
|
+
local fresh = redis.call('XREADGROUP', 'GROUP', group, owner, 'COUNT', 1, 'STREAMS', stream, '>')
|
|
564
|
+
if not fresh and #redis.call('XPENDING', stream, group, '-', '+', 1, owner) == 0 then
|
|
565
|
+
redis.call('XGROUP', 'DELCONSUMER', stream, group, owner)
|
|
566
|
+
end
|
|
567
|
+
if fresh and #fresh > 0 and #fresh[1][2] > 0 then
|
|
568
|
+
return {'1', tostring(i), fresh[1][2][1][1], payloadOf(fresh[1][2][1][2]), unpack(cursors)}
|
|
569
|
+
end
|
|
570
|
+
end
|
|
571
|
+
end
|
|
572
|
+
return {'0', unpack(cursors)}
|
|
573
|
+
`;
|
|
574
|
+
var ASSERT_DRAINED = `
|
|
575
|
+
if redis.call('LLEN', KEYS[1]) > 0 or redis.call('LLEN', KEYS[2]) > 0 or redis.call('HLEN', KEYS[3]) > 0 then
|
|
576
|
+
return redis.error_reply('Legacy queue state requires maintenance conversion before Streams activation')
|
|
577
|
+
end
|
|
578
|
+
return 1
|
|
579
|
+
`;
|
|
580
|
+
var UPDATE_STREAM = `
|
|
581
|
+
local pending = redis.call('XPENDING', KEYS[1], ARGV[1], ARGV[2], ARGV[2], 1)
|
|
582
|
+
if #pending == 0 or pending[1][2] ~= ARGV[3] or tonumber(pending[1][3]) >= tonumber(ARGV[4]) then return 0 end
|
|
583
|
+
if ARGV[7] == '1' then
|
|
584
|
+
local kind = redis.call('TYPE', KEYS[3]).ok
|
|
585
|
+
if kind ~= 'none' and kind ~= 'hash' then return redis.error_reply('Invalid stream control key type') end
|
|
586
|
+
end
|
|
587
|
+
if ARGV[5] == 'renew' then
|
|
588
|
+
local claimed = redis.call('XCLAIM', KEYS[1], ARGV[1], ARGV[3], 0, ARGV[2], 'JUSTID')
|
|
589
|
+
return #claimed
|
|
590
|
+
end
|
|
591
|
+
if ARGV[5] == 'quarantine' then
|
|
592
|
+
local kind = redis.call('TYPE', KEYS[2]).ok
|
|
593
|
+
if kind ~= 'none' and kind ~= 'stream' then return redis.error_reply('Invalid stream quarantine key type') end
|
|
594
|
+
redis.call('XADD', KEYS[2], '*', 'sourceId', ARGV[2], 'payload', ARGV[6])
|
|
595
|
+
end
|
|
596
|
+
local acknowledged = redis.call('XACK', KEYS[1], ARGV[1], ARGV[2])
|
|
597
|
+
if acknowledged == 1 then
|
|
598
|
+
redis.call('XDEL', KEYS[1], ARGV[2])
|
|
599
|
+
if ARGV[7] == '1' then redis.call('DEL', KEYS[3]) end
|
|
600
|
+
end
|
|
601
|
+
if #redis.call('XPENDING', KEYS[1], ARGV[1], '-', '+', 1, ARGV[3]) == 0 then redis.call('XGROUP', 'DELCONSUMER', KEYS[1], ARGV[1], ARGV[3]) end
|
|
602
|
+
return acknowledged
|
|
603
|
+
`;
|
|
604
|
+
var CONVERT_LIST = `
|
|
605
|
+
local kinds = {'list','list','hash','stream'}
|
|
606
|
+
for i=1,4 do
|
|
607
|
+
local kind = redis.call('TYPE', KEYS[i]).ok
|
|
608
|
+
if kind ~= 'none' and kind ~= kinds[i] then return redis.error_reply('Invalid queue conversion key type') end
|
|
609
|
+
end
|
|
610
|
+
if redis.call('LLEN', KEYS[2]) > 0 or redis.call('HLEN', KEYS[3]) > 0 then return redis.error_reply('Drain or recover processing reservations before queue conversion') end
|
|
611
|
+
local converted = 0
|
|
612
|
+
for i=1,tonumber(ARGV[1]) do
|
|
613
|
+
local payload = redis.call('LINDEX', KEYS[1], -1)
|
|
614
|
+
if not payload then break end
|
|
615
|
+
redis.call('XADD', KEYS[4], '*', 'payload', payload)
|
|
616
|
+
redis.call('RPOP', KEYS[1])
|
|
617
|
+
converted = converted + 1
|
|
618
|
+
end
|
|
619
|
+
return converted
|
|
620
|
+
`;
|
|
621
|
+
|
|
622
|
+
// ../../src/core/queue/redisStreams.ts
|
|
623
|
+
var STREAM_GROUP = "strata-workers-v1";
|
|
624
|
+
function streamQueueKey(queueKey) {
|
|
625
|
+
return `${queueKey}:stream`;
|
|
626
|
+
}
|
|
627
|
+
function streamDeadLetterKey(queueKey) {
|
|
628
|
+
return `${streamQueueKey(queueKey)}:invalid`;
|
|
629
|
+
}
|
|
630
|
+
function streamRetryKey(queueKey) {
|
|
631
|
+
return `${streamQueueKey(queueKey)}:retries`;
|
|
632
|
+
}
|
|
633
|
+
function streamControlKey(key, jobId) {
|
|
634
|
+
const prefix = `${streamQueueKey(key)}/`;
|
|
635
|
+
if (!jobId.startsWith(prefix) || !/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/.test(jobId.slice(prefix.length)))
|
|
636
|
+
throw new Error("Invalid cancellable Streams job ID");
|
|
637
|
+
return `${streamQueueKey(key)}:control:${jobId.slice(prefix.length)}`;
|
|
638
|
+
}
|
|
378
639
|
async function cancelStreamJob(client, key, jobId) {
|
|
379
640
|
return Number(await client.send("EVAL", [
|
|
380
641
|
CANCEL_STREAM,
|
|
@@ -428,26 +689,6 @@ function watchStreamDeadline(client, deadlineAtMs, remainingMs, expire, fail) {
|
|
|
428
689
|
clearTimeout(timer);
|
|
429
690
|
};
|
|
430
691
|
}
|
|
431
|
-
var SCHEDULE_RETRY = `
|
|
432
|
-
local pending = redis.call('XPENDING', KEYS[1], ARGV[1], ARGV[2], ARGV[2], 1)
|
|
433
|
-
if #pending == 0 or pending[1][2] ~= ARGV[3] or tonumber(pending[1][3]) >= tonumber(ARGV[4]) then return 0 end
|
|
434
|
-
local kind = redis.call('TYPE', KEYS[2]).ok
|
|
435
|
-
if kind ~= 'none' and kind ~= 'zset' then return redis.error_reply('Invalid stream retry key type') end
|
|
436
|
-
if ARGV[8] == '1' then
|
|
437
|
-
local state = redis.call('HGET', KEYS[3], 'state')
|
|
438
|
-
if state == 'cancelled' then return redis.error_reply('Streams job cancelled') end
|
|
439
|
-
if state ~= 'active' then return redis.error_reply('Missing or invalid Streams control state') end
|
|
440
|
-
end
|
|
441
|
-
local now = redis.call('TIME')
|
|
442
|
-
local due = tonumber(now[1])*1000 + math.floor(tonumber(now[2])/1000) + tonumber(ARGV[5])
|
|
443
|
-
if ARGV[7] ~= '' then due = math.min(due, tonumber(ARGV[7])) end
|
|
444
|
-
redis.call('ZADD', KEYS[2], due, ARGV[6])
|
|
445
|
-
if ARGV[8] == '1' then redis.call('HSET', KEYS[3], 'retryPayload', ARGV[6]) end
|
|
446
|
-
redis.call('XACK', KEYS[1], ARGV[1], ARGV[2])
|
|
447
|
-
redis.call('XDEL', KEYS[1], ARGV[2])
|
|
448
|
-
if #redis.call('XPENDING', KEYS[1], ARGV[1], '-', '+', 1, ARGV[3]) == 0 then redis.call('XGROUP', 'DELCONSUMER', KEYS[1], ARGV[1], ARGV[3]) end
|
|
449
|
-
return 1
|
|
450
|
-
`;
|
|
451
692
|
async function scheduleStreamRetry(client, key, reservation, envelope, delayMs) {
|
|
452
693
|
if (!Number.isSafeInteger(delayMs) || delayMs < 0 || delayMs > Number.MAX_SAFE_INTEGER / 2)
|
|
453
694
|
throw new Error("Invalid Streams retry delay");
|
|
@@ -469,59 +710,6 @@ async function scheduleStreamRetry(client, key, reservation, envelope, delayMs)
|
|
|
469
710
|
if (Number(scheduled) !== 1)
|
|
470
711
|
throw new Error("Queue lease lost while scheduling retry");
|
|
471
712
|
}
|
|
472
|
-
var POLL_STREAM_PRIORITIES = `
|
|
473
|
-
local group = ARGV[1]
|
|
474
|
-
local owner = ARGV[2]
|
|
475
|
-
local visibility = ARGV[3]
|
|
476
|
-
local function payloadOf(fields)
|
|
477
|
-
for index = 1, #fields, 2 do
|
|
478
|
-
if fields[index] == 'payload' and type(fields[index + 1]) == 'string' then return fields[index + 1] end
|
|
479
|
-
end
|
|
480
|
-
return ''
|
|
481
|
-
end
|
|
482
|
-
local cursors = {}
|
|
483
|
-
for i = 1, #KEYS / 2 do cursors[i] = ARGV[3 + i] end
|
|
484
|
-
for i = 1, #KEYS / 2 do
|
|
485
|
-
local stream = KEYS[(i - 1) * 2 + 1]
|
|
486
|
-
local retries = KEYS[(i - 1) * 2 + 2]
|
|
487
|
-
local cursor = ARGV[3 + i]
|
|
488
|
-
local streamKind = redis.call('TYPE', stream).ok
|
|
489
|
-
local retryKind = redis.call('TYPE', retries).ok
|
|
490
|
-
if streamKind ~= 'none' and streamKind ~= 'stream' then return redis.error_reply('Invalid stream retry promotion key type') end
|
|
491
|
-
if retryKind ~= 'none' and retryKind ~= 'zset' then return redis.error_reply('Invalid stream retry promotion key type') end
|
|
492
|
-
if streamKind == 'stream' then
|
|
493
|
-
local now = redis.call('TIME')
|
|
494
|
-
local due = tonumber(now[1]) * 1000 + math.floor(tonumber(now[2]) / 1000)
|
|
495
|
-
local ready = redis.call('ZRANGE', retries, '-inf', due, 'BYSCORE', 'LIMIT', 0, 100)
|
|
496
|
-
for _, payload in ipairs(ready) do
|
|
497
|
-
redis.call('XADD', stream, '*', 'payload', payload)
|
|
498
|
-
redis.call('ZREM', retries, payload)
|
|
499
|
-
end
|
|
500
|
-
local window = redis.call('XPENDING', stream, group, cursor, '+', 10)
|
|
501
|
-
local claimed = redis.call('XAUTOCLAIM', stream, group, owner, visibility, cursor, 'COUNT', 1)
|
|
502
|
-
for _, entry in ipairs(window) do
|
|
503
|
-
if entry[2] ~= owner and #redis.call('XPENDING', stream, group, '-', '+', 1, entry[2]) == 0 then
|
|
504
|
-
redis.call('XGROUP', 'DELCONSUMER', stream, group, entry[2])
|
|
505
|
-
end
|
|
506
|
-
end
|
|
507
|
-
if #redis.call('XPENDING', stream, group, '-', '+', 1, owner) == 0 then
|
|
508
|
-
redis.call('XGROUP', 'DELCONSUMER', stream, group, owner)
|
|
509
|
-
end
|
|
510
|
-
cursors[i] = claimed[1]
|
|
511
|
-
if #claimed[2] > 0 then
|
|
512
|
-
return {'1', tostring(i), claimed[2][1][1], payloadOf(claimed[2][1][2]), unpack(cursors)}
|
|
513
|
-
end
|
|
514
|
-
local fresh = redis.call('XREADGROUP', 'GROUP', group, owner, 'COUNT', 1, 'STREAMS', stream, '>')
|
|
515
|
-
if not fresh and #redis.call('XPENDING', stream, group, '-', '+', 1, owner) == 0 then
|
|
516
|
-
redis.call('XGROUP', 'DELCONSUMER', stream, group, owner)
|
|
517
|
-
end
|
|
518
|
-
if fresh and #fresh > 0 and #fresh[1][2] > 0 then
|
|
519
|
-
return {'1', tostring(i), fresh[1][2][1][1], payloadOf(fresh[1][2][1][2]), unpack(cursors)}
|
|
520
|
-
end
|
|
521
|
-
end
|
|
522
|
-
end
|
|
523
|
-
return {'0', unpack(cursors)}
|
|
524
|
-
`;
|
|
525
713
|
async function pollStreamPriorities(client, queueKeys, owner, cursors) {
|
|
526
714
|
const keys = [];
|
|
527
715
|
for (const key of queueKeys)
|
|
@@ -565,12 +753,6 @@ async function readStreamQueueDepth(client, key) {
|
|
|
565
753
|
streamRetryKey(key)
|
|
566
754
|
]));
|
|
567
755
|
}
|
|
568
|
-
var ASSERT_DRAINED = `
|
|
569
|
-
if redis.call('LLEN', KEYS[1]) > 0 or redis.call('LLEN', KEYS[2]) > 0 or redis.call('HLEN', KEYS[3]) > 0 then
|
|
570
|
-
return redis.error_reply('Legacy queue state requires maintenance conversion before Streams activation')
|
|
571
|
-
end
|
|
572
|
-
return 1
|
|
573
|
-
`;
|
|
574
756
|
async function assertLegacyDrained(client, key) {
|
|
575
757
|
await client.send("EVAL", [
|
|
576
758
|
ASSERT_DRAINED,
|
|
@@ -580,30 +762,6 @@ async function assertLegacyDrained(client, key) {
|
|
|
580
762
|
`${key}:processing:leases`
|
|
581
763
|
]);
|
|
582
764
|
}
|
|
583
|
-
var UPDATE_STREAM = `
|
|
584
|
-
local pending = redis.call('XPENDING', KEYS[1], ARGV[1], ARGV[2], ARGV[2], 1)
|
|
585
|
-
if #pending == 0 or pending[1][2] ~= ARGV[3] or tonumber(pending[1][3]) >= tonumber(ARGV[4]) then return 0 end
|
|
586
|
-
if ARGV[7] == '1' then
|
|
587
|
-
local kind = redis.call('TYPE', KEYS[3]).ok
|
|
588
|
-
if kind ~= 'none' and kind ~= 'hash' then return redis.error_reply('Invalid stream control key type') end
|
|
589
|
-
end
|
|
590
|
-
if ARGV[5] == 'renew' then
|
|
591
|
-
local claimed = redis.call('XCLAIM', KEYS[1], ARGV[1], ARGV[3], 0, ARGV[2], 'JUSTID')
|
|
592
|
-
return #claimed
|
|
593
|
-
end
|
|
594
|
-
if ARGV[5] == 'quarantine' then
|
|
595
|
-
local kind = redis.call('TYPE', KEYS[2]).ok
|
|
596
|
-
if kind ~= 'none' and kind ~= 'stream' then return redis.error_reply('Invalid stream quarantine key type') end
|
|
597
|
-
redis.call('XADD', KEYS[2], '*', 'sourceId', ARGV[2], 'payload', ARGV[6])
|
|
598
|
-
end
|
|
599
|
-
local acknowledged = redis.call('XACK', KEYS[1], ARGV[1], ARGV[2])
|
|
600
|
-
if acknowledged == 1 then
|
|
601
|
-
redis.call('XDEL', KEYS[1], ARGV[2])
|
|
602
|
-
if ARGV[7] == '1' then redis.call('DEL', KEYS[3]) end
|
|
603
|
-
end
|
|
604
|
-
if #redis.call('XPENDING', KEYS[1], ARGV[1], '-', '+', 1, ARGV[3]) == 0 then redis.call('XGROUP', 'DELCONSUMER', KEYS[1], ARGV[1], ARGV[3]) end
|
|
605
|
-
return acknowledged
|
|
606
|
-
`;
|
|
607
765
|
async function ensureStreamGroup(client, key) {
|
|
608
766
|
await assertLegacyDrained(client, key);
|
|
609
767
|
try {
|
|
@@ -913,23 +1071,6 @@ class RedisStreamsWorker {
|
|
|
913
1071
|
}
|
|
914
1072
|
}
|
|
915
1073
|
}
|
|
916
|
-
var CONVERT_LIST = `
|
|
917
|
-
local kinds = {'list','list','hash','stream'}
|
|
918
|
-
for i=1,4 do
|
|
919
|
-
local kind = redis.call('TYPE', KEYS[i]).ok
|
|
920
|
-
if kind ~= 'none' and kind ~= kinds[i] then return redis.error_reply('Invalid queue conversion key type') end
|
|
921
|
-
end
|
|
922
|
-
if redis.call('LLEN', KEYS[2]) > 0 or redis.call('HLEN', KEYS[3]) > 0 then return redis.error_reply('Drain or recover processing reservations before queue conversion') end
|
|
923
|
-
local converted = 0
|
|
924
|
-
for i=1,tonumber(ARGV[1]) do
|
|
925
|
-
local payload = redis.call('LINDEX', KEYS[1], -1)
|
|
926
|
-
if not payload then break end
|
|
927
|
-
redis.call('XADD', KEYS[4], '*', 'payload', payload)
|
|
928
|
-
redis.call('RPOP', KEYS[1])
|
|
929
|
-
converted = converted + 1
|
|
930
|
-
end
|
|
931
|
-
return converted
|
|
932
|
-
`;
|
|
933
1074
|
async function migrateLegacyQueueToStreams(client, options) {
|
|
934
1075
|
if (options.maintenance !== true)
|
|
935
1076
|
throw new Error("Queue conversion requires maintenance mode: stop all producers and workers first.");
|
|
@@ -967,141 +1108,6 @@ function reservationExpired(lease, now, visibilityMs) {
|
|
|
967
1108
|
}
|
|
968
1109
|
return now - reservedAt >= visibilityMs;
|
|
969
1110
|
}
|
|
970
|
-
var SCRIPT_HEADER = `
|
|
971
|
-
local function checkType(key, expected)
|
|
972
|
-
local actual = redis.call('TYPE', key).ok
|
|
973
|
-
if actual ~= 'none' and actual ~= expected then
|
|
974
|
-
return redis.error_reply('Unexpected queue key type: ' .. key)
|
|
975
|
-
end
|
|
976
|
-
end
|
|
977
|
-
local function nowMs()
|
|
978
|
-
local time = redis.call('TIME')
|
|
979
|
-
return tonumber(time[1]) * 1000 + math.floor(tonumber(time[2]) / 1000)
|
|
980
|
-
end
|
|
981
|
-
`;
|
|
982
|
-
var RESERVE_JOB = `${SCRIPT_HEADER}
|
|
983
|
-
local err = checkType(KEYS[1], 'list') or checkType(KEYS[2], 'list') or checkType(KEYS[3], 'hash')
|
|
984
|
-
if err then return err end
|
|
985
|
-
local payload = redis.call('LINDEX', KEYS[1], -1)
|
|
986
|
-
if not payload then return nil end
|
|
987
|
-
local record = cjson.encode({strataReservation=1, id=ARGV[1], owner=ARGV[2], payload=payload})
|
|
988
|
-
local lease = cjson.encode({owner=ARGV[2], expiresAt=nowMs()+tonumber(ARGV[3])})
|
|
989
|
-
redis.call('LPUSH', KEYS[2], record)
|
|
990
|
-
redis.call('HSET', KEYS[3], ARGV[1], lease)
|
|
991
|
-
redis.call('RPOP', KEYS[1])
|
|
992
|
-
return record
|
|
993
|
-
`;
|
|
994
|
-
var RENEW_OR_ACK = `${SCRIPT_HEADER}
|
|
995
|
-
local err = checkType(KEYS[1], 'list') or checkType(KEYS[2], 'hash') or checkType(KEYS[3], 'list')
|
|
996
|
-
if err then return err end
|
|
997
|
-
local raw = redis.call('HGET', KEYS[2], ARGV[1])
|
|
998
|
-
if not raw then return 0 end
|
|
999
|
-
local ok, lease = pcall(cjson.decode, raw)
|
|
1000
|
-
local now = nowMs()
|
|
1001
|
-
if not ok or type(lease) ~= 'table' or lease.owner ~= ARGV[2] or not tonumber(lease.expiresAt) or tonumber(lease.expiresAt) <= now then
|
|
1002
|
-
return 0
|
|
1003
|
-
end
|
|
1004
|
-
if ARGV[4] == 'renew' then
|
|
1005
|
-
redis.call('HSET', KEYS[2], ARGV[1], cjson.encode({owner=ARGV[2], expiresAt=now+tonumber(ARGV[5])}))
|
|
1006
|
-
return 1
|
|
1007
|
-
end
|
|
1008
|
-
if ARGV[4] == 'quarantine' then redis.call('LPUSH', KEYS[3], ARGV[6]) end
|
|
1009
|
-
local removed = redis.call('LREM', KEYS[1], 1, ARGV[3])
|
|
1010
|
-
redis.call('HDEL', KEYS[2], ARGV[1])
|
|
1011
|
-
return removed
|
|
1012
|
-
`;
|
|
1013
|
-
var RECLAIM_JOBS = `${SCRIPT_HEADER}
|
|
1014
|
-
local err = checkType(KEYS[1], 'list') or checkType(KEYS[2], 'list') or checkType(KEYS[3], 'hash')
|
|
1015
|
-
if err then return err end
|
|
1016
|
-
local now = nowMs()
|
|
1017
|
-
local reclaimed = 0
|
|
1018
|
-
for _, record in ipairs(redis.call('LRANGE', KEYS[2], 0, -1)) do
|
|
1019
|
-
local payload, field = record, record
|
|
1020
|
-
local modern = false
|
|
1021
|
-
local ok, reservation = pcall(cjson.decode, record)
|
|
1022
|
-
if ok and type(reservation) == 'table' and reservation.strataReservation == 1 then
|
|
1023
|
-
payload, field, modern = reservation.payload, reservation.id, true
|
|
1024
|
-
end
|
|
1025
|
-
local raw = redis.call('HGET', KEYS[3], field)
|
|
1026
|
-
local expired = true
|
|
1027
|
-
if raw then
|
|
1028
|
-
if modern then
|
|
1029
|
-
local leaseOk, lease = pcall(cjson.decode, raw)
|
|
1030
|
-
expired = not leaseOk or type(lease) ~= 'table' or not tonumber(lease.expiresAt) or tonumber(lease.expiresAt) <= now
|
|
1031
|
-
else
|
|
1032
|
-
local reservedAt = tonumber(raw)
|
|
1033
|
-
expired = not reservedAt or now - reservedAt >= tonumber(ARGV[1])
|
|
1034
|
-
end
|
|
1035
|
-
end
|
|
1036
|
-
if expired then
|
|
1037
|
-
redis.call('LPUSH', KEYS[1], payload)
|
|
1038
|
-
redis.call('LREM', KEYS[2], 1, record)
|
|
1039
|
-
redis.call('HDEL', KEYS[3], field)
|
|
1040
|
-
reclaimed = reclaimed + 1
|
|
1041
|
-
end
|
|
1042
|
-
end
|
|
1043
|
-
return reclaimed
|
|
1044
|
-
`;
|
|
1045
|
-
var POLL_LIST_PRIORITIES = `${SCRIPT_HEADER}
|
|
1046
|
-
local owner = ARGV[1]
|
|
1047
|
-
local visibility = ARGV[2]
|
|
1048
|
-
local reservationId = ARGV[3]
|
|
1049
|
-
local reclaim = ARGV[4] == '1'
|
|
1050
|
-
local function reclaimQueue(listKey, processingKey, leaseKey)
|
|
1051
|
-
local err = checkType(listKey, 'list') or checkType(processingKey, 'list') or checkType(leaseKey, 'hash')
|
|
1052
|
-
if err then return err end
|
|
1053
|
-
local now = nowMs()
|
|
1054
|
-
for _, record in ipairs(redis.call('LRANGE', processingKey, 0, -1)) do
|
|
1055
|
-
local payload, field = record, record
|
|
1056
|
-
local modern = false
|
|
1057
|
-
local ok, reservation = pcall(cjson.decode, record)
|
|
1058
|
-
if ok and type(reservation) == 'table' and reservation.strataReservation == 1 then
|
|
1059
|
-
payload, field, modern = reservation.payload, reservation.id, true
|
|
1060
|
-
end
|
|
1061
|
-
local raw = redis.call('HGET', leaseKey, field)
|
|
1062
|
-
local expired = true
|
|
1063
|
-
if raw then
|
|
1064
|
-
if modern then
|
|
1065
|
-
local leaseOk, lease = pcall(cjson.decode, raw)
|
|
1066
|
-
expired = not leaseOk or type(lease) ~= 'table' or not tonumber(lease.expiresAt) or tonumber(lease.expiresAt) <= now
|
|
1067
|
-
else
|
|
1068
|
-
local reservedAt = tonumber(raw)
|
|
1069
|
-
expired = not reservedAt or now - reservedAt >= tonumber(visibility)
|
|
1070
|
-
end
|
|
1071
|
-
end
|
|
1072
|
-
if expired then
|
|
1073
|
-
redis.call('LPUSH', listKey, payload)
|
|
1074
|
-
redis.call('LREM', processingKey, 1, record)
|
|
1075
|
-
redis.call('HDEL', leaseKey, field)
|
|
1076
|
-
end
|
|
1077
|
-
end
|
|
1078
|
-
end
|
|
1079
|
-
local function reserveQueue(listKey, processingKey, leaseKey)
|
|
1080
|
-
local err = checkType(listKey, 'list') or checkType(processingKey, 'list') or checkType(leaseKey, 'hash')
|
|
1081
|
-
if err then return err end
|
|
1082
|
-
local payload = redis.call('LINDEX', listKey, -1)
|
|
1083
|
-
if not payload then return nil end
|
|
1084
|
-
local record = cjson.encode({strataReservation=1, id=reservationId, owner=owner, payload=payload})
|
|
1085
|
-
local lease = cjson.encode({owner=owner, expiresAt=nowMs()+tonumber(visibility)})
|
|
1086
|
-
redis.call('LPUSH', processingKey, record)
|
|
1087
|
-
redis.call('HSET', leaseKey, reservationId, lease)
|
|
1088
|
-
redis.call('RPOP', listKey)
|
|
1089
|
-
return record
|
|
1090
|
-
end
|
|
1091
|
-
local count = #KEYS / 3
|
|
1092
|
-
if reclaim then
|
|
1093
|
-
for i = 1, count do
|
|
1094
|
-
local err = reclaimQueue(KEYS[(i - 1) * 3 + 1], KEYS[(i - 1) * 3 + 2], KEYS[(i - 1) * 3 + 3])
|
|
1095
|
-
if err then return err end
|
|
1096
|
-
end
|
|
1097
|
-
end
|
|
1098
|
-
for i = 1, count do
|
|
1099
|
-
local record = reserveQueue(KEYS[(i - 1) * 3 + 1], KEYS[(i - 1) * 3 + 2], KEYS[(i - 1) * 3 + 3])
|
|
1100
|
-
if type(record) == 'string' then return {'1', tostring(i), record} end
|
|
1101
|
-
if record then return record end
|
|
1102
|
-
end
|
|
1103
|
-
return {'0'}
|
|
1104
|
-
`;
|
|
1105
1111
|
function queueInvalidKey(queueKey) {
|
|
1106
1112
|
return `${queueKey}:invalid`;
|
|
1107
1113
|
}
|