@getstrata/core 2.2.48 → 2.2.50
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +11 -0
- package/dist/core/audit/exportStore.d.ts +17 -0
- package/dist/core/cache/redisCacheScript.d.ts +2 -0
- package/dist/core/database/baseRepository.d.ts +0 -1
- package/dist/core/database/model.d.ts +6 -113
- package/dist/core/database/modelAttributes.d.ts +17 -0
- package/dist/core/database/modelQuery.d.ts +139 -0
- package/dist/core/database/modelRegistry.d.ts +34 -0
- package/dist/core/database/relationQueries.d.ts +20 -0
- package/dist/core/http/middleware.d.ts +2 -2
- package/dist/core/http/routeMiddleware.d.ts +1 -1
- package/dist/core/mail/mailer.d.ts +1 -4
- package/dist/core/mail/smtpMessage.d.ts +7 -0
- package/dist/core/queue/redisListScripts.d.ts +5 -0
- package/dist/core/queue/redisStreamsScripts.d.ts +11 -0
- package/dist/core/scheduler/leaseScript.d.ts +2 -0
- package/dist/entries/audit/exportAuditLogs.js +63 -58
- package/dist/entries/auth/scimAuthMiddleware.js +31 -20
- package/dist/entries/cache/createCacheStore.js +2 -1
- package/dist/entries/facades.js +103 -101
- package/dist/entries/http/routeMiddleware.js +3 -3
- package/dist/entries/jobs/exportAuditLogsJob.js +63 -58
- package/dist/entries/mail/mailer.js +103 -101
- package/dist/entries/queue/createAppQueue.js +277 -271
- package/dist/entries/queue/publicQueue.js +277 -271
- package/dist/entries/queue/queueMetrics.js +277 -271
- package/dist/entries/queue/redisQueue.js +277 -271
- package/dist/index.js +1482 -1428
- package/package.json +1 -1
|
@@ -157,6 +157,143 @@ async function runQueueJob(envelope, failedJobs, options = {}) {
|
|
|
157
157
|
}
|
|
158
158
|
}
|
|
159
159
|
|
|
160
|
+
// ../../src/core/queue/redisListScripts.ts
|
|
161
|
+
var SCRIPT_HEADER = `
|
|
162
|
+
local function checkType(key, expected)
|
|
163
|
+
local actual = redis.call('TYPE', key).ok
|
|
164
|
+
if actual ~= 'none' and actual ~= expected then
|
|
165
|
+
return redis.error_reply('Unexpected queue key type: ' .. key)
|
|
166
|
+
end
|
|
167
|
+
end
|
|
168
|
+
local function nowMs()
|
|
169
|
+
local time = redis.call('TIME')
|
|
170
|
+
return tonumber(time[1]) * 1000 + math.floor(tonumber(time[2]) / 1000)
|
|
171
|
+
end
|
|
172
|
+
`;
|
|
173
|
+
var RESERVE_JOB = `${SCRIPT_HEADER}
|
|
174
|
+
local err = checkType(KEYS[1], 'list') or checkType(KEYS[2], 'list') or checkType(KEYS[3], 'hash')
|
|
175
|
+
if err then return err end
|
|
176
|
+
local payload = redis.call('LINDEX', KEYS[1], -1)
|
|
177
|
+
if not payload then return nil end
|
|
178
|
+
local record = cjson.encode({strataReservation=1, id=ARGV[1], owner=ARGV[2], payload=payload})
|
|
179
|
+
local lease = cjson.encode({owner=ARGV[2], expiresAt=nowMs()+tonumber(ARGV[3])})
|
|
180
|
+
redis.call('LPUSH', KEYS[2], record)
|
|
181
|
+
redis.call('HSET', KEYS[3], ARGV[1], lease)
|
|
182
|
+
redis.call('RPOP', KEYS[1])
|
|
183
|
+
return record
|
|
184
|
+
`;
|
|
185
|
+
var RENEW_OR_ACK = `${SCRIPT_HEADER}
|
|
186
|
+
local err = checkType(KEYS[1], 'list') or checkType(KEYS[2], 'hash') or checkType(KEYS[3], 'list')
|
|
187
|
+
if err then return err end
|
|
188
|
+
local raw = redis.call('HGET', KEYS[2], ARGV[1])
|
|
189
|
+
if not raw then return 0 end
|
|
190
|
+
local ok, lease = pcall(cjson.decode, raw)
|
|
191
|
+
local now = nowMs()
|
|
192
|
+
if not ok or type(lease) ~= 'table' or lease.owner ~= ARGV[2] or not tonumber(lease.expiresAt) or tonumber(lease.expiresAt) <= now then
|
|
193
|
+
return 0
|
|
194
|
+
end
|
|
195
|
+
if ARGV[4] == 'renew' then
|
|
196
|
+
redis.call('HSET', KEYS[2], ARGV[1], cjson.encode({owner=ARGV[2], expiresAt=now+tonumber(ARGV[5])}))
|
|
197
|
+
return 1
|
|
198
|
+
end
|
|
199
|
+
if ARGV[4] == 'quarantine' then redis.call('LPUSH', KEYS[3], ARGV[6]) end
|
|
200
|
+
local removed = redis.call('LREM', KEYS[1], 1, ARGV[3])
|
|
201
|
+
redis.call('HDEL', KEYS[2], ARGV[1])
|
|
202
|
+
return removed
|
|
203
|
+
`;
|
|
204
|
+
var RECLAIM_JOBS = `${SCRIPT_HEADER}
|
|
205
|
+
local err = checkType(KEYS[1], 'list') or checkType(KEYS[2], 'list') or checkType(KEYS[3], 'hash')
|
|
206
|
+
if err then return err end
|
|
207
|
+
local now = nowMs()
|
|
208
|
+
local reclaimed = 0
|
|
209
|
+
for _, record in ipairs(redis.call('LRANGE', KEYS[2], 0, -1)) do
|
|
210
|
+
local payload, field = record, record
|
|
211
|
+
local modern = false
|
|
212
|
+
local ok, reservation = pcall(cjson.decode, record)
|
|
213
|
+
if ok and type(reservation) == 'table' and reservation.strataReservation == 1 then
|
|
214
|
+
payload, field, modern = reservation.payload, reservation.id, true
|
|
215
|
+
end
|
|
216
|
+
local raw = redis.call('HGET', KEYS[3], field)
|
|
217
|
+
local expired = true
|
|
218
|
+
if raw then
|
|
219
|
+
if modern then
|
|
220
|
+
local leaseOk, lease = pcall(cjson.decode, raw)
|
|
221
|
+
expired = not leaseOk or type(lease) ~= 'table' or not tonumber(lease.expiresAt) or tonumber(lease.expiresAt) <= now
|
|
222
|
+
else
|
|
223
|
+
local reservedAt = tonumber(raw)
|
|
224
|
+
expired = not reservedAt or now - reservedAt >= tonumber(ARGV[1])
|
|
225
|
+
end
|
|
226
|
+
end
|
|
227
|
+
if expired then
|
|
228
|
+
redis.call('LPUSH', KEYS[1], payload)
|
|
229
|
+
redis.call('LREM', KEYS[2], 1, record)
|
|
230
|
+
redis.call('HDEL', KEYS[3], field)
|
|
231
|
+
reclaimed = reclaimed + 1
|
|
232
|
+
end
|
|
233
|
+
end
|
|
234
|
+
return reclaimed
|
|
235
|
+
`;
|
|
236
|
+
var POLL_LIST_PRIORITIES = `${SCRIPT_HEADER}
|
|
237
|
+
local owner = ARGV[1]
|
|
238
|
+
local visibility = ARGV[2]
|
|
239
|
+
local reservationId = ARGV[3]
|
|
240
|
+
local reclaim = ARGV[4] == '1'
|
|
241
|
+
local function reclaimQueue(listKey, processingKey, leaseKey)
|
|
242
|
+
local err = checkType(listKey, 'list') or checkType(processingKey, 'list') or checkType(leaseKey, 'hash')
|
|
243
|
+
if err then return err end
|
|
244
|
+
local now = nowMs()
|
|
245
|
+
for _, record in ipairs(redis.call('LRANGE', processingKey, 0, -1)) do
|
|
246
|
+
local payload, field = record, record
|
|
247
|
+
local modern = false
|
|
248
|
+
local ok, reservation = pcall(cjson.decode, record)
|
|
249
|
+
if ok and type(reservation) == 'table' and reservation.strataReservation == 1 then
|
|
250
|
+
payload, field, modern = reservation.payload, reservation.id, true
|
|
251
|
+
end
|
|
252
|
+
local raw = redis.call('HGET', leaseKey, field)
|
|
253
|
+
local expired = true
|
|
254
|
+
if raw then
|
|
255
|
+
if modern then
|
|
256
|
+
local leaseOk, lease = pcall(cjson.decode, raw)
|
|
257
|
+
expired = not leaseOk or type(lease) ~= 'table' or not tonumber(lease.expiresAt) or tonumber(lease.expiresAt) <= now
|
|
258
|
+
else
|
|
259
|
+
local reservedAt = tonumber(raw)
|
|
260
|
+
expired = not reservedAt or now - reservedAt >= tonumber(visibility)
|
|
261
|
+
end
|
|
262
|
+
end
|
|
263
|
+
if expired then
|
|
264
|
+
redis.call('LPUSH', listKey, payload)
|
|
265
|
+
redis.call('LREM', processingKey, 1, record)
|
|
266
|
+
redis.call('HDEL', leaseKey, field)
|
|
267
|
+
end
|
|
268
|
+
end
|
|
269
|
+
end
|
|
270
|
+
local function reserveQueue(listKey, processingKey, leaseKey)
|
|
271
|
+
local err = checkType(listKey, 'list') or checkType(processingKey, 'list') or checkType(leaseKey, 'hash')
|
|
272
|
+
if err then return err end
|
|
273
|
+
local payload = redis.call('LINDEX', listKey, -1)
|
|
274
|
+
if not payload then return nil end
|
|
275
|
+
local record = cjson.encode({strataReservation=1, id=reservationId, owner=owner, payload=payload})
|
|
276
|
+
local lease = cjson.encode({owner=owner, expiresAt=nowMs()+tonumber(visibility)})
|
|
277
|
+
redis.call('LPUSH', processingKey, record)
|
|
278
|
+
redis.call('HSET', leaseKey, reservationId, lease)
|
|
279
|
+
redis.call('RPOP', listKey)
|
|
280
|
+
return record
|
|
281
|
+
end
|
|
282
|
+
local count = #KEYS / 3
|
|
283
|
+
if reclaim then
|
|
284
|
+
for i = 1, count do
|
|
285
|
+
local err = reclaimQueue(KEYS[(i - 1) * 3 + 1], KEYS[(i - 1) * 3 + 2], KEYS[(i - 1) * 3 + 3])
|
|
286
|
+
if err then return err end
|
|
287
|
+
end
|
|
288
|
+
end
|
|
289
|
+
for i = 1, count do
|
|
290
|
+
local record = reserveQueue(KEYS[(i - 1) * 3 + 1], KEYS[(i - 1) * 3 + 2], KEYS[(i - 1) * 3 + 3])
|
|
291
|
+
if type(record) == 'string' then return {'1', tostring(i), record} end
|
|
292
|
+
if record then return record end
|
|
293
|
+
end
|
|
294
|
+
return {'0'}
|
|
295
|
+
`;
|
|
296
|
+
|
|
160
297
|
// ../../src/core/runtime/appEnv.ts
|
|
161
298
|
var NON_PRODUCTION_APP_ENVS = new Set(["local", "development", "dev", "test", "testing", "ci"]);
|
|
162
299
|
function normalizeEnvValue(value) {
|
|
@@ -273,22 +410,8 @@ function queueKeyForPriority(priority = "default") {
|
|
|
273
410
|
|
|
274
411
|
// ../../src/core/queue/redisStreams.ts
|
|
275
412
|
var {RedisClient } = globalThis.Bun;
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
return `${queueKey}:stream`;
|
|
279
|
-
}
|
|
280
|
-
function streamDeadLetterKey(queueKey) {
|
|
281
|
-
return `${streamQueueKey(queueKey)}:invalid`;
|
|
282
|
-
}
|
|
283
|
-
function streamRetryKey(queueKey) {
|
|
284
|
-
return `${streamQueueKey(queueKey)}:retries`;
|
|
285
|
-
}
|
|
286
|
-
function streamControlKey(key, jobId) {
|
|
287
|
-
const prefix = `${streamQueueKey(key)}/`;
|
|
288
|
-
if (!jobId.startsWith(prefix) || !/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/.test(jobId.slice(prefix.length)))
|
|
289
|
-
throw new Error("Invalid cancellable Streams job ID");
|
|
290
|
-
return `${streamQueueKey(key)}:control:${jobId.slice(prefix.length)}`;
|
|
291
|
-
}
|
|
413
|
+
|
|
414
|
+
// ../../src/core/queue/redisStreamsScripts.ts
|
|
292
415
|
var ENQUEUE_CONTROLLED = `
|
|
293
416
|
local kind = redis.call('TYPE', KEYS[1]).ok
|
|
294
417
|
if kind ~= 'none' and kind ~= 'stream' then return redis.error_reply('Invalid stream admission key type') end
|
|
@@ -307,6 +430,144 @@ if payload and redis.call('ZSCORE', KEYS[2], payload) then redis.call('ZADD', KE
|
|
|
307
430
|
redis.call('HSET', KEYS[1], 'state', 'cancelled')
|
|
308
431
|
return 1
|
|
309
432
|
`;
|
|
433
|
+
var SCHEDULE_RETRY = `
|
|
434
|
+
local pending = redis.call('XPENDING', KEYS[1], ARGV[1], ARGV[2], ARGV[2], 1)
|
|
435
|
+
if #pending == 0 or pending[1][2] ~= ARGV[3] or tonumber(pending[1][3]) >= tonumber(ARGV[4]) then return 0 end
|
|
436
|
+
local kind = redis.call('TYPE', KEYS[2]).ok
|
|
437
|
+
if kind ~= 'none' and kind ~= 'zset' then return redis.error_reply('Invalid stream retry key type') end
|
|
438
|
+
if ARGV[8] == '1' then
|
|
439
|
+
local state = redis.call('HGET', KEYS[3], 'state')
|
|
440
|
+
if state == 'cancelled' then return redis.error_reply('Streams job cancelled') end
|
|
441
|
+
if state ~= 'active' then return redis.error_reply('Missing or invalid Streams control state') end
|
|
442
|
+
end
|
|
443
|
+
local now = redis.call('TIME')
|
|
444
|
+
local due = tonumber(now[1])*1000 + math.floor(tonumber(now[2])/1000) + tonumber(ARGV[5])
|
|
445
|
+
if ARGV[7] ~= '' then due = math.min(due, tonumber(ARGV[7])) end
|
|
446
|
+
redis.call('ZADD', KEYS[2], due, ARGV[6])
|
|
447
|
+
if ARGV[8] == '1' then redis.call('HSET', KEYS[3], 'retryPayload', ARGV[6]) end
|
|
448
|
+
redis.call('XACK', KEYS[1], ARGV[1], ARGV[2])
|
|
449
|
+
redis.call('XDEL', KEYS[1], ARGV[2])
|
|
450
|
+
if #redis.call('XPENDING', KEYS[1], ARGV[1], '-', '+', 1, ARGV[3]) == 0 then redis.call('XGROUP', 'DELCONSUMER', KEYS[1], ARGV[1], ARGV[3]) end
|
|
451
|
+
return 1
|
|
452
|
+
`;
|
|
453
|
+
var POLL_STREAM_PRIORITIES = `
|
|
454
|
+
local group = ARGV[1]
|
|
455
|
+
local owner = ARGV[2]
|
|
456
|
+
local visibility = ARGV[3]
|
|
457
|
+
local function payloadOf(fields)
|
|
458
|
+
for index = 1, #fields, 2 do
|
|
459
|
+
if fields[index] == 'payload' and type(fields[index + 1]) == 'string' then return fields[index + 1] end
|
|
460
|
+
end
|
|
461
|
+
return ''
|
|
462
|
+
end
|
|
463
|
+
local cursors = {}
|
|
464
|
+
for i = 1, #KEYS / 2 do cursors[i] = ARGV[3 + i] end
|
|
465
|
+
for i = 1, #KEYS / 2 do
|
|
466
|
+
local stream = KEYS[(i - 1) * 2 + 1]
|
|
467
|
+
local retries = KEYS[(i - 1) * 2 + 2]
|
|
468
|
+
local cursor = ARGV[3 + i]
|
|
469
|
+
local streamKind = redis.call('TYPE', stream).ok
|
|
470
|
+
local retryKind = redis.call('TYPE', retries).ok
|
|
471
|
+
if streamKind ~= 'none' and streamKind ~= 'stream' then return redis.error_reply('Invalid stream retry promotion key type') end
|
|
472
|
+
if retryKind ~= 'none' and retryKind ~= 'zset' then return redis.error_reply('Invalid stream retry promotion key type') end
|
|
473
|
+
if streamKind == 'stream' then
|
|
474
|
+
local now = redis.call('TIME')
|
|
475
|
+
local due = tonumber(now[1]) * 1000 + math.floor(tonumber(now[2]) / 1000)
|
|
476
|
+
local ready = redis.call('ZRANGE', retries, '-inf', due, 'BYSCORE', 'LIMIT', 0, 100)
|
|
477
|
+
for _, payload in ipairs(ready) do
|
|
478
|
+
redis.call('XADD', stream, '*', 'payload', payload)
|
|
479
|
+
redis.call('ZREM', retries, payload)
|
|
480
|
+
end
|
|
481
|
+
local window = redis.call('XPENDING', stream, group, cursor, '+', 10)
|
|
482
|
+
local claimed = redis.call('XAUTOCLAIM', stream, group, owner, visibility, cursor, 'COUNT', 1)
|
|
483
|
+
for _, entry in ipairs(window) do
|
|
484
|
+
if entry[2] ~= owner and #redis.call('XPENDING', stream, group, '-', '+', 1, entry[2]) == 0 then
|
|
485
|
+
redis.call('XGROUP', 'DELCONSUMER', stream, group, entry[2])
|
|
486
|
+
end
|
|
487
|
+
end
|
|
488
|
+
if #redis.call('XPENDING', stream, group, '-', '+', 1, owner) == 0 then
|
|
489
|
+
redis.call('XGROUP', 'DELCONSUMER', stream, group, owner)
|
|
490
|
+
end
|
|
491
|
+
cursors[i] = claimed[1]
|
|
492
|
+
if #claimed[2] > 0 then
|
|
493
|
+
return {'1', tostring(i), claimed[2][1][1], payloadOf(claimed[2][1][2]), unpack(cursors)}
|
|
494
|
+
end
|
|
495
|
+
local fresh = redis.call('XREADGROUP', 'GROUP', group, owner, 'COUNT', 1, 'STREAMS', stream, '>')
|
|
496
|
+
if not fresh and #redis.call('XPENDING', stream, group, '-', '+', 1, owner) == 0 then
|
|
497
|
+
redis.call('XGROUP', 'DELCONSUMER', stream, group, owner)
|
|
498
|
+
end
|
|
499
|
+
if fresh and #fresh > 0 and #fresh[1][2] > 0 then
|
|
500
|
+
return {'1', tostring(i), fresh[1][2][1][1], payloadOf(fresh[1][2][1][2]), unpack(cursors)}
|
|
501
|
+
end
|
|
502
|
+
end
|
|
503
|
+
end
|
|
504
|
+
return {'0', unpack(cursors)}
|
|
505
|
+
`;
|
|
506
|
+
var ASSERT_DRAINED = `
|
|
507
|
+
if redis.call('LLEN', KEYS[1]) > 0 or redis.call('LLEN', KEYS[2]) > 0 or redis.call('HLEN', KEYS[3]) > 0 then
|
|
508
|
+
return redis.error_reply('Legacy queue state requires maintenance conversion before Streams activation')
|
|
509
|
+
end
|
|
510
|
+
return 1
|
|
511
|
+
`;
|
|
512
|
+
var UPDATE_STREAM = `
|
|
513
|
+
local pending = redis.call('XPENDING', KEYS[1], ARGV[1], ARGV[2], ARGV[2], 1)
|
|
514
|
+
if #pending == 0 or pending[1][2] ~= ARGV[3] or tonumber(pending[1][3]) >= tonumber(ARGV[4]) then return 0 end
|
|
515
|
+
if ARGV[7] == '1' then
|
|
516
|
+
local kind = redis.call('TYPE', KEYS[3]).ok
|
|
517
|
+
if kind ~= 'none' and kind ~= 'hash' then return redis.error_reply('Invalid stream control key type') end
|
|
518
|
+
end
|
|
519
|
+
if ARGV[5] == 'renew' then
|
|
520
|
+
local claimed = redis.call('XCLAIM', KEYS[1], ARGV[1], ARGV[3], 0, ARGV[2], 'JUSTID')
|
|
521
|
+
return #claimed
|
|
522
|
+
end
|
|
523
|
+
if ARGV[5] == 'quarantine' then
|
|
524
|
+
local kind = redis.call('TYPE', KEYS[2]).ok
|
|
525
|
+
if kind ~= 'none' and kind ~= 'stream' then return redis.error_reply('Invalid stream quarantine key type') end
|
|
526
|
+
redis.call('XADD', KEYS[2], '*', 'sourceId', ARGV[2], 'payload', ARGV[6])
|
|
527
|
+
end
|
|
528
|
+
local acknowledged = redis.call('XACK', KEYS[1], ARGV[1], ARGV[2])
|
|
529
|
+
if acknowledged == 1 then
|
|
530
|
+
redis.call('XDEL', KEYS[1], ARGV[2])
|
|
531
|
+
if ARGV[7] == '1' then redis.call('DEL', KEYS[3]) end
|
|
532
|
+
end
|
|
533
|
+
if #redis.call('XPENDING', KEYS[1], ARGV[1], '-', '+', 1, ARGV[3]) == 0 then redis.call('XGROUP', 'DELCONSUMER', KEYS[1], ARGV[1], ARGV[3]) end
|
|
534
|
+
return acknowledged
|
|
535
|
+
`;
|
|
536
|
+
var CONVERT_LIST = `
|
|
537
|
+
local kinds = {'list','list','hash','stream'}
|
|
538
|
+
for i=1,4 do
|
|
539
|
+
local kind = redis.call('TYPE', KEYS[i]).ok
|
|
540
|
+
if kind ~= 'none' and kind ~= kinds[i] then return redis.error_reply('Invalid queue conversion key type') end
|
|
541
|
+
end
|
|
542
|
+
if redis.call('LLEN', KEYS[2]) > 0 or redis.call('HLEN', KEYS[3]) > 0 then return redis.error_reply('Drain or recover processing reservations before queue conversion') end
|
|
543
|
+
local converted = 0
|
|
544
|
+
for i=1,tonumber(ARGV[1]) do
|
|
545
|
+
local payload = redis.call('LINDEX', KEYS[1], -1)
|
|
546
|
+
if not payload then break end
|
|
547
|
+
redis.call('XADD', KEYS[4], '*', 'payload', payload)
|
|
548
|
+
redis.call('RPOP', KEYS[1])
|
|
549
|
+
converted = converted + 1
|
|
550
|
+
end
|
|
551
|
+
return converted
|
|
552
|
+
`;
|
|
553
|
+
|
|
554
|
+
// ../../src/core/queue/redisStreams.ts
|
|
555
|
+
var STREAM_GROUP = "strata-workers-v1";
|
|
556
|
+
function streamQueueKey(queueKey) {
|
|
557
|
+
return `${queueKey}:stream`;
|
|
558
|
+
}
|
|
559
|
+
function streamDeadLetterKey(queueKey) {
|
|
560
|
+
return `${streamQueueKey(queueKey)}:invalid`;
|
|
561
|
+
}
|
|
562
|
+
function streamRetryKey(queueKey) {
|
|
563
|
+
return `${streamQueueKey(queueKey)}:retries`;
|
|
564
|
+
}
|
|
565
|
+
function streamControlKey(key, jobId) {
|
|
566
|
+
const prefix = `${streamQueueKey(key)}/`;
|
|
567
|
+
if (!jobId.startsWith(prefix) || !/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/.test(jobId.slice(prefix.length)))
|
|
568
|
+
throw new Error("Invalid cancellable Streams job ID");
|
|
569
|
+
return `${streamQueueKey(key)}:control:${jobId.slice(prefix.length)}`;
|
|
570
|
+
}
|
|
310
571
|
async function cancelStreamJob(client, key, jobId) {
|
|
311
572
|
return Number(await client.send("EVAL", [
|
|
312
573
|
CANCEL_STREAM,
|
|
@@ -360,26 +621,6 @@ function watchStreamDeadline(client, deadlineAtMs, remainingMs, expire, fail) {
|
|
|
360
621
|
clearTimeout(timer);
|
|
361
622
|
};
|
|
362
623
|
}
|
|
363
|
-
var SCHEDULE_RETRY = `
|
|
364
|
-
local pending = redis.call('XPENDING', KEYS[1], ARGV[1], ARGV[2], ARGV[2], 1)
|
|
365
|
-
if #pending == 0 or pending[1][2] ~= ARGV[3] or tonumber(pending[1][3]) >= tonumber(ARGV[4]) then return 0 end
|
|
366
|
-
local kind = redis.call('TYPE', KEYS[2]).ok
|
|
367
|
-
if kind ~= 'none' and kind ~= 'zset' then return redis.error_reply('Invalid stream retry key type') end
|
|
368
|
-
if ARGV[8] == '1' then
|
|
369
|
-
local state = redis.call('HGET', KEYS[3], 'state')
|
|
370
|
-
if state == 'cancelled' then return redis.error_reply('Streams job cancelled') end
|
|
371
|
-
if state ~= 'active' then return redis.error_reply('Missing or invalid Streams control state') end
|
|
372
|
-
end
|
|
373
|
-
local now = redis.call('TIME')
|
|
374
|
-
local due = tonumber(now[1])*1000 + math.floor(tonumber(now[2])/1000) + tonumber(ARGV[5])
|
|
375
|
-
if ARGV[7] ~= '' then due = math.min(due, tonumber(ARGV[7])) end
|
|
376
|
-
redis.call('ZADD', KEYS[2], due, ARGV[6])
|
|
377
|
-
if ARGV[8] == '1' then redis.call('HSET', KEYS[3], 'retryPayload', ARGV[6]) end
|
|
378
|
-
redis.call('XACK', KEYS[1], ARGV[1], ARGV[2])
|
|
379
|
-
redis.call('XDEL', KEYS[1], ARGV[2])
|
|
380
|
-
if #redis.call('XPENDING', KEYS[1], ARGV[1], '-', '+', 1, ARGV[3]) == 0 then redis.call('XGROUP', 'DELCONSUMER', KEYS[1], ARGV[1], ARGV[3]) end
|
|
381
|
-
return 1
|
|
382
|
-
`;
|
|
383
624
|
async function scheduleStreamRetry(client, key, reservation, envelope, delayMs) {
|
|
384
625
|
if (!Number.isSafeInteger(delayMs) || delayMs < 0 || delayMs > Number.MAX_SAFE_INTEGER / 2)
|
|
385
626
|
throw new Error("Invalid Streams retry delay");
|
|
@@ -401,59 +642,6 @@ async function scheduleStreamRetry(client, key, reservation, envelope, delayMs)
|
|
|
401
642
|
if (Number(scheduled) !== 1)
|
|
402
643
|
throw new Error("Queue lease lost while scheduling retry");
|
|
403
644
|
}
|
|
404
|
-
var POLL_STREAM_PRIORITIES = `
|
|
405
|
-
local group = ARGV[1]
|
|
406
|
-
local owner = ARGV[2]
|
|
407
|
-
local visibility = ARGV[3]
|
|
408
|
-
local function payloadOf(fields)
|
|
409
|
-
for index = 1, #fields, 2 do
|
|
410
|
-
if fields[index] == 'payload' and type(fields[index + 1]) == 'string' then return fields[index + 1] end
|
|
411
|
-
end
|
|
412
|
-
return ''
|
|
413
|
-
end
|
|
414
|
-
local cursors = {}
|
|
415
|
-
for i = 1, #KEYS / 2 do cursors[i] = ARGV[3 + i] end
|
|
416
|
-
for i = 1, #KEYS / 2 do
|
|
417
|
-
local stream = KEYS[(i - 1) * 2 + 1]
|
|
418
|
-
local retries = KEYS[(i - 1) * 2 + 2]
|
|
419
|
-
local cursor = ARGV[3 + i]
|
|
420
|
-
local streamKind = redis.call('TYPE', stream).ok
|
|
421
|
-
local retryKind = redis.call('TYPE', retries).ok
|
|
422
|
-
if streamKind ~= 'none' and streamKind ~= 'stream' then return redis.error_reply('Invalid stream retry promotion key type') end
|
|
423
|
-
if retryKind ~= 'none' and retryKind ~= 'zset' then return redis.error_reply('Invalid stream retry promotion key type') end
|
|
424
|
-
if streamKind == 'stream' then
|
|
425
|
-
local now = redis.call('TIME')
|
|
426
|
-
local due = tonumber(now[1]) * 1000 + math.floor(tonumber(now[2]) / 1000)
|
|
427
|
-
local ready = redis.call('ZRANGE', retries, '-inf', due, 'BYSCORE', 'LIMIT', 0, 100)
|
|
428
|
-
for _, payload in ipairs(ready) do
|
|
429
|
-
redis.call('XADD', stream, '*', 'payload', payload)
|
|
430
|
-
redis.call('ZREM', retries, payload)
|
|
431
|
-
end
|
|
432
|
-
local window = redis.call('XPENDING', stream, group, cursor, '+', 10)
|
|
433
|
-
local claimed = redis.call('XAUTOCLAIM', stream, group, owner, visibility, cursor, 'COUNT', 1)
|
|
434
|
-
for _, entry in ipairs(window) do
|
|
435
|
-
if entry[2] ~= owner and #redis.call('XPENDING', stream, group, '-', '+', 1, entry[2]) == 0 then
|
|
436
|
-
redis.call('XGROUP', 'DELCONSUMER', stream, group, entry[2])
|
|
437
|
-
end
|
|
438
|
-
end
|
|
439
|
-
if #redis.call('XPENDING', stream, group, '-', '+', 1, owner) == 0 then
|
|
440
|
-
redis.call('XGROUP', 'DELCONSUMER', stream, group, owner)
|
|
441
|
-
end
|
|
442
|
-
cursors[i] = claimed[1]
|
|
443
|
-
if #claimed[2] > 0 then
|
|
444
|
-
return {'1', tostring(i), claimed[2][1][1], payloadOf(claimed[2][1][2]), unpack(cursors)}
|
|
445
|
-
end
|
|
446
|
-
local fresh = redis.call('XREADGROUP', 'GROUP', group, owner, 'COUNT', 1, 'STREAMS', stream, '>')
|
|
447
|
-
if not fresh and #redis.call('XPENDING', stream, group, '-', '+', 1, owner) == 0 then
|
|
448
|
-
redis.call('XGROUP', 'DELCONSUMER', stream, group, owner)
|
|
449
|
-
end
|
|
450
|
-
if fresh and #fresh > 0 and #fresh[1][2] > 0 then
|
|
451
|
-
return {'1', tostring(i), fresh[1][2][1][1], payloadOf(fresh[1][2][1][2]), unpack(cursors)}
|
|
452
|
-
end
|
|
453
|
-
end
|
|
454
|
-
end
|
|
455
|
-
return {'0', unpack(cursors)}
|
|
456
|
-
`;
|
|
457
645
|
async function pollStreamPriorities(client, queueKeys, owner, cursors) {
|
|
458
646
|
const keys = [];
|
|
459
647
|
for (const key of queueKeys)
|
|
@@ -497,12 +685,6 @@ async function readStreamQueueDepth(client, key) {
|
|
|
497
685
|
streamRetryKey(key)
|
|
498
686
|
]));
|
|
499
687
|
}
|
|
500
|
-
var ASSERT_DRAINED = `
|
|
501
|
-
if redis.call('LLEN', KEYS[1]) > 0 or redis.call('LLEN', KEYS[2]) > 0 or redis.call('HLEN', KEYS[3]) > 0 then
|
|
502
|
-
return redis.error_reply('Legacy queue state requires maintenance conversion before Streams activation')
|
|
503
|
-
end
|
|
504
|
-
return 1
|
|
505
|
-
`;
|
|
506
688
|
async function assertLegacyDrained(client, key) {
|
|
507
689
|
await client.send("EVAL", [
|
|
508
690
|
ASSERT_DRAINED,
|
|
@@ -512,30 +694,6 @@ async function assertLegacyDrained(client, key) {
|
|
|
512
694
|
`${key}:processing:leases`
|
|
513
695
|
]);
|
|
514
696
|
}
|
|
515
|
-
var UPDATE_STREAM = `
|
|
516
|
-
local pending = redis.call('XPENDING', KEYS[1], ARGV[1], ARGV[2], ARGV[2], 1)
|
|
517
|
-
if #pending == 0 or pending[1][2] ~= ARGV[3] or tonumber(pending[1][3]) >= tonumber(ARGV[4]) then return 0 end
|
|
518
|
-
if ARGV[7] == '1' then
|
|
519
|
-
local kind = redis.call('TYPE', KEYS[3]).ok
|
|
520
|
-
if kind ~= 'none' and kind ~= 'hash' then return redis.error_reply('Invalid stream control key type') end
|
|
521
|
-
end
|
|
522
|
-
if ARGV[5] == 'renew' then
|
|
523
|
-
local claimed = redis.call('XCLAIM', KEYS[1], ARGV[1], ARGV[3], 0, ARGV[2], 'JUSTID')
|
|
524
|
-
return #claimed
|
|
525
|
-
end
|
|
526
|
-
if ARGV[5] == 'quarantine' then
|
|
527
|
-
local kind = redis.call('TYPE', KEYS[2]).ok
|
|
528
|
-
if kind ~= 'none' and kind ~= 'stream' then return redis.error_reply('Invalid stream quarantine key type') end
|
|
529
|
-
redis.call('XADD', KEYS[2], '*', 'sourceId', ARGV[2], 'payload', ARGV[6])
|
|
530
|
-
end
|
|
531
|
-
local acknowledged = redis.call('XACK', KEYS[1], ARGV[1], ARGV[2])
|
|
532
|
-
if acknowledged == 1 then
|
|
533
|
-
redis.call('XDEL', KEYS[1], ARGV[2])
|
|
534
|
-
if ARGV[7] == '1' then redis.call('DEL', KEYS[3]) end
|
|
535
|
-
end
|
|
536
|
-
if #redis.call('XPENDING', KEYS[1], ARGV[1], '-', '+', 1, ARGV[3]) == 0 then redis.call('XGROUP', 'DELCONSUMER', KEYS[1], ARGV[1], ARGV[3]) end
|
|
537
|
-
return acknowledged
|
|
538
|
-
`;
|
|
539
697
|
async function ensureStreamGroup(client, key) {
|
|
540
698
|
await assertLegacyDrained(client, key);
|
|
541
699
|
try {
|
|
@@ -845,23 +1003,6 @@ class RedisStreamsWorker {
|
|
|
845
1003
|
}
|
|
846
1004
|
}
|
|
847
1005
|
}
|
|
848
|
-
var CONVERT_LIST = `
|
|
849
|
-
local kinds = {'list','list','hash','stream'}
|
|
850
|
-
for i=1,4 do
|
|
851
|
-
local kind = redis.call('TYPE', KEYS[i]).ok
|
|
852
|
-
if kind ~= 'none' and kind ~= kinds[i] then return redis.error_reply('Invalid queue conversion key type') end
|
|
853
|
-
end
|
|
854
|
-
if redis.call('LLEN', KEYS[2]) > 0 or redis.call('HLEN', KEYS[3]) > 0 then return redis.error_reply('Drain or recover processing reservations before queue conversion') end
|
|
855
|
-
local converted = 0
|
|
856
|
-
for i=1,tonumber(ARGV[1]) do
|
|
857
|
-
local payload = redis.call('LINDEX', KEYS[1], -1)
|
|
858
|
-
if not payload then break end
|
|
859
|
-
redis.call('XADD', KEYS[4], '*', 'payload', payload)
|
|
860
|
-
redis.call('RPOP', KEYS[1])
|
|
861
|
-
converted = converted + 1
|
|
862
|
-
end
|
|
863
|
-
return converted
|
|
864
|
-
`;
|
|
865
1006
|
async function migrateLegacyQueueToStreams(client, options) {
|
|
866
1007
|
if (options.maintenance !== true)
|
|
867
1008
|
throw new Error("Queue conversion requires maintenance mode: stop all producers and workers first.");
|
|
@@ -899,141 +1040,6 @@ function reservationExpired(lease, now, visibilityMs) {
|
|
|
899
1040
|
}
|
|
900
1041
|
return now - reservedAt >= visibilityMs;
|
|
901
1042
|
}
|
|
902
|
-
var SCRIPT_HEADER = `
|
|
903
|
-
local function checkType(key, expected)
|
|
904
|
-
local actual = redis.call('TYPE', key).ok
|
|
905
|
-
if actual ~= 'none' and actual ~= expected then
|
|
906
|
-
return redis.error_reply('Unexpected queue key type: ' .. key)
|
|
907
|
-
end
|
|
908
|
-
end
|
|
909
|
-
local function nowMs()
|
|
910
|
-
local time = redis.call('TIME')
|
|
911
|
-
return tonumber(time[1]) * 1000 + math.floor(tonumber(time[2]) / 1000)
|
|
912
|
-
end
|
|
913
|
-
`;
|
|
914
|
-
var RESERVE_JOB = `${SCRIPT_HEADER}
|
|
915
|
-
local err = checkType(KEYS[1], 'list') or checkType(KEYS[2], 'list') or checkType(KEYS[3], 'hash')
|
|
916
|
-
if err then return err end
|
|
917
|
-
local payload = redis.call('LINDEX', KEYS[1], -1)
|
|
918
|
-
if not payload then return nil end
|
|
919
|
-
local record = cjson.encode({strataReservation=1, id=ARGV[1], owner=ARGV[2], payload=payload})
|
|
920
|
-
local lease = cjson.encode({owner=ARGV[2], expiresAt=nowMs()+tonumber(ARGV[3])})
|
|
921
|
-
redis.call('LPUSH', KEYS[2], record)
|
|
922
|
-
redis.call('HSET', KEYS[3], ARGV[1], lease)
|
|
923
|
-
redis.call('RPOP', KEYS[1])
|
|
924
|
-
return record
|
|
925
|
-
`;
|
|
926
|
-
var RENEW_OR_ACK = `${SCRIPT_HEADER}
|
|
927
|
-
local err = checkType(KEYS[1], 'list') or checkType(KEYS[2], 'hash') or checkType(KEYS[3], 'list')
|
|
928
|
-
if err then return err end
|
|
929
|
-
local raw = redis.call('HGET', KEYS[2], ARGV[1])
|
|
930
|
-
if not raw then return 0 end
|
|
931
|
-
local ok, lease = pcall(cjson.decode, raw)
|
|
932
|
-
local now = nowMs()
|
|
933
|
-
if not ok or type(lease) ~= 'table' or lease.owner ~= ARGV[2] or not tonumber(lease.expiresAt) or tonumber(lease.expiresAt) <= now then
|
|
934
|
-
return 0
|
|
935
|
-
end
|
|
936
|
-
if ARGV[4] == 'renew' then
|
|
937
|
-
redis.call('HSET', KEYS[2], ARGV[1], cjson.encode({owner=ARGV[2], expiresAt=now+tonumber(ARGV[5])}))
|
|
938
|
-
return 1
|
|
939
|
-
end
|
|
940
|
-
if ARGV[4] == 'quarantine' then redis.call('LPUSH', KEYS[3], ARGV[6]) end
|
|
941
|
-
local removed = redis.call('LREM', KEYS[1], 1, ARGV[3])
|
|
942
|
-
redis.call('HDEL', KEYS[2], ARGV[1])
|
|
943
|
-
return removed
|
|
944
|
-
`;
|
|
945
|
-
var RECLAIM_JOBS = `${SCRIPT_HEADER}
|
|
946
|
-
local err = checkType(KEYS[1], 'list') or checkType(KEYS[2], 'list') or checkType(KEYS[3], 'hash')
|
|
947
|
-
if err then return err end
|
|
948
|
-
local now = nowMs()
|
|
949
|
-
local reclaimed = 0
|
|
950
|
-
for _, record in ipairs(redis.call('LRANGE', KEYS[2], 0, -1)) do
|
|
951
|
-
local payload, field = record, record
|
|
952
|
-
local modern = false
|
|
953
|
-
local ok, reservation = pcall(cjson.decode, record)
|
|
954
|
-
if ok and type(reservation) == 'table' and reservation.strataReservation == 1 then
|
|
955
|
-
payload, field, modern = reservation.payload, reservation.id, true
|
|
956
|
-
end
|
|
957
|
-
local raw = redis.call('HGET', KEYS[3], field)
|
|
958
|
-
local expired = true
|
|
959
|
-
if raw then
|
|
960
|
-
if modern then
|
|
961
|
-
local leaseOk, lease = pcall(cjson.decode, raw)
|
|
962
|
-
expired = not leaseOk or type(lease) ~= 'table' or not tonumber(lease.expiresAt) or tonumber(lease.expiresAt) <= now
|
|
963
|
-
else
|
|
964
|
-
local reservedAt = tonumber(raw)
|
|
965
|
-
expired = not reservedAt or now - reservedAt >= tonumber(ARGV[1])
|
|
966
|
-
end
|
|
967
|
-
end
|
|
968
|
-
if expired then
|
|
969
|
-
redis.call('LPUSH', KEYS[1], payload)
|
|
970
|
-
redis.call('LREM', KEYS[2], 1, record)
|
|
971
|
-
redis.call('HDEL', KEYS[3], field)
|
|
972
|
-
reclaimed = reclaimed + 1
|
|
973
|
-
end
|
|
974
|
-
end
|
|
975
|
-
return reclaimed
|
|
976
|
-
`;
|
|
977
|
-
var POLL_LIST_PRIORITIES = `${SCRIPT_HEADER}
|
|
978
|
-
local owner = ARGV[1]
|
|
979
|
-
local visibility = ARGV[2]
|
|
980
|
-
local reservationId = ARGV[3]
|
|
981
|
-
local reclaim = ARGV[4] == '1'
|
|
982
|
-
local function reclaimQueue(listKey, processingKey, leaseKey)
|
|
983
|
-
local err = checkType(listKey, 'list') or checkType(processingKey, 'list') or checkType(leaseKey, 'hash')
|
|
984
|
-
if err then return err end
|
|
985
|
-
local now = nowMs()
|
|
986
|
-
for _, record in ipairs(redis.call('LRANGE', processingKey, 0, -1)) do
|
|
987
|
-
local payload, field = record, record
|
|
988
|
-
local modern = false
|
|
989
|
-
local ok, reservation = pcall(cjson.decode, record)
|
|
990
|
-
if ok and type(reservation) == 'table' and reservation.strataReservation == 1 then
|
|
991
|
-
payload, field, modern = reservation.payload, reservation.id, true
|
|
992
|
-
end
|
|
993
|
-
local raw = redis.call('HGET', leaseKey, field)
|
|
994
|
-
local expired = true
|
|
995
|
-
if raw then
|
|
996
|
-
if modern then
|
|
997
|
-
local leaseOk, lease = pcall(cjson.decode, raw)
|
|
998
|
-
expired = not leaseOk or type(lease) ~= 'table' or not tonumber(lease.expiresAt) or tonumber(lease.expiresAt) <= now
|
|
999
|
-
else
|
|
1000
|
-
local reservedAt = tonumber(raw)
|
|
1001
|
-
expired = not reservedAt or now - reservedAt >= tonumber(visibility)
|
|
1002
|
-
end
|
|
1003
|
-
end
|
|
1004
|
-
if expired then
|
|
1005
|
-
redis.call('LPUSH', listKey, payload)
|
|
1006
|
-
redis.call('LREM', processingKey, 1, record)
|
|
1007
|
-
redis.call('HDEL', leaseKey, field)
|
|
1008
|
-
end
|
|
1009
|
-
end
|
|
1010
|
-
end
|
|
1011
|
-
local function reserveQueue(listKey, processingKey, leaseKey)
|
|
1012
|
-
local err = checkType(listKey, 'list') or checkType(processingKey, 'list') or checkType(leaseKey, 'hash')
|
|
1013
|
-
if err then return err end
|
|
1014
|
-
local payload = redis.call('LINDEX', listKey, -1)
|
|
1015
|
-
if not payload then return nil end
|
|
1016
|
-
local record = cjson.encode({strataReservation=1, id=reservationId, owner=owner, payload=payload})
|
|
1017
|
-
local lease = cjson.encode({owner=owner, expiresAt=nowMs()+tonumber(visibility)})
|
|
1018
|
-
redis.call('LPUSH', processingKey, record)
|
|
1019
|
-
redis.call('HSET', leaseKey, reservationId, lease)
|
|
1020
|
-
redis.call('RPOP', listKey)
|
|
1021
|
-
return record
|
|
1022
|
-
end
|
|
1023
|
-
local count = #KEYS / 3
|
|
1024
|
-
if reclaim then
|
|
1025
|
-
for i = 1, count do
|
|
1026
|
-
local err = reclaimQueue(KEYS[(i - 1) * 3 + 1], KEYS[(i - 1) * 3 + 2], KEYS[(i - 1) * 3 + 3])
|
|
1027
|
-
if err then return err end
|
|
1028
|
-
end
|
|
1029
|
-
end
|
|
1030
|
-
for i = 1, count do
|
|
1031
|
-
local record = reserveQueue(KEYS[(i - 1) * 3 + 1], KEYS[(i - 1) * 3 + 2], KEYS[(i - 1) * 3 + 3])
|
|
1032
|
-
if type(record) == 'string' then return {'1', tostring(i), record} end
|
|
1033
|
-
if record then return record end
|
|
1034
|
-
end
|
|
1035
|
-
return {'0'}
|
|
1036
|
-
`;
|
|
1037
1043
|
function queueInvalidKey(queueKey) {
|
|
1038
1044
|
return `${queueKey}:invalid`;
|
|
1039
1045
|
}
|