@getstrata/core 2.2.48 → 2.2.50
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +11 -0
- package/dist/core/audit/exportStore.d.ts +17 -0
- package/dist/core/cache/redisCacheScript.d.ts +2 -0
- package/dist/core/database/baseRepository.d.ts +0 -1
- package/dist/core/database/model.d.ts +6 -113
- package/dist/core/database/modelAttributes.d.ts +17 -0
- package/dist/core/database/modelQuery.d.ts +139 -0
- package/dist/core/database/modelRegistry.d.ts +34 -0
- package/dist/core/database/relationQueries.d.ts +20 -0
- package/dist/core/http/middleware.d.ts +2 -2
- package/dist/core/http/routeMiddleware.d.ts +1 -1
- package/dist/core/mail/mailer.d.ts +1 -4
- package/dist/core/mail/smtpMessage.d.ts +7 -0
- package/dist/core/queue/redisListScripts.d.ts +5 -0
- package/dist/core/queue/redisStreamsScripts.d.ts +11 -0
- package/dist/core/scheduler/leaseScript.d.ts +2 -0
- package/dist/entries/audit/exportAuditLogs.js +63 -58
- package/dist/entries/auth/scimAuthMiddleware.js +31 -20
- package/dist/entries/cache/createCacheStore.js +2 -1
- package/dist/entries/facades.js +103 -101
- package/dist/entries/http/routeMiddleware.js +3 -3
- package/dist/entries/jobs/exportAuditLogsJob.js +63 -58
- package/dist/entries/mail/mailer.js +103 -101
- package/dist/entries/queue/createAppQueue.js +277 -271
- package/dist/entries/queue/publicQueue.js +277 -271
- package/dist/entries/queue/queueMetrics.js +277 -271
- package/dist/entries/queue/redisQueue.js +277 -271
- package/dist/index.js +1482 -1428
- package/package.json +1 -1
|
@@ -1142,6 +1142,143 @@ async function runQueueJob(envelope, failedJobs, options = {}) {
|
|
|
1142
1142
|
// ../../src/core/queue/redisQueue.ts
|
|
1143
1143
|
var {RedisClient: RedisClient2 } = globalThis.Bun;
|
|
1144
1144
|
|
|
1145
|
+
// ../../src/core/queue/redisListScripts.ts
|
|
1146
|
+
var SCRIPT_HEADER = `
|
|
1147
|
+
local function checkType(key, expected)
|
|
1148
|
+
local actual = redis.call('TYPE', key).ok
|
|
1149
|
+
if actual ~= 'none' and actual ~= expected then
|
|
1150
|
+
return redis.error_reply('Unexpected queue key type: ' .. key)
|
|
1151
|
+
end
|
|
1152
|
+
end
|
|
1153
|
+
local function nowMs()
|
|
1154
|
+
local time = redis.call('TIME')
|
|
1155
|
+
return tonumber(time[1]) * 1000 + math.floor(tonumber(time[2]) / 1000)
|
|
1156
|
+
end
|
|
1157
|
+
`;
|
|
1158
|
+
var RESERVE_JOB = `${SCRIPT_HEADER}
|
|
1159
|
+
local err = checkType(KEYS[1], 'list') or checkType(KEYS[2], 'list') or checkType(KEYS[3], 'hash')
|
|
1160
|
+
if err then return err end
|
|
1161
|
+
local payload = redis.call('LINDEX', KEYS[1], -1)
|
|
1162
|
+
if not payload then return nil end
|
|
1163
|
+
local record = cjson.encode({strataReservation=1, id=ARGV[1], owner=ARGV[2], payload=payload})
|
|
1164
|
+
local lease = cjson.encode({owner=ARGV[2], expiresAt=nowMs()+tonumber(ARGV[3])})
|
|
1165
|
+
redis.call('LPUSH', KEYS[2], record)
|
|
1166
|
+
redis.call('HSET', KEYS[3], ARGV[1], lease)
|
|
1167
|
+
redis.call('RPOP', KEYS[1])
|
|
1168
|
+
return record
|
|
1169
|
+
`;
|
|
1170
|
+
var RENEW_OR_ACK = `${SCRIPT_HEADER}
|
|
1171
|
+
local err = checkType(KEYS[1], 'list') or checkType(KEYS[2], 'hash') or checkType(KEYS[3], 'list')
|
|
1172
|
+
if err then return err end
|
|
1173
|
+
local raw = redis.call('HGET', KEYS[2], ARGV[1])
|
|
1174
|
+
if not raw then return 0 end
|
|
1175
|
+
local ok, lease = pcall(cjson.decode, raw)
|
|
1176
|
+
local now = nowMs()
|
|
1177
|
+
if not ok or type(lease) ~= 'table' or lease.owner ~= ARGV[2] or not tonumber(lease.expiresAt) or tonumber(lease.expiresAt) <= now then
|
|
1178
|
+
return 0
|
|
1179
|
+
end
|
|
1180
|
+
if ARGV[4] == 'renew' then
|
|
1181
|
+
redis.call('HSET', KEYS[2], ARGV[1], cjson.encode({owner=ARGV[2], expiresAt=now+tonumber(ARGV[5])}))
|
|
1182
|
+
return 1
|
|
1183
|
+
end
|
|
1184
|
+
if ARGV[4] == 'quarantine' then redis.call('LPUSH', KEYS[3], ARGV[6]) end
|
|
1185
|
+
local removed = redis.call('LREM', KEYS[1], 1, ARGV[3])
|
|
1186
|
+
redis.call('HDEL', KEYS[2], ARGV[1])
|
|
1187
|
+
return removed
|
|
1188
|
+
`;
|
|
1189
|
+
var RECLAIM_JOBS = `${SCRIPT_HEADER}
|
|
1190
|
+
local err = checkType(KEYS[1], 'list') or checkType(KEYS[2], 'list') or checkType(KEYS[3], 'hash')
|
|
1191
|
+
if err then return err end
|
|
1192
|
+
local now = nowMs()
|
|
1193
|
+
local reclaimed = 0
|
|
1194
|
+
for _, record in ipairs(redis.call('LRANGE', KEYS[2], 0, -1)) do
|
|
1195
|
+
local payload, field = record, record
|
|
1196
|
+
local modern = false
|
|
1197
|
+
local ok, reservation = pcall(cjson.decode, record)
|
|
1198
|
+
if ok and type(reservation) == 'table' and reservation.strataReservation == 1 then
|
|
1199
|
+
payload, field, modern = reservation.payload, reservation.id, true
|
|
1200
|
+
end
|
|
1201
|
+
local raw = redis.call('HGET', KEYS[3], field)
|
|
1202
|
+
local expired = true
|
|
1203
|
+
if raw then
|
|
1204
|
+
if modern then
|
|
1205
|
+
local leaseOk, lease = pcall(cjson.decode, raw)
|
|
1206
|
+
expired = not leaseOk or type(lease) ~= 'table' or not tonumber(lease.expiresAt) or tonumber(lease.expiresAt) <= now
|
|
1207
|
+
else
|
|
1208
|
+
local reservedAt = tonumber(raw)
|
|
1209
|
+
expired = not reservedAt or now - reservedAt >= tonumber(ARGV[1])
|
|
1210
|
+
end
|
|
1211
|
+
end
|
|
1212
|
+
if expired then
|
|
1213
|
+
redis.call('LPUSH', KEYS[1], payload)
|
|
1214
|
+
redis.call('LREM', KEYS[2], 1, record)
|
|
1215
|
+
redis.call('HDEL', KEYS[3], field)
|
|
1216
|
+
reclaimed = reclaimed + 1
|
|
1217
|
+
end
|
|
1218
|
+
end
|
|
1219
|
+
return reclaimed
|
|
1220
|
+
`;
|
|
1221
|
+
var POLL_LIST_PRIORITIES = `${SCRIPT_HEADER}
|
|
1222
|
+
local owner = ARGV[1]
|
|
1223
|
+
local visibility = ARGV[2]
|
|
1224
|
+
local reservationId = ARGV[3]
|
|
1225
|
+
local reclaim = ARGV[4] == '1'
|
|
1226
|
+
local function reclaimQueue(listKey, processingKey, leaseKey)
|
|
1227
|
+
local err = checkType(listKey, 'list') or checkType(processingKey, 'list') or checkType(leaseKey, 'hash')
|
|
1228
|
+
if err then return err end
|
|
1229
|
+
local now = nowMs()
|
|
1230
|
+
for _, record in ipairs(redis.call('LRANGE', processingKey, 0, -1)) do
|
|
1231
|
+
local payload, field = record, record
|
|
1232
|
+
local modern = false
|
|
1233
|
+
local ok, reservation = pcall(cjson.decode, record)
|
|
1234
|
+
if ok and type(reservation) == 'table' and reservation.strataReservation == 1 then
|
|
1235
|
+
payload, field, modern = reservation.payload, reservation.id, true
|
|
1236
|
+
end
|
|
1237
|
+
local raw = redis.call('HGET', leaseKey, field)
|
|
1238
|
+
local expired = true
|
|
1239
|
+
if raw then
|
|
1240
|
+
if modern then
|
|
1241
|
+
local leaseOk, lease = pcall(cjson.decode, raw)
|
|
1242
|
+
expired = not leaseOk or type(lease) ~= 'table' or not tonumber(lease.expiresAt) or tonumber(lease.expiresAt) <= now
|
|
1243
|
+
else
|
|
1244
|
+
local reservedAt = tonumber(raw)
|
|
1245
|
+
expired = not reservedAt or now - reservedAt >= tonumber(visibility)
|
|
1246
|
+
end
|
|
1247
|
+
end
|
|
1248
|
+
if expired then
|
|
1249
|
+
redis.call('LPUSH', listKey, payload)
|
|
1250
|
+
redis.call('LREM', processingKey, 1, record)
|
|
1251
|
+
redis.call('HDEL', leaseKey, field)
|
|
1252
|
+
end
|
|
1253
|
+
end
|
|
1254
|
+
end
|
|
1255
|
+
local function reserveQueue(listKey, processingKey, leaseKey)
|
|
1256
|
+
local err = checkType(listKey, 'list') or checkType(processingKey, 'list') or checkType(leaseKey, 'hash')
|
|
1257
|
+
if err then return err end
|
|
1258
|
+
local payload = redis.call('LINDEX', listKey, -1)
|
|
1259
|
+
if not payload then return nil end
|
|
1260
|
+
local record = cjson.encode({strataReservation=1, id=reservationId, owner=owner, payload=payload})
|
|
1261
|
+
local lease = cjson.encode({owner=owner, expiresAt=nowMs()+tonumber(visibility)})
|
|
1262
|
+
redis.call('LPUSH', processingKey, record)
|
|
1263
|
+
redis.call('HSET', leaseKey, reservationId, lease)
|
|
1264
|
+
redis.call('RPOP', listKey)
|
|
1265
|
+
return record
|
|
1266
|
+
end
|
|
1267
|
+
local count = #KEYS / 3
|
|
1268
|
+
if reclaim then
|
|
1269
|
+
for i = 1, count do
|
|
1270
|
+
local err = reclaimQueue(KEYS[(i - 1) * 3 + 1], KEYS[(i - 1) * 3 + 2], KEYS[(i - 1) * 3 + 3])
|
|
1271
|
+
if err then return err end
|
|
1272
|
+
end
|
|
1273
|
+
end
|
|
1274
|
+
for i = 1, count do
|
|
1275
|
+
local record = reserveQueue(KEYS[(i - 1) * 3 + 1], KEYS[(i - 1) * 3 + 2], KEYS[(i - 1) * 3 + 3])
|
|
1276
|
+
if type(record) == 'string' then return {'1', tostring(i), record} end
|
|
1277
|
+
if record then return record end
|
|
1278
|
+
end
|
|
1279
|
+
return {'0'}
|
|
1280
|
+
`;
|
|
1281
|
+
|
|
1145
1282
|
// ../../src/core/runtime/appEnv.ts
|
|
1146
1283
|
var NON_PRODUCTION_APP_ENVS = new Set(["local", "development", "dev", "test", "testing", "ci"]);
|
|
1147
1284
|
function normalizeEnvValue(value) {
|
|
@@ -1258,22 +1395,8 @@ function queueKeyForPriority(priority = "default") {
|
|
|
1258
1395
|
|
|
1259
1396
|
// ../../src/core/queue/redisStreams.ts
|
|
1260
1397
|
var {RedisClient } = globalThis.Bun;
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
return `${queueKey}:stream`;
|
|
1264
|
-
}
|
|
1265
|
-
function streamDeadLetterKey(queueKey) {
|
|
1266
|
-
return `${streamQueueKey(queueKey)}:invalid`;
|
|
1267
|
-
}
|
|
1268
|
-
function streamRetryKey(queueKey) {
|
|
1269
|
-
return `${streamQueueKey(queueKey)}:retries`;
|
|
1270
|
-
}
|
|
1271
|
-
function streamControlKey(key, jobId) {
|
|
1272
|
-
const prefix = `${streamQueueKey(key)}/`;
|
|
1273
|
-
if (!jobId.startsWith(prefix) || !/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/.test(jobId.slice(prefix.length)))
|
|
1274
|
-
throw new Error("Invalid cancellable Streams job ID");
|
|
1275
|
-
return `${streamQueueKey(key)}:control:${jobId.slice(prefix.length)}`;
|
|
1276
|
-
}
|
|
1398
|
+
|
|
1399
|
+
// ../../src/core/queue/redisStreamsScripts.ts
|
|
1277
1400
|
var ENQUEUE_CONTROLLED = `
|
|
1278
1401
|
local kind = redis.call('TYPE', KEYS[1]).ok
|
|
1279
1402
|
if kind ~= 'none' and kind ~= 'stream' then return redis.error_reply('Invalid stream admission key type') end
|
|
@@ -1292,6 +1415,144 @@ if payload and redis.call('ZSCORE', KEYS[2], payload) then redis.call('ZADD', KE
|
|
|
1292
1415
|
redis.call('HSET', KEYS[1], 'state', 'cancelled')
|
|
1293
1416
|
return 1
|
|
1294
1417
|
`;
|
|
1418
|
+
var SCHEDULE_RETRY = `
|
|
1419
|
+
local pending = redis.call('XPENDING', KEYS[1], ARGV[1], ARGV[2], ARGV[2], 1)
|
|
1420
|
+
if #pending == 0 or pending[1][2] ~= ARGV[3] or tonumber(pending[1][3]) >= tonumber(ARGV[4]) then return 0 end
|
|
1421
|
+
local kind = redis.call('TYPE', KEYS[2]).ok
|
|
1422
|
+
if kind ~= 'none' and kind ~= 'zset' then return redis.error_reply('Invalid stream retry key type') end
|
|
1423
|
+
if ARGV[8] == '1' then
|
|
1424
|
+
local state = redis.call('HGET', KEYS[3], 'state')
|
|
1425
|
+
if state == 'cancelled' then return redis.error_reply('Streams job cancelled') end
|
|
1426
|
+
if state ~= 'active' then return redis.error_reply('Missing or invalid Streams control state') end
|
|
1427
|
+
end
|
|
1428
|
+
local now = redis.call('TIME')
|
|
1429
|
+
local due = tonumber(now[1])*1000 + math.floor(tonumber(now[2])/1000) + tonumber(ARGV[5])
|
|
1430
|
+
if ARGV[7] ~= '' then due = math.min(due, tonumber(ARGV[7])) end
|
|
1431
|
+
redis.call('ZADD', KEYS[2], due, ARGV[6])
|
|
1432
|
+
if ARGV[8] == '1' then redis.call('HSET', KEYS[3], 'retryPayload', ARGV[6]) end
|
|
1433
|
+
redis.call('XACK', KEYS[1], ARGV[1], ARGV[2])
|
|
1434
|
+
redis.call('XDEL', KEYS[1], ARGV[2])
|
|
1435
|
+
if #redis.call('XPENDING', KEYS[1], ARGV[1], '-', '+', 1, ARGV[3]) == 0 then redis.call('XGROUP', 'DELCONSUMER', KEYS[1], ARGV[1], ARGV[3]) end
|
|
1436
|
+
return 1
|
|
1437
|
+
`;
|
|
1438
|
+
var POLL_STREAM_PRIORITIES = `
|
|
1439
|
+
local group = ARGV[1]
|
|
1440
|
+
local owner = ARGV[2]
|
|
1441
|
+
local visibility = ARGV[3]
|
|
1442
|
+
local function payloadOf(fields)
|
|
1443
|
+
for index = 1, #fields, 2 do
|
|
1444
|
+
if fields[index] == 'payload' and type(fields[index + 1]) == 'string' then return fields[index + 1] end
|
|
1445
|
+
end
|
|
1446
|
+
return ''
|
|
1447
|
+
end
|
|
1448
|
+
local cursors = {}
|
|
1449
|
+
for i = 1, #KEYS / 2 do cursors[i] = ARGV[3 + i] end
|
|
1450
|
+
for i = 1, #KEYS / 2 do
|
|
1451
|
+
local stream = KEYS[(i - 1) * 2 + 1]
|
|
1452
|
+
local retries = KEYS[(i - 1) * 2 + 2]
|
|
1453
|
+
local cursor = ARGV[3 + i]
|
|
1454
|
+
local streamKind = redis.call('TYPE', stream).ok
|
|
1455
|
+
local retryKind = redis.call('TYPE', retries).ok
|
|
1456
|
+
if streamKind ~= 'none' and streamKind ~= 'stream' then return redis.error_reply('Invalid stream retry promotion key type') end
|
|
1457
|
+
if retryKind ~= 'none' and retryKind ~= 'zset' then return redis.error_reply('Invalid stream retry promotion key type') end
|
|
1458
|
+
if streamKind == 'stream' then
|
|
1459
|
+
local now = redis.call('TIME')
|
|
1460
|
+
local due = tonumber(now[1]) * 1000 + math.floor(tonumber(now[2]) / 1000)
|
|
1461
|
+
local ready = redis.call('ZRANGE', retries, '-inf', due, 'BYSCORE', 'LIMIT', 0, 100)
|
|
1462
|
+
for _, payload in ipairs(ready) do
|
|
1463
|
+
redis.call('XADD', stream, '*', 'payload', payload)
|
|
1464
|
+
redis.call('ZREM', retries, payload)
|
|
1465
|
+
end
|
|
1466
|
+
local window = redis.call('XPENDING', stream, group, cursor, '+', 10)
|
|
1467
|
+
local claimed = redis.call('XAUTOCLAIM', stream, group, owner, visibility, cursor, 'COUNT', 1)
|
|
1468
|
+
for _, entry in ipairs(window) do
|
|
1469
|
+
if entry[2] ~= owner and #redis.call('XPENDING', stream, group, '-', '+', 1, entry[2]) == 0 then
|
|
1470
|
+
redis.call('XGROUP', 'DELCONSUMER', stream, group, entry[2])
|
|
1471
|
+
end
|
|
1472
|
+
end
|
|
1473
|
+
if #redis.call('XPENDING', stream, group, '-', '+', 1, owner) == 0 then
|
|
1474
|
+
redis.call('XGROUP', 'DELCONSUMER', stream, group, owner)
|
|
1475
|
+
end
|
|
1476
|
+
cursors[i] = claimed[1]
|
|
1477
|
+
if #claimed[2] > 0 then
|
|
1478
|
+
return {'1', tostring(i), claimed[2][1][1], payloadOf(claimed[2][1][2]), unpack(cursors)}
|
|
1479
|
+
end
|
|
1480
|
+
local fresh = redis.call('XREADGROUP', 'GROUP', group, owner, 'COUNT', 1, 'STREAMS', stream, '>')
|
|
1481
|
+
if not fresh and #redis.call('XPENDING', stream, group, '-', '+', 1, owner) == 0 then
|
|
1482
|
+
redis.call('XGROUP', 'DELCONSUMER', stream, group, owner)
|
|
1483
|
+
end
|
|
1484
|
+
if fresh and #fresh > 0 and #fresh[1][2] > 0 then
|
|
1485
|
+
return {'1', tostring(i), fresh[1][2][1][1], payloadOf(fresh[1][2][1][2]), unpack(cursors)}
|
|
1486
|
+
end
|
|
1487
|
+
end
|
|
1488
|
+
end
|
|
1489
|
+
return {'0', unpack(cursors)}
|
|
1490
|
+
`;
|
|
1491
|
+
var ASSERT_DRAINED = `
|
|
1492
|
+
if redis.call('LLEN', KEYS[1]) > 0 or redis.call('LLEN', KEYS[2]) > 0 or redis.call('HLEN', KEYS[3]) > 0 then
|
|
1493
|
+
return redis.error_reply('Legacy queue state requires maintenance conversion before Streams activation')
|
|
1494
|
+
end
|
|
1495
|
+
return 1
|
|
1496
|
+
`;
|
|
1497
|
+
var UPDATE_STREAM = `
|
|
1498
|
+
local pending = redis.call('XPENDING', KEYS[1], ARGV[1], ARGV[2], ARGV[2], 1)
|
|
1499
|
+
if #pending == 0 or pending[1][2] ~= ARGV[3] or tonumber(pending[1][3]) >= tonumber(ARGV[4]) then return 0 end
|
|
1500
|
+
if ARGV[7] == '1' then
|
|
1501
|
+
local kind = redis.call('TYPE', KEYS[3]).ok
|
|
1502
|
+
if kind ~= 'none' and kind ~= 'hash' then return redis.error_reply('Invalid stream control key type') end
|
|
1503
|
+
end
|
|
1504
|
+
if ARGV[5] == 'renew' then
|
|
1505
|
+
local claimed = redis.call('XCLAIM', KEYS[1], ARGV[1], ARGV[3], 0, ARGV[2], 'JUSTID')
|
|
1506
|
+
return #claimed
|
|
1507
|
+
end
|
|
1508
|
+
if ARGV[5] == 'quarantine' then
|
|
1509
|
+
local kind = redis.call('TYPE', KEYS[2]).ok
|
|
1510
|
+
if kind ~= 'none' and kind ~= 'stream' then return redis.error_reply('Invalid stream quarantine key type') end
|
|
1511
|
+
redis.call('XADD', KEYS[2], '*', 'sourceId', ARGV[2], 'payload', ARGV[6])
|
|
1512
|
+
end
|
|
1513
|
+
local acknowledged = redis.call('XACK', KEYS[1], ARGV[1], ARGV[2])
|
|
1514
|
+
if acknowledged == 1 then
|
|
1515
|
+
redis.call('XDEL', KEYS[1], ARGV[2])
|
|
1516
|
+
if ARGV[7] == '1' then redis.call('DEL', KEYS[3]) end
|
|
1517
|
+
end
|
|
1518
|
+
if #redis.call('XPENDING', KEYS[1], ARGV[1], '-', '+', 1, ARGV[3]) == 0 then redis.call('XGROUP', 'DELCONSUMER', KEYS[1], ARGV[1], ARGV[3]) end
|
|
1519
|
+
return acknowledged
|
|
1520
|
+
`;
|
|
1521
|
+
var CONVERT_LIST = `
|
|
1522
|
+
local kinds = {'list','list','hash','stream'}
|
|
1523
|
+
for i=1,4 do
|
|
1524
|
+
local kind = redis.call('TYPE', KEYS[i]).ok
|
|
1525
|
+
if kind ~= 'none' and kind ~= kinds[i] then return redis.error_reply('Invalid queue conversion key type') end
|
|
1526
|
+
end
|
|
1527
|
+
if redis.call('LLEN', KEYS[2]) > 0 or redis.call('HLEN', KEYS[3]) > 0 then return redis.error_reply('Drain or recover processing reservations before queue conversion') end
|
|
1528
|
+
local converted = 0
|
|
1529
|
+
for i=1,tonumber(ARGV[1]) do
|
|
1530
|
+
local payload = redis.call('LINDEX', KEYS[1], -1)
|
|
1531
|
+
if not payload then break end
|
|
1532
|
+
redis.call('XADD', KEYS[4], '*', 'payload', payload)
|
|
1533
|
+
redis.call('RPOP', KEYS[1])
|
|
1534
|
+
converted = converted + 1
|
|
1535
|
+
end
|
|
1536
|
+
return converted
|
|
1537
|
+
`;
|
|
1538
|
+
|
|
1539
|
+
// ../../src/core/queue/redisStreams.ts
|
|
1540
|
+
var STREAM_GROUP = "strata-workers-v1";
|
|
1541
|
+
function streamQueueKey(queueKey) {
|
|
1542
|
+
return `${queueKey}:stream`;
|
|
1543
|
+
}
|
|
1544
|
+
function streamDeadLetterKey(queueKey) {
|
|
1545
|
+
return `${streamQueueKey(queueKey)}:invalid`;
|
|
1546
|
+
}
|
|
1547
|
+
function streamRetryKey(queueKey) {
|
|
1548
|
+
return `${streamQueueKey(queueKey)}:retries`;
|
|
1549
|
+
}
|
|
1550
|
+
function streamControlKey(key, jobId) {
|
|
1551
|
+
const prefix = `${streamQueueKey(key)}/`;
|
|
1552
|
+
if (!jobId.startsWith(prefix) || !/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/.test(jobId.slice(prefix.length)))
|
|
1553
|
+
throw new Error("Invalid cancellable Streams job ID");
|
|
1554
|
+
return `${streamQueueKey(key)}:control:${jobId.slice(prefix.length)}`;
|
|
1555
|
+
}
|
|
1295
1556
|
async function cancelStreamJob(client, key, jobId) {
|
|
1296
1557
|
return Number(await client.send("EVAL", [
|
|
1297
1558
|
CANCEL_STREAM,
|
|
@@ -1345,26 +1606,6 @@ function watchStreamDeadline(client, deadlineAtMs, remainingMs, expire, fail) {
|
|
|
1345
1606
|
clearTimeout(timer);
|
|
1346
1607
|
};
|
|
1347
1608
|
}
|
|
1348
|
-
var SCHEDULE_RETRY = `
|
|
1349
|
-
local pending = redis.call('XPENDING', KEYS[1], ARGV[1], ARGV[2], ARGV[2], 1)
|
|
1350
|
-
if #pending == 0 or pending[1][2] ~= ARGV[3] or tonumber(pending[1][3]) >= tonumber(ARGV[4]) then return 0 end
|
|
1351
|
-
local kind = redis.call('TYPE', KEYS[2]).ok
|
|
1352
|
-
if kind ~= 'none' and kind ~= 'zset' then return redis.error_reply('Invalid stream retry key type') end
|
|
1353
|
-
if ARGV[8] == '1' then
|
|
1354
|
-
local state = redis.call('HGET', KEYS[3], 'state')
|
|
1355
|
-
if state == 'cancelled' then return redis.error_reply('Streams job cancelled') end
|
|
1356
|
-
if state ~= 'active' then return redis.error_reply('Missing or invalid Streams control state') end
|
|
1357
|
-
end
|
|
1358
|
-
local now = redis.call('TIME')
|
|
1359
|
-
local due = tonumber(now[1])*1000 + math.floor(tonumber(now[2])/1000) + tonumber(ARGV[5])
|
|
1360
|
-
if ARGV[7] ~= '' then due = math.min(due, tonumber(ARGV[7])) end
|
|
1361
|
-
redis.call('ZADD', KEYS[2], due, ARGV[6])
|
|
1362
|
-
if ARGV[8] == '1' then redis.call('HSET', KEYS[3], 'retryPayload', ARGV[6]) end
|
|
1363
|
-
redis.call('XACK', KEYS[1], ARGV[1], ARGV[2])
|
|
1364
|
-
redis.call('XDEL', KEYS[1], ARGV[2])
|
|
1365
|
-
if #redis.call('XPENDING', KEYS[1], ARGV[1], '-', '+', 1, ARGV[3]) == 0 then redis.call('XGROUP', 'DELCONSUMER', KEYS[1], ARGV[1], ARGV[3]) end
|
|
1366
|
-
return 1
|
|
1367
|
-
`;
|
|
1368
1609
|
async function scheduleStreamRetry(client, key, reservation, envelope, delayMs) {
|
|
1369
1610
|
if (!Number.isSafeInteger(delayMs) || delayMs < 0 || delayMs > Number.MAX_SAFE_INTEGER / 2)
|
|
1370
1611
|
throw new Error("Invalid Streams retry delay");
|
|
@@ -1386,59 +1627,6 @@ async function scheduleStreamRetry(client, key, reservation, envelope, delayMs)
|
|
|
1386
1627
|
if (Number(scheduled) !== 1)
|
|
1387
1628
|
throw new Error("Queue lease lost while scheduling retry");
|
|
1388
1629
|
}
|
|
1389
|
-
var POLL_STREAM_PRIORITIES = `
|
|
1390
|
-
local group = ARGV[1]
|
|
1391
|
-
local owner = ARGV[2]
|
|
1392
|
-
local visibility = ARGV[3]
|
|
1393
|
-
local function payloadOf(fields)
|
|
1394
|
-
for index = 1, #fields, 2 do
|
|
1395
|
-
if fields[index] == 'payload' and type(fields[index + 1]) == 'string' then return fields[index + 1] end
|
|
1396
|
-
end
|
|
1397
|
-
return ''
|
|
1398
|
-
end
|
|
1399
|
-
local cursors = {}
|
|
1400
|
-
for i = 1, #KEYS / 2 do cursors[i] = ARGV[3 + i] end
|
|
1401
|
-
for i = 1, #KEYS / 2 do
|
|
1402
|
-
local stream = KEYS[(i - 1) * 2 + 1]
|
|
1403
|
-
local retries = KEYS[(i - 1) * 2 + 2]
|
|
1404
|
-
local cursor = ARGV[3 + i]
|
|
1405
|
-
local streamKind = redis.call('TYPE', stream).ok
|
|
1406
|
-
local retryKind = redis.call('TYPE', retries).ok
|
|
1407
|
-
if streamKind ~= 'none' and streamKind ~= 'stream' then return redis.error_reply('Invalid stream retry promotion key type') end
|
|
1408
|
-
if retryKind ~= 'none' and retryKind ~= 'zset' then return redis.error_reply('Invalid stream retry promotion key type') end
|
|
1409
|
-
if streamKind == 'stream' then
|
|
1410
|
-
local now = redis.call('TIME')
|
|
1411
|
-
local due = tonumber(now[1]) * 1000 + math.floor(tonumber(now[2]) / 1000)
|
|
1412
|
-
local ready = redis.call('ZRANGE', retries, '-inf', due, 'BYSCORE', 'LIMIT', 0, 100)
|
|
1413
|
-
for _, payload in ipairs(ready) do
|
|
1414
|
-
redis.call('XADD', stream, '*', 'payload', payload)
|
|
1415
|
-
redis.call('ZREM', retries, payload)
|
|
1416
|
-
end
|
|
1417
|
-
local window = redis.call('XPENDING', stream, group, cursor, '+', 10)
|
|
1418
|
-
local claimed = redis.call('XAUTOCLAIM', stream, group, owner, visibility, cursor, 'COUNT', 1)
|
|
1419
|
-
for _, entry in ipairs(window) do
|
|
1420
|
-
if entry[2] ~= owner and #redis.call('XPENDING', stream, group, '-', '+', 1, entry[2]) == 0 then
|
|
1421
|
-
redis.call('XGROUP', 'DELCONSUMER', stream, group, entry[2])
|
|
1422
|
-
end
|
|
1423
|
-
end
|
|
1424
|
-
if #redis.call('XPENDING', stream, group, '-', '+', 1, owner) == 0 then
|
|
1425
|
-
redis.call('XGROUP', 'DELCONSUMER', stream, group, owner)
|
|
1426
|
-
end
|
|
1427
|
-
cursors[i] = claimed[1]
|
|
1428
|
-
if #claimed[2] > 0 then
|
|
1429
|
-
return {'1', tostring(i), claimed[2][1][1], payloadOf(claimed[2][1][2]), unpack(cursors)}
|
|
1430
|
-
end
|
|
1431
|
-
local fresh = redis.call('XREADGROUP', 'GROUP', group, owner, 'COUNT', 1, 'STREAMS', stream, '>')
|
|
1432
|
-
if not fresh and #redis.call('XPENDING', stream, group, '-', '+', 1, owner) == 0 then
|
|
1433
|
-
redis.call('XGROUP', 'DELCONSUMER', stream, group, owner)
|
|
1434
|
-
end
|
|
1435
|
-
if fresh and #fresh > 0 and #fresh[1][2] > 0 then
|
|
1436
|
-
return {'1', tostring(i), fresh[1][2][1][1], payloadOf(fresh[1][2][1][2]), unpack(cursors)}
|
|
1437
|
-
end
|
|
1438
|
-
end
|
|
1439
|
-
end
|
|
1440
|
-
return {'0', unpack(cursors)}
|
|
1441
|
-
`;
|
|
1442
1630
|
async function pollStreamPriorities(client, queueKeys, owner, cursors) {
|
|
1443
1631
|
const keys = [];
|
|
1444
1632
|
for (const key of queueKeys)
|
|
@@ -1482,12 +1670,6 @@ async function readStreamQueueDepth(client, key) {
|
|
|
1482
1670
|
streamRetryKey(key)
|
|
1483
1671
|
]));
|
|
1484
1672
|
}
|
|
1485
|
-
var ASSERT_DRAINED = `
|
|
1486
|
-
if redis.call('LLEN', KEYS[1]) > 0 or redis.call('LLEN', KEYS[2]) > 0 or redis.call('HLEN', KEYS[3]) > 0 then
|
|
1487
|
-
return redis.error_reply('Legacy queue state requires maintenance conversion before Streams activation')
|
|
1488
|
-
end
|
|
1489
|
-
return 1
|
|
1490
|
-
`;
|
|
1491
1673
|
async function assertLegacyDrained(client, key) {
|
|
1492
1674
|
await client.send("EVAL", [
|
|
1493
1675
|
ASSERT_DRAINED,
|
|
@@ -1497,30 +1679,6 @@ async function assertLegacyDrained(client, key) {
|
|
|
1497
1679
|
`${key}:processing:leases`
|
|
1498
1680
|
]);
|
|
1499
1681
|
}
|
|
1500
|
-
var UPDATE_STREAM = `
|
|
1501
|
-
local pending = redis.call('XPENDING', KEYS[1], ARGV[1], ARGV[2], ARGV[2], 1)
|
|
1502
|
-
if #pending == 0 or pending[1][2] ~= ARGV[3] or tonumber(pending[1][3]) >= tonumber(ARGV[4]) then return 0 end
|
|
1503
|
-
if ARGV[7] == '1' then
|
|
1504
|
-
local kind = redis.call('TYPE', KEYS[3]).ok
|
|
1505
|
-
if kind ~= 'none' and kind ~= 'hash' then return redis.error_reply('Invalid stream control key type') end
|
|
1506
|
-
end
|
|
1507
|
-
if ARGV[5] == 'renew' then
|
|
1508
|
-
local claimed = redis.call('XCLAIM', KEYS[1], ARGV[1], ARGV[3], 0, ARGV[2], 'JUSTID')
|
|
1509
|
-
return #claimed
|
|
1510
|
-
end
|
|
1511
|
-
if ARGV[5] == 'quarantine' then
|
|
1512
|
-
local kind = redis.call('TYPE', KEYS[2]).ok
|
|
1513
|
-
if kind ~= 'none' and kind ~= 'stream' then return redis.error_reply('Invalid stream quarantine key type') end
|
|
1514
|
-
redis.call('XADD', KEYS[2], '*', 'sourceId', ARGV[2], 'payload', ARGV[6])
|
|
1515
|
-
end
|
|
1516
|
-
local acknowledged = redis.call('XACK', KEYS[1], ARGV[1], ARGV[2])
|
|
1517
|
-
if acknowledged == 1 then
|
|
1518
|
-
redis.call('XDEL', KEYS[1], ARGV[2])
|
|
1519
|
-
if ARGV[7] == '1' then redis.call('DEL', KEYS[3]) end
|
|
1520
|
-
end
|
|
1521
|
-
if #redis.call('XPENDING', KEYS[1], ARGV[1], '-', '+', 1, ARGV[3]) == 0 then redis.call('XGROUP', 'DELCONSUMER', KEYS[1], ARGV[1], ARGV[3]) end
|
|
1522
|
-
return acknowledged
|
|
1523
|
-
`;
|
|
1524
1682
|
async function ensureStreamGroup(client, key) {
|
|
1525
1683
|
await assertLegacyDrained(client, key);
|
|
1526
1684
|
try {
|
|
@@ -1830,23 +1988,6 @@ class RedisStreamsWorker {
|
|
|
1830
1988
|
}
|
|
1831
1989
|
}
|
|
1832
1990
|
}
|
|
1833
|
-
var CONVERT_LIST = `
|
|
1834
|
-
local kinds = {'list','list','hash','stream'}
|
|
1835
|
-
for i=1,4 do
|
|
1836
|
-
local kind = redis.call('TYPE', KEYS[i]).ok
|
|
1837
|
-
if kind ~= 'none' and kind ~= kinds[i] then return redis.error_reply('Invalid queue conversion key type') end
|
|
1838
|
-
end
|
|
1839
|
-
if redis.call('LLEN', KEYS[2]) > 0 or redis.call('HLEN', KEYS[3]) > 0 then return redis.error_reply('Drain or recover processing reservations before queue conversion') end
|
|
1840
|
-
local converted = 0
|
|
1841
|
-
for i=1,tonumber(ARGV[1]) do
|
|
1842
|
-
local payload = redis.call('LINDEX', KEYS[1], -1)
|
|
1843
|
-
if not payload then break end
|
|
1844
|
-
redis.call('XADD', KEYS[4], '*', 'payload', payload)
|
|
1845
|
-
redis.call('RPOP', KEYS[1])
|
|
1846
|
-
converted = converted + 1
|
|
1847
|
-
end
|
|
1848
|
-
return converted
|
|
1849
|
-
`;
|
|
1850
1991
|
async function migrateLegacyQueueToStreams(client, options) {
|
|
1851
1992
|
if (options.maintenance !== true)
|
|
1852
1993
|
throw new Error("Queue conversion requires maintenance mode: stop all producers and workers first.");
|
|
@@ -1884,141 +2025,6 @@ function reservationExpired(lease, now, visibilityMs) {
|
|
|
1884
2025
|
}
|
|
1885
2026
|
return now - reservedAt >= visibilityMs;
|
|
1886
2027
|
}
|
|
1887
|
-
var SCRIPT_HEADER = `
|
|
1888
|
-
local function checkType(key, expected)
|
|
1889
|
-
local actual = redis.call('TYPE', key).ok
|
|
1890
|
-
if actual ~= 'none' and actual ~= expected then
|
|
1891
|
-
return redis.error_reply('Unexpected queue key type: ' .. key)
|
|
1892
|
-
end
|
|
1893
|
-
end
|
|
1894
|
-
local function nowMs()
|
|
1895
|
-
local time = redis.call('TIME')
|
|
1896
|
-
return tonumber(time[1]) * 1000 + math.floor(tonumber(time[2]) / 1000)
|
|
1897
|
-
end
|
|
1898
|
-
`;
|
|
1899
|
-
var RESERVE_JOB = `${SCRIPT_HEADER}
|
|
1900
|
-
local err = checkType(KEYS[1], 'list') or checkType(KEYS[2], 'list') or checkType(KEYS[3], 'hash')
|
|
1901
|
-
if err then return err end
|
|
1902
|
-
local payload = redis.call('LINDEX', KEYS[1], -1)
|
|
1903
|
-
if not payload then return nil end
|
|
1904
|
-
local record = cjson.encode({strataReservation=1, id=ARGV[1], owner=ARGV[2], payload=payload})
|
|
1905
|
-
local lease = cjson.encode({owner=ARGV[2], expiresAt=nowMs()+tonumber(ARGV[3])})
|
|
1906
|
-
redis.call('LPUSH', KEYS[2], record)
|
|
1907
|
-
redis.call('HSET', KEYS[3], ARGV[1], lease)
|
|
1908
|
-
redis.call('RPOP', KEYS[1])
|
|
1909
|
-
return record
|
|
1910
|
-
`;
|
|
1911
|
-
var RENEW_OR_ACK = `${SCRIPT_HEADER}
|
|
1912
|
-
local err = checkType(KEYS[1], 'list') or checkType(KEYS[2], 'hash') or checkType(KEYS[3], 'list')
|
|
1913
|
-
if err then return err end
|
|
1914
|
-
local raw = redis.call('HGET', KEYS[2], ARGV[1])
|
|
1915
|
-
if not raw then return 0 end
|
|
1916
|
-
local ok, lease = pcall(cjson.decode, raw)
|
|
1917
|
-
local now = nowMs()
|
|
1918
|
-
if not ok or type(lease) ~= 'table' or lease.owner ~= ARGV[2] or not tonumber(lease.expiresAt) or tonumber(lease.expiresAt) <= now then
|
|
1919
|
-
return 0
|
|
1920
|
-
end
|
|
1921
|
-
if ARGV[4] == 'renew' then
|
|
1922
|
-
redis.call('HSET', KEYS[2], ARGV[1], cjson.encode({owner=ARGV[2], expiresAt=now+tonumber(ARGV[5])}))
|
|
1923
|
-
return 1
|
|
1924
|
-
end
|
|
1925
|
-
if ARGV[4] == 'quarantine' then redis.call('LPUSH', KEYS[3], ARGV[6]) end
|
|
1926
|
-
local removed = redis.call('LREM', KEYS[1], 1, ARGV[3])
|
|
1927
|
-
redis.call('HDEL', KEYS[2], ARGV[1])
|
|
1928
|
-
return removed
|
|
1929
|
-
`;
|
|
1930
|
-
var RECLAIM_JOBS = `${SCRIPT_HEADER}
|
|
1931
|
-
local err = checkType(KEYS[1], 'list') or checkType(KEYS[2], 'list') or checkType(KEYS[3], 'hash')
|
|
1932
|
-
if err then return err end
|
|
1933
|
-
local now = nowMs()
|
|
1934
|
-
local reclaimed = 0
|
|
1935
|
-
for _, record in ipairs(redis.call('LRANGE', KEYS[2], 0, -1)) do
|
|
1936
|
-
local payload, field = record, record
|
|
1937
|
-
local modern = false
|
|
1938
|
-
local ok, reservation = pcall(cjson.decode, record)
|
|
1939
|
-
if ok and type(reservation) == 'table' and reservation.strataReservation == 1 then
|
|
1940
|
-
payload, field, modern = reservation.payload, reservation.id, true
|
|
1941
|
-
end
|
|
1942
|
-
local raw = redis.call('HGET', KEYS[3], field)
|
|
1943
|
-
local expired = true
|
|
1944
|
-
if raw then
|
|
1945
|
-
if modern then
|
|
1946
|
-
local leaseOk, lease = pcall(cjson.decode, raw)
|
|
1947
|
-
expired = not leaseOk or type(lease) ~= 'table' or not tonumber(lease.expiresAt) or tonumber(lease.expiresAt) <= now
|
|
1948
|
-
else
|
|
1949
|
-
local reservedAt = tonumber(raw)
|
|
1950
|
-
expired = not reservedAt or now - reservedAt >= tonumber(ARGV[1])
|
|
1951
|
-
end
|
|
1952
|
-
end
|
|
1953
|
-
if expired then
|
|
1954
|
-
redis.call('LPUSH', KEYS[1], payload)
|
|
1955
|
-
redis.call('LREM', KEYS[2], 1, record)
|
|
1956
|
-
redis.call('HDEL', KEYS[3], field)
|
|
1957
|
-
reclaimed = reclaimed + 1
|
|
1958
|
-
end
|
|
1959
|
-
end
|
|
1960
|
-
return reclaimed
|
|
1961
|
-
`;
|
|
1962
|
-
var POLL_LIST_PRIORITIES = `${SCRIPT_HEADER}
|
|
1963
|
-
local owner = ARGV[1]
|
|
1964
|
-
local visibility = ARGV[2]
|
|
1965
|
-
local reservationId = ARGV[3]
|
|
1966
|
-
local reclaim = ARGV[4] == '1'
|
|
1967
|
-
local function reclaimQueue(listKey, processingKey, leaseKey)
|
|
1968
|
-
local err = checkType(listKey, 'list') or checkType(processingKey, 'list') or checkType(leaseKey, 'hash')
|
|
1969
|
-
if err then return err end
|
|
1970
|
-
local now = nowMs()
|
|
1971
|
-
for _, record in ipairs(redis.call('LRANGE', processingKey, 0, -1)) do
|
|
1972
|
-
local payload, field = record, record
|
|
1973
|
-
local modern = false
|
|
1974
|
-
local ok, reservation = pcall(cjson.decode, record)
|
|
1975
|
-
if ok and type(reservation) == 'table' and reservation.strataReservation == 1 then
|
|
1976
|
-
payload, field, modern = reservation.payload, reservation.id, true
|
|
1977
|
-
end
|
|
1978
|
-
local raw = redis.call('HGET', leaseKey, field)
|
|
1979
|
-
local expired = true
|
|
1980
|
-
if raw then
|
|
1981
|
-
if modern then
|
|
1982
|
-
local leaseOk, lease = pcall(cjson.decode, raw)
|
|
1983
|
-
expired = not leaseOk or type(lease) ~= 'table' or not tonumber(lease.expiresAt) or tonumber(lease.expiresAt) <= now
|
|
1984
|
-
else
|
|
1985
|
-
local reservedAt = tonumber(raw)
|
|
1986
|
-
expired = not reservedAt or now - reservedAt >= tonumber(visibility)
|
|
1987
|
-
end
|
|
1988
|
-
end
|
|
1989
|
-
if expired then
|
|
1990
|
-
redis.call('LPUSH', listKey, payload)
|
|
1991
|
-
redis.call('LREM', processingKey, 1, record)
|
|
1992
|
-
redis.call('HDEL', leaseKey, field)
|
|
1993
|
-
end
|
|
1994
|
-
end
|
|
1995
|
-
end
|
|
1996
|
-
local function reserveQueue(listKey, processingKey, leaseKey)
|
|
1997
|
-
local err = checkType(listKey, 'list') or checkType(processingKey, 'list') or checkType(leaseKey, 'hash')
|
|
1998
|
-
if err then return err end
|
|
1999
|
-
local payload = redis.call('LINDEX', listKey, -1)
|
|
2000
|
-
if not payload then return nil end
|
|
2001
|
-
local record = cjson.encode({strataReservation=1, id=reservationId, owner=owner, payload=payload})
|
|
2002
|
-
local lease = cjson.encode({owner=owner, expiresAt=nowMs()+tonumber(visibility)})
|
|
2003
|
-
redis.call('LPUSH', processingKey, record)
|
|
2004
|
-
redis.call('HSET', leaseKey, reservationId, lease)
|
|
2005
|
-
redis.call('RPOP', listKey)
|
|
2006
|
-
return record
|
|
2007
|
-
end
|
|
2008
|
-
local count = #KEYS / 3
|
|
2009
|
-
if reclaim then
|
|
2010
|
-
for i = 1, count do
|
|
2011
|
-
local err = reclaimQueue(KEYS[(i - 1) * 3 + 1], KEYS[(i - 1) * 3 + 2], KEYS[(i - 1) * 3 + 3])
|
|
2012
|
-
if err then return err end
|
|
2013
|
-
end
|
|
2014
|
-
end
|
|
2015
|
-
for i = 1, count do
|
|
2016
|
-
local record = reserveQueue(KEYS[(i - 1) * 3 + 1], KEYS[(i - 1) * 3 + 2], KEYS[(i - 1) * 3 + 3])
|
|
2017
|
-
if type(record) == 'string' then return {'1', tostring(i), record} end
|
|
2018
|
-
if record then return record end
|
|
2019
|
-
end
|
|
2020
|
-
return {'0'}
|
|
2021
|
-
`;
|
|
2022
2028
|
function queueInvalidKey(queueKey) {
|
|
2023
2029
|
return `${queueKey}:invalid`;
|
|
2024
2030
|
}
|