@getstrata/core 2.2.14 → 2.2.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +39 -14
- package/dist/core/database/baseRepository.d.ts +1 -0
- package/dist/core/database/defaultConnection.d.ts +2 -1
- package/dist/core/database/relationQuery.d.ts +4 -0
- package/dist/core/queue/redisQueue.d.ts +5 -1
- package/dist/core/queue/redisStreams.d.ts +8 -2
- package/dist/entries/queue/createAppQueue.js +202 -92
- package/dist/entries/queue/publicQueue.js +202 -92
- package/dist/entries/queue/queueMetrics.js +202 -92
- package/dist/entries/queue/redisQueue.js +203 -92
- package/dist/framework/public-api.d.ts +1 -1
- package/dist/index.js +357 -143
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -125,6 +125,10 @@ function getDefaultDatabaseQuery() {
|
|
|
125
125
|
}
|
|
126
126
|
return query;
|
|
127
127
|
}
|
|
128
|
+
function isDefaultDatabaseConnection(connection) {
|
|
129
|
+
const { pool, query } = defaultConnectionState();
|
|
130
|
+
return connection !== null && (connection === pool || connection === query);
|
|
131
|
+
}
|
|
128
132
|
function resetDefaultDatabasePoolForTests() {
|
|
129
133
|
const holder = defaultConnectionState();
|
|
130
134
|
holder.pool = null;
|
|
@@ -5781,6 +5785,31 @@ class RepositoryQuery {
|
|
|
5781
5785
|
// ../../src/core/database/baseRepository.ts
|
|
5782
5786
|
init_transaction();
|
|
5783
5787
|
init_transactionControl();
|
|
5788
|
+
function listOrders(orderBy) {
|
|
5789
|
+
if (!orderBy)
|
|
5790
|
+
return [];
|
|
5791
|
+
if (Array.isArray(orderBy))
|
|
5792
|
+
return orderBy.map(readOrder);
|
|
5793
|
+
if ("column" in orderBy)
|
|
5794
|
+
return [readOrder(orderBy)];
|
|
5795
|
+
return Object.entries(orderBy).flatMap(([column, direction]) => {
|
|
5796
|
+
if (direction === undefined)
|
|
5797
|
+
return [];
|
|
5798
|
+
return [
|
|
5799
|
+
{
|
|
5800
|
+
column,
|
|
5801
|
+
direction: String(direction).toUpperCase() === "DESC" ? "desc" : "asc"
|
|
5802
|
+
}
|
|
5803
|
+
];
|
|
5804
|
+
});
|
|
5805
|
+
}
|
|
5806
|
+
function readOrder(order) {
|
|
5807
|
+
return {
|
|
5808
|
+
column: String(order.column),
|
|
5809
|
+
direction: String(order.direction).toUpperCase() === "DESC" ? "desc" : "asc",
|
|
5810
|
+
...order.nulls === undefined ? {} : { nulls: order.nulls }
|
|
5811
|
+
};
|
|
5812
|
+
}
|
|
5784
5813
|
|
|
5785
5814
|
class BaseRepository {
|
|
5786
5815
|
table;
|
|
@@ -5824,6 +5853,10 @@ class BaseRepository {
|
|
|
5824
5853
|
joins: options.joins,
|
|
5825
5854
|
groupBy: options.groupBy
|
|
5826
5855
|
}, whereNodes);
|
|
5856
|
+
const meta = buildPaginationMeta({ page, perPage, total });
|
|
5857
|
+
if (total === 0) {
|
|
5858
|
+
return { data: [], meta };
|
|
5859
|
+
}
|
|
5827
5860
|
const offset = (page - 1) * perPage;
|
|
5828
5861
|
const data = await this.findAll({
|
|
5829
5862
|
...queryOptions,
|
|
@@ -5834,13 +5867,19 @@ class BaseRepository {
|
|
|
5834
5867
|
});
|
|
5835
5868
|
return {
|
|
5836
5869
|
data,
|
|
5837
|
-
meta
|
|
5870
|
+
meta
|
|
5838
5871
|
};
|
|
5839
5872
|
}
|
|
5840
5873
|
async chunk(count, callback, options = {}) {
|
|
5841
5874
|
if (!Number.isInteger(count) || count <= 0) {
|
|
5842
5875
|
throw new Error("Chunk size must be a positive integer.");
|
|
5843
5876
|
}
|
|
5877
|
+
if (this.primaryKeySeekDirection(options) === "asc") {
|
|
5878
|
+
const seekOptions = { ...options };
|
|
5879
|
+
delete seekOptions.orderBy;
|
|
5880
|
+
await this.chunkById(count, async (rows) => await callback(rows) === false ? false : undefined, seekOptions);
|
|
5881
|
+
return;
|
|
5882
|
+
}
|
|
5844
5883
|
let offset = 0;
|
|
5845
5884
|
while (true) {
|
|
5846
5885
|
const rows = await this.findAll({
|
|
@@ -6125,6 +6164,17 @@ class BaseRepository {
|
|
|
6125
6164
|
return entity;
|
|
6126
6165
|
}));
|
|
6127
6166
|
}
|
|
6167
|
+
primaryKeySeekDirection(options) {
|
|
6168
|
+
if (options.joins?.length || options.groupBy || options.having || options.select?.length || options.offset !== undefined || options.limit !== undefined || options.lock) {
|
|
6169
|
+
return null;
|
|
6170
|
+
}
|
|
6171
|
+
const orders = listOrders(options.orderBy ?? this.table.defaultOrderBy);
|
|
6172
|
+
const order = orders[0];
|
|
6173
|
+
if (orders.length !== 1 || !order || order.nulls || order.column !== this.table.primaryKey) {
|
|
6174
|
+
return null;
|
|
6175
|
+
}
|
|
6176
|
+
return order.direction;
|
|
6177
|
+
}
|
|
6128
6178
|
withConnection(connection) {
|
|
6129
6179
|
const clone = Object.create(Object.getPrototypeOf(this));
|
|
6130
6180
|
Object.assign(clone, this);
|
|
@@ -7599,7 +7649,12 @@ import { isDeepStrictEqual } from "util";
|
|
|
7599
7649
|
init_deferredModelEvents();
|
|
7600
7650
|
|
|
7601
7651
|
// ../../src/core/database/relationQuery.ts
|
|
7652
|
+
init_boundConnection();
|
|
7653
|
+
init_connectionContext();
|
|
7654
|
+
init_defaultConnection();
|
|
7602
7655
|
init_dialect();
|
|
7656
|
+
init_repositoryConnection();
|
|
7657
|
+
init_transaction();
|
|
7603
7658
|
function asWhere(where) {
|
|
7604
7659
|
return where;
|
|
7605
7660
|
}
|
|
@@ -7869,6 +7924,26 @@ class BelongsToRelationQuery {
|
|
|
7869
7924
|
});
|
|
7870
7925
|
}
|
|
7871
7926
|
}
|
|
7927
|
+
function pivotIdKey(id) {
|
|
7928
|
+
return typeof id === "number" || typeof id === "bigint" ? String(id) : id;
|
|
7929
|
+
}
|
|
7930
|
+
function hasRepeatedId(ids) {
|
|
7931
|
+
const seen = new Set;
|
|
7932
|
+
for (const id of ids) {
|
|
7933
|
+
const key = pivotIdKey(id);
|
|
7934
|
+
if (seen.has(key))
|
|
7935
|
+
return true;
|
|
7936
|
+
seen.add(key);
|
|
7937
|
+
}
|
|
7938
|
+
return false;
|
|
7939
|
+
}
|
|
7940
|
+
function canBatchPivotToggle(ids) {
|
|
7941
|
+
return ids.every((id) => typeof id === "bigint" && id >= BigInt(Number.MIN_SAFE_INTEGER) && id <= BigInt(Number.MAX_SAFE_INTEGER) || typeof id === "number" && Number.isSafeInteger(id) || typeof id === "string" && /^(?:0|-?[1-9]\d*)$/.test(id) && Number.isSafeInteger(Number(id)));
|
|
7942
|
+
}
|
|
7943
|
+
function isConfiguredConnection(connection) {
|
|
7944
|
+
const bound = getBoundDatabaseConnection();
|
|
7945
|
+
return connection === repositoryConnection || (bound ? connection === bound : isDefaultDatabaseConnection(connection));
|
|
7946
|
+
}
|
|
7872
7947
|
|
|
7873
7948
|
class BelongsToManyRelationQuery {
|
|
7874
7949
|
parent;
|
|
@@ -7910,7 +7985,20 @@ class BelongsToManyRelationQuery {
|
|
|
7910
7985
|
return { sql, params: extra.params };
|
|
7911
7986
|
}
|
|
7912
7987
|
connection() {
|
|
7913
|
-
|
|
7988
|
+
const connection = this.parent.getRepository().getConnection();
|
|
7989
|
+
return isConfiguredConnection(connection) ? getActiveDatabaseConnection(connection) : connection;
|
|
7990
|
+
}
|
|
7991
|
+
async atomically(work) {
|
|
7992
|
+
const connection = this.connection();
|
|
7993
|
+
if (isConfiguredConnection(connection) || hasActiveDatabaseConnection() && connection === getActiveDatabaseConnection(connection)) {
|
|
7994
|
+
await runInTransaction(work);
|
|
7995
|
+
return;
|
|
7996
|
+
}
|
|
7997
|
+
if (typeof connection.begin !== "function") {
|
|
7998
|
+
await work(connection);
|
|
7999
|
+
return;
|
|
8000
|
+
}
|
|
8001
|
+
await connection.begin(work);
|
|
7914
8002
|
}
|
|
7915
8003
|
async get() {
|
|
7916
8004
|
const query = await this.relatedQuery();
|
|
@@ -7948,36 +8036,72 @@ class BelongsToManyRelationQuery {
|
|
|
7948
8036
|
}
|
|
7949
8037
|
async attach(ids) {
|
|
7950
8038
|
const list = Array.isArray(ids) ? ids : [ids];
|
|
7951
|
-
|
|
7952
|
-
|
|
7953
|
-
|
|
7954
|
-
const extraPlaceholders = extraKeys.map((_, index) => `$${index + 3}`).join(", ");
|
|
7955
|
-
const extraValues = extraKeys.map((key) => this.pivotValues[key]);
|
|
7956
|
-
for (const id of list) {
|
|
7957
|
-
await this.connection().unsafe(extraKeys.length > 0 ? `INSERT INTO ${quoteIdentifier(this.relation.pivotTable)} (${quoteIdentifier(String(this.relation.foreignPivotKey))}, ${quoteIdentifier(String(this.relation.relatedPivotKey))}${extraColumns}) VALUES ($1, $2, ${extraPlaceholders})` : `INSERT INTO ${quoteIdentifier(this.relation.pivotTable)} (${quoteIdentifier(String(this.relation.foreignPivotKey))}, ${quoteIdentifier(String(this.relation.relatedPivotKey))}) VALUES ($1, $2)`, [parentId, id, ...extraValues]);
|
|
7958
|
-
}
|
|
8039
|
+
if (list.length === 0)
|
|
8040
|
+
return;
|
|
8041
|
+
await this.atomically((connection) => this.insertPivotRows(connection, list));
|
|
7959
8042
|
}
|
|
7960
8043
|
async toggle(ids) {
|
|
7961
8044
|
const list = Array.isArray(ids) ? ids : [ids];
|
|
7962
|
-
|
|
7963
|
-
|
|
7964
|
-
|
|
7965
|
-
if (
|
|
7966
|
-
|
|
7967
|
-
|
|
7968
|
-
|
|
8045
|
+
if (list.length === 0)
|
|
8046
|
+
return;
|
|
8047
|
+
await this.atomically(async (connection) => {
|
|
8048
|
+
if (!canBatchPivotToggle(list) || hasRepeatedId(list)) {
|
|
8049
|
+
for (const id of list)
|
|
8050
|
+
await this.toggleOne(connection, id);
|
|
8051
|
+
return;
|
|
7969
8052
|
}
|
|
7970
|
-
|
|
8053
|
+
const parentId = this.parent.get(this.relation.parentKey);
|
|
8054
|
+
const relatedColumn = quoteIdentifier(String(this.relation.relatedPivotKey));
|
|
8055
|
+
const parentColumn = quoteIdentifier(String(this.relation.foreignPivotKey));
|
|
8056
|
+
const placeholders = list.map((_, index) => currentSqlDialect().placeholder(index + 2));
|
|
8057
|
+
const existing = await connection.unsafe(`SELECT ${relatedColumn} FROM ${quoteIdentifier(this.relation.pivotTable)} WHERE ${parentColumn} = ${currentSqlDialect().placeholder(1)} AND ${relatedColumn} IN (${placeholders.join(", ")})`, [parentId, ...list]);
|
|
8058
|
+
const existingIds = new Set(existing.map((row) => pivotIdKey(row[String(this.relation.relatedPivotKey)])));
|
|
8059
|
+
const present = list.filter((id) => existingIds.has(pivotIdKey(id)));
|
|
8060
|
+
const missing = list.filter((id) => !existingIds.has(pivotIdKey(id)));
|
|
8061
|
+
if (present.length > 0)
|
|
8062
|
+
await this.deletePivotRows(connection, present);
|
|
8063
|
+
if (missing.length > 0)
|
|
8064
|
+
await this.insertPivotRows(connection, missing);
|
|
8065
|
+
});
|
|
7971
8066
|
}
|
|
7972
8067
|
async detach(ids) {
|
|
7973
8068
|
const parentId = this.parent.get(this.relation.parentKey);
|
|
7974
8069
|
if (ids === undefined) {
|
|
7975
|
-
await this.connection().unsafe(`DELETE FROM ${quoteIdentifier(this.relation.pivotTable)} WHERE ${quoteIdentifier(String(this.relation.foreignPivotKey))} = $1`, [parentId]);
|
|
8070
|
+
await this.connection().unsafe(`DELETE FROM ${quoteIdentifier(this.relation.pivotTable)} WHERE ${quoteIdentifier(String(this.relation.foreignPivotKey))} = ${currentSqlDialect().placeholder(1)}`, [parentId]);
|
|
7976
8071
|
return;
|
|
7977
8072
|
}
|
|
7978
8073
|
const list = Array.isArray(ids) ? ids : [ids];
|
|
7979
|
-
|
|
7980
|
-
|
|
8074
|
+
if (list.length === 0)
|
|
8075
|
+
return;
|
|
8076
|
+
await this.deletePivotRows(this.connection(), list);
|
|
8077
|
+
}
|
|
8078
|
+
async toggleOne(connection, id) {
|
|
8079
|
+
const parentId = this.parent.get(this.relation.parentKey);
|
|
8080
|
+
const existing = await connection.unsafe(`SELECT 1 FROM ${quoteIdentifier(this.relation.pivotTable)} WHERE ${quoteIdentifier(String(this.relation.foreignPivotKey))} = ${currentSqlDialect().placeholder(1)} AND ${quoteIdentifier(String(this.relation.relatedPivotKey))} = ${currentSqlDialect().placeholder(2)} LIMIT 1`, [parentId, id]);
|
|
8081
|
+
if (existing.length > 0)
|
|
8082
|
+
await this.deletePivotRows(connection, [id]);
|
|
8083
|
+
else
|
|
8084
|
+
await this.insertPivotRows(connection, [id]);
|
|
8085
|
+
}
|
|
8086
|
+
async insertPivotRows(connection, list) {
|
|
8087
|
+
const parentId = this.parent.get(this.relation.parentKey);
|
|
8088
|
+
const extraKeys = Object.keys(this.pivotValues);
|
|
8089
|
+
const extraColumns = extraKeys.length > 0 ? `, ${extraKeys.map((key) => quoteIdentifier(key)).join(", ")}` : "";
|
|
8090
|
+
const extraValues = extraKeys.map((key) => this.pivotValues[key]);
|
|
8091
|
+
const width = 2 + extraKeys.length;
|
|
8092
|
+
const tuples = list.map((_, row) => {
|
|
8093
|
+
const slots = Array.from({ length: width }, (__, column) => currentSqlDialect().placeholder(row * width + column + 1));
|
|
8094
|
+
return `(${slots.join(", ")})`;
|
|
8095
|
+
});
|
|
8096
|
+
const params = [];
|
|
8097
|
+
for (const id of list)
|
|
8098
|
+
params.push(parentId, id, ...extraValues);
|
|
8099
|
+
await connection.unsafe(`INSERT INTO ${quoteIdentifier(this.relation.pivotTable)} (${quoteIdentifier(String(this.relation.foreignPivotKey))}, ${quoteIdentifier(String(this.relation.relatedPivotKey))}${extraColumns}) VALUES ${tuples.join(", ")}`, params);
|
|
8100
|
+
}
|
|
8101
|
+
async deletePivotRows(connection, list) {
|
|
8102
|
+
const parentId = this.parent.get(this.relation.parentKey);
|
|
8103
|
+
const placeholders = list.map((_, index) => currentSqlDialect().placeholder(index + 2));
|
|
8104
|
+
await connection.unsafe(`DELETE FROM ${quoteIdentifier(this.relation.pivotTable)} WHERE ${quoteIdentifier(String(this.relation.foreignPivotKey))} = ${currentSqlDialect().placeholder(1)} AND ${quoteIdentifier(String(this.relation.relatedPivotKey))} IN (${placeholders.join(", ")})`, [parentId, ...list]);
|
|
7981
8105
|
}
|
|
7982
8106
|
async sync(ids) {
|
|
7983
8107
|
await this.detach();
|
|
@@ -8568,10 +8692,20 @@ function applyCasts(values, casts, direction) {
|
|
|
8568
8692
|
return result;
|
|
8569
8693
|
}
|
|
8570
8694
|
return (async () => {
|
|
8695
|
+
const hashes = [];
|
|
8571
8696
|
for (const [key, cast] of entries) {
|
|
8572
|
-
if (key in result
|
|
8573
|
-
|
|
8697
|
+
if (!(key in result) || !cast)
|
|
8698
|
+
continue;
|
|
8699
|
+
if (cast === "hashed") {
|
|
8700
|
+
const current = result[key];
|
|
8701
|
+
hashes.push(hashCastValue(current).then((hashed) => {
|
|
8702
|
+
result[key] = hashed;
|
|
8703
|
+
}));
|
|
8704
|
+
continue;
|
|
8705
|
+
}
|
|
8706
|
+
result[key] = dehydrateValue(result[key], cast);
|
|
8574
8707
|
}
|
|
8708
|
+
await Promise.all(hashes);
|
|
8575
8709
|
return result;
|
|
8576
8710
|
})();
|
|
8577
8711
|
}
|
|
@@ -14184,21 +14318,6 @@ redis.call('XDEL', KEYS[1], ARGV[2])
|
|
|
14184
14318
|
if #redis.call('XPENDING', KEYS[1], ARGV[1], '-', '+', 1, ARGV[3]) == 0 then redis.call('XGROUP', 'DELCONSUMER', KEYS[1], ARGV[1], ARGV[3]) end
|
|
14185
14319
|
return 1
|
|
14186
14320
|
`;
|
|
14187
|
-
var PROMOTE_RETRIES = `
|
|
14188
|
-
local kinds = {'stream','zset'}
|
|
14189
|
-
for i=1,2 do
|
|
14190
|
-
local kind = redis.call('TYPE', KEYS[i]).ok
|
|
14191
|
-
if kind ~= 'none' and kind ~= kinds[i] then return redis.error_reply('Invalid stream retry promotion key type') end
|
|
14192
|
-
end
|
|
14193
|
-
local now = redis.call('TIME')
|
|
14194
|
-
local due = tonumber(now[1])*1000 + math.floor(tonumber(now[2])/1000)
|
|
14195
|
-
local entries = redis.call('ZRANGE', KEYS[2], '-inf', due, 'BYSCORE', 'LIMIT', 0, 100)
|
|
14196
|
-
for _, payload in ipairs(entries) do
|
|
14197
|
-
redis.call('XADD', KEYS[1], '*', 'payload', payload)
|
|
14198
|
-
redis.call('ZREM', KEYS[2], payload)
|
|
14199
|
-
end
|
|
14200
|
-
return #entries
|
|
14201
|
-
`;
|
|
14202
14321
|
async function scheduleStreamRetry(client, key, reservation, envelope, delayMs) {
|
|
14203
14322
|
if (!Number.isSafeInteger(delayMs) || delayMs < 0 || delayMs > Number.MAX_SAFE_INTEGER / 2)
|
|
14204
14323
|
throw new Error("Invalid Streams retry delay");
|
|
@@ -14220,8 +14339,93 @@ async function scheduleStreamRetry(client, key, reservation, envelope, delayMs)
|
|
|
14220
14339
|
if (Number(scheduled) !== 1)
|
|
14221
14340
|
throw new Error("Queue lease lost while scheduling retry");
|
|
14222
14341
|
}
|
|
14223
|
-
|
|
14224
|
-
|
|
14342
|
+
var POLL_STREAM_PRIORITIES = `
|
|
14343
|
+
local group = ARGV[1]
|
|
14344
|
+
local owner = ARGV[2]
|
|
14345
|
+
local visibility = ARGV[3]
|
|
14346
|
+
local function payloadOf(fields)
|
|
14347
|
+
for index = 1, #fields, 2 do
|
|
14348
|
+
if fields[index] == 'payload' and type(fields[index + 1]) == 'string' then return fields[index + 1] end
|
|
14349
|
+
end
|
|
14350
|
+
return ''
|
|
14351
|
+
end
|
|
14352
|
+
local cursors = {}
|
|
14353
|
+
for i = 1, #KEYS / 2 do cursors[i] = ARGV[3 + i] end
|
|
14354
|
+
for i = 1, #KEYS / 2 do
|
|
14355
|
+
local stream = KEYS[(i - 1) * 2 + 1]
|
|
14356
|
+
local retries = KEYS[(i - 1) * 2 + 2]
|
|
14357
|
+
local cursor = ARGV[3 + i]
|
|
14358
|
+
local streamKind = redis.call('TYPE', stream).ok
|
|
14359
|
+
local retryKind = redis.call('TYPE', retries).ok
|
|
14360
|
+
if streamKind ~= 'none' and streamKind ~= 'stream' then return redis.error_reply('Invalid stream retry promotion key type') end
|
|
14361
|
+
if retryKind ~= 'none' and retryKind ~= 'zset' then return redis.error_reply('Invalid stream retry promotion key type') end
|
|
14362
|
+
if streamKind == 'stream' then
|
|
14363
|
+
local now = redis.call('TIME')
|
|
14364
|
+
local due = tonumber(now[1]) * 1000 + math.floor(tonumber(now[2]) / 1000)
|
|
14365
|
+
local ready = redis.call('ZRANGE', retries, '-inf', due, 'BYSCORE', 'LIMIT', 0, 100)
|
|
14366
|
+
for _, payload in ipairs(ready) do
|
|
14367
|
+
redis.call('XADD', stream, '*', 'payload', payload)
|
|
14368
|
+
redis.call('ZREM', retries, payload)
|
|
14369
|
+
end
|
|
14370
|
+
local window = redis.call('XPENDING', stream, group, cursor, '+', 10)
|
|
14371
|
+
local claimed = redis.call('XAUTOCLAIM', stream, group, owner, visibility, cursor, 'COUNT', 1)
|
|
14372
|
+
for _, entry in ipairs(window) do
|
|
14373
|
+
if entry[2] ~= owner and #redis.call('XPENDING', stream, group, '-', '+', 1, entry[2]) == 0 then
|
|
14374
|
+
redis.call('XGROUP', 'DELCONSUMER', stream, group, entry[2])
|
|
14375
|
+
end
|
|
14376
|
+
end
|
|
14377
|
+
if #redis.call('XPENDING', stream, group, '-', '+', 1, owner) == 0 then
|
|
14378
|
+
redis.call('XGROUP', 'DELCONSUMER', stream, group, owner)
|
|
14379
|
+
end
|
|
14380
|
+
cursors[i] = claimed[1]
|
|
14381
|
+
if #claimed[2] > 0 then
|
|
14382
|
+
return {'1', tostring(i), claimed[2][1][1], payloadOf(claimed[2][1][2]), unpack(cursors)}
|
|
14383
|
+
end
|
|
14384
|
+
local fresh = redis.call('XREADGROUP', 'GROUP', group, owner, 'COUNT', 1, 'STREAMS', stream, '>')
|
|
14385
|
+
if not fresh and #redis.call('XPENDING', stream, group, '-', '+', 1, owner) == 0 then
|
|
14386
|
+
redis.call('XGROUP', 'DELCONSUMER', stream, group, owner)
|
|
14387
|
+
end
|
|
14388
|
+
if fresh and #fresh > 0 and #fresh[1][2] > 0 then
|
|
14389
|
+
return {'1', tostring(i), fresh[1][2][1][1], payloadOf(fresh[1][2][1][2]), unpack(cursors)}
|
|
14390
|
+
end
|
|
14391
|
+
end
|
|
14392
|
+
end
|
|
14393
|
+
return {'0', unpack(cursors)}
|
|
14394
|
+
`;
|
|
14395
|
+
async function pollStreamPriorities(client, queueKeys, owner, cursors) {
|
|
14396
|
+
const keys = [];
|
|
14397
|
+
for (const key of queueKeys)
|
|
14398
|
+
keys.push(streamQueueKey(key), streamRetryKey(key));
|
|
14399
|
+
const raw = await client.send("EVAL", [
|
|
14400
|
+
POLL_STREAM_PRIORITIES,
|
|
14401
|
+
String(keys.length),
|
|
14402
|
+
...keys,
|
|
14403
|
+
STREAM_GROUP,
|
|
14404
|
+
owner,
|
|
14405
|
+
String(readQueueVisibilityMs()),
|
|
14406
|
+
...cursors
|
|
14407
|
+
]);
|
|
14408
|
+
if (!Array.isArray(raw))
|
|
14409
|
+
throw new Error("Invalid Redis stream poll response");
|
|
14410
|
+
const flag = String(raw[0]);
|
|
14411
|
+
const cursorCount = queueKeys.length;
|
|
14412
|
+
if (flag === "0") {
|
|
14413
|
+
if (raw.length !== cursorCount + 1)
|
|
14414
|
+
throw new Error("Invalid Redis stream poll response");
|
|
14415
|
+
return { cursors: raw.slice(1).map(String), hit: null };
|
|
14416
|
+
}
|
|
14417
|
+
if (flag !== "1" || raw.length !== cursorCount + 4)
|
|
14418
|
+
throw new Error("Invalid Redis stream poll response");
|
|
14419
|
+
const index = Number(raw[1]);
|
|
14420
|
+
if (!Number.isInteger(index) || index < 1 || index > queueKeys.length)
|
|
14421
|
+
throw new Error("Invalid Redis stream poll response");
|
|
14422
|
+
return {
|
|
14423
|
+
cursors: raw.slice(4).map(String),
|
|
14424
|
+
hit: {
|
|
14425
|
+
index: index - 1,
|
|
14426
|
+
reservation: { id: String(raw[2]), owner, payload: String(raw[3]) }
|
|
14427
|
+
}
|
|
14428
|
+
};
|
|
14225
14429
|
}
|
|
14226
14430
|
async function readStreamQueueDepth(client, key) {
|
|
14227
14431
|
return Number(await client.send("EVAL", [
|
|
@@ -14270,20 +14474,6 @@ end
|
|
|
14270
14474
|
if #redis.call('XPENDING', KEYS[1], ARGV[1], '-', '+', 1, ARGV[3]) == 0 then redis.call('XGROUP', 'DELCONSUMER', KEYS[1], ARGV[1], ARGV[3]) end
|
|
14271
14475
|
return acknowledged
|
|
14272
14476
|
`;
|
|
14273
|
-
var CLAIM_STREAM = `
|
|
14274
|
-
local window = redis.call('XPENDING', KEYS[1], ARGV[1], ARGV[4], '+', 10)
|
|
14275
|
-
local claimed = redis.call('XAUTOCLAIM', KEYS[1], ARGV[1], ARGV[2], ARGV[3], ARGV[4], 'COUNT', 1)
|
|
14276
|
-
for _, entry in ipairs(window) do
|
|
14277
|
-
if entry[2] ~= ARGV[2] and #redis.call('XPENDING', KEYS[1], ARGV[1], '-', '+', 1, entry[2]) == 0 then redis.call('XGROUP', 'DELCONSUMER', KEYS[1], ARGV[1], entry[2]) end
|
|
14278
|
-
end
|
|
14279
|
-
if #redis.call('XPENDING', KEYS[1], ARGV[1], '-', '+', 1, ARGV[2]) == 0 then redis.call('XGROUP', 'DELCONSUMER', KEYS[1], ARGV[1], ARGV[2]) end
|
|
14280
|
-
return claimed
|
|
14281
|
-
`;
|
|
14282
|
-
var READ_STREAM = `
|
|
14283
|
-
local entries = redis.call('XREADGROUP', 'GROUP', ARGV[1], ARGV[2], 'COUNT', 1, 'STREAMS', KEYS[1], '>')
|
|
14284
|
-
if not entries and #redis.call('XPENDING', KEYS[1], ARGV[1], '-', '+', 1, ARGV[2]) == 0 then redis.call('XGROUP', 'DELCONSUMER', KEYS[1], ARGV[1], ARGV[2]) end
|
|
14285
|
-
return entries
|
|
14286
|
-
`;
|
|
14287
14477
|
async function ensureStreamGroup(client, key) {
|
|
14288
14478
|
await assertLegacyDrained(client, key);
|
|
14289
14479
|
try {
|
|
@@ -14293,17 +14483,6 @@ async function ensureStreamGroup(client, key) {
|
|
|
14293
14483
|
throw error;
|
|
14294
14484
|
}
|
|
14295
14485
|
}
|
|
14296
|
-
function readStreamEntry(value, owner) {
|
|
14297
|
-
if (!Array.isArray(value) || typeof value[0] !== "string" || !Array.isArray(value[1]))
|
|
14298
|
-
throw new Error("Invalid Redis stream entry response");
|
|
14299
|
-
const fields = value[1];
|
|
14300
|
-
let payload;
|
|
14301
|
-
for (let index = 0;index < fields.length; index += 2) {
|
|
14302
|
-
if (fields[index] === "payload" && typeof fields[index + 1] === "string")
|
|
14303
|
-
payload = fields[index + 1];
|
|
14304
|
-
}
|
|
14305
|
-
return { id: value[0], owner, payload: payload ?? "" };
|
|
14306
|
-
}
|
|
14307
14486
|
async function updateStreamReservation(client, key, reservation, action, controlKey) {
|
|
14308
14487
|
return Number(await client.send("EVAL", [
|
|
14309
14488
|
UPDATE_STREAM,
|
|
@@ -14320,35 +14499,6 @@ async function updateStreamReservation(client, key, reservation, action, control
|
|
|
14320
14499
|
controlKey ? "1" : "0"
|
|
14321
14500
|
])) === 1;
|
|
14322
14501
|
}
|
|
14323
|
-
async function reserveStreamJob(client, key, owner, cursor = "0-0") {
|
|
14324
|
-
const claimed = await client.send("EVAL", [
|
|
14325
|
-
CLAIM_STREAM,
|
|
14326
|
-
"1",
|
|
14327
|
-
streamQueueKey(key),
|
|
14328
|
-
STREAM_GROUP,
|
|
14329
|
-
owner,
|
|
14330
|
-
String(readQueueVisibilityMs()),
|
|
14331
|
-
cursor
|
|
14332
|
-
]);
|
|
14333
|
-
if (!Array.isArray(claimed) || typeof claimed[0] !== "string" || !Array.isArray(claimed[1]))
|
|
14334
|
-
throw new Error("Invalid Redis XAUTOCLAIM response");
|
|
14335
|
-
const nextCursor = claimed[0];
|
|
14336
|
-
if (claimed[1].length > 0)
|
|
14337
|
-
return { reservation: readStreamEntry(claimed[1][0], owner), cursor: nextCursor };
|
|
14338
|
-
const fresh = await client.send("EVAL", [
|
|
14339
|
-
READ_STREAM,
|
|
14340
|
-
"1",
|
|
14341
|
-
streamQueueKey(key),
|
|
14342
|
-
STREAM_GROUP,
|
|
14343
|
-
owner
|
|
14344
|
-
]);
|
|
14345
|
-
if (fresh === null)
|
|
14346
|
-
return { reservation: null, cursor: nextCursor };
|
|
14347
|
-
if (!Array.isArray(fresh) || !Array.isArray(fresh[0]) || !Array.isArray(fresh[0][1]))
|
|
14348
|
-
throw new Error("Invalid Redis XREADGROUP response");
|
|
14349
|
-
return { reservation: readStreamEntry(fresh[0][1][0], owner), cursor: nextCursor };
|
|
14350
|
-
}
|
|
14351
|
-
|
|
14352
14502
|
class RedisStreamsQueue {
|
|
14353
14503
|
client;
|
|
14354
14504
|
constructor(redisUrl) {
|
|
@@ -14446,12 +14596,6 @@ class RedisStreamsWorker {
|
|
|
14446
14596
|
await ensureStreamGroup(this.client, key);
|
|
14447
14597
|
this.initialized = true;
|
|
14448
14598
|
}
|
|
14449
|
-
async reserve(key) {
|
|
14450
|
-
await promoteStreamRetries(this.client, key);
|
|
14451
|
-
const result = await reserveStreamJob(this.client, key, this.owner, this.cursors.get(key));
|
|
14452
|
-
this.cursors.set(key, result.cursor);
|
|
14453
|
-
return result.reservation;
|
|
14454
|
-
}
|
|
14455
14599
|
async processNext() {
|
|
14456
14600
|
if (this.processing)
|
|
14457
14601
|
throw new Error("Queue worker already processing a job.");
|
|
@@ -14468,13 +14612,19 @@ class RedisStreamsWorker {
|
|
|
14468
14612
|
await this.initialize();
|
|
14469
14613
|
const deadline = performance.now() + this.timeoutSeconds * 1000;
|
|
14470
14614
|
do {
|
|
14471
|
-
|
|
14472
|
-
|
|
14473
|
-
|
|
14474
|
-
|
|
14475
|
-
|
|
14476
|
-
|
|
14477
|
-
|
|
14615
|
+
if (this.stopping)
|
|
14616
|
+
return false;
|
|
14617
|
+
const polled = await pollStreamPriorities(this.client, this.queueKeys, this.owner, this.queueKeys.map((key) => this.cursors.get(key) ?? "0-0"));
|
|
14618
|
+
this.queueKeys.forEach((key, index) => {
|
|
14619
|
+
const cursor = polled.cursors[index];
|
|
14620
|
+
if (cursor)
|
|
14621
|
+
this.cursors.set(key, cursor);
|
|
14622
|
+
});
|
|
14623
|
+
if (polled.hit) {
|
|
14624
|
+
const key = this.queueKeys[polled.hit.index];
|
|
14625
|
+
if (!key)
|
|
14626
|
+
throw new Error("Invalid Redis stream poll response");
|
|
14627
|
+
await this.handle(key, polled.hit.reservation);
|
|
14478
14628
|
return true;
|
|
14479
14629
|
}
|
|
14480
14630
|
if (performance.now() >= deadline || this.stopping)
|
|
@@ -14701,25 +14851,69 @@ for _, record in ipairs(redis.call('LRANGE', KEYS[2], 0, -1)) do
|
|
|
14701
14851
|
end
|
|
14702
14852
|
return reclaimed
|
|
14703
14853
|
`;
|
|
14854
|
+
var POLL_LIST_PRIORITIES = `${SCRIPT_HEADER}
|
|
14855
|
+
local owner = ARGV[1]
|
|
14856
|
+
local visibility = ARGV[2]
|
|
14857
|
+
local reservationId = ARGV[3]
|
|
14858
|
+
local reclaim = ARGV[4] == '1'
|
|
14859
|
+
local function reclaimQueue(listKey, processingKey, leaseKey)
|
|
14860
|
+
local err = checkType(listKey, 'list') or checkType(processingKey, 'list') or checkType(leaseKey, 'hash')
|
|
14861
|
+
if err then return err end
|
|
14862
|
+
local now = nowMs()
|
|
14863
|
+
for _, record in ipairs(redis.call('LRANGE', processingKey, 0, -1)) do
|
|
14864
|
+
local payload, field = record, record
|
|
14865
|
+
local modern = false
|
|
14866
|
+
local ok, reservation = pcall(cjson.decode, record)
|
|
14867
|
+
if ok and type(reservation) == 'table' and reservation.strataReservation == 1 then
|
|
14868
|
+
payload, field, modern = reservation.payload, reservation.id, true
|
|
14869
|
+
end
|
|
14870
|
+
local raw = redis.call('HGET', leaseKey, field)
|
|
14871
|
+
local expired = true
|
|
14872
|
+
if raw then
|
|
14873
|
+
if modern then
|
|
14874
|
+
local leaseOk, lease = pcall(cjson.decode, raw)
|
|
14875
|
+
expired = not leaseOk or type(lease) ~= 'table' or not tonumber(lease.expiresAt) or tonumber(lease.expiresAt) <= now
|
|
14876
|
+
else
|
|
14877
|
+
local reservedAt = tonumber(raw)
|
|
14878
|
+
expired = not reservedAt or now - reservedAt >= tonumber(visibility)
|
|
14879
|
+
end
|
|
14880
|
+
end
|
|
14881
|
+
if expired then
|
|
14882
|
+
redis.call('LPUSH', listKey, payload)
|
|
14883
|
+
redis.call('LREM', processingKey, 1, record)
|
|
14884
|
+
redis.call('HDEL', leaseKey, field)
|
|
14885
|
+
end
|
|
14886
|
+
end
|
|
14887
|
+
end
|
|
14888
|
+
local function reserveQueue(listKey, processingKey, leaseKey)
|
|
14889
|
+
local err = checkType(listKey, 'list') or checkType(processingKey, 'list') or checkType(leaseKey, 'hash')
|
|
14890
|
+
if err then return err end
|
|
14891
|
+
local payload = redis.call('LINDEX', listKey, -1)
|
|
14892
|
+
if not payload then return nil end
|
|
14893
|
+
local record = cjson.encode({strataReservation=1, id=reservationId, owner=owner, payload=payload})
|
|
14894
|
+
local lease = cjson.encode({owner=owner, expiresAt=nowMs()+tonumber(visibility)})
|
|
14895
|
+
redis.call('LPUSH', processingKey, record)
|
|
14896
|
+
redis.call('HSET', leaseKey, reservationId, lease)
|
|
14897
|
+
redis.call('RPOP', listKey)
|
|
14898
|
+
return record
|
|
14899
|
+
end
|
|
14900
|
+
local count = #KEYS / 3
|
|
14901
|
+
if reclaim then
|
|
14902
|
+
for i = 1, count do
|
|
14903
|
+
local err = reclaimQueue(KEYS[(i - 1) * 3 + 1], KEYS[(i - 1) * 3 + 2], KEYS[(i - 1) * 3 + 3])
|
|
14904
|
+
if err then return err end
|
|
14905
|
+
end
|
|
14906
|
+
end
|
|
14907
|
+
for i = 1, count do
|
|
14908
|
+
local record = reserveQueue(KEYS[(i - 1) * 3 + 1], KEYS[(i - 1) * 3 + 2], KEYS[(i - 1) * 3 + 3])
|
|
14909
|
+
if type(record) == 'string' then return {'1', tostring(i), record} end
|
|
14910
|
+
if record then return record end
|
|
14911
|
+
end
|
|
14912
|
+
return {'0'}
|
|
14913
|
+
`;
|
|
14704
14914
|
function queueInvalidKey(queueKey) {
|
|
14705
14915
|
return `${queueKey}:invalid`;
|
|
14706
14916
|
}
|
|
14707
|
-
async function reserveQueueJob(client, queueKey, owner) {
|
|
14708
|
-
const raw = await client.send("EVAL", [
|
|
14709
|
-
RESERVE_JOB,
|
|
14710
|
-
"3",
|
|
14711
|
-
queueKey,
|
|
14712
|
-
queueProcessingKey(queueKey),
|
|
14713
|
-
queueProcessingLeaseKey(queueKey),
|
|
14714
|
-
crypto.randomUUID(),
|
|
14715
|
-
owner,
|
|
14716
|
-
String(readQueueVisibilityMs())
|
|
14717
|
-
]);
|
|
14718
|
-
if (typeof raw !== "string")
|
|
14719
|
-
return null;
|
|
14720
|
-
const reservation = JSON.parse(raw);
|
|
14721
|
-
return { ...reservation, record: raw };
|
|
14722
|
-
}
|
|
14723
14917
|
async function updateQueueReservation(client, queueKey, reservation, action) {
|
|
14724
14918
|
return Number(await client.send("EVAL", [
|
|
14725
14919
|
RENEW_OR_ACK,
|
|
@@ -14735,21 +14929,41 @@ async function updateQueueReservation(client, queueKey, reservation, action) {
|
|
|
14735
14929
|
reservation.payload
|
|
14736
14930
|
])) === 1;
|
|
14737
14931
|
}
|
|
14738
|
-
async function
|
|
14739
|
-
|
|
14932
|
+
async function pollListPriorities(client, queueKeys, owner, reservationId, reclaim) {
|
|
14933
|
+
const keys = [];
|
|
14740
14934
|
for (const key of queueKeys) {
|
|
14741
|
-
|
|
14742
|
-
RECLAIM_JOBS,
|
|
14743
|
-
"3",
|
|
14744
|
-
key,
|
|
14745
|
-
queueProcessingKey(key),
|
|
14746
|
-
queueProcessingLeaseKey(key),
|
|
14747
|
-
String(readQueueVisibilityMs())
|
|
14748
|
-
]));
|
|
14935
|
+
keys.push(key, queueProcessingKey(key), queueProcessingLeaseKey(key));
|
|
14749
14936
|
}
|
|
14750
|
-
|
|
14937
|
+
const raw = await client.send("EVAL", [
|
|
14938
|
+
POLL_LIST_PRIORITIES,
|
|
14939
|
+
String(keys.length),
|
|
14940
|
+
...keys,
|
|
14941
|
+
owner,
|
|
14942
|
+
String(readQueueVisibilityMs()),
|
|
14943
|
+
reservationId,
|
|
14944
|
+
reclaim ? "1" : "0"
|
|
14945
|
+
]);
|
|
14946
|
+
if (!Array.isArray(raw))
|
|
14947
|
+
throw new Error("Invalid Redis list poll response");
|
|
14948
|
+
const flag = String(raw[0]);
|
|
14949
|
+
if (flag === "0") {
|
|
14950
|
+
if (raw.length !== 1)
|
|
14951
|
+
throw new Error("Invalid Redis list poll response");
|
|
14952
|
+
return null;
|
|
14953
|
+
}
|
|
14954
|
+
if (flag !== "1" || raw.length !== 3)
|
|
14955
|
+
throw new Error("Invalid Redis list poll response");
|
|
14956
|
+
const index = Number(raw[1]);
|
|
14957
|
+
const record = raw[2];
|
|
14958
|
+
if (!Number.isInteger(index) || index < 1 || index > queueKeys.length || typeof record !== "string") {
|
|
14959
|
+
throw new Error("Invalid Redis list poll response");
|
|
14960
|
+
}
|
|
14961
|
+
const queueKey = queueKeys[index - 1];
|
|
14962
|
+
if (!queueKey)
|
|
14963
|
+
throw new Error("Invalid Redis list poll response");
|
|
14964
|
+
const reservation = JSON.parse(record);
|
|
14965
|
+
return { queueKey, reservation: { ...reservation, record } };
|
|
14751
14966
|
}
|
|
14752
|
-
|
|
14753
14967
|
class RedisQueue {
|
|
14754
14968
|
client;
|
|
14755
14969
|
constructor(redisUrl) {
|
|
@@ -14795,14 +15009,14 @@ class QueueWorker {
|
|
|
14795
15009
|
}
|
|
14796
15010
|
async reserve() {
|
|
14797
15011
|
const deadline = performance.now() + this.timeoutSeconds * 1000;
|
|
15012
|
+
let reclaim = true;
|
|
14798
15013
|
do {
|
|
14799
|
-
|
|
14800
|
-
|
|
14801
|
-
|
|
14802
|
-
|
|
14803
|
-
|
|
14804
|
-
|
|
14805
|
-
}
|
|
15014
|
+
if (this.stopping)
|
|
15015
|
+
return null;
|
|
15016
|
+
const reserved = await pollListPriorities(this.client, this.queueKeys, this.owner, crypto.randomUUID(), reclaim);
|
|
15017
|
+
reclaim = false;
|
|
15018
|
+
if (reserved)
|
|
15019
|
+
return reserved;
|
|
14806
15020
|
if (this.stopping || performance.now() >= deadline)
|
|
14807
15021
|
return null;
|
|
14808
15022
|
await Bun.sleep(Math.min(100, Math.max(1, deadline - performance.now())));
|
|
@@ -14812,7 +15026,6 @@ class QueueWorker {
|
|
|
14812
15026
|
async processNext() {
|
|
14813
15027
|
if (this.stopping)
|
|
14814
15028
|
return false;
|
|
14815
|
-
await reclaimExpiredQueueReservations(this.client, this.queueKeys);
|
|
14816
15029
|
const reserved = await this.reserve();
|
|
14817
15030
|
if (!reserved)
|
|
14818
15031
|
return false;
|
|
@@ -16579,6 +16792,7 @@ export {
|
|
|
16579
16792
|
installGracefulShutdownSignals,
|
|
16580
16793
|
isBlockedHostname,
|
|
16581
16794
|
isBlockedIpAddress,
|
|
16795
|
+
isDefaultDatabaseConnection,
|
|
16582
16796
|
isEtagEnabled,
|
|
16583
16797
|
isGlobalAdmin,
|
|
16584
16798
|
isHtmxRequest,
|