@getstrata/core 2.2.14 → 2.2.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +39 -14
- package/dist/core/database/baseRepository.d.ts +1 -0
- package/dist/core/database/defaultConnection.d.ts +2 -1
- package/dist/core/database/relationQuery.d.ts +4 -0
- package/dist/core/queue/redisQueue.d.ts +5 -1
- package/dist/core/queue/redisStreams.d.ts +8 -2
- package/dist/entries/queue/createAppQueue.js +202 -92
- package/dist/entries/queue/publicQueue.js +202 -92
- package/dist/entries/queue/queueMetrics.js +202 -92
- package/dist/entries/queue/redisQueue.js +203 -92
- package/dist/framework/public-api.d.ts +1 -1
- package/dist/index.js +357 -143
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,22 @@
|
|
|
1
1
|
# @getstrata/core changelog
|
|
2
2
|
|
|
3
|
+
## 2.2.16
|
|
4
|
+
|
|
5
|
+
Allow managed outbox workers to reuse a configured instance, preserving application tenant resolution, listener registration and delivery settings. Keep generated listeners-only defaults compatible, reject ambiguous worker sources, and cover configured SQL delivery, cancellation, cleanup and TypeScript contracts.
|
|
6
|
+
|
|
7
|
+
## 2.2.15
|
|
8
|
+
|
|
9
|
+
- Batch legacy list-queue priority polling while retaining recovery and ownership contracts.
|
|
10
|
+
- Batch dialect-portable pivot writes with framework transaction/savepoint composition and consistent numeric/string ID toggle semantics. Noncanonical or textual IDs use sequential database comparisons to preserve coercion and collation behavior.
|
|
11
|
+
- New Redis scaffolds explicitly select Streams; existing unset transport settings continue to select lists.
|
|
12
|
+
- Generate the SQL outbox migration, listener registry and managed worker command. Durable publication and cache-invalidation activation remain explicit.
|
|
13
|
+
|
|
14
|
+
- Primary-key ascending `chunk` queries reuse the existing keyset walk, including default primary-key ordering; projections, joins, grouping, locks and other orders retain OFFSET behavior.
|
|
15
|
+
- Empty filtered pagination totals skip the row query. Nonempty page-number pagination preserves OFFSET semantics and its count-plus-row-query budget; explicit cursor/keyset APIs remain available for deep traversal.
|
|
16
|
+
- Independent hashed write casts run concurrently. Deterministic regressions prove concurrent start, complete awaiting and failure before model observers or SQL writes; hydration and existing-hash behavior are preserved.
|
|
17
|
+
- Streams workers poll retry promotion, pending recovery and new delivery across priorities in one bounded Redis script, preserving priority, ownership fencing and at-least-once delivery contracts.
|
|
18
|
+
- Clarify historical 2.1.0–2.2.13 release notes and database/queue documentation. Remove the consolidated 1.x-to-2.x migration guide; existing projects are upgraded individually.
|
|
19
|
+
|
|
3
20
|
## 2.2.14
|
|
4
21
|
|
|
5
22
|
Complete the scoped HTML rendering starter adoption: generated apps inject ViewRenderer through createViewProvider instead of copying renderPage; welcome pages opt into HTMX fragments with full-page fallbacks, auth retains existing contracts, and CORS preserves representation Vary headers. Includes generator, runtime auth and cache-header regressions.
|
|
@@ -8,7 +25,8 @@ Add optional typed resourceRoutes and strict mergeRouteMaps composition, preserv
|
|
|
8
25
|
|
|
9
26
|
## 2.2.13
|
|
10
27
|
|
|
11
|
-
-
|
|
28
|
+
- Add a typed per-app HTML renderer on top of the native response path.
|
|
29
|
+
- Add explicit HTMX page and fragment responses. A fragment can fall back to the full page. Layout and auth response contracts stay the same.
|
|
12
30
|
|
|
13
31
|
## 2.2.12
|
|
14
32
|
|
|
@@ -30,42 +48,47 @@ Add independently interruptible read-only SQLite and MySQL observations; extend
|
|
|
30
48
|
|
|
31
49
|
## 2.2.9
|
|
32
50
|
|
|
33
|
-
-
|
|
51
|
+
- Pool checkout can be cancelled. `reserve({ signal })` on the native driver aborts while it is still waiting for a connection. Drivers without that method are not given a fake one.
|
|
34
52
|
|
|
35
53
|
## 2.2.8
|
|
36
54
|
|
|
37
|
-
- Add bounded read-only Postgres outbox metrics
|
|
55
|
+
- Add bounded, read-only Postgres outbox metrics.
|
|
56
|
+
- Add bounded, opt-in Redis queue metrics. List and Streams depth stay separate.
|
|
38
57
|
|
|
39
58
|
## 2.2.7
|
|
40
59
|
|
|
41
|
-
-
|
|
60
|
+
- `ORDER BY` accepts explicit `nulls: "first" | "last"`. Postgres and SQLite emit `NULLS FIRST` / `NULLS LAST`. MySQL has no `NULLS FIRST` / `NULLS LAST` clause, so it orders nulls with an `IS NULL` discriminator and leaves the rest of the sort unchanged.
|
|
42
61
|
|
|
43
62
|
## 2.2.6
|
|
44
63
|
|
|
45
|
-
-
|
|
64
|
+
- The CLI, when it supervises the app or worker as a child, forwards shutdown signals to that child and waits for it. The container launcher no longer exits while the child is still running.
|
|
46
65
|
|
|
47
66
|
## 2.2.5
|
|
48
67
|
|
|
49
|
-
-
|
|
68
|
+
- Container readiness no longer means the API is admitting traffic. Database readiness no longer provisions roles or creates a missing database. A failed configured admin connection does not fall back to a generated password.
|
|
50
69
|
|
|
51
70
|
## 2.2.4
|
|
52
71
|
|
|
53
|
-
-
|
|
72
|
+
- Composite keyset pages on Postgres seek with a row comparison (`(a, b) > (cursor)`), so the planner can start at the index bound instead of expanding the cursor into a chain of OR predicates.
|
|
54
73
|
|
|
55
74
|
## 2.2.3
|
|
56
75
|
|
|
57
|
-
-
|
|
76
|
+
- Add scoped composite keyset pagination. The order must end with the primary key. Global scopes and caller filters stay in force across the cursor. Joins, grouping, and offset queries are rejected.
|
|
58
77
|
|
|
59
78
|
## 2.2.2
|
|
60
79
|
|
|
61
|
-
-
|
|
80
|
+
- Schema `bigId` emits an explicit 64-bit generated identity.
|
|
81
|
+
- Repository writes can be conditional and set-based.
|
|
82
|
+
- Repositories and models can take a typed row lock inside the current transaction.
|
|
83
|
+
- Generated Postgres query handles keep tagged-SQL typing.
|
|
62
84
|
|
|
63
85
|
## 2.2.1
|
|
64
86
|
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
87
|
+
- Opt-in Redis Streams (`QUEUE_REDIS_TRANSPORT=streams`). Lists stay the default. Streams adds consumer-group recovery, persisted retry schedules, total job deadlines, opt-in cancellation, and a stable job id across replay. Conversion from lists is a stopped maintenance step.
|
|
88
|
+
- Failed jobs stay recorded until retry admission succeeds. Manual replay keeps `failed_job.job_id` when the row has one.
|
|
89
|
+
- Public reads and tenant host selection are separate. Unknown hosts fail closed. Forwarded hosts are accepted only from configured trusted proxies.
|
|
90
|
+
- Generated apps can provision one initial administrator with `auth:provision-admin` and a one-time claim migration. Demo seeding stays explicit and is rejected in production and staging.
|
|
91
|
+
- Quota and in-memory throttle docs match the code: memory throttles stay process-local, and a Redis failure does not fall back to them.
|
|
69
92
|
|
|
70
93
|
## 2.2.0
|
|
71
94
|
|
|
@@ -102,7 +125,9 @@ Bounded memory throttling.
|
|
|
102
125
|
|
|
103
126
|
## 2.1.0
|
|
104
127
|
|
|
105
|
-
-
|
|
128
|
+
- Generated apps reuse the shared config store and base repositories instead of a copied stack.
|
|
129
|
+
- Password casts are awaited before the write reaches SQL or observers. Hydration does not hash.
|
|
130
|
+
- Model queries keep the concrete model type in their results. Service tokens, route handlers, and the starter's generated infrastructure were tightened in the same release.
|
|
106
131
|
|
|
107
132
|
## 2.0.9
|
|
108
133
|
|
|
@@ -69,6 +69,7 @@ declare class BaseRepository<TEntity extends object, PrimaryKey extends keyof TE
|
|
|
69
69
|
softDeleteById(id: TEntity[PrimaryKey]): Promise<boolean>;
|
|
70
70
|
forceDeleteById(id: TEntity[PrimaryKey]): Promise<boolean>;
|
|
71
71
|
restoreById(id: TEntity[PrimaryKey]): Promise<TEntity | null>;
|
|
72
|
+
private primaryKeySeekDirection;
|
|
72
73
|
withConnection(connection: DatabaseConnection): this;
|
|
73
74
|
getConnection(): DatabaseConnection;
|
|
74
75
|
getTable(): TableDefinition<TEntity, PrimaryKey>;
|
|
@@ -2,5 +2,6 @@ import type { SqlDatabaseConnection } from "./baseRepository";
|
|
|
2
2
|
declare function registerDefaultDatabasePool(connection: SqlDatabaseConnection): void;
|
|
3
3
|
declare function getDefaultDatabasePool(): SqlDatabaseConnection;
|
|
4
4
|
declare function getDefaultDatabaseQuery(): SqlDatabaseConnection;
|
|
5
|
+
declare function isDefaultDatabaseConnection(connection: unknown): boolean;
|
|
5
6
|
declare function resetDefaultDatabasePoolForTests(): void;
|
|
6
|
-
export { getDefaultDatabasePool, getDefaultDatabaseQuery, registerDefaultDatabasePool, resetDefaultDatabasePoolForTests, };
|
|
7
|
+
export { getDefaultDatabasePool, getDefaultDatabaseQuery, isDefaultDatabaseConnection, registerDefaultDatabasePool, resetDefaultDatabasePoolForTests, };
|
|
@@ -112,6 +112,7 @@ declare class BelongsToManyRelationQuery<TParent extends object, ParentKey exten
|
|
|
112
112
|
hydrateEager(row: Record<string, unknown>, alias: string): unknown;
|
|
113
113
|
toExistsClause(parentTable: string): ExistsClause;
|
|
114
114
|
private connection;
|
|
115
|
+
private atomically;
|
|
115
116
|
get(): Promise<RelatedRecord[]>;
|
|
116
117
|
first(): Promise<RelatedRecord | null>;
|
|
117
118
|
count(): Promise<number>;
|
|
@@ -123,6 +124,9 @@ declare class BelongsToManyRelationQuery<TParent extends object, ParentKey exten
|
|
|
123
124
|
attach(ids: unknown | readonly unknown[]): Promise<void>;
|
|
124
125
|
toggle(ids: unknown | readonly unknown[]): Promise<void>;
|
|
125
126
|
detach(ids?: unknown | readonly unknown[]): Promise<void>;
|
|
127
|
+
private toggleOne;
|
|
128
|
+
private insertPivotRows;
|
|
129
|
+
private deletePivotRows;
|
|
126
130
|
sync(ids: readonly unknown[]): Promise<void>;
|
|
127
131
|
create(attributes?: Record<string, unknown>): Promise<RelatedRecord>;
|
|
128
132
|
}
|
|
@@ -15,6 +15,10 @@ type QueueReservation = {
|
|
|
15
15
|
declare function queueInvalidKey(queueKey: string): string;
|
|
16
16
|
declare function reserveQueueJob(client: RedisClient, queueKey: string, owner: string): Promise<QueueReservation | null>;
|
|
17
17
|
declare function updateQueueReservation(client: RedisClient, queueKey: string, reservation: QueueReservation, action: "renew" | "ack" | "quarantine"): Promise<boolean>;
|
|
18
|
+
declare function pollListPriorities(client: RedisClient, queueKeys: readonly string[], owner: string, reservationId: string, reclaim: boolean): Promise<{
|
|
19
|
+
queueKey: string;
|
|
20
|
+
reservation: QueueReservation;
|
|
21
|
+
} | null>;
|
|
18
22
|
declare function reclaimExpiredQueueReservations(client: RedisClient, queueKeys?: readonly string[]): Promise<number>;
|
|
19
23
|
declare class RedisQueue implements Queue {
|
|
20
24
|
private readonly client;
|
|
@@ -42,4 +46,4 @@ declare function countPendingQueueJobs(redisUrl: string): Promise<number>;
|
|
|
42
46
|
export type { StreamReservation, StreamsEnqueueOptions } from "./redisStreams";
|
|
43
47
|
export { migrateLegacyQueueToStreams, RedisStreamsQueue, RedisStreamsWorker, STREAM_GROUP, streamDeadLetterKey, streamQueueKey, updateStreamReservation, } from "./redisStreams";
|
|
44
48
|
export type { QueueJobEnvelope, QueueReservation };
|
|
45
|
-
export { countPendingQueueJobs, parseQueueJobEnvelope, QUEUE_HIGH_KEY, QUEUE_LIST_KEY, QUEUE_LOW_KEY, QueueWorker, queueInvalidKey, queueKeyForPriority, queueProcessingKey, queueProcessingLeaseKey, RedisQueue, reclaimExpiredQueueReservations, reservationExpired, reserveQueueJob, updateQueueReservation, };
|
|
49
|
+
export { countPendingQueueJobs, parseQueueJobEnvelope, pollListPriorities, QUEUE_HIGH_KEY, QUEUE_LIST_KEY, QUEUE_LOW_KEY, QueueWorker, queueInvalidKey, queueKeyForPriority, queueProcessingKey, queueProcessingLeaseKey, RedisQueue, reclaimExpiredQueueReservations, reservationExpired, reserveQueueJob, updateQueueReservation, };
|
|
@@ -18,6 +18,13 @@ declare function readStreamTime(client: RedisClient): Promise<number>;
|
|
|
18
18
|
declare function watchStreamDeadline(client: RedisClient, deadlineAtMs: number, remainingMs: number, expire: () => void, fail: (error: unknown) => void): () => void;
|
|
19
19
|
declare function scheduleStreamRetry(client: RedisClient, key: string, reservation: StreamReservation, envelope: QueueJobEnvelope, delayMs: number): Promise<void>;
|
|
20
20
|
declare function promoteStreamRetries(client: RedisClient, key: string): Promise<number>;
|
|
21
|
+
declare function pollStreamPriorities(client: RedisClient, queueKeys: readonly string[], owner: string, cursors: readonly string[]): Promise<{
|
|
22
|
+
cursors: string[];
|
|
23
|
+
hit: {
|
|
24
|
+
index: number;
|
|
25
|
+
reservation: StreamReservation;
|
|
26
|
+
} | null;
|
|
27
|
+
}>;
|
|
21
28
|
declare function readStreamQueueDepth(client: RedisClient, key: string): Promise<number>;
|
|
22
29
|
type StreamReservation = {
|
|
23
30
|
id: string;
|
|
@@ -58,7 +65,6 @@ declare class RedisStreamsWorker {
|
|
|
58
65
|
isRunning(): boolean;
|
|
59
66
|
close(): void;
|
|
60
67
|
private initialize;
|
|
61
|
-
private reserve;
|
|
62
68
|
processNext(): Promise<boolean>;
|
|
63
69
|
private processOne;
|
|
64
70
|
private handle;
|
|
@@ -70,4 +76,4 @@ declare function migrateLegacyQueueToStreams(client: RedisClient, options: {
|
|
|
70
76
|
queueKeys?: readonly string[];
|
|
71
77
|
}): Promise<number>;
|
|
72
78
|
export type { StreamReservation, StreamsEnqueueOptions };
|
|
73
|
-
export { cancelStreamJob, ensureStreamGroup, migrateLegacyQueueToStreams, promoteStreamRetries, RedisStreamsQueue, RedisStreamsWorker, readStreamCancellation, readStreamEntry, readStreamQueueDepth, readStreamTime, reserveStreamJob, STREAM_GROUP, scheduleStreamRetry, streamControlKey, streamDeadLetterKey, streamQueueKey, streamRetryKey, updateStreamReservation, watchStreamDeadline, };
|
|
79
|
+
export { cancelStreamJob, ensureStreamGroup, migrateLegacyQueueToStreams, pollStreamPriorities, promoteStreamRetries, RedisStreamsQueue, RedisStreamsWorker, readStreamCancellation, readStreamEntry, readStreamQueueDepth, readStreamTime, reserveStreamJob, STREAM_GROUP, scheduleStreamRetry, streamControlKey, streamDeadLetterKey, streamQueueKey, streamRetryKey, updateStreamReservation, watchStreamDeadline, };
|
|
@@ -448,21 +448,6 @@ redis.call('XDEL', KEYS[1], ARGV[2])
|
|
|
448
448
|
if #redis.call('XPENDING', KEYS[1], ARGV[1], '-', '+', 1, ARGV[3]) == 0 then redis.call('XGROUP', 'DELCONSUMER', KEYS[1], ARGV[1], ARGV[3]) end
|
|
449
449
|
return 1
|
|
450
450
|
`;
|
|
451
|
-
var PROMOTE_RETRIES = `
|
|
452
|
-
local kinds = {'stream','zset'}
|
|
453
|
-
for i=1,2 do
|
|
454
|
-
local kind = redis.call('TYPE', KEYS[i]).ok
|
|
455
|
-
if kind ~= 'none' and kind ~= kinds[i] then return redis.error_reply('Invalid stream retry promotion key type') end
|
|
456
|
-
end
|
|
457
|
-
local now = redis.call('TIME')
|
|
458
|
-
local due = tonumber(now[1])*1000 + math.floor(tonumber(now[2])/1000)
|
|
459
|
-
local entries = redis.call('ZRANGE', KEYS[2], '-inf', due, 'BYSCORE', 'LIMIT', 0, 100)
|
|
460
|
-
for _, payload in ipairs(entries) do
|
|
461
|
-
redis.call('XADD', KEYS[1], '*', 'payload', payload)
|
|
462
|
-
redis.call('ZREM', KEYS[2], payload)
|
|
463
|
-
end
|
|
464
|
-
return #entries
|
|
465
|
-
`;
|
|
466
451
|
async function scheduleStreamRetry(client, key, reservation, envelope, delayMs) {
|
|
467
452
|
if (!Number.isSafeInteger(delayMs) || delayMs < 0 || delayMs > Number.MAX_SAFE_INTEGER / 2)
|
|
468
453
|
throw new Error("Invalid Streams retry delay");
|
|
@@ -484,8 +469,93 @@ async function scheduleStreamRetry(client, key, reservation, envelope, delayMs)
|
|
|
484
469
|
if (Number(scheduled) !== 1)
|
|
485
470
|
throw new Error("Queue lease lost while scheduling retry");
|
|
486
471
|
}
|
|
487
|
-
|
|
488
|
-
|
|
472
|
+
var POLL_STREAM_PRIORITIES = `
|
|
473
|
+
local group = ARGV[1]
|
|
474
|
+
local owner = ARGV[2]
|
|
475
|
+
local visibility = ARGV[3]
|
|
476
|
+
local function payloadOf(fields)
|
|
477
|
+
for index = 1, #fields, 2 do
|
|
478
|
+
if fields[index] == 'payload' and type(fields[index + 1]) == 'string' then return fields[index + 1] end
|
|
479
|
+
end
|
|
480
|
+
return ''
|
|
481
|
+
end
|
|
482
|
+
local cursors = {}
|
|
483
|
+
for i = 1, #KEYS / 2 do cursors[i] = ARGV[3 + i] end
|
|
484
|
+
for i = 1, #KEYS / 2 do
|
|
485
|
+
local stream = KEYS[(i - 1) * 2 + 1]
|
|
486
|
+
local retries = KEYS[(i - 1) * 2 + 2]
|
|
487
|
+
local cursor = ARGV[3 + i]
|
|
488
|
+
local streamKind = redis.call('TYPE', stream).ok
|
|
489
|
+
local retryKind = redis.call('TYPE', retries).ok
|
|
490
|
+
if streamKind ~= 'none' and streamKind ~= 'stream' then return redis.error_reply('Invalid stream retry promotion key type') end
|
|
491
|
+
if retryKind ~= 'none' and retryKind ~= 'zset' then return redis.error_reply('Invalid stream retry promotion key type') end
|
|
492
|
+
if streamKind == 'stream' then
|
|
493
|
+
local now = redis.call('TIME')
|
|
494
|
+
local due = tonumber(now[1]) * 1000 + math.floor(tonumber(now[2]) / 1000)
|
|
495
|
+
local ready = redis.call('ZRANGE', retries, '-inf', due, 'BYSCORE', 'LIMIT', 0, 100)
|
|
496
|
+
for _, payload in ipairs(ready) do
|
|
497
|
+
redis.call('XADD', stream, '*', 'payload', payload)
|
|
498
|
+
redis.call('ZREM', retries, payload)
|
|
499
|
+
end
|
|
500
|
+
local window = redis.call('XPENDING', stream, group, cursor, '+', 10)
|
|
501
|
+
local claimed = redis.call('XAUTOCLAIM', stream, group, owner, visibility, cursor, 'COUNT', 1)
|
|
502
|
+
for _, entry in ipairs(window) do
|
|
503
|
+
if entry[2] ~= owner and #redis.call('XPENDING', stream, group, '-', '+', 1, entry[2]) == 0 then
|
|
504
|
+
redis.call('XGROUP', 'DELCONSUMER', stream, group, entry[2])
|
|
505
|
+
end
|
|
506
|
+
end
|
|
507
|
+
if #redis.call('XPENDING', stream, group, '-', '+', 1, owner) == 0 then
|
|
508
|
+
redis.call('XGROUP', 'DELCONSUMER', stream, group, owner)
|
|
509
|
+
end
|
|
510
|
+
cursors[i] = claimed[1]
|
|
511
|
+
if #claimed[2] > 0 then
|
|
512
|
+
return {'1', tostring(i), claimed[2][1][1], payloadOf(claimed[2][1][2]), unpack(cursors)}
|
|
513
|
+
end
|
|
514
|
+
local fresh = redis.call('XREADGROUP', 'GROUP', group, owner, 'COUNT', 1, 'STREAMS', stream, '>')
|
|
515
|
+
if not fresh and #redis.call('XPENDING', stream, group, '-', '+', 1, owner) == 0 then
|
|
516
|
+
redis.call('XGROUP', 'DELCONSUMER', stream, group, owner)
|
|
517
|
+
end
|
|
518
|
+
if fresh and #fresh > 0 and #fresh[1][2] > 0 then
|
|
519
|
+
return {'1', tostring(i), fresh[1][2][1][1], payloadOf(fresh[1][2][1][2]), unpack(cursors)}
|
|
520
|
+
end
|
|
521
|
+
end
|
|
522
|
+
end
|
|
523
|
+
return {'0', unpack(cursors)}
|
|
524
|
+
`;
|
|
525
|
+
async function pollStreamPriorities(client, queueKeys, owner, cursors) {
|
|
526
|
+
const keys = [];
|
|
527
|
+
for (const key of queueKeys)
|
|
528
|
+
keys.push(streamQueueKey(key), streamRetryKey(key));
|
|
529
|
+
const raw = await client.send("EVAL", [
|
|
530
|
+
POLL_STREAM_PRIORITIES,
|
|
531
|
+
String(keys.length),
|
|
532
|
+
...keys,
|
|
533
|
+
STREAM_GROUP,
|
|
534
|
+
owner,
|
|
535
|
+
String(readQueueVisibilityMs()),
|
|
536
|
+
...cursors
|
|
537
|
+
]);
|
|
538
|
+
if (!Array.isArray(raw))
|
|
539
|
+
throw new Error("Invalid Redis stream poll response");
|
|
540
|
+
const flag = String(raw[0]);
|
|
541
|
+
const cursorCount = queueKeys.length;
|
|
542
|
+
if (flag === "0") {
|
|
543
|
+
if (raw.length !== cursorCount + 1)
|
|
544
|
+
throw new Error("Invalid Redis stream poll response");
|
|
545
|
+
return { cursors: raw.slice(1).map(String), hit: null };
|
|
546
|
+
}
|
|
547
|
+
if (flag !== "1" || raw.length !== cursorCount + 4)
|
|
548
|
+
throw new Error("Invalid Redis stream poll response");
|
|
549
|
+
const index = Number(raw[1]);
|
|
550
|
+
if (!Number.isInteger(index) || index < 1 || index > queueKeys.length)
|
|
551
|
+
throw new Error("Invalid Redis stream poll response");
|
|
552
|
+
return {
|
|
553
|
+
cursors: raw.slice(4).map(String),
|
|
554
|
+
hit: {
|
|
555
|
+
index: index - 1,
|
|
556
|
+
reservation: { id: String(raw[2]), owner, payload: String(raw[3]) }
|
|
557
|
+
}
|
|
558
|
+
};
|
|
489
559
|
}
|
|
490
560
|
async function readStreamQueueDepth(client, key) {
|
|
491
561
|
return Number(await client.send("EVAL", [
|
|
@@ -534,20 +604,6 @@ end
|
|
|
534
604
|
if #redis.call('XPENDING', KEYS[1], ARGV[1], '-', '+', 1, ARGV[3]) == 0 then redis.call('XGROUP', 'DELCONSUMER', KEYS[1], ARGV[1], ARGV[3]) end
|
|
535
605
|
return acknowledged
|
|
536
606
|
`;
|
|
537
|
-
var CLAIM_STREAM = `
|
|
538
|
-
local window = redis.call('XPENDING', KEYS[1], ARGV[1], ARGV[4], '+', 10)
|
|
539
|
-
local claimed = redis.call('XAUTOCLAIM', KEYS[1], ARGV[1], ARGV[2], ARGV[3], ARGV[4], 'COUNT', 1)
|
|
540
|
-
for _, entry in ipairs(window) do
|
|
541
|
-
if entry[2] ~= ARGV[2] and #redis.call('XPENDING', KEYS[1], ARGV[1], '-', '+', 1, entry[2]) == 0 then redis.call('XGROUP', 'DELCONSUMER', KEYS[1], ARGV[1], entry[2]) end
|
|
542
|
-
end
|
|
543
|
-
if #redis.call('XPENDING', KEYS[1], ARGV[1], '-', '+', 1, ARGV[2]) == 0 then redis.call('XGROUP', 'DELCONSUMER', KEYS[1], ARGV[1], ARGV[2]) end
|
|
544
|
-
return claimed
|
|
545
|
-
`;
|
|
546
|
-
var READ_STREAM = `
|
|
547
|
-
local entries = redis.call('XREADGROUP', 'GROUP', ARGV[1], ARGV[2], 'COUNT', 1, 'STREAMS', KEYS[1], '>')
|
|
548
|
-
if not entries and #redis.call('XPENDING', KEYS[1], ARGV[1], '-', '+', 1, ARGV[2]) == 0 then redis.call('XGROUP', 'DELCONSUMER', KEYS[1], ARGV[1], ARGV[2]) end
|
|
549
|
-
return entries
|
|
550
|
-
`;
|
|
551
607
|
async function ensureStreamGroup(client, key) {
|
|
552
608
|
await assertLegacyDrained(client, key);
|
|
553
609
|
try {
|
|
@@ -557,17 +613,6 @@ async function ensureStreamGroup(client, key) {
|
|
|
557
613
|
throw error;
|
|
558
614
|
}
|
|
559
615
|
}
|
|
560
|
-
function readStreamEntry(value, owner) {
|
|
561
|
-
if (!Array.isArray(value) || typeof value[0] !== "string" || !Array.isArray(value[1]))
|
|
562
|
-
throw new Error("Invalid Redis stream entry response");
|
|
563
|
-
const fields = value[1];
|
|
564
|
-
let payload;
|
|
565
|
-
for (let index = 0;index < fields.length; index += 2) {
|
|
566
|
-
if (fields[index] === "payload" && typeof fields[index + 1] === "string")
|
|
567
|
-
payload = fields[index + 1];
|
|
568
|
-
}
|
|
569
|
-
return { id: value[0], owner, payload: payload ?? "" };
|
|
570
|
-
}
|
|
571
616
|
async function updateStreamReservation(client, key, reservation, action, controlKey) {
|
|
572
617
|
return Number(await client.send("EVAL", [
|
|
573
618
|
UPDATE_STREAM,
|
|
@@ -584,35 +629,6 @@ async function updateStreamReservation(client, key, reservation, action, control
|
|
|
584
629
|
controlKey ? "1" : "0"
|
|
585
630
|
])) === 1;
|
|
586
631
|
}
|
|
587
|
-
async function reserveStreamJob(client, key, owner, cursor = "0-0") {
|
|
588
|
-
const claimed = await client.send("EVAL", [
|
|
589
|
-
CLAIM_STREAM,
|
|
590
|
-
"1",
|
|
591
|
-
streamQueueKey(key),
|
|
592
|
-
STREAM_GROUP,
|
|
593
|
-
owner,
|
|
594
|
-
String(readQueueVisibilityMs()),
|
|
595
|
-
cursor
|
|
596
|
-
]);
|
|
597
|
-
if (!Array.isArray(claimed) || typeof claimed[0] !== "string" || !Array.isArray(claimed[1]))
|
|
598
|
-
throw new Error("Invalid Redis XAUTOCLAIM response");
|
|
599
|
-
const nextCursor = claimed[0];
|
|
600
|
-
if (claimed[1].length > 0)
|
|
601
|
-
return { reservation: readStreamEntry(claimed[1][0], owner), cursor: nextCursor };
|
|
602
|
-
const fresh = await client.send("EVAL", [
|
|
603
|
-
READ_STREAM,
|
|
604
|
-
"1",
|
|
605
|
-
streamQueueKey(key),
|
|
606
|
-
STREAM_GROUP,
|
|
607
|
-
owner
|
|
608
|
-
]);
|
|
609
|
-
if (fresh === null)
|
|
610
|
-
return { reservation: null, cursor: nextCursor };
|
|
611
|
-
if (!Array.isArray(fresh) || !Array.isArray(fresh[0]) || !Array.isArray(fresh[0][1]))
|
|
612
|
-
throw new Error("Invalid Redis XREADGROUP response");
|
|
613
|
-
return { reservation: readStreamEntry(fresh[0][1][0], owner), cursor: nextCursor };
|
|
614
|
-
}
|
|
615
|
-
|
|
616
632
|
class RedisStreamsQueue {
|
|
617
633
|
client;
|
|
618
634
|
constructor(redisUrl) {
|
|
@@ -710,12 +726,6 @@ class RedisStreamsWorker {
|
|
|
710
726
|
await ensureStreamGroup(this.client, key);
|
|
711
727
|
this.initialized = true;
|
|
712
728
|
}
|
|
713
|
-
async reserve(key) {
|
|
714
|
-
await promoteStreamRetries(this.client, key);
|
|
715
|
-
const result = await reserveStreamJob(this.client, key, this.owner, this.cursors.get(key));
|
|
716
|
-
this.cursors.set(key, result.cursor);
|
|
717
|
-
return result.reservation;
|
|
718
|
-
}
|
|
719
729
|
async processNext() {
|
|
720
730
|
if (this.processing)
|
|
721
731
|
throw new Error("Queue worker already processing a job.");
|
|
@@ -732,13 +742,19 @@ class RedisStreamsWorker {
|
|
|
732
742
|
await this.initialize();
|
|
733
743
|
const deadline = performance.now() + this.timeoutSeconds * 1000;
|
|
734
744
|
do {
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
745
|
+
if (this.stopping)
|
|
746
|
+
return false;
|
|
747
|
+
const polled = await pollStreamPriorities(this.client, this.queueKeys, this.owner, this.queueKeys.map((key) => this.cursors.get(key) ?? "0-0"));
|
|
748
|
+
this.queueKeys.forEach((key, index) => {
|
|
749
|
+
const cursor = polled.cursors[index];
|
|
750
|
+
if (cursor)
|
|
751
|
+
this.cursors.set(key, cursor);
|
|
752
|
+
});
|
|
753
|
+
if (polled.hit) {
|
|
754
|
+
const key = this.queueKeys[polled.hit.index];
|
|
755
|
+
if (!key)
|
|
756
|
+
throw new Error("Invalid Redis stream poll response");
|
|
757
|
+
await this.handle(key, polled.hit.reservation);
|
|
742
758
|
return true;
|
|
743
759
|
}
|
|
744
760
|
if (performance.now() >= deadline || this.stopping)
|
|
@@ -1011,6 +1027,66 @@ for _, record in ipairs(redis.call('LRANGE', KEYS[2], 0, -1)) do
|
|
|
1011
1027
|
end
|
|
1012
1028
|
return reclaimed
|
|
1013
1029
|
`;
|
|
1030
|
+
var POLL_LIST_PRIORITIES = `${SCRIPT_HEADER}
|
|
1031
|
+
local owner = ARGV[1]
|
|
1032
|
+
local visibility = ARGV[2]
|
|
1033
|
+
local reservationId = ARGV[3]
|
|
1034
|
+
local reclaim = ARGV[4] == '1'
|
|
1035
|
+
local function reclaimQueue(listKey, processingKey, leaseKey)
|
|
1036
|
+
local err = checkType(listKey, 'list') or checkType(processingKey, 'list') or checkType(leaseKey, 'hash')
|
|
1037
|
+
if err then return err end
|
|
1038
|
+
local now = nowMs()
|
|
1039
|
+
for _, record in ipairs(redis.call('LRANGE', processingKey, 0, -1)) do
|
|
1040
|
+
local payload, field = record, record
|
|
1041
|
+
local modern = false
|
|
1042
|
+
local ok, reservation = pcall(cjson.decode, record)
|
|
1043
|
+
if ok and type(reservation) == 'table' and reservation.strataReservation == 1 then
|
|
1044
|
+
payload, field, modern = reservation.payload, reservation.id, true
|
|
1045
|
+
end
|
|
1046
|
+
local raw = redis.call('HGET', leaseKey, field)
|
|
1047
|
+
local expired = true
|
|
1048
|
+
if raw then
|
|
1049
|
+
if modern then
|
|
1050
|
+
local leaseOk, lease = pcall(cjson.decode, raw)
|
|
1051
|
+
expired = not leaseOk or type(lease) ~= 'table' or not tonumber(lease.expiresAt) or tonumber(lease.expiresAt) <= now
|
|
1052
|
+
else
|
|
1053
|
+
local reservedAt = tonumber(raw)
|
|
1054
|
+
expired = not reservedAt or now - reservedAt >= tonumber(visibility)
|
|
1055
|
+
end
|
|
1056
|
+
end
|
|
1057
|
+
if expired then
|
|
1058
|
+
redis.call('LPUSH', listKey, payload)
|
|
1059
|
+
redis.call('LREM', processingKey, 1, record)
|
|
1060
|
+
redis.call('HDEL', leaseKey, field)
|
|
1061
|
+
end
|
|
1062
|
+
end
|
|
1063
|
+
end
|
|
1064
|
+
local function reserveQueue(listKey, processingKey, leaseKey)
|
|
1065
|
+
local err = checkType(listKey, 'list') or checkType(processingKey, 'list') or checkType(leaseKey, 'hash')
|
|
1066
|
+
if err then return err end
|
|
1067
|
+
local payload = redis.call('LINDEX', listKey, -1)
|
|
1068
|
+
if not payload then return nil end
|
|
1069
|
+
local record = cjson.encode({strataReservation=1, id=reservationId, owner=owner, payload=payload})
|
|
1070
|
+
local lease = cjson.encode({owner=owner, expiresAt=nowMs()+tonumber(visibility)})
|
|
1071
|
+
redis.call('LPUSH', processingKey, record)
|
|
1072
|
+
redis.call('HSET', leaseKey, reservationId, lease)
|
|
1073
|
+
redis.call('RPOP', listKey)
|
|
1074
|
+
return record
|
|
1075
|
+
end
|
|
1076
|
+
local count = #KEYS / 3
|
|
1077
|
+
if reclaim then
|
|
1078
|
+
for i = 1, count do
|
|
1079
|
+
local err = reclaimQueue(KEYS[(i - 1) * 3 + 1], KEYS[(i - 1) * 3 + 2], KEYS[(i - 1) * 3 + 3])
|
|
1080
|
+
if err then return err end
|
|
1081
|
+
end
|
|
1082
|
+
end
|
|
1083
|
+
for i = 1, count do
|
|
1084
|
+
local record = reserveQueue(KEYS[(i - 1) * 3 + 1], KEYS[(i - 1) * 3 + 2], KEYS[(i - 1) * 3 + 3])
|
|
1085
|
+
if type(record) == 'string' then return {'1', tostring(i), record} end
|
|
1086
|
+
if record then return record end
|
|
1087
|
+
end
|
|
1088
|
+
return {'0'}
|
|
1089
|
+
`;
|
|
1014
1090
|
function queueInvalidKey(queueKey) {
|
|
1015
1091
|
return `${queueKey}:invalid`;
|
|
1016
1092
|
}
|
|
@@ -1045,6 +1121,41 @@ async function updateQueueReservation(client, queueKey, reservation, action) {
|
|
|
1045
1121
|
reservation.payload
|
|
1046
1122
|
])) === 1;
|
|
1047
1123
|
}
|
|
1124
|
+
async function pollListPriorities(client, queueKeys, owner, reservationId, reclaim) {
|
|
1125
|
+
const keys = [];
|
|
1126
|
+
for (const key of queueKeys) {
|
|
1127
|
+
keys.push(key, queueProcessingKey(key), queueProcessingLeaseKey(key));
|
|
1128
|
+
}
|
|
1129
|
+
const raw = await client.send("EVAL", [
|
|
1130
|
+
POLL_LIST_PRIORITIES,
|
|
1131
|
+
String(keys.length),
|
|
1132
|
+
...keys,
|
|
1133
|
+
owner,
|
|
1134
|
+
String(readQueueVisibilityMs()),
|
|
1135
|
+
reservationId,
|
|
1136
|
+
reclaim ? "1" : "0"
|
|
1137
|
+
]);
|
|
1138
|
+
if (!Array.isArray(raw))
|
|
1139
|
+
throw new Error("Invalid Redis list poll response");
|
|
1140
|
+
const flag = String(raw[0]);
|
|
1141
|
+
if (flag === "0") {
|
|
1142
|
+
if (raw.length !== 1)
|
|
1143
|
+
throw new Error("Invalid Redis list poll response");
|
|
1144
|
+
return null;
|
|
1145
|
+
}
|
|
1146
|
+
if (flag !== "1" || raw.length !== 3)
|
|
1147
|
+
throw new Error("Invalid Redis list poll response");
|
|
1148
|
+
const index = Number(raw[1]);
|
|
1149
|
+
const record = raw[2];
|
|
1150
|
+
if (!Number.isInteger(index) || index < 1 || index > queueKeys.length || typeof record !== "string") {
|
|
1151
|
+
throw new Error("Invalid Redis list poll response");
|
|
1152
|
+
}
|
|
1153
|
+
const queueKey = queueKeys[index - 1];
|
|
1154
|
+
if (!queueKey)
|
|
1155
|
+
throw new Error("Invalid Redis list poll response");
|
|
1156
|
+
const reservation = JSON.parse(record);
|
|
1157
|
+
return { queueKey, reservation: { ...reservation, record } };
|
|
1158
|
+
}
|
|
1048
1159
|
async function reclaimExpiredQueueReservations(client, queueKeys = QUEUE_KEYS) {
|
|
1049
1160
|
let reclaimed = 0;
|
|
1050
1161
|
for (const key of queueKeys) {
|
|
@@ -1105,14 +1216,14 @@ class QueueWorker {
|
|
|
1105
1216
|
}
|
|
1106
1217
|
async reserve() {
|
|
1107
1218
|
const deadline = performance.now() + this.timeoutSeconds * 1000;
|
|
1219
|
+
let reclaim = true;
|
|
1108
1220
|
do {
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
}
|
|
1221
|
+
if (this.stopping)
|
|
1222
|
+
return null;
|
|
1223
|
+
const reserved = await pollListPriorities(this.client, this.queueKeys, this.owner, crypto.randomUUID(), reclaim);
|
|
1224
|
+
reclaim = false;
|
|
1225
|
+
if (reserved)
|
|
1226
|
+
return reserved;
|
|
1116
1227
|
if (this.stopping || performance.now() >= deadline)
|
|
1117
1228
|
return null;
|
|
1118
1229
|
await Bun.sleep(Math.min(100, Math.max(1, deadline - performance.now())));
|
|
@@ -1122,7 +1233,6 @@ class QueueWorker {
|
|
|
1122
1233
|
async processNext() {
|
|
1123
1234
|
if (this.stopping)
|
|
1124
1235
|
return false;
|
|
1125
|
-
await reclaimExpiredQueueReservations(this.client, this.queueKeys);
|
|
1126
1236
|
const reserved = await this.reserve();
|
|
1127
1237
|
if (!reserved)
|
|
1128
1238
|
return false;
|