bunqueue 2.9.0 → 2.9.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -3
- package/dist/application/background/recovery/active.d.ts +1 -0
- package/dist/application/background/recovery/active.js +12 -0
- package/dist/application/background/recovery/index.js +3 -1
- package/dist/application/background/recovery/pending.js +1 -0
- package/dist/application/cleanupTasks.js +4 -16
- package/dist/application/contextFactory.js +3 -0
- package/dist/application/dlqManager.js +1 -0
- package/dist/application/dlqRetry.js +1 -0
- package/dist/application/eventsManager.d.ts +8 -1
- package/dist/application/eventsManager.js +76 -29
- package/dist/application/operations/ackHelpers.d.ts +1 -4
- package/dist/application/operations/ackHelpers.js +39 -14
- package/dist/application/operations/flowPush.js +1 -0
- package/dist/application/operations/flowValidation.js +4 -4
- package/dist/application/operations/pull.d.ts +7 -2
- package/dist/application/operations/pull.js +15 -57
- package/dist/application/operations/pullFinalization.d.ts +6 -0
- package/dist/application/operations/pullFinalization.js +69 -0
- package/dist/application/operations/pullStateTransition.d.ts +8 -7
- package/dist/application/operations/pullStateTransition.js +25 -18
- package/dist/application/operations/push.js +3 -0
- package/dist/application/operations/pushBatch.js +14 -7
- package/dist/application/operations/pushContext.d.ts +2 -1
- package/dist/application/postgres-queue-manager/control.d.ts +4 -0
- package/dist/application/postgres-queue-manager/control.js +24 -0
- package/dist/application/postgres-queue-manager/delivery.d.ts +5 -4
- package/dist/application/postgres-queue-manager/delivery.js +13 -10
- package/dist/application/postgres-queue-manager/groupQueries.d.ts +10 -0
- package/dist/application/postgres-queue-manager/groupQueries.js +40 -0
- package/dist/application/postgres-queue-manager/queries.d.ts +2 -2
- package/dist/application/postgres-queue-manager/queries.js +2 -2
- package/dist/application/queue-manager/control.js +1 -0
- package/dist/application/queue-manager/delivery.d.ts +5 -4
- package/dist/application/queue-manager/delivery.js +15 -8
- package/dist/application/queue-manager/limits.d.ts +11 -0
- package/dist/application/queue-manager/limits.js +61 -0
- package/dist/application/queue-manager/state.js +1 -1
- package/dist/application/queueTelemetryJournal.d.ts +2 -0
- package/dist/application/queueTelemetryJournal.js +23 -0
- package/dist/application/types/ack.d.ts +3 -1
- package/dist/cli/commandRouter.d.ts +1 -1
- package/dist/client/flowPush.js +6 -0
- package/dist/client/groupId.d.ts +1 -0
- package/dist/client/groupId.js +1 -0
- package/dist/client/index.d.ts +1 -1
- package/dist/client/jobHelpers.js +1 -0
- package/dist/client/queue/operations/add/bulk.js +4 -4
- package/dist/client/queue/operations/add/payload.d.ts +1 -0
- package/dist/client/queue/operations/add/payload.js +7 -1
- package/dist/client/queue/operations/add/single.js +3 -3
- package/dist/client/queue/operations/groups.d.ts +21 -0
- package/dist/client/queue/operations/groups.js +88 -0
- package/dist/client/queue/runtime/queries.d.ts +13 -0
- package/dist/client/queue/runtime/queries.js +31 -0
- package/dist/client/queue/runtime/state.d.ts +4 -0
- package/dist/client/queue/runtime/state.js +1 -0
- package/dist/client/types/index.d.ts +2 -2
- package/dist/client/types/options.d.ts +4 -0
- package/dist/client/types/worker.d.ts +8 -0
- package/dist/client/worker/runtime/execution.js +1 -0
- package/dist/client/worker/runtime/options.js +3 -0
- package/dist/client/worker/types/options.d.ts +2 -1
- package/dist/client/worker/workerPull.d.ts +2 -0
- package/dist/client/worker/workerPull.js +6 -4
- package/dist/domain/job/groupFifoOrder.d.ts +9 -0
- package/dist/domain/job/groupFifoOrder.js +26 -0
- package/dist/domain/queue/groupLimiterManager.d.ts +23 -0
- package/dist/domain/queue/groupLimiterManager.js +107 -0
- package/dist/domain/queue/groupScheduler.d.ts +31 -0
- package/dist/domain/queue/groupScheduler.js +256 -0
- package/dist/domain/queue/groupSchedulerOrder.d.ts +6 -0
- package/dist/domain/queue/groupSchedulerOrder.js +30 -0
- package/dist/domain/queue/priorityQueue.d.ts +13 -23
- package/dist/domain/queue/priorityQueue.js +53 -42
- package/dist/domain/queue/shard/keys.d.ts +1 -0
- package/dist/domain/queue/shard/keys.js +25 -2
- package/dist/domain/queue/shard/lifecycle.js +4 -0
- package/dist/domain/queue/shard/limits.d.ts +20 -1
- package/dist/domain/queue/shard/limits.js +56 -0
- package/dist/domain/queue/shard/state.d.ts +5 -0
- package/dist/domain/queue/shard/state.js +9 -1
- package/dist/domain/types/commands/core.d.ts +3 -0
- package/dist/domain/types/commands/limits.d.ts +41 -0
- package/dist/domain/types/commands/union.d.ts +1 -1
- package/dist/domain/types/group.d.ts +21 -0
- package/dist/domain/types/group.js +78 -0
- package/dist/domain/types/priorityQueue.d.ts +1 -0
- package/dist/infrastructure/persistence/jobOptionsBlob.d.ts +1 -0
- package/dist/infrastructure/persistence/jobOptionsBlob.js +10 -2
- package/dist/infrastructure/persistence/migrations.js +11 -0
- package/dist/infrastructure/persistence/postgres/admission.js +5 -2
- package/dist/infrastructure/persistence/postgres/batchAdmission.js +22 -4
- package/dist/infrastructure/persistence/postgres/claimBatch.d.ts +3 -0
- package/dist/infrastructure/persistence/postgres/claimBatch.js +16 -3
- package/dist/infrastructure/persistence/postgres/claimSelection.d.ts +8 -1
- package/dist/infrastructure/persistence/postgres/claimSelection.js +73 -131
- package/dist/infrastructure/persistence/postgres/claims.d.ts +8 -1
- package/dist/infrastructure/persistence/postgres/claims.js +19 -3
- package/dist/infrastructure/persistence/postgres/eventCatchupCursors.d.ts +32 -0
- package/dist/infrastructure/persistence/postgres/eventCatchupCursors.js +66 -0
- package/dist/infrastructure/persistence/postgres/eventJournalSchema.d.ts +2 -2
- package/dist/infrastructure/persistence/postgres/eventJournalSchema.js +16 -2
- package/dist/infrastructure/persistence/postgres/eventPruneWatermarks.d.ts +2 -0
- package/dist/infrastructure/persistence/postgres/eventPruneWatermarks.js +21 -3
- package/dist/infrastructure/persistence/postgres/events.d.ts +3 -3
- package/dist/infrastructure/persistence/postgres/events.js +44 -17
- package/dist/infrastructure/persistence/postgres/groupClaims.d.ts +15 -0
- package/dist/infrastructure/persistence/postgres/groupClaims.js +95 -0
- package/dist/infrastructure/persistence/postgres/groupSchema.d.ts +2 -0
- package/dist/infrastructure/persistence/postgres/groupSchema.js +155 -0
- package/dist/infrastructure/persistence/postgres/groupSchemaFingerprint.d.ts +3 -0
- package/dist/infrastructure/persistence/postgres/groupSchemaFingerprint.js +127 -0
- package/dist/infrastructure/persistence/postgres/groupStateRetention.d.ts +3 -0
- package/dist/infrastructure/persistence/postgres/groupStateRetention.js +93 -0
- package/dist/infrastructure/persistence/postgres/groups.d.ts +11 -0
- package/dist/infrastructure/persistence/postgres/groups.js +117 -0
- package/dist/infrastructure/persistence/postgres/queueDestruction.js +4 -0
- package/dist/infrastructure/persistence/postgres/runtime.d.ts +0 -1
- package/dist/infrastructure/persistence/postgres/runtime.js +3 -1
- package/dist/infrastructure/persistence/postgres/schema.d.ts +2 -2
- package/dist/infrastructure/persistence/postgres/schema.js +8 -4
- package/dist/infrastructure/persistence/postgres/schemaInitialization.js +6 -1
- package/dist/infrastructure/persistence/postgres/store.d.ts +11 -1
- package/dist/infrastructure/persistence/postgres/store.js +47 -3
- package/dist/infrastructure/persistence/postgres/telemetry.d.ts +2 -0
- package/dist/infrastructure/persistence/postgres/telemetry.js +33 -21
- package/dist/infrastructure/persistence/postgres/types.d.ts +1 -0
- package/dist/infrastructure/persistence/schema.d.ts +2 -2
- package/dist/infrastructure/persistence/schema.js +12 -1
- package/dist/infrastructure/persistence/sqlite/control.d.ts +14 -0
- package/dist/infrastructure/persistence/sqlite/control.js +69 -0
- package/dist/infrastructure/persistence/sqlite/jobs.d.ts +12 -0
- package/dist/infrastructure/persistence/sqlite/jobs.js +36 -0
- package/dist/infrastructure/persistence/sqlite/telemetry.d.ts +2 -1
- package/dist/infrastructure/persistence/sqlite/telemetry.js +5 -64
- package/dist/infrastructure/persistence/sqlite/telemetryWrites.d.ts +4 -0
- package/dist/infrastructure/persistence/sqlite/telemetryWrites.js +123 -0
- package/dist/infrastructure/persistence/sqliteSerializer.js +2 -0
- package/dist/infrastructure/server/handler-routes/control.js +13 -0
- package/dist/infrastructure/server/handlers/core.d.ts +0 -5
- package/dist/infrastructure/server/handlers/core.js +15 -18
- package/dist/infrastructure/server/handlers/groups.d.ts +11 -0
- package/dist/infrastructure/server/handlers/groups.js +55 -0
- package/dist/infrastructure/server/handlers/pushBatchValidation.js +4 -1
- package/dist/infrastructure/server/protocol/validation.d.ts +1 -0
- package/dist/infrastructure/server/protocol/validation.js +1 -0
- package/dist/infrastructure/server/protocol.d.ts +1 -1
- package/dist/infrastructure/server/protocol.js +1 -1
- package/package.json +2 -1
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { getGroupFifoOrder } from '../../domain/job/groupFifoOrder';
|
|
1
2
|
import { storageLog } from '../../shared/logger';
|
|
2
3
|
import { decodeMessagePack, encodeMessagePack } from '../../shared/msgpack';
|
|
3
4
|
const DEFAULTS = {
|
|
@@ -12,10 +13,14 @@ const DEFAULTS = {
|
|
|
12
13
|
debounceId: null,
|
|
13
14
|
debounceTtl: null,
|
|
14
15
|
durable: false,
|
|
16
|
+
groupFifoOrder: null,
|
|
15
17
|
};
|
|
18
|
+
function validStoredOrder(value) {
|
|
19
|
+
return typeof value === 'string' && /^[1-9]\d*$/.test(value) ? value : null;
|
|
20
|
+
}
|
|
16
21
|
/** Persist generation policies that do not have dedicated legacy columns. */
|
|
17
22
|
export function encodeJobOptions(job) {
|
|
18
|
-
|
|
23
|
+
const options = {
|
|
19
24
|
backoffConfig: job.backoffConfig,
|
|
20
25
|
repeat: job.repeat,
|
|
21
26
|
stackTraceLimit: job.stackTraceLimit,
|
|
@@ -27,7 +32,9 @@ export function encodeJobOptions(job) {
|
|
|
27
32
|
debounceId: job.debounceId,
|
|
28
33
|
debounceTtl: job.debounceTtl,
|
|
29
34
|
durable: job.durable ?? false,
|
|
30
|
-
}
|
|
35
|
+
};
|
|
36
|
+
const order = getGroupFifoOrder(job);
|
|
37
|
+
return encodeMessagePack(order === null ? options : { ...options, groupFifoOrder: String(order) });
|
|
31
38
|
}
|
|
32
39
|
/** Decode current blobs while giving pre-v34 rows their historical defaults. */
|
|
33
40
|
export function decodeJobOptions(blob, context) {
|
|
@@ -47,6 +54,7 @@ export function decodeJobOptions(blob, context) {
|
|
|
47
54
|
debounceId: decoded.debounceId ?? null,
|
|
48
55
|
debounceTtl: decoded.debounceTtl ?? null,
|
|
49
56
|
durable: decoded.durable ?? false,
|
|
57
|
+
groupFifoOrder: validStoredOrder(decoded.groupFifoOrder),
|
|
50
58
|
};
|
|
51
59
|
}
|
|
52
60
|
catch (error) {
|
|
@@ -117,4 +117,15 @@ CREATE INDEX IF NOT EXISTS idx_queue_metric_buckets_window
|
|
|
117
117
|
`,
|
|
118
118
|
// Persist repeat-chain and advanced per-generation job policies.
|
|
119
119
|
34: 'ALTER TABLE jobs ADD COLUMN extended_options BLOB;',
|
|
120
|
+
// Persist per-group rate and concurrency overrides.
|
|
121
|
+
35: `
|
|
122
|
+
CREATE TABLE IF NOT EXISTS group_state (
|
|
123
|
+
queue TEXT NOT NULL,
|
|
124
|
+
group_id TEXT NOT NULL,
|
|
125
|
+
rate_limit INTEGER,
|
|
126
|
+
rate_duration INTEGER,
|
|
127
|
+
concurrency_limit INTEGER,
|
|
128
|
+
PRIMARY KEY (queue, group_id)
|
|
129
|
+
);
|
|
130
|
+
`,
|
|
120
131
|
};
|
|
@@ -75,14 +75,17 @@ async function insertRow(tx, ctx, job, state, now) {
|
|
|
75
75
|
INSERT INTO bunqueue_jobs (
|
|
76
76
|
namespace, id, queue, payload, state, priority, lifo, run_at, created_at,
|
|
77
77
|
started_at, completed_at, attempts, max_attempts, ttl, timeout, unique_key,
|
|
78
|
-
unique_expires_at, custom_id, group_id, parent_id
|
|
78
|
+
unique_expires_at, custom_id, group_id, group_order, parent_id
|
|
79
79
|
) VALUES (
|
|
80
80
|
${ctx.config.namespace}, ${String(job.id)}, ${job.queue}, ${encodePostgresValue(job)},
|
|
81
81
|
${state}, ${job.priority}, ${job.lifo}, ${job.runAt}, ${job.createdAt},
|
|
82
82
|
${job.startedAt}, ${job.completedAt}, ${job.attempts}, ${job.maxAttempts}, ${job.ttl},
|
|
83
83
|
${job.timeout}, ${job.uniqueKey},
|
|
84
84
|
${job.uniqueKey && job.deduplicationTtl !== null ? now + job.deduplicationTtl : null},
|
|
85
|
-
${job.customId}, ${job.groupId},
|
|
85
|
+
${job.customId}, ${job.groupId},
|
|
86
|
+
CASE WHEN CAST(${job.groupId} AS TEXT) IS NULL
|
|
87
|
+
THEN NULL ELSE nextval('bunqueue_group_order_seq') END,
|
|
88
|
+
${job.parentId}
|
|
86
89
|
)
|
|
87
90
|
`;
|
|
88
91
|
await insertDependencies(tx, ctx, job);
|
|
@@ -57,8 +57,8 @@ function prepareAdmissions(jobs, now) {
|
|
|
57
57
|
return { job: preparedJob, state };
|
|
58
58
|
});
|
|
59
59
|
}
|
|
60
|
-
function toRecords(prepared, namespace, now) {
|
|
61
|
-
return prepared.map(({ job, state }) => ({
|
|
60
|
+
function toRecords(prepared, namespace, now, groupOrders) {
|
|
61
|
+
return prepared.map(({ job, state }, position) => ({
|
|
62
62
|
namespace,
|
|
63
63
|
id: String(job.id),
|
|
64
64
|
queue: job.queue,
|
|
@@ -78,15 +78,33 @@ function toRecords(prepared, namespace, now) {
|
|
|
78
78
|
unique_expires_at: job.uniqueKey && job.deduplicationTtl !== null ? now + job.deduplicationTtl : null,
|
|
79
79
|
custom_id: job.customId,
|
|
80
80
|
group_id: job.groupId,
|
|
81
|
+
group_order: groupOrders.get(position) ?? null,
|
|
81
82
|
}));
|
|
82
83
|
}
|
|
84
|
+
async function allocateGroupOrders(tx, prepared) {
|
|
85
|
+
const positions = prepared.flatMap(({ job }, position) => job.groupId === null ? [] : [position]);
|
|
86
|
+
if (positions.length === 0)
|
|
87
|
+
return new Map();
|
|
88
|
+
const rows = await tx `
|
|
89
|
+
WITH ordered AS MATERIALIZED (
|
|
90
|
+
SELECT position
|
|
91
|
+
FROM unnest(${tx.array(positions, 'INTEGER')}) AS requested(position)
|
|
92
|
+
ORDER BY position
|
|
93
|
+
)
|
|
94
|
+
SELECT position, nextval('bunqueue_group_order_seq') AS group_order
|
|
95
|
+
FROM ordered
|
|
96
|
+
ORDER BY position
|
|
97
|
+
`;
|
|
98
|
+
return new Map(rows.map(({ position, group_order }) => [position, group_order]));
|
|
99
|
+
}
|
|
83
100
|
async function insertRows(tx, ctx, prepared, now) {
|
|
84
|
-
const
|
|
101
|
+
const groupOrders = await allocateGroupOrders(tx, prepared);
|
|
102
|
+
const records = toRecords(prepared, ctx.config.namespace, now, groupOrders);
|
|
85
103
|
let insertedCount = 0;
|
|
86
104
|
for (let offset = 0; offset < records.length; offset += 1000) {
|
|
87
105
|
const batch = records.slice(offset, offset + 1000);
|
|
88
106
|
const inserted = await tx `
|
|
89
|
-
INSERT INTO bunqueue_jobs ${tx(batch, 'namespace', 'id', 'queue', 'payload', 'state', 'priority', 'lifo', 'run_at', 'created_at', 'started_at', 'completed_at', 'attempts', 'max_attempts', 'ttl', 'timeout', 'unique_key', 'unique_expires_at', 'custom_id', 'group_id')}
|
|
107
|
+
INSERT INTO bunqueue_jobs ${tx(batch, 'namespace', 'id', 'queue', 'payload', 'state', 'priority', 'lifo', 'run_at', 'created_at', 'started_at', 'completed_at', 'attempts', 'max_attempts', 'ttl', 'timeout', 'unique_key', 'unique_expires_at', 'custom_id', 'group_id', 'group_order')}
|
|
90
108
|
ON CONFLICT DO NOTHING
|
|
91
109
|
RETURNING id
|
|
92
110
|
`;
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { TransactionSQL } from 'bun';
|
|
2
|
+
import type { GroupPullOptions } from '../../../domain/types/group';
|
|
2
3
|
import type { PostgresContext } from './context';
|
|
3
4
|
import type { ClaimedPostgresJob } from './types';
|
|
4
5
|
/** Claim one queue batch after its queue-state row has been locked. */
|
|
@@ -8,4 +9,6 @@ export declare function claimPostgresCandidates(tx: TransactionSQL, ctx: Postgre
|
|
|
8
9
|
owner: string;
|
|
9
10
|
requestedLeaseMs: number;
|
|
10
11
|
now: number;
|
|
12
|
+
groupOptions?: GroupPullOptions;
|
|
13
|
+
groupSequence: number;
|
|
11
14
|
}): Promise<ClaimedPostgresJob[]>;
|
|
@@ -2,6 +2,7 @@ import { generateLockToken, MAX_TIMELINE_ENTRIES } from '../../../domain/types/j
|
|
|
2
2
|
import { recordPostgresJobEvents } from './batchEvents';
|
|
3
3
|
import { selectPostgresClaimCandidates } from './claimSelection';
|
|
4
4
|
import { decodePostgresJob, encodePostgresValue, postgresByteaBase64 } from './codec';
|
|
5
|
+
import { commitPostgresGroupClaims } from './groupClaims';
|
|
5
6
|
function prepareClaims(rows, input) {
|
|
6
7
|
const { owner, requestedLeaseMs, now, brokerId } = input;
|
|
7
8
|
return rows.map((row) => {
|
|
@@ -42,15 +43,27 @@ async function persistClaims(tx, ctx, claimed, now) {
|
|
|
42
43
|
}
|
|
43
44
|
/** Claim one queue batch after its queue-state row has been locked. */
|
|
44
45
|
export async function claimPostgresCandidates(tx, ctx, input) {
|
|
45
|
-
const { queue, capacity, owner, requestedLeaseMs, now } = input;
|
|
46
|
-
const rows = await selectPostgresClaimCandidates(tx, ctx,
|
|
46
|
+
const { queue, capacity, owner, requestedLeaseMs, now, groupOptions, groupSequence } = input;
|
|
47
|
+
const rows = await selectPostgresClaimCandidates(tx, ctx, {
|
|
48
|
+
queue,
|
|
49
|
+
now,
|
|
50
|
+
capacity,
|
|
51
|
+
options: groupOptions,
|
|
52
|
+
});
|
|
47
53
|
const claimed = prepareClaims(rows, {
|
|
48
54
|
owner,
|
|
49
55
|
requestedLeaseMs,
|
|
50
56
|
now,
|
|
51
57
|
brokerId: ctx.config.brokerId,
|
|
52
58
|
});
|
|
53
|
-
if (claimed.length > 0)
|
|
59
|
+
if (claimed.length > 0) {
|
|
60
|
+
await commitPostgresGroupClaims(tx, ctx, {
|
|
61
|
+
queue,
|
|
62
|
+
rows,
|
|
63
|
+
options: groupOptions,
|
|
64
|
+
sequence: groupSequence,
|
|
65
|
+
});
|
|
54
66
|
await persistClaims(tx, ctx, claimed, now);
|
|
67
|
+
}
|
|
55
68
|
return claimed;
|
|
56
69
|
}
|
|
@@ -1,4 +1,11 @@
|
|
|
1
1
|
import type { TransactionSQL } from 'bun';
|
|
2
|
+
import type { GroupPullOptions } from '../../../domain/types/group';
|
|
2
3
|
import type { PostgresContext } from './context';
|
|
3
4
|
import type { PostgresJobRow } from './types';
|
|
4
|
-
export declare function
|
|
5
|
+
export declare function hasClaimablePostgresGroup(tx: TransactionSQL, ctx: PostgresContext, queue: string, now: number): Promise<boolean>;
|
|
6
|
+
export declare function selectPostgresClaimCandidates(tx: TransactionSQL, ctx: PostgresContext, input: {
|
|
7
|
+
queue: string;
|
|
8
|
+
now: number;
|
|
9
|
+
capacity: number;
|
|
10
|
+
options?: GroupPullOptions;
|
|
11
|
+
}): Promise<PostgresJobRow[]>;
|
|
@@ -1,143 +1,75 @@
|
|
|
1
|
-
async function
|
|
1
|
+
export async function hasClaimablePostgresGroup(tx, ctx, queue, now) {
|
|
2
2
|
const [row] = await tx `
|
|
3
3
|
SELECT EXISTS (
|
|
4
|
-
SELECT 1
|
|
5
|
-
|
|
6
|
-
WHERE job.namespace = ${ctx.config.namespace}
|
|
7
|
-
AND job.queue = ${queue}
|
|
4
|
+
SELECT 1 FROM bunqueue_jobs AS job
|
|
5
|
+
WHERE job.namespace = ${ctx.config.namespace} AND job.queue = ${queue}
|
|
8
6
|
AND job.group_id IS NOT NULL
|
|
9
7
|
AND job.state IN ('waiting', 'prioritized', 'delayed')
|
|
10
8
|
AND job.run_at <= ${now}
|
|
11
9
|
AND (job.ttl IS NULL OR job.created_at + job.ttl >= ${now})
|
|
12
10
|
AND NOT EXISTS (
|
|
13
|
-
SELECT 1
|
|
14
|
-
|
|
15
|
-
WHERE dependency.namespace = job.namespace
|
|
16
|
-
AND dependency.job_id = job.id
|
|
11
|
+
SELECT 1 FROM bunqueue_dependencies AS dependency
|
|
12
|
+
WHERE dependency.namespace = job.namespace AND dependency.job_id = job.id
|
|
17
13
|
AND NOT EXISTS (
|
|
18
14
|
SELECT 1 FROM bunqueue_completions AS completion
|
|
19
15
|
WHERE completion.namespace = dependency.namespace
|
|
20
16
|
AND completion.job_id = dependency.dependency_id
|
|
21
17
|
)
|
|
22
18
|
)
|
|
23
|
-
AND NOT EXISTS (
|
|
24
|
-
SELECT 1 FROM bunqueue_jobs AS active_group
|
|
25
|
-
WHERE active_group.namespace = job.namespace
|
|
26
|
-
AND active_group.queue = job.queue
|
|
27
|
-
AND active_group.group_id = job.group_id
|
|
28
|
-
AND active_group.state = 'active'
|
|
29
|
-
AND active_group.lease_until > ${now}
|
|
30
|
-
)
|
|
31
|
-
) AS exists
|
|
32
|
-
`;
|
|
33
|
-
return row.exists;
|
|
34
|
-
}
|
|
35
|
-
async function hasClaimableLifoJob(tx, ctx, queue, now) {
|
|
36
|
-
const [row] = await tx `
|
|
37
|
-
SELECT EXISTS (
|
|
38
|
-
SELECT 1 FROM bunqueue_jobs AS job
|
|
39
|
-
WHERE job.namespace = ${ctx.config.namespace}
|
|
40
|
-
AND job.queue = ${queue}
|
|
41
|
-
AND job.group_id IS NULL
|
|
42
|
-
AND job.lifo
|
|
43
|
-
AND job.state IN ('waiting', 'prioritized', 'delayed')
|
|
44
|
-
AND job.run_at <= ${now}
|
|
45
|
-
AND (job.ttl IS NULL OR job.created_at + job.ttl >= ${now})
|
|
46
19
|
) AS exists
|
|
47
20
|
`;
|
|
48
21
|
return row.exists;
|
|
49
22
|
}
|
|
50
|
-
async function selectFifoCandidates(tx, ctx, queue, now, capacity) {
|
|
51
|
-
return await tx `
|
|
52
|
-
WITH selected AS (
|
|
53
|
-
SELECT job.id, job.priority, job.run_at
|
|
54
|
-
FROM bunqueue_jobs AS job
|
|
55
|
-
WHERE job.namespace = ${ctx.config.namespace}
|
|
56
|
-
AND job.queue = ${queue}
|
|
57
|
-
AND job.group_id IS NULL
|
|
58
|
-
AND NOT job.lifo
|
|
59
|
-
AND job.state IN ('waiting', 'prioritized', 'delayed')
|
|
60
|
-
AND job.run_at <= ${now}
|
|
61
|
-
AND (job.ttl IS NULL OR job.created_at + job.ttl >= ${now})
|
|
62
|
-
AND NOT EXISTS (
|
|
63
|
-
SELECT 1 FROM bunqueue_dependencies AS dependency
|
|
64
|
-
WHERE dependency.namespace = job.namespace
|
|
65
|
-
AND dependency.job_id = job.id
|
|
66
|
-
AND NOT EXISTS (
|
|
67
|
-
SELECT 1 FROM bunqueue_completions AS completion
|
|
68
|
-
WHERE completion.namespace = dependency.namespace
|
|
69
|
-
AND completion.job_id = dependency.dependency_id
|
|
70
|
-
)
|
|
71
|
-
)
|
|
72
|
-
ORDER BY job.priority DESC, job.run_at ASC, job.id ASC
|
|
73
|
-
LIMIT ${capacity}
|
|
74
|
-
FOR UPDATE OF job SKIP LOCKED
|
|
75
|
-
)
|
|
76
|
-
SELECT job.*
|
|
77
|
-
FROM bunqueue_jobs AS job
|
|
78
|
-
JOIN selected ON selected.id = job.id
|
|
79
|
-
WHERE job.namespace = ${ctx.config.namespace}
|
|
80
|
-
ORDER BY selected.priority DESC, selected.run_at ASC, selected.id ASC
|
|
81
|
-
`;
|
|
82
|
-
}
|
|
83
23
|
async function selectUngroupedCandidates(tx, ctx, queue, now, capacity) {
|
|
84
24
|
return await tx `
|
|
85
25
|
WITH selected AS (
|
|
86
26
|
SELECT job.id, job.priority, job.lifo, job.run_at
|
|
87
27
|
FROM bunqueue_jobs AS job
|
|
88
|
-
WHERE job.namespace = ${ctx.config.namespace}
|
|
89
|
-
AND job.queue = ${queue}
|
|
28
|
+
WHERE job.namespace = ${ctx.config.namespace} AND job.queue = ${queue}
|
|
90
29
|
AND job.group_id IS NULL
|
|
91
30
|
AND job.state IN ('waiting', 'prioritized', 'delayed')
|
|
92
31
|
AND job.run_at <= ${now}
|
|
93
32
|
AND (job.ttl IS NULL OR job.created_at + job.ttl >= ${now})
|
|
94
33
|
AND NOT EXISTS (
|
|
95
34
|
SELECT 1 FROM bunqueue_dependencies AS dependency
|
|
96
|
-
WHERE dependency.namespace = job.namespace
|
|
97
|
-
AND dependency.job_id = job.id
|
|
35
|
+
WHERE dependency.namespace = job.namespace AND dependency.job_id = job.id
|
|
98
36
|
AND NOT EXISTS (
|
|
99
37
|
SELECT 1 FROM bunqueue_completions AS completion
|
|
100
38
|
WHERE completion.namespace = dependency.namespace
|
|
101
39
|
AND completion.job_id = dependency.dependency_id
|
|
102
40
|
)
|
|
103
41
|
)
|
|
104
|
-
ORDER BY
|
|
105
|
-
job.
|
|
106
|
-
CASE WHEN job.lifo THEN 0 ELSE 1 END ASC,
|
|
42
|
+
ORDER BY job.priority DESC,
|
|
43
|
+
CASE WHEN job.lifo THEN 0 ELSE 1 END,
|
|
107
44
|
CASE WHEN job.lifo THEN job.id END DESC,
|
|
108
|
-
CASE WHEN NOT job.lifo THEN job.run_at END
|
|
109
|
-
job.id
|
|
45
|
+
CASE WHEN NOT job.lifo THEN job.run_at END,
|
|
46
|
+
job.id
|
|
110
47
|
LIMIT ${capacity}
|
|
111
48
|
FOR UPDATE OF job SKIP LOCKED
|
|
112
49
|
)
|
|
113
|
-
SELECT job.*
|
|
114
|
-
FROM bunqueue_jobs AS job
|
|
50
|
+
SELECT job.* FROM bunqueue_jobs AS job
|
|
115
51
|
JOIN selected ON selected.id = job.id
|
|
116
52
|
WHERE job.namespace = ${ctx.config.namespace}
|
|
117
|
-
ORDER BY
|
|
118
|
-
selected.
|
|
119
|
-
CASE WHEN selected.lifo THEN 0 ELSE 1 END ASC,
|
|
53
|
+
ORDER BY selected.priority DESC,
|
|
54
|
+
CASE WHEN selected.lifo THEN 0 ELSE 1 END,
|
|
120
55
|
CASE WHEN selected.lifo THEN selected.id END DESC,
|
|
121
|
-
CASE WHEN NOT selected.lifo THEN selected.run_at END
|
|
122
|
-
selected.id
|
|
56
|
+
CASE WHEN NOT selected.lifo THEN selected.run_at END,
|
|
57
|
+
selected.id
|
|
123
58
|
`;
|
|
124
59
|
}
|
|
125
|
-
async function selectGroupedCandidates(tx, ctx,
|
|
60
|
+
async function selectGroupedCandidates(tx, ctx, input) {
|
|
61
|
+
const { queue, now, capacity, options } = input;
|
|
62
|
+
const concurrency = options?.concurrency ?? null;
|
|
63
|
+
const rateMax = options?.limit?.max ?? null;
|
|
126
64
|
return await tx `
|
|
127
|
-
WITH
|
|
128
|
-
SELECT job.id, job.
|
|
65
|
+
WITH ready AS (
|
|
66
|
+
SELECT job.id, job.group_id,
|
|
129
67
|
row_number() OVER (
|
|
130
|
-
PARTITION BY
|
|
131
|
-
WHEN job.group_id IS NULL THEN 'job:' || job.id ELSE 'group:' || job.group_id
|
|
132
|
-
END
|
|
133
|
-
ORDER BY job.priority DESC,
|
|
134
|
-
CASE WHEN job.lifo THEN 0 ELSE 1 END ASC,
|
|
135
|
-
CASE WHEN job.lifo THEN job.id END DESC,
|
|
136
|
-
CASE WHEN NOT job.lifo THEN job.run_at END ASC, job.id ASC
|
|
68
|
+
PARTITION BY job.group_id ORDER BY job.run_at, job.group_order, job.id
|
|
137
69
|
) AS group_position
|
|
138
70
|
FROM bunqueue_jobs AS job
|
|
139
|
-
WHERE job.namespace = ${ctx.config.namespace}
|
|
140
|
-
AND job.
|
|
71
|
+
WHERE job.namespace = ${ctx.config.namespace} AND job.queue = ${queue}
|
|
72
|
+
AND job.group_id IS NOT NULL
|
|
141
73
|
AND job.state IN ('waiting', 'prioritized', 'delayed')
|
|
142
74
|
AND job.run_at <= ${now}
|
|
143
75
|
AND (job.ttl IS NULL OR job.created_at + job.ttl >= ${now})
|
|
@@ -150,49 +82,59 @@ async function selectGroupedCandidates(tx, ctx, queue, now, capacity) {
|
|
|
150
82
|
AND completion.job_id = dependency.dependency_id
|
|
151
83
|
)
|
|
152
84
|
)
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
85
|
+
), active AS (
|
|
86
|
+
SELECT group_id, COUNT(*)::integer AS count
|
|
87
|
+
FROM bunqueue_jobs
|
|
88
|
+
WHERE namespace = ${ctx.config.namespace} AND queue = ${queue}
|
|
89
|
+
AND group_id IS NOT NULL AND state = 'active' AND lease_until > ${now}
|
|
90
|
+
GROUP BY group_id
|
|
91
|
+
), capacities AS (
|
|
92
|
+
SELECT groups.group_id, groups.last_served,
|
|
93
|
+
LEAST(
|
|
94
|
+
${capacity},
|
|
95
|
+
CASE WHEN CAST(${concurrency} AS BIGINT) IS NULL THEN ${capacity}
|
|
96
|
+
ELSE GREATEST(
|
|
97
|
+
0,
|
|
98
|
+
COALESCE(groups.concurrency_limit, CAST(${concurrency} AS BIGINT))
|
|
99
|
+
- COALESCE(active.count, 0)
|
|
100
|
+
)
|
|
101
|
+
END,
|
|
102
|
+
CASE WHEN CAST(${rateMax} AS BIGINT) IS NULL THEN ${capacity}
|
|
103
|
+
ELSE GREATEST(
|
|
104
|
+
0,
|
|
105
|
+
COALESCE(groups.rate_limit, CAST(${rateMax} AS BIGINT)) - groups.rate_count
|
|
106
|
+
)
|
|
107
|
+
END
|
|
108
|
+
) AS available
|
|
109
|
+
FROM bunqueue_group_state AS groups
|
|
110
|
+
LEFT JOIN active ON active.group_id = groups.group_id
|
|
111
|
+
WHERE groups.namespace = ${ctx.config.namespace} AND groups.queue = ${queue}
|
|
112
|
+
), selected AS (
|
|
113
|
+
SELECT ready.id, ready.group_id, ready.group_position, capacities.last_served
|
|
114
|
+
FROM ready JOIN capacities ON capacities.group_id = ready.group_id
|
|
115
|
+
WHERE ready.group_position <= capacities.available
|
|
116
|
+
ORDER BY ready.group_position, capacities.last_served NULLS FIRST, ready.group_id
|
|
117
|
+
LIMIT ${capacity}
|
|
159
118
|
)
|
|
160
|
-
SELECT job.*
|
|
161
|
-
|
|
162
|
-
JOIN eligible ON eligible.id = job.id
|
|
119
|
+
SELECT job.* FROM bunqueue_jobs AS job
|
|
120
|
+
JOIN selected ON selected.id = job.id
|
|
163
121
|
WHERE job.namespace = ${ctx.config.namespace}
|
|
164
|
-
AND eligible.group_position = 1
|
|
165
122
|
AND job.state IN ('waiting', 'prioritized', 'delayed')
|
|
166
123
|
AND job.run_at <= ${now}
|
|
167
|
-
|
|
168
|
-
AND NOT EXISTS (
|
|
169
|
-
SELECT 1 FROM bunqueue_dependencies AS dependency
|
|
170
|
-
WHERE dependency.namespace = job.namespace AND dependency.job_id = job.id
|
|
171
|
-
AND NOT EXISTS (
|
|
172
|
-
SELECT 1 FROM bunqueue_completions AS completion
|
|
173
|
-
WHERE completion.namespace = dependency.namespace
|
|
174
|
-
AND completion.job_id = dependency.dependency_id
|
|
175
|
-
)
|
|
176
|
-
)
|
|
177
|
-
AND (job.group_id IS NULL OR NOT EXISTS (
|
|
178
|
-
SELECT 1 FROM bunqueue_jobs AS active_group
|
|
179
|
-
WHERE active_group.namespace = job.namespace AND active_group.queue = job.queue
|
|
180
|
-
AND active_group.group_id = job.group_id AND active_group.state = 'active'
|
|
181
|
-
AND active_group.lease_until > ${now}
|
|
182
|
-
))
|
|
183
|
-
ORDER BY job.priority DESC,
|
|
184
|
-
CASE WHEN job.lifo THEN 0 ELSE 1 END ASC,
|
|
185
|
-
CASE WHEN job.lifo THEN job.id END DESC,
|
|
186
|
-
CASE WHEN NOT job.lifo THEN job.run_at END ASC, job.id ASC
|
|
187
|
-
LIMIT ${capacity}
|
|
124
|
+
ORDER BY selected.group_position, selected.last_served NULLS FIRST, selected.group_id
|
|
188
125
|
FOR UPDATE OF job SKIP LOCKED
|
|
189
126
|
`;
|
|
190
127
|
}
|
|
191
|
-
export async function selectPostgresClaimCandidates(tx, ctx,
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
128
|
+
export async function selectPostgresClaimCandidates(tx, ctx, input) {
|
|
129
|
+
const { queue, now, capacity, options } = input;
|
|
130
|
+
const ungrouped = await selectUngroupedCandidates(tx, ctx, queue, now, capacity);
|
|
131
|
+
if (ungrouped.length >= capacity)
|
|
132
|
+
return ungrouped;
|
|
133
|
+
const grouped = await selectGroupedCandidates(tx, ctx, {
|
|
134
|
+
queue,
|
|
135
|
+
now,
|
|
136
|
+
capacity: capacity - ungrouped.length,
|
|
137
|
+
options,
|
|
138
|
+
});
|
|
139
|
+
return [...ungrouped, ...grouped];
|
|
198
140
|
}
|
|
@@ -1,3 +1,10 @@
|
|
|
1
|
+
import type { GroupPullOptions } from '../../../domain/types/group';
|
|
1
2
|
import { type PostgresContext } from './context';
|
|
2
3
|
import type { ClaimedPostgresJob } from './types';
|
|
3
|
-
export declare function claimPostgresJobs(ctx: PostgresContext,
|
|
4
|
+
export declare function claimPostgresJobs(ctx: PostgresContext, input: {
|
|
5
|
+
queue: string;
|
|
6
|
+
count: number;
|
|
7
|
+
owner: string;
|
|
8
|
+
leaseDurationMs: number;
|
|
9
|
+
groupOptions?: GroupPullOptions;
|
|
10
|
+
}): Promise<ClaimedPostgresJob[]>;
|
|
@@ -7,6 +7,8 @@ import { expirePendingPostgresJobs } from './expiry';
|
|
|
7
7
|
import { repairResolvedPostgresDependencyConsumers } from './dependencyPromotion';
|
|
8
8
|
import { lockPostgresBrokerSession } from './brokerSessions';
|
|
9
9
|
import { runPostgresTransactionWithRetry } from './transactionRetry';
|
|
10
|
+
import { hasClaimablePostgresGroup } from './claimSelection';
|
|
11
|
+
import { ensurePostgresGroupStates, refreshPostgresGroupRateWindows } from './groupClaims';
|
|
10
12
|
function toQueueState(row) {
|
|
11
13
|
return {
|
|
12
14
|
queue: row.queue,
|
|
@@ -19,20 +21,23 @@ function toQueueState(row) {
|
|
|
19
21
|
concurrencyLimit: row.concurrency_limit,
|
|
20
22
|
stallConfig: decodePostgresValue(row.stall_config, DEFAULT_STALL_CONFIG, 'stallConfig'),
|
|
21
23
|
dlqConfig: decodePostgresValue(row.dlq_config, DEFAULT_DLQ_CONFIG, 'dlqConfig'),
|
|
24
|
+
groupSequence: Number(row.group_sequence),
|
|
22
25
|
};
|
|
23
26
|
}
|
|
24
27
|
async function lockQueueState(tx, ctx, queue, exclusive) {
|
|
25
28
|
const select = async () => exclusive
|
|
26
29
|
? await tx `
|
|
27
30
|
SELECT queue, paused, rate_limit, rate_duration_ms, rate_window_started_at,
|
|
28
|
-
rate_expires_at, rate_count, concurrency_limit, stall_config, dlq_config
|
|
31
|
+
rate_expires_at, rate_count, concurrency_limit, stall_config, dlq_config,
|
|
32
|
+
group_sequence
|
|
29
33
|
FROM bunqueue_queue_state
|
|
30
34
|
WHERE namespace = ${ctx.config.namespace} AND queue = ${queue}
|
|
31
35
|
FOR UPDATE
|
|
32
36
|
`
|
|
33
37
|
: await tx `
|
|
34
38
|
SELECT queue, paused, rate_limit, rate_duration_ms, rate_window_started_at,
|
|
35
|
-
rate_expires_at, rate_count, concurrency_limit, stall_config, dlq_config
|
|
39
|
+
rate_expires_at, rate_count, concurrency_limit, stall_config, dlq_config,
|
|
40
|
+
group_sequence
|
|
36
41
|
FROM bunqueue_queue_state
|
|
37
42
|
WHERE namespace = ${ctx.config.namespace} AND queue = ${queue}
|
|
38
43
|
FOR SHARE
|
|
@@ -92,7 +97,8 @@ async function activeCount(tx, ctx, queue, now) {
|
|
|
92
97
|
`;
|
|
93
98
|
return Number(row.count);
|
|
94
99
|
}
|
|
95
|
-
export async function claimPostgresJobs(ctx,
|
|
100
|
+
export async function claimPostgresJobs(ctx, input) {
|
|
101
|
+
const { queue, count, owner, leaseDurationMs, groupOptions } = input;
|
|
96
102
|
if (count <= 0)
|
|
97
103
|
return [];
|
|
98
104
|
await expirePendingPostgresJobs(ctx, queue);
|
|
@@ -114,12 +120,22 @@ export async function claimPostgresJobs(ctx, queue, count, owner, leaseDurationM
|
|
|
114
120
|
if (capacity <= 0)
|
|
115
121
|
return { claimed: [], retryExclusive: false };
|
|
116
122
|
await repairResolvedPostgresDependencyConsumers(tx, ctx, queue, now);
|
|
123
|
+
const hasGroups = await hasClaimablePostgresGroup(tx, ctx, queue, now);
|
|
124
|
+
if (!exclusive && hasGroups)
|
|
125
|
+
return { claimed: [], retryExclusive: true };
|
|
126
|
+
let groupSequence = state.groupSequence;
|
|
127
|
+
if (hasGroups) {
|
|
128
|
+
groupSequence = await ensurePostgresGroupStates(tx, ctx, queue, groupSequence);
|
|
129
|
+
await refreshPostgresGroupRateWindows(tx, ctx, queue, groupOptions, now);
|
|
130
|
+
}
|
|
117
131
|
const claimed = await claimPostgresCandidates(tx, ctx, {
|
|
118
132
|
queue,
|
|
119
133
|
capacity,
|
|
120
134
|
owner,
|
|
121
135
|
requestedLeaseMs: leaseDurationMs,
|
|
122
136
|
now,
|
|
137
|
+
groupOptions,
|
|
138
|
+
groupSequence,
|
|
123
139
|
});
|
|
124
140
|
if (state.rateLimit !== null && claimed.length > 0) {
|
|
125
141
|
await tx `
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import type { PostgresEventPruneWatermark } from './eventPruneWatermarks';
|
|
2
|
+
/** Per-queue catch-up bookkeeping for commit-ordered durable event delivery. */
|
|
3
|
+
export declare class PostgresEventCatchupCursors {
|
|
4
|
+
private readonly appliedCommitSeqs;
|
|
5
|
+
private readonly handledPruneWatermarks;
|
|
6
|
+
private baselineCommitSeq;
|
|
7
|
+
reset(baselineCommitSeq: number): void;
|
|
8
|
+
appliedThrough(queue: string): number;
|
|
9
|
+
applied(queue: string, commitSeq: number): void;
|
|
10
|
+
/** Record a prune watermark as accounted for and report whether it was new. */
|
|
11
|
+
remember(watermark: PostgresEventPruneWatermark): boolean;
|
|
12
|
+
/** Report whether a commit pruned its own events without this reader knowing. */
|
|
13
|
+
private hasUnhandledSelfPrune;
|
|
14
|
+
/**
|
|
15
|
+
* Report whether a prune watermark requires an authoritative queue refresh.
|
|
16
|
+
*
|
|
17
|
+
* Two cases need one. A reader at or behind the pruned frontier may have
|
|
18
|
+
* missed discarded history. And a commit that writes more queue events than
|
|
19
|
+
* the retained window prunes its own older events before any reader can
|
|
20
|
+
* observe them, so every reader must reload once per such commit, even one
|
|
21
|
+
* whose applied position has already moved past it. `self_pruned_commit_seq`
|
|
22
|
+
* carries that evidence forward per queue, so a superseding watermark does
|
|
23
|
+
* not drop it; queue destruction erases it with the queue itself, which is
|
|
24
|
+
* safe because the same transaction also invalidates the queue and commit
|
|
25
|
+
* sequence numbers never rewind. A watermark already accounted for never
|
|
26
|
+
* refreshes again. The `<=` comparison is deliberately wider than the
|
|
27
|
+
* observed cases: it also covers a commit split across drain batches, where
|
|
28
|
+
* the applied cursor already equals the pruned frontier while the rest of
|
|
29
|
+
* that commit is still unread.
|
|
30
|
+
*/
|
|
31
|
+
requiresRefresh(watermark: PostgresEventPruneWatermark): boolean;
|
|
32
|
+
}
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
/** Per-queue catch-up bookkeeping for commit-ordered durable event delivery. */
|
|
2
|
+
export class PostgresEventCatchupCursors {
|
|
3
|
+
appliedCommitSeqs = new Map();
|
|
4
|
+
handledPruneWatermarks = new Map();
|
|
5
|
+
baselineCommitSeq = 0;
|
|
6
|
+
reset(baselineCommitSeq) {
|
|
7
|
+
this.baselineCommitSeq = baselineCommitSeq;
|
|
8
|
+
this.appliedCommitSeqs.clear();
|
|
9
|
+
this.handledPruneWatermarks.clear();
|
|
10
|
+
}
|
|
11
|
+
appliedThrough(queue) {
|
|
12
|
+
return this.appliedCommitSeqs.get(queue) ?? this.baselineCommitSeq;
|
|
13
|
+
}
|
|
14
|
+
applied(queue, commitSeq) {
|
|
15
|
+
this.appliedCommitSeqs.set(queue, Math.max(this.appliedThrough(queue), commitSeq));
|
|
16
|
+
}
|
|
17
|
+
/** Record a prune watermark as accounted for and report whether it was new. */
|
|
18
|
+
remember(watermark) {
|
|
19
|
+
const { queue, commitSeq, prunedThrough, selfPrunedCommitSeq } = watermark;
|
|
20
|
+
const handled = this.handledPruneWatermarks.get(queue);
|
|
21
|
+
if (handled !== undefined &&
|
|
22
|
+
handled.commitSeq >= commitSeq &&
|
|
23
|
+
handled.prunedThrough >= prunedThrough &&
|
|
24
|
+
handled.selfPrunedCommitSeq >= selfPrunedCommitSeq) {
|
|
25
|
+
return false;
|
|
26
|
+
}
|
|
27
|
+
this.handledPruneWatermarks.set(queue, {
|
|
28
|
+
commitSeq: Math.max(handled?.commitSeq ?? 0, commitSeq),
|
|
29
|
+
prunedThrough: Math.max(handled?.prunedThrough ?? 0, prunedThrough),
|
|
30
|
+
selfPrunedCommitSeq: Math.max(handled?.selfPrunedCommitSeq ?? 0, selfPrunedCommitSeq),
|
|
31
|
+
});
|
|
32
|
+
return true;
|
|
33
|
+
}
|
|
34
|
+
/** Report whether a commit pruned its own events without this reader knowing. */
|
|
35
|
+
hasUnhandledSelfPrune(watermark) {
|
|
36
|
+
const handled = this.handledPruneWatermarks.get(watermark.queue);
|
|
37
|
+
return watermark.selfPrunedCommitSeq > (handled?.selfPrunedCommitSeq ?? 0);
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Report whether a prune watermark requires an authoritative queue refresh.
|
|
41
|
+
*
|
|
42
|
+
* Two cases need one. A reader at or behind the pruned frontier may have
|
|
43
|
+
* missed discarded history. And a commit that writes more queue events than
|
|
44
|
+
* the retained window prunes its own older events before any reader can
|
|
45
|
+
* observe them, so every reader must reload once per such commit, even one
|
|
46
|
+
* whose applied position has already moved past it. `self_pruned_commit_seq`
|
|
47
|
+
* carries that evidence forward per queue, so a superseding watermark does
|
|
48
|
+
* not drop it; queue destruction erases it with the queue itself, which is
|
|
49
|
+
* safe because the same transaction also invalidates the queue and commit
|
|
50
|
+
* sequence numbers never rewind. A watermark already accounted for never
|
|
51
|
+
* refreshes again. The `<=` comparison is deliberately wider than the
|
|
52
|
+
* observed cases: it also covers a commit split across drain batches, where
|
|
53
|
+
* the applied cursor already equals the pruned frontier while the rest of
|
|
54
|
+
* that commit is still unread.
|
|
55
|
+
*/
|
|
56
|
+
requiresRefresh(watermark) {
|
|
57
|
+
const behind = this.appliedThrough(watermark.queue) <= watermark.prunedCommitSeq;
|
|
58
|
+
const selfPruned = this.hasUnhandledSelfPrune(watermark);
|
|
59
|
+
if (!this.remember(watermark))
|
|
60
|
+
return false;
|
|
61
|
+
if (!behind && !selfPruned)
|
|
62
|
+
return false;
|
|
63
|
+
this.applied(watermark.queue, watermark.commitSeq);
|
|
64
|
+
return true;
|
|
65
|
+
}
|
|
66
|
+
}
|