@relaybus/postgres 0.0.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +149 -46
- package/dist/drizzle.d.cts +1 -1
- package/dist/drizzle.d.ts +1 -1
- package/dist/{executor-CSM7ZOhz.d.cts → executor-VZGmsMX4.d.cts} +12 -2
- package/dist/{executor-CSM7ZOhz.d.ts → executor-VZGmsMX4.d.ts} +12 -2
- package/dist/index.cjs +2034 -396
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +132 -31
- package/dist/index.d.ts +132 -31
- package/dist/index.js +2035 -397
- package/dist/index.js.map +1 -1
- package/dist/knex.d.cts +1 -1
- package/dist/knex.d.ts +1 -1
- package/dist/kysely.d.cts +1 -1
- package/dist/kysely.d.ts +1 -1
- package/dist/prisma.d.cts +1 -1
- package/dist/prisma.d.ts +1 -1
- package/dist/typeorm.d.cts +1 -1
- package/dist/typeorm.d.ts +1 -1
- package/package.json +24 -11
package/dist/index.cjs
CHANGED
|
@@ -1,13 +1,19 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
|
|
3
|
+
var crypto = require('crypto');
|
|
3
4
|
var core = require('@relaybus/core');
|
|
4
5
|
var pg = require('pg');
|
|
5
6
|
|
|
6
7
|
// src/pg-executor.ts
|
|
8
|
+
var FOREIGN_SESSION_ERRORS = /* @__PURE__ */ new Set(["26000", "42P05"]);
|
|
7
9
|
var PgExecutor = class {
|
|
8
10
|
#pool;
|
|
9
|
-
|
|
11
|
+
#logger;
|
|
12
|
+
#names = /* @__PURE__ */ new Map();
|
|
13
|
+
#preparing = true;
|
|
14
|
+
constructor(pool, options = {}) {
|
|
10
15
|
this.#pool = pool;
|
|
16
|
+
this.#logger = options.logger ?? console;
|
|
11
17
|
}
|
|
12
18
|
async transaction(work) {
|
|
13
19
|
const client = await this.#pool.connect();
|
|
@@ -22,9 +28,38 @@ var PgExecutor = class {
|
|
|
22
28
|
throw error;
|
|
23
29
|
}
|
|
24
30
|
}
|
|
25
|
-
async query(sql, parameters, transaction) {
|
|
26
|
-
const
|
|
27
|
-
|
|
31
|
+
async query(sql, parameters, transaction, options) {
|
|
32
|
+
const queryable = transaction ?? this.#pool;
|
|
33
|
+
const values = [...parameters];
|
|
34
|
+
if (options?.prepare !== true || !this.#preparing) {
|
|
35
|
+
return (await queryable.query(sql, values)).rows;
|
|
36
|
+
}
|
|
37
|
+
try {
|
|
38
|
+
return (await queryable.query({ name: this.#nameOf(sql), text: sql, values })).rows;
|
|
39
|
+
} catch (error) {
|
|
40
|
+
if (!FOREIGN_SESSION_ERRORS.has(error.code)) {
|
|
41
|
+
throw error;
|
|
42
|
+
}
|
|
43
|
+
if (this.#preparing) {
|
|
44
|
+
this.#preparing = false;
|
|
45
|
+
this.#logger.warn(
|
|
46
|
+
`[PgExecutor](query) The server does not keep prepared statements per connection, statements are sent unprepared from now on; set preparedStatements: false to skip this check | code=${String(error.code)}`
|
|
47
|
+
);
|
|
48
|
+
}
|
|
49
|
+
if (transaction !== void 0) {
|
|
50
|
+
throw error;
|
|
51
|
+
}
|
|
52
|
+
return (await queryable.query(sql, values)).rows;
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
/** `relaybus_` and 24 hex digits of the SHA-256 of the text: well under the 63 bytes of an identifier. */
|
|
56
|
+
#nameOf(sql) {
|
|
57
|
+
let name = this.#names.get(sql);
|
|
58
|
+
if (name === void 0) {
|
|
59
|
+
name = `relaybus_${crypto.createHash("sha256").update(sql).digest("hex").slice(0, 24)}`;
|
|
60
|
+
this.#names.set(sql, name);
|
|
61
|
+
}
|
|
62
|
+
return name;
|
|
28
63
|
}
|
|
29
64
|
/** A client whose rollback fails is in an unknown state, so it is destroyed instead of reused. */
|
|
30
65
|
async #rollbackAndRelease(client) {
|
|
@@ -50,7 +85,9 @@ function quoteIdentifier(value) {
|
|
|
50
85
|
}
|
|
51
86
|
|
|
52
87
|
// src/names.ts
|
|
88
|
+
var LANE_COUNT = 256;
|
|
53
89
|
var DEFAULT_SCHEMA = "public";
|
|
90
|
+
var MAXIMUM_CHANNEL_LENGTH = 63;
|
|
54
91
|
var DEFAULT_TABLE_PREFIX = "relaybus_";
|
|
55
92
|
var TABLE_PREFIX_PATTERN = /^[a-z0-9_]*$/;
|
|
56
93
|
function resolvePostgresNames(options = {}) {
|
|
@@ -76,34 +113,47 @@ function resolvePostgresNames(options = {}) {
|
|
|
76
113
|
events: table("events"),
|
|
77
114
|
deliveries: table("deliveries"),
|
|
78
115
|
keySequences: table("key_sequences"),
|
|
116
|
+
lanes: table("lanes"),
|
|
117
|
+
lanesTableName: name("lanes"),
|
|
118
|
+
eventCompletions: table("event_completions"),
|
|
119
|
+
completions: table("completed_deliveries"),
|
|
120
|
+
completionHistory: options.completionHistory ?? false,
|
|
121
|
+
notificationChannel: channelName(`${schema}.${tablePrefix}deliveries`),
|
|
79
122
|
indexes: {
|
|
80
123
|
deliveryEventHandler: index("deliveries_event_handler_unique"),
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
deliveryHandlerCreated: index("deliveries_handler_created_idx"),
|
|
124
|
+
deliveryLaneDue: index("deliveries_lane_due_idx"),
|
|
125
|
+
deliveryKeyPending: index("deliveries_key_pending_idx"),
|
|
84
126
|
eventCreated: index("events_created_idx"),
|
|
85
|
-
deliveryKeyUnfinished: index("deliveries_key_unfinished_idx"),
|
|
86
|
-
deliveryKeyProcessing: index("deliveries_key_processing_idx"),
|
|
87
|
-
deliveryStatusCreated: index("deliveries_status_created_idx"),
|
|
88
127
|
deliveryCompleted: index("deliveries_completed_idx"),
|
|
89
|
-
deliveryDead: index("deliveries_dead_idx")
|
|
128
|
+
deliveryDead: index("deliveries_dead_idx"),
|
|
129
|
+
completionCompleted: index("completed_deliveries_completed_idx"),
|
|
130
|
+
eventCompletionsCompleted: index("event_completions_completed_idx"),
|
|
131
|
+
obsolete: [
|
|
132
|
+
index("deliveries_due_idx"),
|
|
133
|
+
index("deliveries_due_unkeyed_idx"),
|
|
134
|
+
index("deliveries_handler_created_idx"),
|
|
135
|
+
index("deliveries_key_unfinished_idx"),
|
|
136
|
+
index("deliveries_key_processing_idx"),
|
|
137
|
+
index("deliveries_status_created_idx")
|
|
138
|
+
]
|
|
90
139
|
}
|
|
91
140
|
};
|
|
92
141
|
}
|
|
142
|
+
function channelName(readable) {
|
|
143
|
+
return readable.length <= MAXIMUM_CHANNEL_LENGTH ? readable : `relaybus_${crypto.createHash("sha256").update(readable).digest("hex").slice(0, 32)}`;
|
|
144
|
+
}
|
|
93
145
|
|
|
94
146
|
// src/sql-fragments.ts
|
|
95
|
-
function dueAtExpression(alias) {
|
|
96
|
-
const prefix = alias === void 0 ? "" : `${alias}.`;
|
|
97
|
-
return `CASE WHEN ${prefix}status = 'pending' THEN ${prefix}available_at ELSE ${prefix}lease_expires_at END`;
|
|
98
|
-
}
|
|
99
147
|
function isoText(expression) {
|
|
100
148
|
return `to_char(${expression} AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')`;
|
|
101
149
|
}
|
|
150
|
+
function laneExpression(keyExpression, eventIdExpression) {
|
|
151
|
+
return `(hashtextextended(COALESCE(${keyExpression}, ${eventIdExpression}), 0) & ${LANE_COUNT - 1})::smallint`;
|
|
152
|
+
}
|
|
102
153
|
|
|
103
154
|
// src/schema.ts
|
|
104
155
|
function createPostgresSchemaStatements(options = {}) {
|
|
105
156
|
const names = resolvePostgresNames(options);
|
|
106
|
-
const unfinished = `status IN ('pending', 'processing')`;
|
|
107
157
|
const statements = [
|
|
108
158
|
`CREATE TABLE IF NOT EXISTS ${names.events} (
|
|
109
159
|
id text PRIMARY KEY,
|
|
@@ -116,7 +166,8 @@ function createPostgresSchemaStatements(options = {}) {
|
|
|
116
166
|
correlation_id text,
|
|
117
167
|
causation_id text,
|
|
118
168
|
occurred_at timestamptz NOT NULL,
|
|
119
|
-
created_at timestamptz NOT NULL
|
|
169
|
+
created_at timestamptz NOT NULL,
|
|
170
|
+
handlers text[] NOT NULL DEFAULT '{}'
|
|
120
171
|
)`,
|
|
121
172
|
`CREATE INDEX IF NOT EXISTS ${names.indexes.eventCreated} ON ${names.events} (created_at, id)`,
|
|
122
173
|
`CREATE TABLE IF NOT EXISTS ${names.keySequences} (
|
|
@@ -127,12 +178,11 @@ function createPostgresSchemaStatements(options = {}) {
|
|
|
127
178
|
id text COLLATE "C" PRIMARY KEY,
|
|
128
179
|
event_id text NOT NULL REFERENCES ${names.events} (id) ON DELETE CASCADE,
|
|
129
180
|
handler text NOT NULL,
|
|
181
|
+
lane smallint NOT NULL,
|
|
182
|
+
claims integer NOT NULL DEFAULT 0,
|
|
130
183
|
status text NOT NULL CHECK (status IN ('pending', 'processing', 'completed', 'dead')),
|
|
131
184
|
attempts integer NOT NULL DEFAULT 0,
|
|
132
185
|
available_at timestamptz NOT NULL,
|
|
133
|
-
lease_owner text,
|
|
134
|
-
lease_token text,
|
|
135
|
-
lease_expires_at timestamptz,
|
|
136
186
|
last_error jsonb,
|
|
137
187
|
key text,
|
|
138
188
|
sequence bigint,
|
|
@@ -140,26 +190,180 @@ function createPostgresSchemaStatements(options = {}) {
|
|
|
140
190
|
updated_at timestamptz NOT NULL,
|
|
141
191
|
completed_at timestamptz,
|
|
142
192
|
CONSTRAINT ${names.indexes.deliveryEventHandler} UNIQUE (event_id, handler)
|
|
143
|
-
)`,
|
|
144
|
-
|
|
145
|
-
`
|
|
146
|
-
`
|
|
147
|
-
`
|
|
148
|
-
`CREATE INDEX IF NOT EXISTS ${names.indexes.
|
|
149
|
-
`CREATE INDEX IF NOT EXISTS ${names.indexes.
|
|
193
|
+
) WITH (autovacuum_vacuum_scale_factor = 0.02, autovacuum_analyze_scale_factor = 0.05)`,
|
|
194
|
+
upgradeFromVersion010(names),
|
|
195
|
+
`ALTER TABLE ${names.deliveries} ADD COLUMN IF NOT EXISTS claims integer NOT NULL DEFAULT 0`,
|
|
196
|
+
`ALTER TABLE ${names.deliveries} SET (autovacuum_vacuum_scale_factor = 0.02, autovacuum_analyze_scale_factor = 0.05)`,
|
|
197
|
+
...names.indexes.obsolete.map((index) => `DROP INDEX IF EXISTS ${names.quotedSchema}.${index}`),
|
|
198
|
+
`CREATE INDEX IF NOT EXISTS ${names.indexes.deliveryLaneDue} ON ${names.deliveries} (lane, available_at, id) WHERE status = 'pending'`,
|
|
199
|
+
`CREATE INDEX IF NOT EXISTS ${names.indexes.deliveryKeyPending} ON ${names.deliveries} (lane, handler, key, sequence) WHERE key IS NOT NULL AND status = 'pending'`,
|
|
150
200
|
`CREATE INDEX IF NOT EXISTS ${names.indexes.deliveryCompleted} ON ${names.deliveries} (completed_at, id) WHERE status = 'completed'`,
|
|
151
|
-
`CREATE INDEX IF NOT EXISTS ${names.indexes.deliveryDead} ON ${names.deliveries} (updated_at, id) WHERE status = 'dead'
|
|
201
|
+
`CREATE INDEX IF NOT EXISTS ${names.indexes.deliveryDead} ON ${names.deliveries} (updated_at, id) WHERE status = 'dead'`,
|
|
202
|
+
`CREATE TABLE IF NOT EXISTS ${names.lanes} (
|
|
203
|
+
lane smallint PRIMARY KEY CHECK (lane >= 0 AND lane < ${LANE_COUNT}),
|
|
204
|
+
epoch bigint NOT NULL DEFAULT 0,
|
|
205
|
+
lease_owner text,
|
|
206
|
+
lease_expires_at timestamptz,
|
|
207
|
+
in_flight text[] NOT NULL DEFAULT '{}',
|
|
208
|
+
claim_floor bigint NOT NULL DEFAULT 0
|
|
209
|
+
) WITH (fillfactor = 50)`,
|
|
210
|
+
`ALTER TABLE ${names.lanes} ADD COLUMN IF NOT EXISTS claim_floor bigint NOT NULL DEFAULT 0`,
|
|
211
|
+
`CREATE TABLE IF NOT EXISTS ${names.eventCompletions} (
|
|
212
|
+
event_id text PRIMARY KEY REFERENCES ${names.events} (id) ON DELETE CASCADE,
|
|
213
|
+
completed_at timestamptz NOT NULL
|
|
214
|
+
)`,
|
|
215
|
+
`CREATE INDEX IF NOT EXISTS ${names.indexes.eventCompletionsCompleted} ON ${names.eventCompletions} (completed_at, event_id)`,
|
|
216
|
+
`INSERT INTO ${names.lanes} (lane) SELECT generate_series(0, ${LANE_COUNT - 1})::smallint ON CONFLICT (lane) DO NOTHING`
|
|
152
217
|
];
|
|
218
|
+
if (names.completionHistory) {
|
|
219
|
+
statements.push(
|
|
220
|
+
`CREATE TABLE IF NOT EXISTS ${names.completions} (
|
|
221
|
+
id text COLLATE "C" PRIMARY KEY,
|
|
222
|
+
event_id text NOT NULL REFERENCES ${names.events} (id) ON DELETE CASCADE,
|
|
223
|
+
handler text NOT NULL,
|
|
224
|
+
attempts integer NOT NULL,
|
|
225
|
+
available_at timestamptz NOT NULL,
|
|
226
|
+
created_at timestamptz NOT NULL,
|
|
227
|
+
completed_at timestamptz NOT NULL
|
|
228
|
+
)`,
|
|
229
|
+
`CREATE INDEX IF NOT EXISTS ${names.indexes.completionCompleted} ON ${names.completions} (completed_at, id)`
|
|
230
|
+
);
|
|
231
|
+
}
|
|
153
232
|
if (names.schema !== "public") {
|
|
154
233
|
statements.unshift(`CREATE SCHEMA IF NOT EXISTS ${names.quotedSchema}`);
|
|
155
234
|
}
|
|
235
|
+
statements.push(`COMMENT ON TABLE ${names.lanes} IS '${postgresSchemaMarker(statements)}'`);
|
|
156
236
|
return statements;
|
|
157
237
|
}
|
|
238
|
+
function postgresSchemaMarker(statements) {
|
|
239
|
+
return `relaybus schema ${crypto.createHash("sha256").update(statements.join(";")).digest("hex").slice(0, 32)}`;
|
|
240
|
+
}
|
|
241
|
+
function upgradeFromVersion010(names) {
|
|
242
|
+
return `DO $relaybus_upgrade$
|
|
243
|
+
BEGIN
|
|
244
|
+
IF EXISTS (
|
|
245
|
+
SELECT 1 FROM pg_attribute
|
|
246
|
+
WHERE attrelid = '${names.deliveries}'::regclass AND attname = 'lease_token' AND NOT attisdropped
|
|
247
|
+
) THEN
|
|
248
|
+
ALTER TABLE ${names.events} ADD COLUMN IF NOT EXISTS handlers text[] NOT NULL DEFAULT '{}';
|
|
249
|
+
UPDATE ${names.events} AS event SET handlers = known.handlers
|
|
250
|
+
FROM (
|
|
251
|
+
SELECT event_id, array_agg(handler) AS handlers FROM ${names.deliveries} GROUP BY event_id
|
|
252
|
+
) AS known
|
|
253
|
+
WHERE event.id = known.event_id;
|
|
254
|
+
ALTER TABLE ${names.deliveries} ADD COLUMN IF NOT EXISTS lane smallint;
|
|
255
|
+
UPDATE ${names.deliveries} SET lane = ${laneExpression("key", "event_id")} WHERE lane IS NULL;
|
|
256
|
+
ALTER TABLE ${names.deliveries} ALTER COLUMN lane SET NOT NULL;
|
|
257
|
+
UPDATE ${names.deliveries} SET status = 'pending' WHERE status = 'processing';
|
|
258
|
+
ALTER TABLE ${names.deliveries}
|
|
259
|
+
DROP COLUMN lease_owner, DROP COLUMN lease_token, DROP COLUMN lease_expires_at;
|
|
260
|
+
END IF;
|
|
261
|
+
END
|
|
262
|
+
$relaybus_upgrade$`;
|
|
263
|
+
}
|
|
158
264
|
function createPostgresSchemaSql(options = {}) {
|
|
159
265
|
return `${createPostgresSchemaStatements(options).join(";\n\n")};
|
|
160
266
|
`;
|
|
161
267
|
}
|
|
162
268
|
|
|
269
|
+
// src/notification-listener.ts
|
|
270
|
+
var RECONNECT_DELAY_MS = 1e3;
|
|
271
|
+
var NotificationListener = class {
|
|
272
|
+
#connect;
|
|
273
|
+
#channel;
|
|
274
|
+
#listener;
|
|
275
|
+
#logger;
|
|
276
|
+
/** Ends the current connection, listening or still starting; undefined while there is none. */
|
|
277
|
+
#drop;
|
|
278
|
+
#reconnectTimer;
|
|
279
|
+
#closed = false;
|
|
280
|
+
constructor(connect, channel, listener, logger) {
|
|
281
|
+
this.#connect = connect;
|
|
282
|
+
this.#channel = channel;
|
|
283
|
+
this.#listener = listener;
|
|
284
|
+
this.#logger = logger;
|
|
285
|
+
}
|
|
286
|
+
/** Connects and listens. Rejects when that fails; a connection lost later is replaced. */
|
|
287
|
+
async start() {
|
|
288
|
+
const connection = await this.#connect();
|
|
289
|
+
let ended = false;
|
|
290
|
+
let ending = Promise.resolve();
|
|
291
|
+
const end = () => {
|
|
292
|
+
if (!ended) {
|
|
293
|
+
ended = true;
|
|
294
|
+
ending = connection.end().catch(() => void 0);
|
|
295
|
+
}
|
|
296
|
+
return ending;
|
|
297
|
+
};
|
|
298
|
+
if (this.#closed) {
|
|
299
|
+
await end();
|
|
300
|
+
return;
|
|
301
|
+
}
|
|
302
|
+
this.#drop = end;
|
|
303
|
+
let listening = false;
|
|
304
|
+
const onLost = (error) => {
|
|
305
|
+
if (!listening || ended) {
|
|
306
|
+
return;
|
|
307
|
+
}
|
|
308
|
+
void end();
|
|
309
|
+
this.#drop = void 0;
|
|
310
|
+
this.#logger.warn(
|
|
311
|
+
`[NotificationListener](start) Listening connection lost, opening another | channel=${this.#channel} | error=${error?.message ?? "connection ended"}`
|
|
312
|
+
);
|
|
313
|
+
this.#scheduleReconnect();
|
|
314
|
+
};
|
|
315
|
+
connection.on("error", onLost);
|
|
316
|
+
connection.on("end", () => onLost());
|
|
317
|
+
connection.on("notification", () => this.#listener());
|
|
318
|
+
try {
|
|
319
|
+
await connection.query(`LISTEN "${this.#channel}"`);
|
|
320
|
+
} catch (error) {
|
|
321
|
+
if (this.#drop === end) {
|
|
322
|
+
this.#drop = void 0;
|
|
323
|
+
}
|
|
324
|
+
await end();
|
|
325
|
+
if (this.#closed) {
|
|
326
|
+
return;
|
|
327
|
+
}
|
|
328
|
+
throw error;
|
|
329
|
+
}
|
|
330
|
+
if (this.#closed) {
|
|
331
|
+
await end();
|
|
332
|
+
return;
|
|
333
|
+
}
|
|
334
|
+
listening = true;
|
|
335
|
+
}
|
|
336
|
+
#scheduleReconnect() {
|
|
337
|
+
this.#reconnectTimer = setTimeout(() => {
|
|
338
|
+
this.#reconnectTimer = void 0;
|
|
339
|
+
this.start().then(
|
|
340
|
+
() => {
|
|
341
|
+
if (!this.#closed) {
|
|
342
|
+
this.#listener();
|
|
343
|
+
}
|
|
344
|
+
},
|
|
345
|
+
(error) => {
|
|
346
|
+
this.#logger.warn(
|
|
347
|
+
`[NotificationListener](scheduleReconnect) Listening again failed, retrying | channel=${this.#channel} | error=${error instanceof Error ? error.message : String(error)}`
|
|
348
|
+
);
|
|
349
|
+
if (!this.#closed) {
|
|
350
|
+
this.#scheduleReconnect();
|
|
351
|
+
}
|
|
352
|
+
}
|
|
353
|
+
);
|
|
354
|
+
}, RECONNECT_DELAY_MS);
|
|
355
|
+
this.#reconnectTimer.unref();
|
|
356
|
+
}
|
|
357
|
+
/** Stops listening for good and ends the connection. */
|
|
358
|
+
async close() {
|
|
359
|
+
this.#closed = true;
|
|
360
|
+
clearTimeout(this.#reconnectTimer);
|
|
361
|
+
const drop = this.#drop;
|
|
362
|
+
this.#drop = void 0;
|
|
363
|
+
await drop?.();
|
|
364
|
+
}
|
|
365
|
+
};
|
|
366
|
+
|
|
163
367
|
// src/row-mapping.ts
|
|
164
368
|
function text(row, column) {
|
|
165
369
|
return row[column];
|
|
@@ -187,12 +391,27 @@ function mapEventEnvelope(row) {
|
|
|
187
391
|
occurredAt: new Date(text(row, "event_occurred_at"))
|
|
188
392
|
};
|
|
189
393
|
}
|
|
394
|
+
function formatLeaseToken(lane, epoch, claims) {
|
|
395
|
+
return `${lane}:${epoch}:${claims}`;
|
|
396
|
+
}
|
|
397
|
+
var LEASE_TOKEN_PATTERN = /^(\d{1,3}):(\d{1,18}):(\d{1,9})$/;
|
|
398
|
+
function parseLeaseToken(token) {
|
|
399
|
+
const match = LEASE_TOKEN_PATTERN.exec(token);
|
|
400
|
+
if (match === null || Number(match[1]) >= LANE_COUNT) {
|
|
401
|
+
return void 0;
|
|
402
|
+
}
|
|
403
|
+
return { lane: Number(match[1]), epoch: match[2], claims: Number(match[3]) };
|
|
404
|
+
}
|
|
190
405
|
function mapClaimedDelivery(row, leaseExpiresAt) {
|
|
191
406
|
return {
|
|
192
407
|
deliveryId: text(row, "delivery_id"),
|
|
193
|
-
leaseToken:
|
|
408
|
+
leaseToken: formatLeaseToken(
|
|
409
|
+
readCount(row, "lane"),
|
|
410
|
+
text(row, "epoch"),
|
|
411
|
+
readCount(row, "claims")
|
|
412
|
+
),
|
|
194
413
|
handler: text(row, "handler"),
|
|
195
|
-
attempt: readCount(row, "
|
|
414
|
+
attempt: readCount(row, "attempt"),
|
|
196
415
|
leaseExpiresAt,
|
|
197
416
|
event: mapEventEnvelope(row)
|
|
198
417
|
};
|
|
@@ -215,7 +434,7 @@ function mapDeliveryRecord(row) {
|
|
|
215
434
|
};
|
|
216
435
|
}
|
|
217
436
|
function mapOldestDueAt(rows) {
|
|
218
|
-
return
|
|
437
|
+
return nullableDate(rows[0], "oldest_due_at");
|
|
219
438
|
}
|
|
220
439
|
|
|
221
440
|
// src/statements.ts
|
|
@@ -235,9 +454,12 @@ function countChanged(dml) {
|
|
|
235
454
|
return `WITH changed AS (${dml} RETURNING 1) SELECT count(*)::int AS count FROM changed`;
|
|
236
455
|
}
|
|
237
456
|
var CLAIM_SCAN_WINDOW = 256;
|
|
238
|
-
var
|
|
457
|
+
var LANE_SCAN_WINDOW = 64;
|
|
458
|
+
var LANE_DEPTH_CAP = 16;
|
|
459
|
+
var INITIAL_FREE_LANES = 4;
|
|
460
|
+
var FREE_LANE_SURPLUS = 16;
|
|
239
461
|
var HANDLER_PREDICATE = `delivery.handler = ANY (ARRAY(SELECT jsonb_array_elements_text($2::jsonb)))`;
|
|
240
|
-
var
|
|
462
|
+
var LANE_LIST = `ARRAY(SELECT jsonb_array_elements_text($4::jsonb)::smallint)`;
|
|
241
463
|
var PostgresStatements = class {
|
|
242
464
|
#names;
|
|
243
465
|
constructor(names) {
|
|
@@ -251,318 +473,1131 @@ var PostgresStatements = class {
|
|
|
251
473
|
get setupLockKey() {
|
|
252
474
|
return `relaybus:${this.#names.schema}`;
|
|
253
475
|
}
|
|
476
|
+
/**
|
|
477
|
+
* Whether the lane table carries `marker` as its comment, that is whether the schema statements
|
|
478
|
+
* that produce that marker already ran. Takes no table lock. Parameters: `[marker]`.
|
|
479
|
+
*/
|
|
480
|
+
get schemaCurrent() {
|
|
481
|
+
return `SELECT EXISTS (
|
|
482
|
+
SELECT 1 FROM pg_catalog.pg_class AS c
|
|
483
|
+
JOIN pg_catalog.pg_namespace AS n ON n.oid = c.relnamespace
|
|
484
|
+
JOIN pg_catalog.pg_description AS d
|
|
485
|
+
ON d.objoid = c.oid AND d.classoid = 'pg_catalog.pg_class'::regclass AND d.objsubid = 0
|
|
486
|
+
WHERE n.nspname = '${this.#names.schema}' AND c.relname = '${this.#names.lanesTableName}' AND d.description = $1::text
|
|
487
|
+
) AS current`;
|
|
488
|
+
}
|
|
254
489
|
get setupLock() {
|
|
255
490
|
return "SELECT pg_advisory_xact_lock(hashtextextended($1::text, 0))";
|
|
256
491
|
}
|
|
257
492
|
/**
|
|
258
|
-
*
|
|
259
|
-
*
|
|
260
|
-
*
|
|
261
|
-
*
|
|
493
|
+
* Publishes in one statement: reserves the sequences of the keys (`[{ key, count }]` in `$4`, locked in
|
|
494
|
+
* key order: each row lock is held until the publishing transaction ends, which makes sequence order equal commit order for a key), numbers the keyed events in entry order (`ordinal`), inserts
|
|
495
|
+
* the events of `$1` that do not exist yet and the deliveries of `$3` that belong to them, and sends one
|
|
496
|
+
* wake-up notification when a delivery it inserted is due at `$5`. Parameters:
|
|
497
|
+
* `[eventsJson, createdAt, deliveriesJson, keyCountsJson, now]`. Returns one row per event of `$1`: its
|
|
498
|
+
* id, its sequence, whether this statement inserted it, and (on every row) how many deliveries it
|
|
499
|
+
* inserted. Deliveries of events that already existed are left to {@link insertDeliveries}.
|
|
500
|
+
*/
|
|
501
|
+
get insertFreshEvents() {
|
|
502
|
+
const events = this.#names.events;
|
|
503
|
+
return `WITH reserved AS (
|
|
504
|
+
INSERT INTO ${this.#names.keySequences} AS reserved (key, sequence)
|
|
505
|
+
SELECT requested.key, requested.count
|
|
506
|
+
FROM jsonb_to_recordset($4::jsonb) AS requested (key text, count bigint)
|
|
507
|
+
ORDER BY requested.key
|
|
508
|
+
ON CONFLICT (key) DO UPDATE SET sequence = reserved.sequence + EXCLUDED.sequence
|
|
509
|
+
RETURNING reserved.key AS key, reserved.sequence AS sequence
|
|
510
|
+
), incoming AS (
|
|
511
|
+
SELECT incoming.*,
|
|
512
|
+
CASE WHEN incoming.key IS NULL THEN NULL ELSE
|
|
513
|
+
reserved.sequence - count(*) OVER (PARTITION BY incoming.key)
|
|
514
|
+
+ row_number() OVER (PARTITION BY incoming.key ORDER BY incoming.ordinal)
|
|
515
|
+
END AS sequence
|
|
516
|
+
FROM jsonb_to_recordset($1::jsonb) AS incoming (
|
|
517
|
+
ordinal int, id text, type text, source text, key text, payload jsonb, headers jsonb,
|
|
518
|
+
correlation_id text, causation_id text, occurred_at timestamptz, handlers text[])
|
|
519
|
+
LEFT JOIN reserved ON reserved.key = incoming.key
|
|
520
|
+
), inserted_events AS (
|
|
521
|
+
INSERT INTO ${events}
|
|
522
|
+
(id, type, source, key, sequence, payload, headers, correlation_id, causation_id, occurred_at, created_at, handlers)
|
|
523
|
+
SELECT incoming.id, incoming.type, incoming.source, incoming.key, incoming.sequence,
|
|
524
|
+
COALESCE(incoming.payload, 'null'::jsonb), COALESCE(incoming.headers, '{}'::jsonb),
|
|
525
|
+
incoming.correlation_id, incoming.causation_id, incoming.occurred_at, $2::timestamptz,
|
|
526
|
+
COALESCE(incoming.handlers, '{}')
|
|
527
|
+
FROM incoming
|
|
528
|
+
ORDER BY incoming.id
|
|
529
|
+
ON CONFLICT (id) DO NOTHING
|
|
530
|
+
RETURNING id, key, sequence
|
|
531
|
+
), inserted_deliveries AS (
|
|
532
|
+
INSERT INTO ${this.#names.deliveries}
|
|
533
|
+
(id, event_id, handler, lane, status, attempts, available_at, key, sequence, created_at, updated_at)
|
|
534
|
+
SELECT delivery.id, delivery.event_id, delivery.handler, ${laneExpression("event.key", "event.id")}, 'pending', 0,
|
|
535
|
+
delivery.available_at, event.key, event.sequence, $2::timestamptz, $2::timestamptz
|
|
536
|
+
FROM jsonb_to_recordset($3::jsonb) AS delivery (id text, event_id text, handler text, available_at timestamptz)
|
|
537
|
+
JOIN inserted_events AS event ON event.id = delivery.event_id
|
|
538
|
+
ORDER BY delivery.event_id, delivery.handler
|
|
539
|
+
ON CONFLICT (event_id, handler) DO NOTHING
|
|
540
|
+
RETURNING available_at
|
|
541
|
+
), ${this.#wakeUp("inserted_deliveries", "$5")}
|
|
542
|
+
SELECT incoming.id, incoming.sequence::text AS sequence,
|
|
543
|
+
incoming.id IN (SELECT inserted_events.id FROM inserted_events) AS inserted,
|
|
544
|
+
(SELECT count(*) FROM inserted_deliveries)::int AS deliveries
|
|
545
|
+
FROM incoming
|
|
546
|
+
WHERE (SELECT count(*) FROM woken) >= 0`;
|
|
547
|
+
}
|
|
548
|
+
/** One wake-up notification on {@link PostgresNames.notificationChannel}, outside any publishing transaction. */
|
|
549
|
+
get notify() {
|
|
550
|
+
return `SELECT pg_notify('${this.#names.notificationChannel}', '')`;
|
|
551
|
+
}
|
|
552
|
+
/**
|
|
553
|
+
* A CTE `woken` that sends one notification on {@link PostgresNames.notificationChannel} when a row of
|
|
554
|
+
* `inserted` is due at `now`. PostgreSQL delivers it when the transaction commits, merges the same
|
|
555
|
+
* notification sent twice in one transaction, and drops it on rollback. The statement must reference
|
|
556
|
+
* `woken` for it to run.
|
|
262
557
|
*/
|
|
263
|
-
|
|
264
|
-
return `
|
|
265
|
-
SELECT
|
|
266
|
-
FROM
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
RETURNING reserved.key AS key, reserved.sequence::text AS sequence`;
|
|
558
|
+
#wakeUp(inserted, now) {
|
|
559
|
+
return `woken AS (
|
|
560
|
+
SELECT pg_notify('${this.#names.notificationChannel}', '') FROM (
|
|
561
|
+
SELECT 1 FROM ${inserted} WHERE ${inserted}.available_at <= ${now}::timestamptz LIMIT 1
|
|
562
|
+
) AS due
|
|
563
|
+
)`;
|
|
270
564
|
}
|
|
271
|
-
/**
|
|
565
|
+
/**
|
|
566
|
+
* Parameters: `[eventsJson, createdAt]`. Returns the ids that were inserted.
|
|
567
|
+
* `handlers` is the list of handlers the event gets deliveries for: it
|
|
568
|
+
* outlives the deliveries, which are deleted when they complete.
|
|
569
|
+
*/
|
|
272
570
|
get insertEvents() {
|
|
273
571
|
return `INSERT INTO ${this.#names.events}
|
|
274
|
-
(id, type, source, key, sequence, payload, headers, correlation_id, causation_id, occurred_at, created_at)
|
|
572
|
+
(id, type, source, key, sequence, payload, headers, correlation_id, causation_id, occurred_at, created_at, handlers)
|
|
275
573
|
SELECT incoming.id, incoming.type, incoming.source, incoming.key, incoming.sequence,
|
|
276
574
|
COALESCE(incoming.payload, 'null'::jsonb), COALESCE(incoming.headers, '{}'::jsonb),
|
|
277
|
-
incoming.correlation_id, incoming.causation_id, incoming.occurred_at, $2::timestamptz
|
|
575
|
+
incoming.correlation_id, incoming.causation_id, incoming.occurred_at, $2::timestamptz,
|
|
576
|
+
COALESCE(incoming.handlers, '{}')
|
|
278
577
|
FROM jsonb_to_recordset($1::jsonb) AS incoming (
|
|
279
578
|
id text, type text, source text, key text, sequence bigint, payload jsonb, headers jsonb,
|
|
280
|
-
correlation_id text, causation_id text, occurred_at timestamptz)
|
|
579
|
+
correlation_id text, causation_id text, occurred_at timestamptz, handlers text[])
|
|
281
580
|
ORDER BY incoming.id
|
|
282
581
|
ON CONFLICT (id) DO NOTHING
|
|
283
582
|
RETURNING id`;
|
|
284
583
|
}
|
|
285
584
|
/**
|
|
286
|
-
*
|
|
287
|
-
*
|
|
288
|
-
*
|
|
585
|
+
* Locks events that were not inserted by the current append, in id order, so
|
|
586
|
+
* that appends adding handlers to the same event run one after the other.
|
|
587
|
+
* Parameters: `[eventIdsJson]`. `FOR NO KEY UPDATE` conflicts with itself and
|
|
588
|
+
* not with the foreign key checks of other inserts.
|
|
589
|
+
*/
|
|
590
|
+
get lockExistingEvents() {
|
|
591
|
+
return `SELECT id FROM ${this.#names.events}
|
|
592
|
+
WHERE id = ANY (ARRAY(SELECT jsonb_array_elements_text($1::jsonb)))
|
|
593
|
+
ORDER BY id
|
|
594
|
+
FOR NO KEY UPDATE`;
|
|
595
|
+
}
|
|
596
|
+
/**
|
|
597
|
+
* Parameters: `[deliveriesJson, createdAt, insertedEventIdsJson, now]`. Key and
|
|
598
|
+
* sequence are copied from the stored event, so a delivery added later to an
|
|
599
|
+
* existing event keeps the event's position in its key and lands in the same
|
|
600
|
+
* lane. Returns the ids that were inserted.
|
|
601
|
+
*
|
|
602
|
+
* The deliveries of events this call just inserted are all new. For an event
|
|
603
|
+
* that already existed, a handler gets a delivery only if the event does not
|
|
604
|
+
* list it yet: a completed delivery is deleted, but its handler stays listed,
|
|
605
|
+
* so a redelivered or republished event never runs a handler twice. The
|
|
606
|
+
* handlers added to an existing event are appended to its list.
|
|
289
607
|
*/
|
|
290
608
|
get insertDeliveries() {
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
FROM jsonb_to_recordset($1::jsonb) AS incoming (id text, event_id text, handler text, available_at timestamptz)
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
609
|
+
const events = this.#names.events;
|
|
610
|
+
const insertedEvents = `ARRAY(SELECT jsonb_array_elements_text($3::jsonb))`;
|
|
611
|
+
return `WITH incoming AS (
|
|
612
|
+
SELECT incoming.id, incoming.event_id, incoming.handler, incoming.available_at
|
|
613
|
+
FROM jsonb_to_recordset($1::jsonb) AS incoming (id text, event_id text, handler text, available_at timestamptz)
|
|
614
|
+
), fresh AS (
|
|
615
|
+
SELECT incoming.*, event.key, event.sequence, ${laneExpression("event.key", "event.id")} AS lane
|
|
616
|
+
FROM incoming
|
|
617
|
+
JOIN ${events} AS event ON event.id = incoming.event_id
|
|
618
|
+
WHERE event.id = ANY (${insertedEvents}) OR NOT incoming.handler = ANY (event.handlers)
|
|
619
|
+
), inserted AS (
|
|
620
|
+
INSERT INTO ${this.#names.deliveries}
|
|
621
|
+
(id, event_id, handler, lane, status, attempts, available_at, key, sequence, created_at, updated_at)
|
|
622
|
+
SELECT fresh.id, fresh.event_id, fresh.handler, fresh.lane, 'pending', 0, fresh.available_at,
|
|
623
|
+
fresh.key, fresh.sequence, $2::timestamptz, $2::timestamptz
|
|
624
|
+
FROM fresh
|
|
625
|
+
ORDER BY fresh.event_id, fresh.handler
|
|
626
|
+
ON CONFLICT (event_id, handler) DO NOTHING
|
|
627
|
+
RETURNING id, event_id, handler, available_at
|
|
628
|
+
), extended AS (
|
|
629
|
+
UPDATE ${events} AS event SET handlers = event.handlers || added.handlers
|
|
630
|
+
FROM (
|
|
631
|
+
SELECT inserted.event_id, array_agg(inserted.handler) AS handlers FROM inserted
|
|
632
|
+
WHERE inserted.event_id <> ALL (${insertedEvents}) GROUP BY inserted.event_id
|
|
633
|
+
) AS added
|
|
634
|
+
WHERE event.id = added.event_id
|
|
635
|
+
RETURNING event.id
|
|
636
|
+
), ${this.#wakeUp("inserted", "$4")}
|
|
637
|
+
SELECT id FROM inserted WHERE (SELECT count(*) FROM woken) >= 0`;
|
|
300
638
|
}
|
|
301
639
|
/**
|
|
302
|
-
*
|
|
303
|
-
*
|
|
304
|
-
*
|
|
305
|
-
* `
|
|
640
|
+
* A claim in one statement, the only one in the common case. Parameters:
|
|
641
|
+
* `[now, handlersJson, limit, leaseOwner, leaseExpiresAt, generation]`.
|
|
642
|
+
*
|
|
643
|
+
* 1. **Held lanes.** The lanes this owner holds under a live lease whose `claim_floor` is exactly the
|
|
644
|
+
* generation are locked `FOR NO KEY UPDATE` in lane order. A lane above it was raised by a
|
|
645
|
+
* reconciliation, so a statement of a claim whose caller gave up (a client timeout does not cancel
|
|
646
|
+
* the statement on the server) and that reaches the server late records nothing; a lane below it was
|
|
647
|
+
* taken by such a late statement after the reconciliation, and its listed deliveries were never
|
|
648
|
+
* handed out. A lane that another transaction of this owner is changing makes the statement wait;
|
|
649
|
+
* the statement then reports `stale` and does nothing (see the `stale` CTE).
|
|
650
|
+
* 2. **Free lanes**, only when the held lanes yield fewer than `limit`: the free lanes with a due
|
|
651
|
+
* delivery (one index probe per free lane) are ranked by their oldest due delivery; the first
|
|
652
|
+
* {@link INITIAL_FREE_LANES} that can give something now, and further ones until what they can give
|
|
653
|
+
* (estimated from their first {@link LANE_DEPTH_CAP} due deliveries) covers what is still wanted, are locked
|
|
654
|
+
* `FOR UPDATE SKIP LOCKED`, so a lane another claimer has is skipped, never waited
|
|
655
|
+
* for. A lane that still runs deliveries of an expired lease is left to the transaction of
|
|
656
|
+
* {@link takeFreeLanes}, which charges their lost attempts (`takeover_waiting`); so is any lane whose
|
|
657
|
+
* locked row differs from what the statement's snapshot saw. A lane that runs nothing and did not
|
|
658
|
+
* change cannot have had a delivery written since the snapshot, so its deliveries can be read here.
|
|
659
|
+
* 3. **Picks** follow the rules of {@link claim}: the fast window of each lane, and for lanes whose
|
|
660
|
+
* window filled up while the batch is not full, the heads of all keys (a loose index scan per lane).
|
|
661
|
+
* 4. **Writes**: one update per lane. A held lane records its picks; a taken lane gets a new epoch, the
|
|
662
|
+
* owner, the lease and its picks; a taken lane that yielded nothing is not written. Held lanes that
|
|
663
|
+
* picked nothing, run nothing and have no full scan window are freed.
|
|
306
664
|
*
|
|
307
|
-
* Always returns one row per
|
|
308
|
-
*
|
|
665
|
+
* Always returns one row per picked delivery, or one row whose `delivery_id` is null. `running_ids`
|
|
666
|
+
* (JSON text) lists the deliveries the held lanes still run, `examined_lanes` the held lanes.
|
|
667
|
+
*/
|
|
668
|
+
get claimLanes() {
|
|
669
|
+
return this.#claimLanes(true);
|
|
670
|
+
}
|
|
671
|
+
/**
|
|
672
|
+
* {@link claimLanes} limited to the lanes the owner holds: the same parameters, rows and writes, without
|
|
673
|
+
* looking at free lanes (`takeover_waiting` and `unexamined_free_lanes` are false). A worker draining a
|
|
674
|
+
* backlog finds its batch in the lanes it holds; this statement is about half as costly to plan, and
|
|
675
|
+
* the caller runs {@link claimLanes} after it only when it came up short.
|
|
676
|
+
*/
|
|
677
|
+
get claimHeldLanes() {
|
|
678
|
+
return this.#claimLanes(false);
|
|
679
|
+
}
|
|
680
|
+
#claimLanes(includeFreeLanes) {
|
|
681
|
+
const deliveries = this.#names.deliveries;
|
|
682
|
+
const lanes = this.#names.lanes;
|
|
683
|
+
const freeLane = (alias) => `(${alias}.lease_expires_at IS NULL OR ${alias}.lease_expires_at <= $1::timestamptz
|
|
684
|
+
OR (${alias}.lease_owner = $4::text AND ${alias}.claim_floor < $6::bigint))`;
|
|
685
|
+
const free = includeFreeLanes ? `), free_lanes AS MATERIALIZED (
|
|
686
|
+
-- The oldest due delivery of each free lane: one index probe per free lane, bounded by the due time,
|
|
687
|
+
-- so deliveries scheduled for later are never read.
|
|
688
|
+
SELECT lane.lane, lane.epoch, lane.in_flight, lane.lease_owner, lane.lease_expires_at,
|
|
689
|
+
oldest.available_at, oldest.id, NOT ${this.#noRunningDelivery("lane")} AS running
|
|
690
|
+
FROM ${lanes} AS lane
|
|
691
|
+
CROSS JOIN LATERAL (
|
|
692
|
+
SELECT delivery.available_at, delivery.id
|
|
693
|
+
FROM ${deliveries} AS delivery
|
|
694
|
+
WHERE delivery.lane = lane.lane AND delivery.status = 'pending'
|
|
695
|
+
AND delivery.available_at <= $1::timestamptz AND ${HANDLER_PREDICATE}
|
|
696
|
+
ORDER BY delivery.available_at, delivery.id
|
|
697
|
+
LIMIT 1
|
|
698
|
+
) AS oldest
|
|
699
|
+
WHERE (SELECT count FROM wanted) > 0 AND ${freeLane("lane")}
|
|
700
|
+
ORDER BY oldest.available_at, oldest.id
|
|
701
|
+
LIMIT (SELECT count FROM wanted) + ${FREE_LANE_SURPLUS}
|
|
702
|
+
), free_depth AS MATERIALIZED (
|
|
703
|
+
-- What each candidate lane can give now, from its first due deliveries: unkeyed ones, and of a key its
|
|
704
|
+
-- lowest one when nothing earlier of the key is pending. Lanes that can give nothing are passed over;
|
|
705
|
+
-- of the others, the first ${INITIAL_FREE_LANES} are taken, and further ones until they cover what is
|
|
706
|
+
-- still wanted, so the earliest due work of the free lanes comes first.
|
|
707
|
+
SELECT runnable.lane,
|
|
708
|
+
row_number() OVER (ORDER BY runnable.available_at, runnable.id) AS rank,
|
|
709
|
+
sum(runnable.count) OVER (ORDER BY runnable.available_at, runnable.id ROWS UNBOUNDED PRECEDING) - runnable.count AS due_before
|
|
710
|
+
FROM (
|
|
711
|
+
SELECT lane_candidate.lane, lane_candidate.available_at, lane_candidate.id, depth.count
|
|
712
|
+
FROM free_lanes AS lane_candidate
|
|
713
|
+
CROSS JOIN LATERAL (
|
|
714
|
+
-- A full window that shows nothing runnable may hide runnable work behind it: such a lane counts
|
|
715
|
+
-- once, and the claim looks behind its blocked deliveries.
|
|
716
|
+
SELECT GREATEST(count(*) FILTER (
|
|
717
|
+
WHERE due.key IS NULL OR (due.position = 1 AND NOT EXISTS (
|
|
718
|
+
SELECT 1 FROM ${deliveries} AS earlier
|
|
719
|
+
WHERE earlier.lane = due.lane AND earlier.handler = due.handler AND earlier.key = due.key
|
|
720
|
+
AND earlier.status = 'pending' AND earlier.key IS NOT NULL AND earlier.sequence < due.sequence
|
|
721
|
+
))
|
|
722
|
+
), CASE WHEN count(*) >= ${LANE_DEPTH_CAP} THEN 1 ELSE 0 END) AS count
|
|
723
|
+
FROM (
|
|
724
|
+
SELECT window_rows.*,
|
|
725
|
+
row_number() OVER (PARTITION BY window_rows.handler, window_rows.key ORDER BY window_rows.sequence) AS position
|
|
726
|
+
FROM (
|
|
727
|
+
SELECT delivery.lane, delivery.handler, delivery.key, delivery.sequence FROM ${deliveries} AS delivery
|
|
728
|
+
WHERE delivery.lane = lane_candidate.lane AND delivery.status = 'pending'
|
|
729
|
+
AND delivery.available_at <= $1::timestamptz AND ${HANDLER_PREDICATE}
|
|
730
|
+
ORDER BY delivery.available_at, delivery.id
|
|
731
|
+
LIMIT ${LANE_DEPTH_CAP}
|
|
732
|
+
) AS window_rows
|
|
733
|
+
) AS due
|
|
734
|
+
) AS depth
|
|
735
|
+
WHERE NOT lane_candidate.running AND depth.count > 0
|
|
736
|
+
) AS runnable
|
|
737
|
+
), free_locked AS MATERIALIZED (
|
|
738
|
+
SELECT lane.lane, lane.epoch, lane.in_flight, lane.lease_owner, lane.lease_expires_at
|
|
739
|
+
FROM ${lanes} AS lane
|
|
740
|
+
WHERE lane.lane IN (
|
|
741
|
+
SELECT chosen.lane FROM free_depth AS chosen
|
|
742
|
+
WHERE chosen.rank <= ${INITIAL_FREE_LANES} OR chosen.due_before < (SELECT count FROM wanted)
|
|
743
|
+
)
|
|
744
|
+
AND ${freeLane("lane")}
|
|
745
|
+
ORDER BY lane.lane
|
|
746
|
+
FOR UPDATE SKIP LOCKED
|
|
747
|
+
), free_scope AS MATERIALIZED (
|
|
748
|
+
-- A lane whose locked row is not the one the snapshot saw changed hands meanwhile: left for a later claim.
|
|
749
|
+
SELECT locked.lane, locked.epoch + 1 AS epoch, '{}'::text[] AS in_flight
|
|
750
|
+
FROM free_locked AS locked
|
|
751
|
+
JOIN free_lanes AS seen ON seen.lane = locked.lane
|
|
752
|
+
WHERE locked.epoch = seen.epoch AND locked.in_flight = seen.in_flight
|
|
753
|
+
AND locked.lease_owner IS NOT DISTINCT FROM seen.lease_owner
|
|
754
|
+
AND locked.lease_expires_at IS NOT DISTINCT FROM seen.lease_expires_at
|
|
755
|
+
), ${this.#candidates("free", "free_scope", null, "(SELECT count FROM wanted)")}` : `), free_lanes AS (
|
|
756
|
+
SELECT NULL::smallint AS lane, NULL::bigint AS epoch, NULL::text[] AS in_flight, false AS running WHERE false
|
|
757
|
+
), free_scope AS (
|
|
758
|
+
SELECT NULL::smallint AS lane, NULL::bigint AS epoch WHERE false
|
|
759
|
+
), free_candidates AS (
|
|
760
|
+
SELECT NULL::text AS id, NULL::smallint AS lane, NULL::timestamptz AS available_at WHERE false
|
|
761
|
+
)`;
|
|
762
|
+
return `WITH held AS MATERIALIZED (
|
|
763
|
+
SELECT lane.lane, lane.epoch, lane.in_flight
|
|
764
|
+
FROM ${lanes} AS lane
|
|
765
|
+
WHERE lane.lease_owner = $4::text AND lane.lease_expires_at > $1::timestamptz
|
|
766
|
+
AND lane.claim_floor = $6::bigint
|
|
767
|
+
ORDER BY lane.lane
|
|
768
|
+
FOR NO KEY UPDATE
|
|
769
|
+
), stale AS (
|
|
770
|
+
-- A claim that waited for a lane lock sees the new lane row but reads deliveries with the snapshot it took
|
|
771
|
+
-- before the wait: whatever changed the lane meanwhile (a reschedule, a release) may also have changed
|
|
772
|
+
-- deliveries it cannot see. The statement then does nothing and reports it, and the caller runs it again.
|
|
773
|
+
SELECT EXISTS (
|
|
774
|
+
SELECT 1 FROM held
|
|
775
|
+
JOIN ${lanes} AS before ON before.lane = held.lane
|
|
776
|
+
WHERE before.epoch <> held.epoch OR before.in_flight IS DISTINCT FROM held.in_flight
|
|
777
|
+
) AS value
|
|
778
|
+
), held_scope AS MATERIALIZED (
|
|
779
|
+
SELECT held.lane, held.epoch, held.in_flight FROM held WHERE NOT (SELECT value FROM stale)
|
|
780
|
+
), held_running AS MATERIALIZED (
|
|
781
|
+
SELECT delivery.handler, delivery.key
|
|
782
|
+
FROM held_scope
|
|
783
|
+
CROSS JOIN LATERAL unnest(held_scope.in_flight) AS flying (id)
|
|
784
|
+
JOIN ${deliveries} AS delivery ON delivery.id = flying.id
|
|
785
|
+
WHERE delivery.status = 'pending' AND delivery.key IS NOT NULL
|
|
786
|
+
), ${this.#candidates("held", "held_scope", "held_running", "$3::int")}, wanted AS MATERIALIZED (
|
|
787
|
+
SELECT CASE WHEN (SELECT value FROM stale) THEN 0
|
|
788
|
+
ELSE $3::int - (SELECT count(*) FROM held_candidates)::int END AS count
|
|
789
|
+
${free}, claim_scope AS MATERIALIZED (
|
|
790
|
+
SELECT held_scope.lane, held_scope.epoch FROM held_scope
|
|
791
|
+
UNION ALL
|
|
792
|
+
SELECT free_scope.lane, free_scope.epoch FROM free_scope
|
|
793
|
+
), picked_rows AS MATERIALIZED (
|
|
794
|
+
SELECT candidates.available_at AS due_at, delivery.id AS delivery_id, delivery.lane::int AS lane,
|
|
795
|
+
scope.epoch::text AS epoch, delivery.claims AS claims, delivery.handler, delivery.key AS delivery_key, delivery.sequence::text AS sequence, (delivery.attempts + 1)::int AS attempt,
|
|
796
|
+
${eventColumns("event")}
|
|
797
|
+
FROM (
|
|
798
|
+
SELECT held_candidates.id, held_candidates.lane, held_candidates.available_at FROM held_candidates
|
|
799
|
+
UNION ALL
|
|
800
|
+
SELECT free_candidates.id, free_candidates.lane, free_candidates.available_at FROM free_candidates
|
|
801
|
+
) AS candidates
|
|
802
|
+
JOIN claim_scope AS scope ON scope.lane = candidates.lane
|
|
803
|
+
JOIN ${deliveries} AS delivery ON delivery.id = candidates.id
|
|
804
|
+
JOIN ${this.#names.events} AS event ON event.id = delivery.event_id
|
|
805
|
+
), by_lane AS (
|
|
806
|
+
SELECT picked_rows.lane, array_agg(picked_rows.delivery_id) AS ids FROM picked_rows GROUP BY picked_rows.lane
|
|
807
|
+
), still_running AS (
|
|
808
|
+
-- The retained ids come from the locked \`held\` rows, not from the lane table: a claim that waited for the lock
|
|
809
|
+
-- must see the in-flight list the claim before it recorded, which the statement snapshot may not.
|
|
810
|
+
SELECT held.lane, array_agg(delivery.id) AS ids
|
|
811
|
+
FROM by_lane
|
|
812
|
+
JOIN held ON held.lane = by_lane.lane
|
|
813
|
+
CROSS JOIN LATERAL unnest(held.in_flight) AS flying (id)
|
|
814
|
+
JOIN ${deliveries} AS delivery ON delivery.id = flying.id AND delivery.status = 'pending'
|
|
815
|
+
GROUP BY held.lane
|
|
816
|
+
), extended AS (
|
|
817
|
+
UPDATE ${lanes} AS lane
|
|
818
|
+
SET in_flight = COALESCE(still_running.ids, '{}') || by_lane.ids,
|
|
819
|
+
lease_expires_at = GREATEST(lane.lease_expires_at, $5::timestamptz)
|
|
820
|
+
FROM by_lane
|
|
821
|
+
JOIN held_scope ON held_scope.lane = by_lane.lane
|
|
822
|
+
LEFT JOIN still_running ON still_running.lane = by_lane.lane
|
|
823
|
+
WHERE lane.lane = by_lane.lane
|
|
824
|
+
RETURNING lane.lane
|
|
825
|
+
), taken AS (
|
|
826
|
+
UPDATE ${lanes} AS lane
|
|
827
|
+
SET epoch = free_scope.epoch, lease_owner = $4::text, lease_expires_at = $5::timestamptz,
|
|
828
|
+
in_flight = by_lane.ids, claim_floor = $6::bigint
|
|
829
|
+
FROM free_scope
|
|
830
|
+
JOIN by_lane ON by_lane.lane = free_scope.lane
|
|
831
|
+
WHERE lane.lane = free_scope.lane
|
|
832
|
+
RETURNING lane.lane
|
|
833
|
+
), saturated_lanes AS (
|
|
834
|
+
SELECT held_window.lane FROM held_window GROUP BY held_window.lane HAVING count(*) >= ${LANE_SCAN_WINDOW}
|
|
835
|
+
), freed AS (
|
|
836
|
+
UPDATE ${lanes} AS lane
|
|
837
|
+
SET lease_owner = NULL, lease_expires_at = NULL, in_flight = '{}'
|
|
838
|
+
FROM held
|
|
839
|
+
WHERE lane.lane = held.lane
|
|
840
|
+
AND lane.lane NOT IN (SELECT lane FROM by_lane)
|
|
841
|
+
AND lane.lane NOT IN (SELECT lane FROM saturated_lanes)
|
|
842
|
+
AND (SELECT count(*) FROM held_window) < ${CLAIM_SCAN_WINDOW}
|
|
843
|
+
AND NOT (SELECT value FROM stale)
|
|
844
|
+
AND ${this.#noRunningDelivery("lane")}
|
|
845
|
+
RETURNING lane.lane
|
|
846
|
+
)
|
|
847
|
+
SELECT (SELECT value FROM stale) AS stale,
|
|
848
|
+
EXISTS (SELECT 1 FROM free_lanes WHERE free_lanes.running) AS takeover_waiting,
|
|
849
|
+
(SELECT count(*) FROM free_lanes) = (SELECT count FROM wanted) + ${FREE_LANE_SURPLUS} AS unexamined_free_lanes,
|
|
850
|
+
(
|
|
851
|
+
SELECT COALESCE(jsonb_agg(delivery.id), '[]'::jsonb)
|
|
852
|
+
FROM held
|
|
853
|
+
CROSS JOIN LATERAL unnest(held.in_flight) AS flying (id)
|
|
854
|
+
JOIN ${deliveries} AS delivery ON delivery.id = flying.id AND delivery.status = 'pending'
|
|
855
|
+
)::text AS running_ids,
|
|
856
|
+
(SELECT COALESCE(jsonb_agg(held.lane), '[]'::jsonb) FROM held)::text AS examined_lanes,
|
|
857
|
+
picked.*
|
|
858
|
+
FROM (SELECT 1) AS anchor
|
|
859
|
+
LEFT JOIN picked_rows AS picked ON true
|
|
860
|
+
ORDER BY picked.due_at, picked.delivery_id`;
|
|
861
|
+
}
|
|
862
|
+
/**
|
|
863
|
+
* The head (lowest pending sequence) of every key of every wanted handler in the lane `scope.lane`,
|
|
864
|
+
* as a subquery returning `(id, lane, available_at, handler, key)`. A loose index scan on the
|
|
865
|
+
* pending-key index hops from key to key of a handler, and after its last key on to the first key of
|
|
866
|
+
* the next wanted handler, so the cost follows the number of keys, not the number of followers. The
|
|
867
|
+
* handlers are walked inside the recursion instead of joined to it: a join multiplied the planner's
|
|
868
|
+
* cost estimate by the handler count, which pushed the claim statements over `jit_above_cost`, and
|
|
869
|
+
* compiling them took longer than running them.
|
|
870
|
+
*/
|
|
871
|
+
#keyHeads(scope) {
|
|
872
|
+
const deliveries = this.#names.deliveries;
|
|
873
|
+
const pending = `candidate.lane = ${scope}.lane AND candidate.status = 'pending'`;
|
|
874
|
+
const nextHandler = (after) => `(SELECT min(wanted.handler) FROM jsonb_array_elements_text($2::jsonb) AS wanted (handler)${after === "" ? "" : ` WHERE wanted.handler > ${after}`})`;
|
|
875
|
+
return `
|
|
876
|
+
WITH RECURSIVE hops (handler, key) AS (
|
|
877
|
+
SELECT ${nextHandler("")}, NULL::text
|
|
878
|
+
UNION ALL
|
|
879
|
+
SELECT CASE WHEN next.key IS NULL THEN ${nextHandler("hops.handler")} ELSE hops.handler END, next.key
|
|
880
|
+
FROM hops
|
|
881
|
+
CROSS JOIN LATERAL (
|
|
882
|
+
SELECT CASE WHEN hops.key IS NULL
|
|
883
|
+
THEN (SELECT candidate.key FROM ${deliveries} AS candidate
|
|
884
|
+
WHERE ${pending} AND candidate.handler = hops.handler AND candidate.key IS NOT NULL
|
|
885
|
+
ORDER BY candidate.key LIMIT 1)
|
|
886
|
+
ELSE (SELECT candidate.key FROM ${deliveries} AS candidate
|
|
887
|
+
WHERE ${pending} AND candidate.handler = hops.handler AND candidate.key > hops.key
|
|
888
|
+
ORDER BY candidate.key LIMIT 1)
|
|
889
|
+
END AS key
|
|
890
|
+
OFFSET 0
|
|
891
|
+
) AS next
|
|
892
|
+
WHERE hops.handler IS NOT NULL
|
|
893
|
+
)
|
|
894
|
+
SELECT lowest.id, lowest.lane, lowest.available_at, lowest.handler, lowest.key
|
|
895
|
+
FROM hops
|
|
896
|
+
CROSS JOIN LATERAL (
|
|
897
|
+
SELECT candidate.id, candidate.lane, candidate.available_at, candidate.handler, candidate.key
|
|
898
|
+
FROM ${deliveries} AS candidate
|
|
899
|
+
WHERE ${pending} AND candidate.handler = hops.handler AND candidate.key = hops.key
|
|
900
|
+
ORDER BY candidate.sequence LIMIT 1
|
|
901
|
+
) AS lowest
|
|
902
|
+
WHERE hops.key IS NOT NULL
|
|
903
|
+
`;
|
|
904
|
+
}
|
|
905
|
+
/**
|
|
906
|
+
* CTEs that pick up to `limit` claimable deliveries from the lanes of the CTE `scope` (columns `lane`,
|
|
907
|
+
* `epoch`, `in_flight`), named after `prefix`; the result is `<prefix>_candidates (id, lane, available_at)`.
|
|
908
|
+
* The rules are those of {@link claim}. The fast window reads the first {@link LANE_SCAN_WINDOW} due
|
|
909
|
+
* deliveries of each lane (at most {@link CLAIM_SCAN_WINDOW} together); of a key, only the lowest sequence
|
|
910
|
+
* in the window can be its head, so only that one is checked for an earlier pending delivery. Only when
|
|
911
|
+
* that yields fewer than `limit` are the lanes whose window filled up looked at again with a loose index
|
|
912
|
+
* scan over the heads of all their keys. `running` names the CTE of the keyed deliveries running in the
|
|
913
|
+
* lanes, or is null when nothing runs there.
|
|
914
|
+
*/
|
|
915
|
+
#candidates(prefix, scope, running, limit) {
|
|
916
|
+
const deliveries = this.#names.deliveries;
|
|
917
|
+
const notRunning = (alias) => running === null ? "true" : `NOT EXISTS (SELECT 1 FROM ${running} AS running WHERE running.handler = ${alias}.handler AND running.key = ${alias}.key)`;
|
|
918
|
+
return `${prefix}_window AS MATERIALIZED (
|
|
919
|
+
SELECT recent.id, recent.lane, recent.available_at, recent.handler, recent.key, recent.sequence
|
|
920
|
+
FROM ${scope}
|
|
921
|
+
CROSS JOIN LATERAL (
|
|
922
|
+
SELECT delivery.id, delivery.lane, delivery.available_at, delivery.handler, delivery.key, delivery.sequence
|
|
923
|
+
FROM ${deliveries} AS delivery
|
|
924
|
+
WHERE delivery.lane = ${scope}.lane AND delivery.status = 'pending'
|
|
925
|
+
AND delivery.available_at <= $1::timestamptz AND ${HANDLER_PREDICATE}
|
|
926
|
+
AND delivery.id <> ALL (${scope}.in_flight)
|
|
927
|
+
ORDER BY delivery.available_at, delivery.id
|
|
928
|
+
LIMIT ${LANE_SCAN_WINDOW}
|
|
929
|
+
) AS recent
|
|
930
|
+
ORDER BY recent.available_at, recent.id
|
|
931
|
+
LIMIT ${CLAIM_SCAN_WINDOW}
|
|
932
|
+
), ${prefix}_fast AS MATERIALIZED (
|
|
933
|
+
SELECT picked.id, picked.lane, picked.available_at FROM (
|
|
934
|
+
SELECT unkeyed.id, unkeyed.lane, unkeyed.available_at FROM ${prefix}_window AS unkeyed WHERE unkeyed.key IS NULL
|
|
935
|
+
UNION ALL
|
|
936
|
+
SELECT head.id, head.lane, head.available_at
|
|
937
|
+
FROM (
|
|
938
|
+
SELECT DISTINCT ON (keyed.handler, keyed.key) keyed.id, keyed.lane, keyed.available_at, keyed.handler, keyed.key, keyed.sequence
|
|
939
|
+
FROM ${prefix}_window AS keyed
|
|
940
|
+
WHERE keyed.key IS NOT NULL
|
|
941
|
+
ORDER BY keyed.handler, keyed.key, keyed.sequence
|
|
942
|
+
) AS head
|
|
943
|
+
WHERE NOT EXISTS (
|
|
944
|
+
SELECT 1 FROM ${deliveries} AS earlier
|
|
945
|
+
WHERE earlier.lane = head.lane AND earlier.handler = head.handler AND earlier.key = head.key
|
|
946
|
+
AND earlier.status = 'pending' AND earlier.key IS NOT NULL AND earlier.sequence < head.sequence
|
|
947
|
+
) AND ${notRunning("head")}
|
|
948
|
+
) AS picked
|
|
949
|
+
ORDER BY picked.available_at, picked.id
|
|
950
|
+
LIMIT ${limit}
|
|
951
|
+
), ${prefix}_saturated AS MATERIALIZED (
|
|
952
|
+
SELECT ${scope}.lane, ${scope}.in_flight FROM ${scope}
|
|
953
|
+
WHERE (SELECT count(*) FROM ${prefix}_fast) < ${limit}
|
|
954
|
+
AND ((SELECT count(*) FROM ${prefix}_window) >= ${CLAIM_SCAN_WINDOW}
|
|
955
|
+
OR ${scope}.lane IN (
|
|
956
|
+
SELECT full_lane.lane FROM ${prefix}_window AS full_lane
|
|
957
|
+
GROUP BY full_lane.lane HAVING count(*) >= ${LANE_SCAN_WINDOW}
|
|
958
|
+
))
|
|
959
|
+
), ${prefix}_heads AS MATERIALIZED (
|
|
960
|
+
SELECT head.id, head.lane, head.available_at, head.handler, head.key
|
|
961
|
+
FROM ${prefix}_saturated AS scope
|
|
962
|
+
CROSS JOIN LATERAL (${this.#keyHeads("scope")}) AS head
|
|
963
|
+
), ${prefix}_keyed AS MATERIALIZED (
|
|
964
|
+
SELECT heads.id, heads.lane, heads.available_at
|
|
965
|
+
FROM ${prefix}_heads AS heads
|
|
966
|
+
JOIN ${prefix}_saturated AS scope ON scope.lane = heads.lane
|
|
967
|
+
WHERE heads.available_at <= $1::timestamptz AND heads.id <> ALL (scope.in_flight) AND ${notRunning("heads")}
|
|
968
|
+
ORDER BY heads.available_at, heads.id
|
|
969
|
+
LIMIT ${limit}
|
|
970
|
+
), ${prefix}_unkeyed AS MATERIALIZED (
|
|
971
|
+
SELECT recent.id, recent.lane, recent.available_at
|
|
972
|
+
FROM ${prefix}_saturated AS scope
|
|
973
|
+
CROSS JOIN LATERAL (
|
|
974
|
+
SELECT delivery.id, delivery.lane, delivery.available_at
|
|
975
|
+
FROM ${deliveries} AS delivery
|
|
976
|
+
WHERE delivery.lane = scope.lane AND delivery.key IS NULL AND delivery.status = 'pending'
|
|
977
|
+
AND delivery.available_at <= $1::timestamptz AND ${HANDLER_PREDICATE}
|
|
978
|
+
AND delivery.id <> ALL (scope.in_flight)
|
|
979
|
+
ORDER BY delivery.available_at, delivery.id
|
|
980
|
+
LIMIT ${limit}
|
|
981
|
+
) AS recent
|
|
982
|
+
ORDER BY recent.available_at, recent.id
|
|
983
|
+
LIMIT ${limit}
|
|
984
|
+
), ${prefix}_candidates AS MATERIALIZED (
|
|
985
|
+
SELECT chosen.id, chosen.lane, chosen.available_at FROM (
|
|
986
|
+
SELECT fast.id, fast.lane, fast.available_at FROM ${prefix}_fast AS fast
|
|
987
|
+
UNION
|
|
988
|
+
SELECT keyed.id, keyed.lane, keyed.available_at FROM ${prefix}_keyed AS keyed
|
|
989
|
+
UNION
|
|
990
|
+
SELECT unkeyed.id, unkeyed.lane, unkeyed.available_at FROM ${prefix}_unkeyed AS unkeyed
|
|
991
|
+
) AS chosen
|
|
992
|
+
ORDER BY chosen.available_at, chosen.id
|
|
993
|
+
LIMIT ${limit}
|
|
994
|
+
)`;
|
|
995
|
+
}
|
|
996
|
+
/**
|
|
997
|
+
* Takes up to `$3` free lanes (no lease, or an expired one) that have a due
|
|
998
|
+
* delivery of the requested handlers, the ones with the oldest due delivery
|
|
999
|
+
* first. Parameters: `[now, handlersJson, laneCount, leaseOwner, leaseExpiresAt, generation]`.
|
|
1000
|
+
* The lane gets the owner's `generation` as its `claim_floor`. A lane the owner itself holds under a live
|
|
1001
|
+
* lease with a lower floor was taken by a claim statement that reached the server after its caller gave up
|
|
1002
|
+
* and the owner reconciled (or by an earlier incarnation of the same owner): it is taken over like the lane of a
|
|
1003
|
+
* crashed worker, so its listed deliveries are charged an attempt.
|
|
1004
|
+
*
|
|
1005
|
+
* A taken lane gets a new epoch, which fences everything the previous owner
|
|
1006
|
+
* still holds. Deliveries the previous owner had in flight are the ones whose
|
|
1007
|
+
* attempt was never recorded (a claim writes no delivery row), so each of
|
|
1008
|
+
* them is charged one attempt here: a worker that crashes with a poison
|
|
1009
|
+
* message still runs it into the retry limit. Lanes another claimer has locked
|
|
1010
|
+
* are skipped, never waited for.
|
|
1011
|
+
*/
|
|
1012
|
+
get takeFreeLanes() {
|
|
1013
|
+
const deliveries = this.#names.deliveries;
|
|
1014
|
+
const lanes = this.#names.lanes;
|
|
1015
|
+
return `WITH picked AS (
|
|
1016
|
+
-- in_flight is read here, from the locked version of the lane: a settlement that committed after the
|
|
1017
|
+
-- statement took its snapshot has taken its delivery out of the list already and must not be charged again.
|
|
1018
|
+
SELECT free.lane, free.in_flight
|
|
1019
|
+
FROM ${lanes} AS free
|
|
1020
|
+
CROSS JOIN LATERAL (
|
|
1021
|
+
SELECT delivery.available_at, delivery.id
|
|
1022
|
+
FROM ${deliveries} AS delivery
|
|
1023
|
+
WHERE delivery.lane = free.lane AND delivery.status = 'pending'
|
|
1024
|
+
AND delivery.available_at <= $1::timestamptz AND ${HANDLER_PREDICATE}
|
|
1025
|
+
ORDER BY delivery.available_at, delivery.id
|
|
1026
|
+
LIMIT 1
|
|
1027
|
+
) AS oldest
|
|
1028
|
+
WHERE free.lease_expires_at IS NULL OR free.lease_expires_at <= $1::timestamptz
|
|
1029
|
+
OR (free.lease_owner = $4::text AND free.claim_floor < $6::bigint)
|
|
1030
|
+
ORDER BY oldest.available_at, oldest.id
|
|
1031
|
+
LIMIT $3::int
|
|
1032
|
+
FOR UPDATE OF free SKIP LOCKED
|
|
1033
|
+
), charged AS (
|
|
1034
|
+
UPDATE ${deliveries} AS delivery SET attempts = delivery.attempts + 1, claims = delivery.claims + 1
|
|
1035
|
+
FROM (
|
|
1036
|
+
SELECT unnest(picked.in_flight) AS id FROM picked
|
|
1037
|
+
) AS lost
|
|
1038
|
+
WHERE delivery.id = lost.id AND delivery.status = 'pending'
|
|
1039
|
+
RETURNING delivery.id
|
|
1040
|
+
), taken AS (
|
|
1041
|
+
UPDATE ${lanes} AS lane
|
|
1042
|
+
SET epoch = lane.epoch + 1, lease_owner = $4::text, lease_expires_at = $5::timestamptz, in_flight = '{}',
|
|
1043
|
+
claim_floor = $6::bigint
|
|
1044
|
+
FROM picked WHERE lane.lane = picked.lane
|
|
1045
|
+
RETURNING lane.lane, lane.epoch
|
|
1046
|
+
)
|
|
1047
|
+
SELECT lane::int AS lane, epoch::text AS epoch FROM taken ORDER BY lane`;
|
|
1048
|
+
}
|
|
1049
|
+
/**
|
|
1050
|
+
* Fast claim: picks from the first {@link CLAIM_SCAN_WINDOW} due deliveries of
|
|
1051
|
+
* the given lanes in due order, which is everything the claim needs while the
|
|
1052
|
+
* front of the queue is not clogged with blocked followers of busy keys.
|
|
1053
|
+
* Parameters: `[now, handlersJson, limit, lanesJson]`. The lanes must be held
|
|
1054
|
+
* and locked by the caller.
|
|
1055
|
+
*
|
|
1056
|
+
* Always returns one row per picked delivery, or a single row whose
|
|
1057
|
+
* `delivery_id` is null when nothing was picked. `window_saturated` says
|
|
309
1058
|
* whether the window was full: when it is and fewer than `limit` deliveries
|
|
310
|
-
* were
|
|
1059
|
+
* were picked, deliveries beyond the window may still be claimable and the
|
|
311
1060
|
* caller runs {@link claimBehindBlockedKeys}. When the window is not
|
|
312
1061
|
* saturated every due delivery was examined, so the result is complete.
|
|
313
1062
|
*
|
|
314
|
-
*
|
|
315
|
-
*
|
|
316
|
-
*
|
|
317
|
-
*
|
|
318
|
-
* other delivery of that handler and key
|
|
319
|
-
* delivery that re-enters the key behind a running one
|
|
320
|
-
* added to an old event)
|
|
321
|
-
* is presumed dead and the key must not stall behind it.
|
|
1063
|
+
* A delivery is a candidate when it is pending, due, and not in flight. A
|
|
1064
|
+
* keyed delivery is a candidate only when it is the head of its (handler,
|
|
1065
|
+
* key), that is (a) no earlier delivery of that handler and key is pending
|
|
1066
|
+
* (an earlier one in flight is still a pending row, which is the ordering
|
|
1067
|
+
* guarantee), and (b) no other delivery of that handler and key is in
|
|
1068
|
+
* flight, which covers a delivery that re-enters the key behind a running one
|
|
1069
|
+
* (requeue, or a handler added to an old event).
|
|
322
1070
|
*
|
|
323
|
-
*
|
|
324
|
-
*
|
|
325
|
-
* for it either skip it (the other holds the row lock) or, if the other
|
|
326
|
-
* already committed, re-check the row's new version (now processing and not
|
|
327
|
-
* due), which excludes it. Followers stay blocked because their predecessor
|
|
328
|
-
* remains pending or processing in every snapshot until it completes or is
|
|
329
|
-
* buried, and that commit happens-before the next claim that sees it.
|
|
1071
|
+
* No row is locked or written here: the lanes' exclusive leases are the
|
|
1072
|
+
* mutual exclusion, so two claimers never see the same lane.
|
|
330
1073
|
*/
|
|
331
1074
|
get claim() {
|
|
1075
|
+
return `WITH ${this.#scopeAndRunning(`SELECT lane.lane, lane.epoch, lane.in_flight FROM ${this.#names.lanes} AS lane WHERE lane.lane = ANY (${LANE_LIST})`)},
|
|
1076
|
+
${this.#fastWindow()}
|
|
1077
|
+
)
|
|
1078
|
+
${this.#pickedRows(`((SELECT count(*) FROM due_window) >= ${CLAIM_SCAN_WINDOW} OR EXISTS (SELECT 1 FROM due_window GROUP BY lane HAVING count(*) >= ${LANE_SCAN_WINDOW}))`)}`;
|
|
1079
|
+
}
|
|
1080
|
+
/** The `due_window` and `candidates` CTEs of the fast claim, up to (not including) the closing parenthesis of `candidates`. */
|
|
1081
|
+
#fastWindow() {
|
|
332
1082
|
const deliveries = this.#names.deliveries;
|
|
333
|
-
return `
|
|
334
|
-
SELECT
|
|
335
|
-
FROM
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
1083
|
+
return `due_window AS MATERIALIZED (
|
|
1084
|
+
SELECT recent.id, recent.lane, recent.available_at
|
|
1085
|
+
FROM scope
|
|
1086
|
+
CROSS JOIN LATERAL (
|
|
1087
|
+
SELECT delivery.id, delivery.lane, delivery.available_at
|
|
1088
|
+
FROM ${deliveries} AS delivery
|
|
1089
|
+
WHERE delivery.lane = scope.lane AND delivery.status = 'pending'
|
|
1090
|
+
AND delivery.available_at <= $1::timestamptz AND ${HANDLER_PREDICATE}
|
|
1091
|
+
AND delivery.id <> ALL (scope.in_flight)
|
|
1092
|
+
ORDER BY delivery.available_at, delivery.id
|
|
1093
|
+
LIMIT ${LANE_SCAN_WINDOW}
|
|
1094
|
+
) AS recent
|
|
1095
|
+
ORDER BY recent.available_at, recent.id
|
|
340
1096
|
LIMIT ${CLAIM_SCAN_WINDOW}
|
|
341
1097
|
), candidates AS MATERIALIZED (
|
|
342
|
-
SELECT delivery.id,
|
|
1098
|
+
SELECT delivery.id, delivery.lane, delivery.available_at
|
|
343
1099
|
FROM due_window
|
|
344
1100
|
JOIN ${deliveries} AS delivery ON delivery.id = due_window.id
|
|
345
|
-
WHERE ${
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
delivery.key IS NULL
|
|
349
|
-
OR NOT (
|
|
350
|
-
EXISTS (
|
|
351
|
-
SELECT 1 FROM ${deliveries} AS earlier
|
|
352
|
-
WHERE earlier.key IS NOT NULL AND earlier.handler = delivery.handler AND earlier.key = delivery.key
|
|
353
|
-
AND ${UNFINISHED_PREDICATE("earlier")}
|
|
354
|
-
AND earlier.sequence < delivery.sequence
|
|
355
|
-
)
|
|
356
|
-
OR ${this.#liveLeaseElsewhere()}
|
|
357
|
-
)
|
|
358
|
-
)
|
|
359
|
-
ORDER BY due_at, delivery.id
|
|
360
|
-
LIMIT $3::int
|
|
361
|
-
FOR UPDATE OF delivery SKIP LOCKED
|
|
362
|
-
), ${this.#leaseCandidates()}
|
|
363
|
-
SELECT (SELECT count(*) FROM due_window) >= ${CLAIM_SCAN_WINDOW} AS window_saturated, leased.*
|
|
364
|
-
FROM (SELECT 1) AS anchor
|
|
365
|
-
LEFT JOIN (${this.#leasedRows()}) AS leased ON true
|
|
366
|
-
ORDER BY leased.due_at, leased.delivery_id`;
|
|
1101
|
+
WHERE delivery.key IS NULL OR NOT (${this.#blockedKey("delivery")})
|
|
1102
|
+
ORDER BY delivery.available_at, delivery.id
|
|
1103
|
+
LIMIT $3::int`;
|
|
367
1104
|
}
|
|
368
1105
|
/**
|
|
369
|
-
* Slow claim, run when {@link claim}
|
|
1106
|
+
* Slow claim, run when {@link claim} picked too few deliveries from a full
|
|
370
1107
|
* window: the front of the due queue is mostly followers of blocked keys.
|
|
371
|
-
* Same parameters
|
|
372
|
-
*
|
|
373
|
-
* cost follows the number of
|
|
374
|
-
*
|
|
1108
|
+
* Same parameters and rules as {@link claim}, but heads are found with
|
|
1109
|
+
* a loose index scan over the pending-key index of each lane (it hops from key
|
|
1110
|
+
* to key), so the cost follows the number of pending keys in the lanes
|
|
1111
|
+
* instead of one probe per due follower.
|
|
375
1112
|
*/
|
|
376
1113
|
get claimBehindBlockedKeys() {
|
|
377
1114
|
const deliveries = this.#names.deliveries;
|
|
378
|
-
return `WITH
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
AND head.handler = ANY (ARRAY(SELECT jsonb_array_elements_text($2::jsonb)))
|
|
384
|
-
ORDER BY head.handler, head.key, head.sequence
|
|
1115
|
+
return `WITH ${this.#scopeAndRunning(`SELECT lane.lane, lane.epoch, lane.in_flight FROM ${this.#names.lanes} AS lane WHERE lane.lane = ANY (${LANE_LIST})`)},
|
|
1116
|
+
heads AS MATERIALIZED (
|
|
1117
|
+
SELECT head.id, head.lane, head.available_at, head.handler, head.key
|
|
1118
|
+
FROM scope
|
|
1119
|
+
CROSS JOIN LATERAL (${this.#keyHeads("scope")}) AS head
|
|
385
1120
|
), keyed AS MATERIALIZED (
|
|
386
|
-
SELECT
|
|
1121
|
+
SELECT heads.id, heads.lane, heads.available_at
|
|
387
1122
|
FROM heads
|
|
388
|
-
JOIN
|
|
389
|
-
WHERE $
|
|
390
|
-
AND
|
|
391
|
-
|
|
392
|
-
ORDER BY due_at, delivery.id
|
|
1123
|
+
JOIN scope ON scope.lane = heads.lane
|
|
1124
|
+
WHERE heads.available_at <= $1::timestamptz AND heads.id <> ALL (scope.in_flight)
|
|
1125
|
+
AND NOT EXISTS (SELECT 1 FROM running WHERE running.handler = heads.handler AND running.key = heads.key)
|
|
1126
|
+
ORDER BY heads.available_at, heads.id
|
|
393
1127
|
LIMIT $3::int
|
|
394
|
-
FOR UPDATE OF delivery SKIP LOCKED
|
|
395
1128
|
), unkeyed AS MATERIALIZED (
|
|
396
|
-
SELECT
|
|
397
|
-
FROM
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
AND
|
|
402
|
-
|
|
1129
|
+
SELECT recent.id, recent.lane, recent.available_at
|
|
1130
|
+
FROM scope
|
|
1131
|
+
CROSS JOIN LATERAL (
|
|
1132
|
+
SELECT delivery.id, delivery.lane, delivery.available_at
|
|
1133
|
+
FROM ${deliveries} AS delivery
|
|
1134
|
+
WHERE delivery.lane = scope.lane AND delivery.key IS NULL AND delivery.status = 'pending'
|
|
1135
|
+
AND delivery.available_at <= $1::timestamptz AND ${HANDLER_PREDICATE}
|
|
1136
|
+
AND delivery.id <> ALL (scope.in_flight)
|
|
1137
|
+
ORDER BY delivery.available_at, delivery.id
|
|
1138
|
+
LIMIT $3::int
|
|
1139
|
+
) AS recent
|
|
1140
|
+
ORDER BY recent.available_at, recent.id
|
|
403
1141
|
LIMIT $3::int
|
|
404
|
-
FOR UPDATE OF delivery SKIP LOCKED
|
|
405
1142
|
), candidates AS MATERIALIZED (
|
|
406
|
-
SELECT id,
|
|
407
|
-
SELECT id,
|
|
1143
|
+
SELECT id, lane, available_at FROM (
|
|
1144
|
+
SELECT id, lane, available_at FROM keyed
|
|
408
1145
|
UNION ALL
|
|
409
|
-
SELECT id,
|
|
1146
|
+
SELECT id, lane, available_at FROM unkeyed
|
|
410
1147
|
) AS both_sets
|
|
411
|
-
ORDER BY
|
|
1148
|
+
ORDER BY available_at, id
|
|
412
1149
|
LIMIT $3::int
|
|
413
|
-
)
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
1150
|
+
)
|
|
1151
|
+
${this.#pickedRows("false")}`;
|
|
1152
|
+
}
|
|
1153
|
+
/** The held lanes with their epochs and in-flight ids, and the keyed deliveries running in them. */
|
|
1154
|
+
#scopeAndRunning(scopeQuery) {
|
|
1155
|
+
const deliveries = this.#names.deliveries;
|
|
1156
|
+
return `scope AS MATERIALIZED (
|
|
1157
|
+
${scopeQuery}
|
|
1158
|
+
), running AS MATERIALIZED (
|
|
1159
|
+
SELECT delivery.handler, delivery.key
|
|
1160
|
+
FROM scope
|
|
1161
|
+
CROSS JOIN LATERAL unnest(scope.in_flight) AS flying (id)
|
|
1162
|
+
JOIN ${deliveries} AS delivery ON delivery.id = flying.id
|
|
1163
|
+
WHERE delivery.status = 'pending' AND delivery.key IS NOT NULL
|
|
1164
|
+
)`;
|
|
1165
|
+
}
|
|
1166
|
+
/** True when an earlier delivery of the candidate's handler and key is pending, or another one is running. */
|
|
1167
|
+
#blockedKey(alias) {
|
|
1168
|
+
return `EXISTS (
|
|
1169
|
+
SELECT 1 FROM ${this.#names.deliveries} AS earlier
|
|
1170
|
+
WHERE earlier.lane = ${alias}.lane AND earlier.handler = ${alias}.handler AND earlier.key = ${alias}.key
|
|
1171
|
+
AND earlier.status = 'pending' AND earlier.key IS NOT NULL
|
|
1172
|
+
AND earlier.sequence < ${alias}.sequence
|
|
1173
|
+
) OR ${this.#runningElsewhere(alias)}`;
|
|
418
1174
|
}
|
|
419
|
-
/** True when
|
|
420
|
-
#
|
|
1175
|
+
/** True when a delivery of the candidate's handler and key is in flight. The candidate itself never is. */
|
|
1176
|
+
#runningElsewhere(alias) {
|
|
421
1177
|
return `EXISTS (
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
1178
|
+
SELECT 1 FROM running
|
|
1179
|
+
WHERE running.handler = ${alias}.handler AND running.key = ${alias}.key
|
|
1180
|
+
)`;
|
|
1181
|
+
}
|
|
1182
|
+
/** Final SELECT shared by both claims: the picked deliveries with their lane epoch and event. */
|
|
1183
|
+
#pickedRows(saturated) {
|
|
1184
|
+
return `SELECT ${saturated} AS window_saturated, picked.*
|
|
1185
|
+
FROM (SELECT 1) AS anchor
|
|
1186
|
+
LEFT JOIN (
|
|
1187
|
+
${this.#pickedRowsSelect()}
|
|
1188
|
+
) AS picked ON true
|
|
1189
|
+
ORDER BY picked.due_at, picked.delivery_id`;
|
|
1190
|
+
}
|
|
1191
|
+
/** The picked deliveries joined with the lane epoch and their event, as the claim result columns. */
|
|
1192
|
+
#pickedRowsSelect() {
|
|
1193
|
+
return `SELECT candidates.available_at AS due_at, delivery.id AS delivery_id, delivery.lane::int AS lane,
|
|
1194
|
+
scope.epoch::text AS epoch, delivery.claims AS claims, delivery.handler, delivery.key AS delivery_key, delivery.sequence::text AS sequence, (delivery.attempts + 1)::int AS attempt,
|
|
1195
|
+
${eventColumns("event")}
|
|
439
1196
|
FROM candidates
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
1197
|
+
JOIN scope ON scope.lane = candidates.lane
|
|
1198
|
+
JOIN ${this.#names.deliveries} AS delivery ON delivery.id = candidates.id
|
|
1199
|
+
JOIN ${this.#names.events} AS event ON event.id = delivery.event_id`;
|
|
443
1200
|
}
|
|
444
|
-
/**
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
1201
|
+
/**
|
|
1202
|
+
* Records the deliveries a claim picked in their lanes and frees the lanes it
|
|
1203
|
+
* took or held for nothing. Parameters:
|
|
1204
|
+
* `[pickedJson, idleLanesJson, leaseOwner, leaseExpiresAt]` with
|
|
1205
|
+
* `pickedJson = [{ lane, ids }]`.
|
|
1206
|
+
*
|
|
1207
|
+
* A picked lane lists the ids that are still in flight (listed ids whose row
|
|
1208
|
+
* still exists) plus the new ones, and its lease reaches `leaseExpiresAt`.
|
|
1209
|
+
* An idle lane is freed when no delivery of it is in flight, so other workers
|
|
1210
|
+
* need not wait for the lease to expire.
|
|
1211
|
+
*/
|
|
1212
|
+
get recordClaim() {
|
|
1213
|
+
const deliveries = this.#names.deliveries;
|
|
1214
|
+
const lanes = this.#names.lanes;
|
|
1215
|
+
return `WITH picked AS (
|
|
1216
|
+
SELECT recorded.lane, recorded.ids
|
|
1217
|
+
FROM jsonb_to_recordset($1::jsonb) AS recorded (lane smallint, ids text[])
|
|
1218
|
+
), still_running AS (
|
|
1219
|
+
SELECT lane.lane, array_agg(delivery.id) AS ids
|
|
1220
|
+
FROM picked
|
|
1221
|
+
JOIN ${lanes} AS lane ON lane.lane = picked.lane
|
|
1222
|
+
CROSS JOIN LATERAL unnest(lane.in_flight) AS flying (id)
|
|
1223
|
+
JOIN ${deliveries} AS delivery ON delivery.id = flying.id AND delivery.status = 'pending'
|
|
1224
|
+
GROUP BY lane.lane
|
|
1225
|
+
), extended AS (
|
|
1226
|
+
UPDATE ${lanes} AS lane
|
|
1227
|
+
SET in_flight = COALESCE(still_running.ids, '{}') || picked.ids,
|
|
1228
|
+
lease_expires_at = GREATEST(lane.lease_expires_at, $4::timestamptz)
|
|
1229
|
+
FROM picked
|
|
1230
|
+
LEFT JOIN still_running ON still_running.lane = picked.lane
|
|
1231
|
+
WHERE lane.lane = picked.lane
|
|
1232
|
+
RETURNING lane.lane
|
|
1233
|
+
), freed AS (
|
|
1234
|
+
UPDATE ${lanes} AS lane
|
|
1235
|
+
SET lease_owner = NULL, lease_expires_at = NULL, in_flight = '{}'
|
|
1236
|
+
WHERE lane.lane = ANY (ARRAY(SELECT jsonb_array_elements_text($2::jsonb)::smallint))
|
|
1237
|
+
AND lane.lease_owner = $3::text
|
|
1238
|
+
AND ${this.#noRunningDelivery("lane")}
|
|
1239
|
+
RETURNING lane.lane
|
|
1240
|
+
)
|
|
1241
|
+
SELECT (SELECT count(*) FROM extended)::int AS extended, (SELECT count(*) FROM freed)::int AS freed`;
|
|
1242
|
+
}
|
|
1243
|
+
/** True when none of the lane's listed deliveries exists as a pending row. */
|
|
1244
|
+
#noRunningDelivery(laneAlias) {
|
|
1245
|
+
return `NOT EXISTS (
|
|
1246
|
+
SELECT 1 FROM unnest(${laneAlias}.in_flight) AS flying (id)
|
|
1247
|
+
JOIN ${this.#names.deliveries} AS delivery ON delivery.id = flying.id AND delivery.status = 'pending'
|
|
1248
|
+
)`;
|
|
1249
|
+
}
|
|
1250
|
+
/**
|
|
1251
|
+
* Keeps in the in-flight lists of the lanes `$1` holds only the claims in `$2`
|
|
1252
|
+
* (`[{ id, lane, epoch, claims }]`, the ones this store handed out and nobody
|
|
1253
|
+
* settled yet) and takes every other delivery out. That is a batch a claim
|
|
1254
|
+
* recorded but whose answer never arrived: nothing runs it and the lane lease,
|
|
1255
|
+
* renewed by the owner's other deliveries, would keep it away from everybody for
|
|
1256
|
+
* ever. A claim matches only with the lane epoch and the delivery generation it was
|
|
1257
|
+
* handed out with. Parameters: `[leaseOwner, handedOutJson, generation]`; a lane whose `claim_floor` is above
|
|
1258
|
+
* the generation was reconciled by a newer call, so an older (late) reconciliation leaves it alone.
|
|
1259
|
+
* The lanes are locked in lane order first (see the lock order on {@link PostgresStatements}).
|
|
1260
|
+
*/
|
|
1261
|
+
get reconcileClaims() {
|
|
1262
|
+
const deliveries = this.#names.deliveries;
|
|
1263
|
+
const handedOut = `jsonb_to_recordset($2::jsonb) AS handed (id text, lane smallint, epoch bigint, claims int)`;
|
|
1264
|
+
const kept = `SELECT 1 FROM ${handedOut} JOIN ${deliveries} AS delivery ON delivery.id = handed.id
|
|
1265
|
+
WHERE handed.id = flying.id AND handed.lane = lane.lane AND handed.epoch = lane.epoch AND delivery.claims = handed.claims`;
|
|
1266
|
+
return `WITH owned AS MATERIALIZED (
|
|
1267
|
+
SELECT lane.lane FROM ${this.#names.lanes} AS lane
|
|
1268
|
+
WHERE lane.lease_owner = $1::text AND lane.claim_floor <= $3::bigint
|
|
1269
|
+
ORDER BY lane.lane
|
|
1270
|
+
FOR NO KEY UPDATE
|
|
1271
|
+
)
|
|
1272
|
+
UPDATE ${this.#names.lanes} AS lane
|
|
1273
|
+
SET in_flight = ARRAY(SELECT flying.id FROM unnest(lane.in_flight) AS flying (id) WHERE EXISTS (${kept})),
|
|
1274
|
+
claim_floor = $3::bigint
|
|
1275
|
+
FROM owned
|
|
1276
|
+
WHERE lane.lane = owned.lane AND lane.lease_owner = $1::text AND lane.claim_floor <= $3::bigint`;
|
|
1277
|
+
}
|
|
1278
|
+
/**
|
|
1279
|
+
* Frees every lane `$1` holds that has no running delivery, locking them in lane order first.
|
|
1280
|
+
* Parameters: `[leaseOwner]`.
|
|
1281
|
+
*/
|
|
1282
|
+
get relinquish() {
|
|
1283
|
+
return `WITH owned AS MATERIALIZED (
|
|
1284
|
+
SELECT lane.lane FROM ${this.#names.lanes} AS lane
|
|
1285
|
+
WHERE lane.lease_owner = $1::text
|
|
1286
|
+
ORDER BY lane.lane
|
|
1287
|
+
FOR NO KEY UPDATE
|
|
1288
|
+
)
|
|
1289
|
+
UPDATE ${this.#names.lanes} AS lane
|
|
1290
|
+
SET lease_owner = NULL, lease_expires_at = NULL, in_flight = '{}'
|
|
1291
|
+
FROM owned
|
|
1292
|
+
WHERE lane.lane = owned.lane AND lane.lease_owner = $1::text AND ${this.#noRunningDelivery("lane")}`;
|
|
450
1293
|
}
|
|
451
|
-
/**
|
|
1294
|
+
/**
|
|
1295
|
+
* Extends the lease of the lane a running delivery belongs to; the lane lease
|
|
1296
|
+
* never shrinks. Parameters: `[deliveryId, lane, epoch, claims, leaseExpiresAt]`.
|
|
1297
|
+
*/
|
|
452
1298
|
get extendLease() {
|
|
453
|
-
return countChanged(`UPDATE ${this.#names.
|
|
454
|
-
SET lease_expires_at =
|
|
455
|
-
WHERE
|
|
1299
|
+
return countChanged(`UPDATE ${this.#names.lanes} AS lane
|
|
1300
|
+
SET lease_expires_at = GREATEST(lane.lease_expires_at, $5::timestamptz)
|
|
1301
|
+
WHERE lane.lane = $2::smallint AND lane.epoch = $3::bigint AND $1::text = ANY (lane.in_flight)
|
|
1302
|
+
AND EXISTS (
|
|
1303
|
+
SELECT 1 FROM ${this.#names.deliveries} AS delivery
|
|
1304
|
+
WHERE delivery.id = $1::text AND delivery.status = 'pending' AND delivery.claims = $4::int
|
|
1305
|
+
)`);
|
|
456
1306
|
}
|
|
457
|
-
/**
|
|
1307
|
+
/**
|
|
1308
|
+
* Deletes the running deliveries whose lane lease is still the one they were
|
|
1309
|
+
* claimed under. Parameters: `[leasesJson, now]` with
|
|
1310
|
+
* `leasesJson = [{ ord, id, lane, epoch, claims }]`. Returns the `ord` of every lease that was completed, so
|
|
1311
|
+
* results are matched to leases and not to delivery ids. A delivery is only deleted by the claim that
|
|
1312
|
+
* is still its current one (`claims`), which makes a stale lease harmless even when the same owner re-claimed
|
|
1313
|
+
* the delivery in the same lane epoch. The event gets a completion time for retention.
|
|
1314
|
+
*
|
|
1315
|
+
* The lane rows are locked `FOR KEY SHARE` first, in lane order, and the lock
|
|
1316
|
+
* is held until the surrounding transaction ends. A claimer that takes the
|
|
1317
|
+
* lane over locks it `FOR UPDATE`, which conflicts with that lock, so it
|
|
1318
|
+
* skips the lane until this completion commits or rolls back: a completion
|
|
1319
|
+
* and a takeover cannot both succeed for one delivery, which is what makes a
|
|
1320
|
+
* transactional handler's writes effectively-once. The weakest lock that
|
|
1321
|
+
* conflicts with a takeover is used on purpose: it does not conflict with the
|
|
1322
|
+
* owner's own claim and heartbeats (plain updates of the lane row), which
|
|
1323
|
+
* would otherwise wait for every completion in progress. With a completion history the deleted rows are copied to
|
|
1324
|
+
* it in the same statement.
|
|
1325
|
+
*/
|
|
458
1326
|
get complete() {
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
1327
|
+
const deliveries = this.#names.deliveries;
|
|
1328
|
+
const history = this.#names.completionHistory;
|
|
1329
|
+
return `WITH leases AS (
|
|
1330
|
+
SELECT lease.ord, lease.id, lease.lane, lease.epoch, lease.claims
|
|
1331
|
+
FROM jsonb_to_recordset($1::jsonb) AS lease (ord int, id text, lane smallint, epoch bigint, claims int)
|
|
1332
|
+
), fence AS MATERIALIZED (
|
|
1333
|
+
SELECT lane.lane, lane.epoch, lane.in_flight
|
|
1334
|
+
FROM ${this.#names.lanes} AS lane
|
|
1335
|
+
WHERE lane.lane IN (SELECT lane FROM leases)
|
|
1336
|
+
ORDER BY lane.lane
|
|
1337
|
+
FOR KEY SHARE
|
|
1338
|
+
), completed AS (
|
|
1339
|
+
DELETE FROM ${deliveries} AS delivery
|
|
1340
|
+
USING leases JOIN fence ON fence.lane = leases.lane AND fence.epoch = leases.epoch
|
|
1341
|
+
WHERE delivery.id = leases.id AND delivery.lane = leases.lane AND delivery.status = 'pending'
|
|
1342
|
+
AND delivery.claims = leases.claims AND leases.id = ANY (fence.in_flight)
|
|
1343
|
+
RETURNING leases.ord, delivery.id, delivery.event_id, delivery.handler, delivery.attempts, delivery.available_at, delivery.created_at
|
|
1344
|
+
), stamped AS (
|
|
1345
|
+
INSERT INTO ${this.#names.eventCompletions} (event_id, completed_at)
|
|
1346
|
+
SELECT DISTINCT completed.event_id, $2::timestamptz FROM completed
|
|
1347
|
+
-- Event id order: completions of other handlers of the same events lock these rows too.
|
|
1348
|
+
ORDER BY completed.event_id
|
|
1349
|
+
ON CONFLICT (event_id) DO UPDATE SET completed_at = GREATEST(${this.#names.eventCompletions}.completed_at, EXCLUDED.completed_at)
|
|
1350
|
+
RETURNING event_id
|
|
1351
|
+
)${history ? `, archived AS (
|
|
1352
|
+
INSERT INTO ${this.#names.completions} (id, event_id, handler, attempts, available_at, created_at, completed_at)
|
|
1353
|
+
SELECT id, event_id, handler, attempts + 1, available_at, created_at, $2::timestamptz FROM completed
|
|
1354
|
+
RETURNING id
|
|
1355
|
+
)` : ""}
|
|
1356
|
+
SELECT ord FROM completed`;
|
|
462
1357
|
}
|
|
463
|
-
/**
|
|
1358
|
+
/**
|
|
1359
|
+
* Returns a running delivery to pending, due at `availableAt`, and takes it
|
|
1360
|
+
* out of the lane's in-flight list. Parameters:
|
|
1361
|
+
* `[deliveryId, lane, epoch, claims, availableAt, errorJson, now]`. The attempt that
|
|
1362
|
+
* just ended is counted.
|
|
1363
|
+
*/
|
|
464
1364
|
get reschedule() {
|
|
465
|
-
return
|
|
466
|
-
SET
|
|
467
|
-
|
|
1365
|
+
return this.#leaveFlight(
|
|
1366
|
+
`SET available_at = $5::timestamptz, last_error = $6::jsonb, attempts = delivery.attempts + 1, claims = delivery.claims + 1, updated_at = $7::timestamptz`
|
|
1367
|
+
);
|
|
468
1368
|
}
|
|
469
|
-
/** Parameters: `[deliveryId,
|
|
1369
|
+
/** Parameters: `[deliveryId, lane, epoch, claims, errorJson, now]`. The attempt that just ended is counted. */
|
|
470
1370
|
get bury() {
|
|
471
|
-
return
|
|
472
|
-
SET status = 'dead', last_error = $
|
|
473
|
-
|
|
1371
|
+
return this.#leaveFlight(
|
|
1372
|
+
`SET status = 'dead', last_error = $5::jsonb, attempts = delivery.attempts + 1, claims = delivery.claims + 1, updated_at = $6::timestamptz`
|
|
1373
|
+
);
|
|
474
1374
|
}
|
|
475
1375
|
/**
|
|
476
|
-
* Gives a
|
|
477
|
-
*
|
|
1376
|
+
* Gives a running delivery back without spending its attempt (an attempt is
|
|
1377
|
+
* only recorded when it ends in failure or a lost lease): pending, due `now`.
|
|
1378
|
+
* Parameters: `[deliveryId, lane, epoch, claims, now]`.
|
|
478
1379
|
*/
|
|
479
1380
|
get release() {
|
|
480
|
-
return
|
|
481
|
-
SET
|
|
482
|
-
|
|
1381
|
+
return this.#leaveFlight(
|
|
1382
|
+
`SET available_at = $5::timestamptz, claims = delivery.claims + 1, updated_at = $5::timestamptz`
|
|
1383
|
+
);
|
|
1384
|
+
}
|
|
1385
|
+
/**
|
|
1386
|
+
* Applies `assignments` to a running delivery and removes it from its lane's
|
|
1387
|
+
* in-flight list, if the lane still has the caller's epoch. The lane row is
|
|
1388
|
+
* locked first (`FOR NO KEY UPDATE`, which conflicts with a takeover's `FOR UPDATE`), so a takeover cannot interleave. Parameters
|
|
1389
|
+
* `$1`..`$4` are always `[deliveryId, lane, epoch, claims]`; ending a claim bumps `claims`.
|
|
1390
|
+
*/
|
|
1391
|
+
#leaveFlight(assignments) {
|
|
1392
|
+
const lanes = this.#names.lanes;
|
|
1393
|
+
return `WITH fence AS MATERIALIZED (
|
|
1394
|
+
SELECT lane.lane FROM ${lanes} AS lane
|
|
1395
|
+
WHERE lane.lane = $2::smallint AND lane.epoch = $3::bigint AND $1::text = ANY (lane.in_flight)
|
|
1396
|
+
FOR NO KEY UPDATE
|
|
1397
|
+
), changed AS (
|
|
1398
|
+
UPDATE ${this.#names.deliveries} AS delivery ${assignments}
|
|
1399
|
+
FROM fence
|
|
1400
|
+
WHERE delivery.id = $1::text AND delivery.lane = fence.lane AND delivery.status = 'pending'
|
|
1401
|
+
AND delivery.claims = $4::int
|
|
1402
|
+
RETURNING delivery.id
|
|
1403
|
+
), trimmed AS (
|
|
1404
|
+
UPDATE ${lanes} AS lane SET in_flight = array_remove(lane.in_flight, $1::text)
|
|
1405
|
+
FROM changed WHERE lane.lane = $2::smallint
|
|
1406
|
+
RETURNING lane.lane
|
|
1407
|
+
)
|
|
1408
|
+
SELECT count(*)::int AS count FROM changed`;
|
|
483
1409
|
}
|
|
484
|
-
/** Parameters: `[deliveryIdsJson, now]`. */
|
|
1410
|
+
/** Parameters: `[deliveryIdsJson, now]`. The dead rows are locked in id order first. */
|
|
485
1411
|
get requeue() {
|
|
486
|
-
|
|
1412
|
+
const deliveries = this.#names.deliveries;
|
|
1413
|
+
return countChanged(`WITH dead AS MATERIALIZED (
|
|
1414
|
+
SELECT delivery.id FROM ${deliveries} AS delivery
|
|
1415
|
+
WHERE delivery.id = ANY (ARRAY(SELECT jsonb_array_elements_text($1::jsonb))) AND delivery.status = 'dead'
|
|
1416
|
+
ORDER BY delivery.id
|
|
1417
|
+
FOR NO KEY UPDATE
|
|
1418
|
+
)
|
|
1419
|
+
UPDATE ${deliveries} AS delivery
|
|
487
1420
|
SET status = 'pending', attempts = 0, available_at = $2::timestamptz, updated_at = $2::timestamptz
|
|
488
|
-
|
|
1421
|
+
FROM dead
|
|
1422
|
+
WHERE delivery.id = dead.id AND delivery.status = 'dead'`);
|
|
489
1423
|
}
|
|
490
|
-
/**
|
|
1424
|
+
/**
|
|
1425
|
+
* Builds the filtered, creation-ordered listing. A delivery is `processing`
|
|
1426
|
+
* while its row exists as `pending` and its lane lists its id; its attempt
|
|
1427
|
+
* count, lease owner and lease expiry then come from the claim and the lane.
|
|
1428
|
+
* Records of the completion history (when kept) are listed as `completed`.
|
|
1429
|
+
*/
|
|
491
1430
|
listDeliveries(query) {
|
|
492
1431
|
const parameters = [];
|
|
493
|
-
const conditions = [];
|
|
494
1432
|
const bind = (value) => {
|
|
495
1433
|
parameters.push(value);
|
|
496
1434
|
return `$${parameters.length}::${typeof value === "number" ? "int" : "text"}`;
|
|
497
1435
|
};
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
}
|
|
504
|
-
if (query.eventType !== void 0) {
|
|
505
|
-
conditions.push(`event.type = ${bind(query.eventType)}`);
|
|
506
|
-
}
|
|
507
|
-
if (query.eventId !== void 0) {
|
|
508
|
-
conditions.push(`delivery.event_id = ${bind(query.eventId)}`);
|
|
509
|
-
}
|
|
510
|
-
const where = conditions.length === 0 ? "" : `WHERE ${conditions.join(" AND ")}`;
|
|
1436
|
+
const filters = {
|
|
1437
|
+
handler: query.handler === void 0 ? void 0 : bind(query.handler),
|
|
1438
|
+
eventType: query.eventType === void 0 ? void 0 : bind(query.eventType),
|
|
1439
|
+
eventId: query.eventId === void 0 ? void 0 : bind(query.eventId),
|
|
1440
|
+
status: query.status === void 0 ? void 0 : bind(query.status)
|
|
1441
|
+
};
|
|
511
1442
|
const limit = bind(query.limit ?? DEFAULT_LIST_LIMIT);
|
|
512
|
-
const
|
|
1443
|
+
const common = (alias) => [
|
|
1444
|
+
...filters.handler === void 0 ? [] : [`${alias}.handler = ${filters.handler}`],
|
|
1445
|
+
...filters.eventType === void 0 ? [] : [`event.type = ${filters.eventType}`],
|
|
1446
|
+
...filters.eventId === void 0 ? [] : [`${alias}.event_id = ${filters.eventId}`]
|
|
1447
|
+
];
|
|
1448
|
+
const inFlight = `(delivery.status = 'pending' AND delivery.id = ANY (lane.in_flight))`;
|
|
1449
|
+
const whereCurrent = [
|
|
1450
|
+
...common("delivery"),
|
|
1451
|
+
...filters.status === void 0 ? [] : [
|
|
1452
|
+
`(CASE WHEN ${inFlight} THEN 'processing' ELSE delivery.status END) = ${filters.status}`
|
|
1453
|
+
]
|
|
1454
|
+
];
|
|
1455
|
+
const current = `SELECT delivery.id AS delivery_id, delivery.handler,
|
|
1456
|
+
CASE WHEN ${inFlight} THEN 'processing' ELSE delivery.status END AS status,
|
|
1457
|
+
(delivery.attempts + CASE WHEN ${inFlight} THEN 1 ELSE 0 END)::int AS attempts,
|
|
513
1458
|
${isoText("delivery.available_at")} AS available_at,
|
|
514
|
-
|
|
515
|
-
${isoText("
|
|
1459
|
+
CASE WHEN ${inFlight} THEN lane.lease_owner END AS lease_owner,
|
|
1460
|
+
CASE WHEN ${inFlight} THEN ${isoText("lane.lease_expires_at")} END AS lease_expires_at,
|
|
516
1461
|
delivery.last_error::text AS last_error,
|
|
517
1462
|
${isoText("delivery.created_at")} AS created_at,
|
|
518
1463
|
${isoText("delivery.updated_at")} AS updated_at,
|
|
519
1464
|
${isoText("delivery.completed_at")} AS completed_at,
|
|
520
1465
|
${eventColumns("event")}
|
|
521
1466
|
FROM ${this.#names.deliveries} AS delivery
|
|
1467
|
+
JOIN ${this.#names.lanes} AS lane ON lane.lane = delivery.lane
|
|
522
1468
|
JOIN ${this.#names.events} AS event ON event.id = delivery.event_id
|
|
523
|
-
${
|
|
1469
|
+
${whereCurrent.length === 0 ? "" : `WHERE ${whereCurrent.join(" AND ")}`}`;
|
|
1470
|
+
const includeHistory = this.#names.completionHistory && (query.status === void 0 || query.status === "completed");
|
|
1471
|
+
if (!includeHistory) {
|
|
1472
|
+
return {
|
|
1473
|
+
sql: `${current}
|
|
524
1474
|
ORDER BY delivery.created_at, delivery.id
|
|
525
|
-
LIMIT ${limit}
|
|
526
|
-
|
|
1475
|
+
LIMIT ${limit}`,
|
|
1476
|
+
parameters
|
|
1477
|
+
};
|
|
1478
|
+
}
|
|
1479
|
+
const history = `SELECT done.id AS delivery_id, done.handler, 'completed' AS status, done.attempts,
|
|
1480
|
+
${isoText("done.available_at")} AS available_at,
|
|
1481
|
+
NULL AS lease_owner,
|
|
1482
|
+
NULL AS lease_expires_at,
|
|
1483
|
+
NULL AS last_error,
|
|
1484
|
+
${isoText("done.created_at")} AS created_at,
|
|
1485
|
+
${isoText("done.completed_at")} AS updated_at,
|
|
1486
|
+
${isoText("done.completed_at")} AS completed_at,
|
|
1487
|
+
${eventColumns("event")}
|
|
1488
|
+
FROM ${this.#names.completions} AS done
|
|
1489
|
+
JOIN ${this.#names.events} AS event ON event.id = done.event_id
|
|
1490
|
+
${common("done").length === 0 ? "" : `WHERE ${common("done").join(" AND ")}`}`;
|
|
1491
|
+
return {
|
|
1492
|
+
sql: `SELECT * FROM (
|
|
1493
|
+
${current}
|
|
1494
|
+
UNION ALL
|
|
1495
|
+
${history}
|
|
1496
|
+
) AS listed
|
|
1497
|
+
ORDER BY created_at, delivery_id
|
|
1498
|
+
LIMIT ${limit}`,
|
|
1499
|
+
parameters
|
|
1500
|
+
};
|
|
527
1501
|
}
|
|
1502
|
+
/**
|
|
1503
|
+
* Counts of rows by stored status (`pending` includes running deliveries;
|
|
1504
|
+
* `completed` rows exist only after an upgrade from 0.1.0). Each count is
|
|
1505
|
+
* answered from a partial index.
|
|
1506
|
+
*/
|
|
528
1507
|
get countByStatus() {
|
|
529
|
-
|
|
1508
|
+
const deliveries = this.#names.deliveries;
|
|
1509
|
+
return ["pending", "dead", "completed"].map(
|
|
1510
|
+
(status) => `SELECT '${status}' AS status, count(*)::text AS count FROM ${deliveries} WHERE status = '${status}'`
|
|
1511
|
+
).join("\nUNION ALL\n");
|
|
1512
|
+
}
|
|
1513
|
+
/** Deliveries that are running: listed by a lane and still pending as rows. */
|
|
1514
|
+
get countRunning() {
|
|
1515
|
+
return `SELECT count(*)::text AS count
|
|
1516
|
+
FROM ${this.#names.lanes} AS lane
|
|
1517
|
+
CROSS JOIN LATERAL unnest(lane.in_flight) AS flying (id)
|
|
1518
|
+
JOIN ${this.#names.deliveries} AS delivery ON delivery.id = flying.id AND delivery.status = 'pending'`;
|
|
530
1519
|
}
|
|
531
|
-
|
|
1520
|
+
get countCompletionHistory() {
|
|
1521
|
+
return `SELECT count(*)::text AS count FROM ${this.#names.completions}`;
|
|
1522
|
+
}
|
|
1523
|
+
/**
|
|
1524
|
+
* Oldest instant at which a delivery became claimable: `available_at` of a
|
|
1525
|
+
* pending delivery that nobody runs, or the lease expiry of a lane that runs
|
|
1526
|
+
* deliveries. Parameters: `[now]`.
|
|
1527
|
+
*/
|
|
532
1528
|
get oldestDue() {
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
1529
|
+
const deliveries = this.#names.deliveries;
|
|
1530
|
+
const lanes = this.#names.lanes;
|
|
1531
|
+
return `SELECT ${isoText("min(due.due_at)")} AS oldest_due_at FROM (
|
|
1532
|
+
SELECT waiting.available_at AS due_at
|
|
1533
|
+
FROM ${lanes} AS lane
|
|
1534
|
+
CROSS JOIN LATERAL (
|
|
1535
|
+
SELECT delivery.available_at FROM ${deliveries} AS delivery
|
|
1536
|
+
WHERE delivery.lane = lane.lane AND delivery.status = 'pending'
|
|
1537
|
+
AND delivery.available_at <= $1::timestamptz AND delivery.id <> ALL (lane.in_flight)
|
|
1538
|
+
ORDER BY delivery.available_at LIMIT 1
|
|
1539
|
+
) AS waiting
|
|
1540
|
+
UNION ALL
|
|
1541
|
+
SELECT lane.lease_expires_at AS due_at
|
|
1542
|
+
FROM ${lanes} AS lane
|
|
1543
|
+
WHERE lane.lease_expires_at <= $1::timestamptz AND NOT ${this.#noRunningDelivery("lane")}
|
|
1544
|
+
) AS due`;
|
|
540
1545
|
}
|
|
541
1546
|
/**
|
|
542
|
-
* Deletes one bounded batch of purgeable deliveries
|
|
543
|
-
* of each deleted
|
|
544
|
-
* `FOR UPDATE SKIP LOCKED` keeps purge from waiting
|
|
1547
|
+
* Deletes one bounded batch of purgeable deliveries (or of the completion
|
|
1548
|
+
* history) and returns the event id of each deleted row. Parameters:
|
|
1549
|
+
* `[cutoff, batchSize]`. `FOR UPDATE SKIP LOCKED` keeps purge from waiting
|
|
1550
|
+
* on, or being waited on by, anything.
|
|
545
1551
|
*/
|
|
546
1552
|
purgeCompletedBatch() {
|
|
547
1553
|
return this.#purgeBatch(
|
|
1554
|
+
this.#names.deliveries,
|
|
548
1555
|
`status = 'completed' AND completed_at < $1::timestamptz`,
|
|
549
1556
|
"completed_at"
|
|
550
1557
|
);
|
|
551
1558
|
}
|
|
1559
|
+
purgeCompletionHistoryBatch() {
|
|
1560
|
+
return this.#purgeBatch(
|
|
1561
|
+
this.#names.completions,
|
|
1562
|
+
`completed_at < $1::timestamptz`,
|
|
1563
|
+
"completed_at"
|
|
1564
|
+
);
|
|
1565
|
+
}
|
|
552
1566
|
purgeDeadBatch() {
|
|
553
|
-
return this.#purgeBatch(
|
|
1567
|
+
return this.#purgeBatch(
|
|
1568
|
+
this.#names.deliveries,
|
|
1569
|
+
`status = 'dead' AND updated_at < $1::timestamptz`,
|
|
1570
|
+
"updated_at"
|
|
1571
|
+
);
|
|
554
1572
|
}
|
|
555
1573
|
/**
|
|
556
1574
|
* Selects and locks one bounded batch of events created before the cutoff
|
|
557
|
-
* that no delivery references
|
|
558
|
-
*
|
|
1575
|
+
* that no delivery references and that never had a delivery complete (those
|
|
1576
|
+
* are measured from their completion, see {@link lockCompletedOrphanEventsBatch}).
|
|
1577
|
+
* Parameters: `[cutoff, batchSize]`. The ids go through
|
|
1578
|
+
* {@link deleteOrphanEventsWithoutCompletion} or {@link deleteExpiredCompletedOrphanEvents}, which re-check under a fresh snapshot.
|
|
559
1579
|
*/
|
|
560
1580
|
get lockOrphanEventsBatch() {
|
|
561
1581
|
return `SELECT event.id FROM ${this.#names.events} AS event
|
|
562
1582
|
WHERE event.created_at < $1::timestamptz
|
|
563
|
-
AND
|
|
1583
|
+
AND ${this.#unreferenced("event")}
|
|
1584
|
+
AND NOT EXISTS (SELECT 1 FROM ${this.#names.eventCompletions} AS completion WHERE completion.event_id = event.id)
|
|
564
1585
|
ORDER BY event.created_at, event.id
|
|
565
1586
|
LIMIT $2::int
|
|
1587
|
+
FOR UPDATE OF event SKIP LOCKED`;
|
|
1588
|
+
}
|
|
1589
|
+
/**
|
|
1590
|
+
* Like {@link lockOrphanEventsBatch} for events that had a delivery complete:
|
|
1591
|
+
* they go once their last completion is older than the cutoff, however old the
|
|
1592
|
+
* event itself is. Parameters: `[cutoff, batchSize]`.
|
|
1593
|
+
*/
|
|
1594
|
+
get lockCompletedOrphanEventsBatch() {
|
|
1595
|
+
return `SELECT event.id FROM ${this.#names.events} AS event
|
|
1596
|
+
JOIN ${this.#names.eventCompletions} AS completion ON completion.event_id = event.id
|
|
1597
|
+
WHERE completion.completed_at < $1::timestamptz
|
|
1598
|
+
AND ${this.#unreferenced("event")}
|
|
1599
|
+
ORDER BY completion.completed_at, event.id
|
|
1600
|
+
LIMIT $2::int
|
|
566
1601
|
FOR UPDATE OF event SKIP LOCKED`;
|
|
567
1602
|
}
|
|
568
1603
|
/**
|
|
@@ -576,42 +1611,90 @@ WHERE id = ANY (ARRAY(SELECT jsonb_array_elements_text($1::jsonb)))
|
|
|
576
1611
|
FOR UPDATE SKIP LOCKED`;
|
|
577
1612
|
}
|
|
578
1613
|
/**
|
|
579
|
-
* Deletes
|
|
580
|
-
*
|
|
581
|
-
*
|
|
582
|
-
*
|
|
583
|
-
* Parameters: `[eventIdsJson]`.
|
|
1614
|
+
* Deletes events that no delivery references and whose last completion is older than
|
|
1615
|
+
* `$2`, checked again under the fresh snapshot of this statement: a completion recorded
|
|
1616
|
+
* after the sweep selected the event moves its time and keeps the event. Parameters:
|
|
1617
|
+
* `[eventIdsJson, cutoff]`.
|
|
584
1618
|
*/
|
|
585
|
-
get
|
|
586
|
-
|
|
587
|
-
return countChanged(`DELETE FROM ${events} AS event
|
|
1619
|
+
get deleteExpiredCompletedOrphanEvents() {
|
|
1620
|
+
return countChanged(`DELETE FROM ${this.#names.events} AS event
|
|
588
1621
|
WHERE event.id = ANY (ARRAY(SELECT jsonb_array_elements_text($1::jsonb)))
|
|
589
|
-
AND
|
|
1622
|
+
AND ${this.#unreferenced("event")}
|
|
1623
|
+
AND EXISTS (
|
|
1624
|
+
SELECT 1 FROM ${this.#names.eventCompletions} AS completion
|
|
1625
|
+
WHERE completion.event_id = event.id AND completion.completed_at < $2::timestamptz
|
|
1626
|
+
)`);
|
|
590
1627
|
}
|
|
591
|
-
|
|
592
|
-
|
|
1628
|
+
/**
|
|
1629
|
+
* Deletes locked events that no delivery references and that never had a delivery complete.
|
|
1630
|
+
* It must run as a statement of its own after {@link lockEvents}: it then takes a fresh
|
|
1631
|
+
* snapshot that includes every delivery committed before the lock was granted, so an event
|
|
1632
|
+
* that just gained a delivery is never cascaded away. Keeps events that had a delivery complete:
|
|
1633
|
+
* deleting dead letters must not shorten the retention of completed work.
|
|
1634
|
+
*/
|
|
1635
|
+
get deleteOrphanEventsWithoutCompletion() {
|
|
1636
|
+
return countChanged(`DELETE FROM ${this.#names.events} AS event
|
|
1637
|
+
WHERE event.id = ANY (ARRAY(SELECT jsonb_array_elements_text($1::jsonb)))
|
|
1638
|
+
AND ${this.#unreferenced("event")}
|
|
1639
|
+
AND NOT EXISTS (SELECT 1 FROM ${this.#names.eventCompletions} AS completion WHERE completion.event_id = event.id)`);
|
|
1640
|
+
}
|
|
1641
|
+
/** True when no delivery and no completion history record references the event. */
|
|
1642
|
+
#unreferenced(eventAlias) {
|
|
1643
|
+
const history = this.#names.completionHistory ? ` AND NOT EXISTS (SELECT 1 FROM ${this.#names.completions} AS done WHERE done.event_id = ${eventAlias}.id)` : "";
|
|
1644
|
+
return `NOT EXISTS (SELECT 1 FROM ${this.#names.deliveries} AS delivery WHERE delivery.event_id = ${eventAlias}.id)${history}`;
|
|
1645
|
+
}
|
|
1646
|
+
#purgeBatch(table, condition, orderColumn) {
|
|
593
1647
|
return `WITH batch AS MATERIALIZED (
|
|
594
|
-
SELECT id FROM ${
|
|
1648
|
+
SELECT id FROM ${table}
|
|
595
1649
|
WHERE ${condition}
|
|
596
1650
|
ORDER BY ${orderColumn}, id
|
|
597
1651
|
LIMIT $2::int
|
|
598
1652
|
FOR UPDATE SKIP LOCKED
|
|
599
1653
|
), deleted AS (
|
|
600
|
-
DELETE FROM ${
|
|
1654
|
+
DELETE FROM ${table} AS delivery USING batch WHERE delivery.id = batch.id RETURNING delivery.event_id
|
|
601
1655
|
)
|
|
602
1656
|
SELECT event_id FROM deleted`;
|
|
603
1657
|
}
|
|
604
1658
|
};
|
|
605
1659
|
|
|
606
1660
|
// src/postgres-event-store.ts
|
|
1661
|
+
var WAKE_UPS = ["notify", "poll"];
|
|
607
1662
|
var DEFAULT_PURGE_BATCH_SIZE = 1e3;
|
|
1663
|
+
var INITIAL_LANES_PER_ROUND = 4;
|
|
1664
|
+
var MAXIMUM_STALE_CLAIM_ATTEMPTS = 3;
|
|
1665
|
+
var NOTIFICATION_INTERVAL_MS = 10;
|
|
1666
|
+
function handedOutKey(deliveryId, leaseToken) {
|
|
1667
|
+
return `${deliveryId}\0${leaseToken}`;
|
|
1668
|
+
}
|
|
608
1669
|
var PostgresEventStore = class {
|
|
609
1670
|
#executor;
|
|
610
1671
|
#statements;
|
|
611
1672
|
#schemaStatements;
|
|
612
1673
|
#setupLockKey;
|
|
1674
|
+
#schemaMarker;
|
|
613
1675
|
#purgeBatchSize;
|
|
1676
|
+
#completionHistory;
|
|
1677
|
+
#logger;
|
|
1678
|
+
/** Passed with every statement whose text is fixed: asks the executor to prepare it (`preparedStatements`). */
|
|
1679
|
+
#queryOptions;
|
|
1680
|
+
#notifies;
|
|
1681
|
+
/** Every claim handed to a caller and not settled yet, by delivery id and exact lease token. */
|
|
1682
|
+
#handedOut = /* @__PURE__ */ new Map();
|
|
1683
|
+
#unreconciled = /* @__PURE__ */ new Set();
|
|
1684
|
+
#generations = /* @__PURE__ */ new Map();
|
|
1685
|
+
#ownerQueues = /* @__PURE__ */ new Map();
|
|
1686
|
+
/** Owners whose last claim was filled from the lanes they held. */
|
|
1687
|
+
#drainingOwners = /* @__PURE__ */ new Set();
|
|
614
1688
|
#ownedPool;
|
|
1689
|
+
/** The `pg` pool behind the store, when it has one: listening for notifications needs a connection of its own. */
|
|
1690
|
+
#pool;
|
|
1691
|
+
#notificationChannel;
|
|
1692
|
+
#lastNotificationAt = Number.NEGATIVE_INFINITY;
|
|
1693
|
+
#trailingNotification;
|
|
1694
|
+
/** Set by `close()`: a transaction that commits afterwards schedules no wake-up. */
|
|
1695
|
+
#closed = false;
|
|
1696
|
+
/** The transactions `transaction` runs right now, and whether one of their appends left its wake-up for after the commit. */
|
|
1697
|
+
#ownTransactions = /* @__PURE__ */ new Map();
|
|
615
1698
|
constructor(options) {
|
|
616
1699
|
const sources = [options.pool, options.connectionString, options.executor].filter(
|
|
617
1700
|
(source) => source !== void 0
|
|
@@ -627,13 +1710,30 @@ var PostgresEventStore = class {
|
|
|
627
1710
|
`purgeBatchSize must be a positive integer, received ${String(purgeBatchSize)}`
|
|
628
1711
|
);
|
|
629
1712
|
}
|
|
1713
|
+
const wakeUp = options.wakeUp ?? "notify";
|
|
1714
|
+
if (!WAKE_UPS.includes(wakeUp)) {
|
|
1715
|
+
throw new core.ConfigurationError(`wakeUp must be 'notify' or 'poll', received ${String(wakeUp)}`);
|
|
1716
|
+
}
|
|
1717
|
+
const preparedStatements = options.preparedStatements ?? true;
|
|
1718
|
+
if (typeof preparedStatements !== "boolean") {
|
|
1719
|
+
throw new core.ConfigurationError(
|
|
1720
|
+
`preparedStatements must be a boolean, received ${String(preparedStatements)}`
|
|
1721
|
+
);
|
|
1722
|
+
}
|
|
1723
|
+
this.#notifies = wakeUp === "notify";
|
|
1724
|
+
this.#queryOptions = preparedStatements ? { prepare: true } : void 0;
|
|
630
1725
|
const names = resolvePostgresNames(options);
|
|
631
1726
|
this.#statements = new PostgresStatements(names);
|
|
632
1727
|
this.#schemaStatements = createPostgresSchemaStatements(options);
|
|
633
1728
|
this.#setupLockKey = this.#statements.setupLockKey;
|
|
1729
|
+
this.#schemaMarker = postgresSchemaMarker(this.#schemaStatements.slice(0, -1));
|
|
634
1730
|
this.#purgeBatchSize = purgeBatchSize;
|
|
1731
|
+
this.#completionHistory = names.completionHistory;
|
|
1732
|
+
this.#logger = options.logger ?? console;
|
|
1733
|
+
this.#notificationChannel = names.notificationChannel;
|
|
635
1734
|
if (options.executor !== void 0) {
|
|
636
1735
|
this.#executor = options.executor;
|
|
1736
|
+
this.#pool = void 0;
|
|
637
1737
|
} else {
|
|
638
1738
|
const pool = options.pool ?? new pg.Pool({ ...options.poolConfig, connectionString: options.connectionString });
|
|
639
1739
|
if (options.pool === void 0) {
|
|
@@ -645,53 +1745,125 @@ var PostgresEventStore = class {
|
|
|
645
1745
|
});
|
|
646
1746
|
this.#ownedPool = pool;
|
|
647
1747
|
}
|
|
648
|
-
this.#
|
|
1748
|
+
this.#pool = pool;
|
|
1749
|
+
this.#executor = new PgExecutor(pool, {
|
|
1750
|
+
logger: this.#logger
|
|
1751
|
+
});
|
|
649
1752
|
}
|
|
650
1753
|
}
|
|
651
1754
|
/**
|
|
652
1755
|
* Creates the schema, tables and indexes if they do not exist. Safe to call
|
|
653
1756
|
* from many processes at once: a transaction-scoped advisory lock serializes
|
|
654
1757
|
* the DDL, so racing `CREATE TABLE IF NOT EXISTS` statements cannot collide.
|
|
1758
|
+
* When the lane table already carries the marker of the current statements
|
|
1759
|
+
* ({@link postgresSchemaMarker}) nothing runs and no table is locked, so instances
|
|
1760
|
+
* that start next to running dispatchers cannot deadlock with their claims.
|
|
655
1761
|
* Skip it (`setupStore: false`) when a migration tool applies
|
|
656
1762
|
* `createPostgresSchemaSql()` instead.
|
|
657
1763
|
*/
|
|
658
1764
|
async setup() {
|
|
1765
|
+
if (await this.#schemaIsCurrent()) {
|
|
1766
|
+
return;
|
|
1767
|
+
}
|
|
659
1768
|
await this.#executor.transaction(async (transaction) => {
|
|
660
1769
|
await this.#executor.query(this.#statements.setupLock, [this.#setupLockKey], transaction);
|
|
1770
|
+
if (await this.#schemaIsCurrent(transaction)) {
|
|
1771
|
+
return;
|
|
1772
|
+
}
|
|
661
1773
|
for (const statement of this.#schemaStatements) {
|
|
662
1774
|
await this.#executor.query(statement, [], transaction);
|
|
663
1775
|
}
|
|
664
1776
|
});
|
|
665
1777
|
}
|
|
666
|
-
transaction
|
|
667
|
-
|
|
1778
|
+
async #schemaIsCurrent(transaction) {
|
|
1779
|
+
const rows = await this.#executor.query(
|
|
1780
|
+
this.#statements.schemaCurrent,
|
|
1781
|
+
[this.#schemaMarker],
|
|
1782
|
+
transaction
|
|
1783
|
+
);
|
|
1784
|
+
return rows[0]["current"] === true;
|
|
1785
|
+
}
|
|
1786
|
+
async transaction(work) {
|
|
1787
|
+
const own = { wake: false };
|
|
1788
|
+
const result = await this.#executor.transaction(async (transaction) => {
|
|
1789
|
+
this.#ownTransactions.set(transaction, own);
|
|
1790
|
+
try {
|
|
1791
|
+
return await work(transaction);
|
|
1792
|
+
} finally {
|
|
1793
|
+
this.#ownTransactions.delete(transaction);
|
|
1794
|
+
}
|
|
1795
|
+
});
|
|
1796
|
+
if (own.wake) {
|
|
1797
|
+
this.#wakeAfterCommit();
|
|
1798
|
+
}
|
|
1799
|
+
return result;
|
|
668
1800
|
}
|
|
669
1801
|
/**
|
|
670
|
-
* Writes the entries in `transaction
|
|
671
|
-
*
|
|
672
|
-
* deadlock inside one call), then
|
|
673
|
-
*
|
|
674
|
-
*
|
|
675
|
-
*
|
|
1802
|
+
* Writes the entries in `transaction`, normally with one statement
|
|
1803
|
+
* ({@link PostgresStatements.insertFreshEvents}): keyed events reserve their sequence numbers (locking
|
|
1804
|
+
* the key rows in sorted order, so publishers cannot deadlock inside one call), then the new events and
|
|
1805
|
+
* their deliveries are inserted, ordered by primary key, and one notification wakes the dispatchers
|
|
1806
|
+
* when a delivery is due now. Events that already existed get their missing handlers' deliveries in
|
|
1807
|
+
* two more statements. Two transactions that append keys in opposite order across several calls can
|
|
1808
|
+
* still deadlock (40P01); the database rolls one back and the caller retries.
|
|
676
1809
|
* A duplicate event id still consumes a sequence number; gaps are harmless
|
|
677
1810
|
* because only the order of sequences matters.
|
|
678
1811
|
*/
|
|
679
1812
|
async append(transaction, entries, now) {
|
|
680
1813
|
const plan = planAppend(entries);
|
|
1814
|
+
if (plan.events.length === 0) {
|
|
1815
|
+
return { appendedEventIds: [], duplicateEventIds: [], appendedDeliveryCount: 0 };
|
|
1816
|
+
}
|
|
681
1817
|
const createdAt = now.toISOString();
|
|
682
|
-
const
|
|
1818
|
+
const wakeAt = this.#wakeAt(plan.deliveries, now, transaction);
|
|
1819
|
+
const handlersByEvent = /* @__PURE__ */ new Map();
|
|
1820
|
+
for (const delivery of plan.deliveries) {
|
|
1821
|
+
handlersByEvent.set(delivery.event_id, [
|
|
1822
|
+
...handlersByEvent.get(delivery.event_id) ?? [],
|
|
1823
|
+
delivery.handler
|
|
1824
|
+
]);
|
|
1825
|
+
}
|
|
1826
|
+
const keyCounts = /* @__PURE__ */ new Map();
|
|
1827
|
+
for (const event of plan.events) {
|
|
1828
|
+
if (event.key !== null) {
|
|
1829
|
+
keyCounts.set(event.key, (keyCounts.get(event.key) ?? 0) + 1);
|
|
1830
|
+
}
|
|
1831
|
+
}
|
|
1832
|
+
const fresh = await this.#query(
|
|
1833
|
+
this.#statements.insertFreshEvents,
|
|
1834
|
+
[
|
|
1835
|
+
JSON.stringify(
|
|
1836
|
+
plan.events.map((event, ordinal) => ({
|
|
1837
|
+
ordinal,
|
|
1838
|
+
...toEventRow(event, null, handlersByEvent.get(event.id) ?? [])
|
|
1839
|
+
}))
|
|
1840
|
+
),
|
|
1841
|
+
createdAt,
|
|
1842
|
+
JSON.stringify(plan.deliveries),
|
|
1843
|
+
JSON.stringify([...keyCounts].map(([key, count]) => ({ key, count }))),
|
|
1844
|
+
wakeAt
|
|
1845
|
+
],
|
|
1846
|
+
transaction
|
|
1847
|
+
);
|
|
683
1848
|
const insertedEventIds = new Set(
|
|
684
|
-
|
|
685
|
-
this.#statements.insertEvents,
|
|
686
|
-
[JSON.stringify(toEventRows(plan.events, sequences)), createdAt],
|
|
687
|
-
transaction
|
|
688
|
-
)).map((row) => row["id"])
|
|
1849
|
+
fresh.filter((row) => row["inserted"] === true).map((row) => row["id"])
|
|
689
1850
|
);
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
transaction
|
|
1851
|
+
let appendedDeliveryCount = readCount(fresh[0], "deliveries");
|
|
1852
|
+
const remaining = plan.deliveries.filter(
|
|
1853
|
+
(delivery) => !insertedEventIds.has(delivery.event_id)
|
|
694
1854
|
);
|
|
1855
|
+
if (remaining.length > 0) {
|
|
1856
|
+
appendedDeliveryCount += await this.#appendToExistingEvents(
|
|
1857
|
+
transaction,
|
|
1858
|
+
plan,
|
|
1859
|
+
fresh,
|
|
1860
|
+
remaining,
|
|
1861
|
+
handlersByEvent,
|
|
1862
|
+
insertedEventIds,
|
|
1863
|
+
createdAt,
|
|
1864
|
+
wakeAt
|
|
1865
|
+
);
|
|
1866
|
+
}
|
|
695
1867
|
const appendedEventIds = [];
|
|
696
1868
|
const duplicateEventIds = [];
|
|
697
1869
|
for (const { eventId, repeated } of plan.outcomes) {
|
|
@@ -699,81 +1871,486 @@ var PostgresEventStore = class {
|
|
|
699
1871
|
eventId
|
|
700
1872
|
);
|
|
701
1873
|
}
|
|
702
|
-
return {
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
1874
|
+
return { appendedEventIds, duplicateEventIds, appendedDeliveryCount };
|
|
1875
|
+
}
|
|
1876
|
+
/**
|
|
1877
|
+
* The time the publishing statements notify the dispatchers for (deliveries due by then wake them),
|
|
1878
|
+
* or `null` for no notification. An append in a transaction of the caller's always notifies, at its
|
|
1879
|
+
* commit. In a transaction of this store, only the first append within {@link NOTIFICATION_INTERVAL_MS}
|
|
1880
|
+
* does; the others leave the wake-up to {@link #wakeAfterCommit}, which runs once their transaction
|
|
1881
|
+
* committed, so no committed delivery waits for the poll.
|
|
1882
|
+
*/
|
|
1883
|
+
#wakeAt(deliveries, now, transaction) {
|
|
1884
|
+
if (!this.#notifies || !deliveries.some((delivery) => Date.parse(delivery.available_at) <= now.getTime())) {
|
|
1885
|
+
return null;
|
|
1886
|
+
}
|
|
1887
|
+
const own = this.#ownTransactions.get(transaction);
|
|
1888
|
+
if (own !== void 0 && Date.now() - this.#lastNotificationAt < NOTIFICATION_INTERVAL_MS) {
|
|
1889
|
+
own.wake = true;
|
|
1890
|
+
return null;
|
|
1891
|
+
}
|
|
1892
|
+
this.#lastNotificationAt = Date.now();
|
|
1893
|
+
return now.toISOString();
|
|
1894
|
+
}
|
|
1895
|
+
/**
|
|
1896
|
+
* Wakes the dispatchers for a committed transaction whose appends sent no notification: at once when
|
|
1897
|
+
* none was sent within {@link NOTIFICATION_INTERVAL_MS}, otherwise once when the interval ends. A
|
|
1898
|
+
* wake-up already scheduled covers this transaction too, since it is sent after this commit. Nothing is
|
|
1899
|
+
* sent once the store is closed or the `pg` pool behind it is ending: no dispatcher can be woken through
|
|
1900
|
+
* it then, and the deliveries are found by polling.
|
|
1901
|
+
*/
|
|
1902
|
+
#wakeAfterCommit() {
|
|
1903
|
+
if (this.#trailingNotification !== void 0 || this.#closed) {
|
|
1904
|
+
return;
|
|
1905
|
+
}
|
|
1906
|
+
this.#trailingNotification = setTimeout(
|
|
1907
|
+
() => {
|
|
1908
|
+
this.#trailingNotification = void 0;
|
|
1909
|
+
if (this.#pool?.ending === true) {
|
|
1910
|
+
return;
|
|
1911
|
+
}
|
|
1912
|
+
this.#lastNotificationAt = Date.now();
|
|
1913
|
+
this.#query(this.#statements.notify, []).catch((error) => {
|
|
1914
|
+
this.#logger.error(
|
|
1915
|
+
`[PostgresEventStore](wakeAfterCommit) Waking the dispatchers failed, they find the deliveries when they poll | error=${error instanceof Error ? error.message : String(error)}`
|
|
1916
|
+
);
|
|
1917
|
+
});
|
|
1918
|
+
},
|
|
1919
|
+
Math.max(0, this.#lastNotificationAt + NOTIFICATION_INTERVAL_MS - Date.now())
|
|
1920
|
+
);
|
|
1921
|
+
}
|
|
1922
|
+
/**
|
|
1923
|
+
* Adds the deliveries of events the publishing statement found already stored. An event that already
|
|
1924
|
+
* exists may be getting the same new handler from another transaction at this very moment. Whoever
|
|
1925
|
+
* holds the event decides, so the freshness check of the insert sees what the other transaction
|
|
1926
|
+
* committed, even if its delivery has completed since. An existing event that a purge removed
|
|
1927
|
+
* meanwhile is inserted again (with the sequence reserved for it) instead of being skipped; its id is
|
|
1928
|
+
* added to `insertedEventIds`. Returns how many deliveries were inserted.
|
|
1929
|
+
*/
|
|
1930
|
+
async #appendToExistingEvents(transaction, plan, fresh, deliveries, handlersByEvent, insertedEventIds, createdAt, wakeAt) {
|
|
1931
|
+
const sequences = new Map(
|
|
1932
|
+
fresh.map((row) => [row["id"], row["sequence"]])
|
|
1933
|
+
);
|
|
1934
|
+
const reinserted = [];
|
|
1935
|
+
let existing = plan.events.filter((event) => !insertedEventIds.has(event.id)).map(
|
|
1936
|
+
(event) => toEventRow(event, sequences.get(event.id), handlersByEvent.get(event.id) ?? [])
|
|
1937
|
+
);
|
|
1938
|
+
while (existing.length > 0) {
|
|
1939
|
+
const locked = new Set(
|
|
1940
|
+
(await this.#query(
|
|
1941
|
+
this.#statements.lockExistingEvents,
|
|
1942
|
+
[JSON.stringify(existing.map((row) => row.id).sort())],
|
|
1943
|
+
transaction
|
|
1944
|
+
)).map((row) => row["id"])
|
|
1945
|
+
);
|
|
1946
|
+
const gone = existing.filter((row) => !locked.has(row.id));
|
|
1947
|
+
if (gone.length > 0) {
|
|
1948
|
+
const inserted2 = await this.#query(
|
|
1949
|
+
this.#statements.insertEvents,
|
|
1950
|
+
[JSON.stringify(gone), createdAt],
|
|
1951
|
+
transaction
|
|
1952
|
+
);
|
|
1953
|
+
for (const row of inserted2) {
|
|
1954
|
+
insertedEventIds.add(row["id"]);
|
|
1955
|
+
reinserted.push(row["id"]);
|
|
1956
|
+
}
|
|
1957
|
+
}
|
|
1958
|
+
existing = gone.filter((row) => !insertedEventIds.has(row.id));
|
|
1959
|
+
}
|
|
1960
|
+
const inserted = await this.#query(
|
|
1961
|
+
this.#statements.insertDeliveries,
|
|
1962
|
+
[JSON.stringify(deliveries), createdAt, JSON.stringify(reinserted), wakeAt],
|
|
1963
|
+
transaction
|
|
1964
|
+
);
|
|
1965
|
+
return inserted.length;
|
|
707
1966
|
}
|
|
708
1967
|
/**
|
|
709
|
-
*
|
|
710
|
-
*
|
|
711
|
-
*
|
|
712
|
-
*
|
|
713
|
-
*
|
|
1968
|
+
* Claims up to `limit` due deliveries, normally with one statement
|
|
1969
|
+
* ({@link PostgresStatements.claimLanes}): from the lanes this owner already holds, then from free lanes
|
|
1970
|
+
* it takes in the same statement, recording the batch in the lane rows. A claim that would have to take
|
|
1971
|
+
* over a lane whose previous holder still runs deliveries under an expired lease continues in a
|
|
1972
|
+
* transaction that charges their lost attempts first, taking free lanes a few at a time (4, 8, 16, ...);
|
|
1973
|
+
* so does a claim whose statement kept seeing a lane it holds change under it, and one that came up
|
|
1974
|
+
* short with more free lanes left than it looked at. The step that fails after
|
|
1975
|
+
* the first one has recorded deliveries returns those instead of failing: a recorded delivery nobody
|
|
1976
|
+
* received would be held for a whole lease. See {@link PostgresStatements.claim} for the rules a delivery
|
|
1977
|
+
* must satisfy.
|
|
714
1978
|
*/
|
|
715
|
-
|
|
1979
|
+
claim(request) {
|
|
716
1980
|
if (request.handlers.length === 0 || !(request.limit > 0)) {
|
|
717
|
-
return [];
|
|
1981
|
+
return Promise.resolve([]);
|
|
1982
|
+
}
|
|
1983
|
+
return this.#serialized(request.leaseOwner, () => this.#claim(request));
|
|
1984
|
+
}
|
|
1985
|
+
/**
|
|
1986
|
+
* Runs `work` after the earlier claims of the same owner on this store settled. The
|
|
1987
|
+
* bookkeeping of what was handed out and the reconciliation of a failed claim need
|
|
1988
|
+
* one claim at a time per owner (the dispatcher claims one batch at a time anyway;
|
|
1989
|
+
* `runOnce` next to a running dispatcher is what this guards against).
|
|
1990
|
+
*/
|
|
1991
|
+
#serialized(owner, work) {
|
|
1992
|
+
const previous = this.#ownerQueues.get(owner) ?? Promise.resolve();
|
|
1993
|
+
const run = previous.then(work, work);
|
|
1994
|
+
const tail = run.then(
|
|
1995
|
+
() => void 0,
|
|
1996
|
+
() => void 0
|
|
1997
|
+
);
|
|
1998
|
+
this.#ownerQueues.set(owner, tail);
|
|
1999
|
+
void tail.then(() => {
|
|
2000
|
+
if (this.#ownerQueues.get(owner) === tail) {
|
|
2001
|
+
this.#ownerQueues.delete(owner);
|
|
2002
|
+
}
|
|
2003
|
+
});
|
|
2004
|
+
return run;
|
|
2005
|
+
}
|
|
2006
|
+
async #claim(request) {
|
|
2007
|
+
if (this.#unreconciled.has(request.leaseOwner) && !await this.#reconcile(request.leaseOwner)) {
|
|
2008
|
+
throw new Error(
|
|
2009
|
+
`[PostgresEventStore](claim) Giving back unreturned deliveries still fails, so no claim is made | owner=${request.leaseOwner}`
|
|
2010
|
+
);
|
|
2011
|
+
}
|
|
2012
|
+
const heldFirst = this.#drainingOwners.has(request.leaseOwner);
|
|
2013
|
+
let rows;
|
|
2014
|
+
try {
|
|
2015
|
+
rows = await this.#claimInOneStatement(
|
|
2016
|
+
request,
|
|
2017
|
+
heldFirst ? this.#statements.claimHeldLanes : this.#statements.claimLanes
|
|
2018
|
+
);
|
|
2019
|
+
} catch (error) {
|
|
2020
|
+
await this.#reconcile(request.leaseOwner);
|
|
2021
|
+
throw error;
|
|
2022
|
+
}
|
|
2023
|
+
const heldLanes = new Set(JSON.parse(rows[0]["examined_lanes"]));
|
|
2024
|
+
const claimed = this.#takeClaimed(rows, request.leaseOwner);
|
|
2025
|
+
if (heldFirst && claimed.length < request.limit && rows[0]["stale"] !== true) {
|
|
2026
|
+
try {
|
|
2027
|
+
rows = await this.#claimInOneStatement(
|
|
2028
|
+
{ ...request, limit: request.limit - claimed.length },
|
|
2029
|
+
this.#statements.claimLanes
|
|
2030
|
+
);
|
|
2031
|
+
} catch (error) {
|
|
2032
|
+
await this.#reconcile(request.leaseOwner);
|
|
2033
|
+
if (claimed.length === 0) {
|
|
2034
|
+
this.#drainingOwners.delete(request.leaseOwner);
|
|
2035
|
+
throw error;
|
|
2036
|
+
}
|
|
2037
|
+
this.#logger.error(
|
|
2038
|
+
`[PostgresEventStore](claim) Claiming from free lanes failed, returning the deliveries already claimed | claimed=${claimed.length} | error=${error instanceof Error ? error.message : String(error)}`
|
|
2039
|
+
);
|
|
2040
|
+
return this.#handOutInDueOrder(claimed, request, heldLanes);
|
|
2041
|
+
}
|
|
2042
|
+
claimed.push(...this.#takeClaimed(rows, request.leaseOwner));
|
|
2043
|
+
}
|
|
2044
|
+
if (claimed.length < request.limit && (rows[0]["takeover_waiting"] === true || rows[0]["unexamined_free_lanes"] === true || rows[0]["stale"] === true)) {
|
|
2045
|
+
try {
|
|
2046
|
+
const more = await this.#claimMore(request, claimed.length);
|
|
2047
|
+
this.#handOut(more, request.leaseOwner);
|
|
2048
|
+
claimed.push(...more);
|
|
2049
|
+
} catch (error) {
|
|
2050
|
+
await this.#reconcile(request.leaseOwner);
|
|
2051
|
+
if (claimed.length === 0) {
|
|
2052
|
+
throw error;
|
|
2053
|
+
}
|
|
2054
|
+
this.#logger.error(
|
|
2055
|
+
`[PostgresEventStore](claim) Taking more lanes failed, returning the deliveries already claimed | claimed=${claimed.length} | error=${error instanceof Error ? error.message : String(error)}`
|
|
2056
|
+
);
|
|
2057
|
+
}
|
|
2058
|
+
}
|
|
2059
|
+
return this.#handOutInDueOrder(claimed, request, heldLanes);
|
|
2060
|
+
}
|
|
2061
|
+
/** The deliveries a claim statement picked, registered as handed out to `leaseOwner`. */
|
|
2062
|
+
#takeClaimed(rows, leaseOwner) {
|
|
2063
|
+
const claimed = rows.filter((row) => row["delivery_id"] !== null);
|
|
2064
|
+
this.#forgetFinished(rows[0], leaseOwner, claimed);
|
|
2065
|
+
this.#handOut(claimed, leaseOwner);
|
|
2066
|
+
return claimed;
|
|
2067
|
+
}
|
|
2068
|
+
#handOutInDueOrder(claimed, request, heldLanes) {
|
|
2069
|
+
if (claimed.length >= request.limit && claimed.every((row) => heldLanes.has(readCount(row, "lane")))) {
|
|
2070
|
+
this.#drainingOwners.add(request.leaseOwner);
|
|
2071
|
+
} else {
|
|
2072
|
+
this.#drainingOwners.delete(request.leaseOwner);
|
|
718
2073
|
}
|
|
719
|
-
|
|
2074
|
+
claimed.sort(compareDue);
|
|
2075
|
+
return claimed.map((row) => mapClaimedDelivery(row, request.leaseExpiresAt));
|
|
2076
|
+
}
|
|
2077
|
+
/**
|
|
2078
|
+
* Runs `statement` ({@link PostgresStatements.claimLanes} or its held-lane form), again when it reports that it waited for a lane lock and
|
|
2079
|
+
* read deliveries with an older snapshot than the lane it got. A held lane is only locked by an
|
|
2080
|
+
* operation of its own owner (another claim, a reschedule), so a few attempts are enough.
|
|
2081
|
+
*/
|
|
2082
|
+
async #claimInOneStatement(request, statement) {
|
|
2083
|
+
const parameters = [
|
|
720
2084
|
request.now.toISOString(),
|
|
721
2085
|
JSON.stringify(request.handlers),
|
|
722
|
-
limit,
|
|
2086
|
+
request.limit,
|
|
723
2087
|
request.leaseOwner,
|
|
724
|
-
request.leaseExpiresAt.toISOString()
|
|
2088
|
+
request.leaseExpiresAt.toISOString(),
|
|
2089
|
+
this.#generation(request.leaseOwner).toString()
|
|
725
2090
|
];
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
leased.push(
|
|
730
|
-
...(await this.#executor.query(
|
|
731
|
-
this.#statements.claimBehindBlockedKeys,
|
|
732
|
-
parameters(request.limit - leased.length)
|
|
733
|
-
)).filter((row) => row["delivery_id"] !== null)
|
|
734
|
-
);
|
|
2091
|
+
let rows = await this.#query(statement, parameters);
|
|
2092
|
+
for (let attempt = 1; attempt < MAXIMUM_STALE_CLAIM_ATTEMPTS && rows[0]["stale"] === true; attempt += 1) {
|
|
2093
|
+
rows = await this.#query(statement, parameters);
|
|
735
2094
|
}
|
|
736
|
-
return
|
|
2095
|
+
return rows;
|
|
737
2096
|
}
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
2097
|
+
/**
|
|
2098
|
+
* Second step of {@link PostgresEventStore.claim}, in one transaction: takes free lanes a few at a
|
|
2099
|
+
* time, lanes to take over included (their lost attempts are charged first), and records the batch.
|
|
2100
|
+
* `alreadyClaimed` deliveries of the first step are in flight, so they are not picked again.
|
|
2101
|
+
*/
|
|
2102
|
+
async #claimMore(request, alreadyClaimed) {
|
|
2103
|
+
const now = request.now.toISOString();
|
|
2104
|
+
const leaseExpiresAt = request.leaseExpiresAt.toISOString();
|
|
2105
|
+
const handlers = JSON.stringify(request.handlers);
|
|
2106
|
+
const generation = this.#generation(request.leaseOwner).toString();
|
|
2107
|
+
return this.#executor.transaction(async (transaction) => {
|
|
2108
|
+
const claimed = [];
|
|
2109
|
+
const remaining = () => request.limit - alreadyClaimed - claimed.length;
|
|
2110
|
+
const takenLanes = /* @__PURE__ */ new Set();
|
|
2111
|
+
for (let laneCount = INITIAL_LANES_PER_ROUND; remaining() > 0; ) {
|
|
2112
|
+
const taken = await this.#query(
|
|
2113
|
+
this.#statements.takeFreeLanes,
|
|
2114
|
+
[now, handlers, laneCount, request.leaseOwner, leaseExpiresAt, generation],
|
|
2115
|
+
transaction
|
|
2116
|
+
);
|
|
2117
|
+
if (taken.length === 0) {
|
|
2118
|
+
break;
|
|
2119
|
+
}
|
|
2120
|
+
const lanes = taken.map((row) => readCount(row, "lane"));
|
|
2121
|
+
for (const lane of lanes) {
|
|
2122
|
+
takenLanes.add(lane);
|
|
2123
|
+
}
|
|
2124
|
+
claimed.push(
|
|
2125
|
+
...await this.#pickDeliveries(
|
|
2126
|
+
[now, handlers, remaining(), JSON.stringify(lanes)],
|
|
2127
|
+
transaction
|
|
2128
|
+
)
|
|
2129
|
+
);
|
|
2130
|
+
laneCount *= 2;
|
|
2131
|
+
}
|
|
2132
|
+
await this.#recordClaim(claimed, takenLanes, request, transaction);
|
|
2133
|
+
return claimed;
|
|
2134
|
+
});
|
|
2135
|
+
}
|
|
2136
|
+
/**
|
|
2137
|
+
* The generation of the owner's claim statements. It only grows, also across restarts (it starts at the
|
|
2138
|
+
* clock), and a reconciliation moves it up, which fences off every earlier statement still in flight.
|
|
2139
|
+
*/
|
|
2140
|
+
#generation(owner, advance = false) {
|
|
2141
|
+
const current = this.#generations.get(owner) ?? BigInt(Date.now());
|
|
2142
|
+
const next = advance ? current + 1n > BigInt(Date.now()) ? current + 1n : BigInt(Date.now()) : current;
|
|
2143
|
+
this.#generations.set(owner, next);
|
|
2144
|
+
return next;
|
|
2145
|
+
}
|
|
2146
|
+
/** Forgets a handed-out lease once the store answered about it (any answer: the claim is over or fenced out). */
|
|
2147
|
+
async #settled(lease, work) {
|
|
2148
|
+
const result = await work;
|
|
2149
|
+
this.#handedOut.delete(handedOutKey(lease.deliveryId, lease.leaseToken));
|
|
2150
|
+
return result;
|
|
2151
|
+
}
|
|
2152
|
+
/** Remembers the claims handed to the caller of `claim` until they are settled. */
|
|
2153
|
+
#handOut(rows, leaseOwner) {
|
|
2154
|
+
for (const row of rows) {
|
|
2155
|
+
const claim = mapClaimedDelivery(row, /* @__PURE__ */ new Date(0));
|
|
2156
|
+
this.#handedOut.set(handedOutKey(claim.deliveryId, claim.leaseToken), {
|
|
2157
|
+
owner: leaseOwner,
|
|
2158
|
+
deliveryId: claim.deliveryId,
|
|
2159
|
+
...parseLeaseToken(claim.leaseToken)
|
|
2160
|
+
});
|
|
2161
|
+
}
|
|
2162
|
+
}
|
|
2163
|
+
/**
|
|
2164
|
+
* Drops the registrations of the owner's held lanes whose delivery no longer runs there
|
|
2165
|
+
* (completed inside the caller's transaction, deleted, or listed no more). Registrations of
|
|
2166
|
+
* lanes this claim did not see stay: their tokens may still be valid.
|
|
2167
|
+
*/
|
|
2168
|
+
#forgetFinished(row, leaseOwner, claimed) {
|
|
2169
|
+
const heldLanes = new Set(JSON.parse(row["examined_lanes"]));
|
|
2170
|
+
const running = /* @__PURE__ */ new Set([
|
|
2171
|
+
...JSON.parse(row["running_ids"]),
|
|
2172
|
+
...claimed.map((picked) => picked["delivery_id"])
|
|
744
2173
|
]);
|
|
2174
|
+
for (const [key, entry] of this.#handedOut) {
|
|
2175
|
+
if (entry.owner === leaseOwner && heldLanes.has(entry.lane) && !running.has(entry.deliveryId)) {
|
|
2176
|
+
this.#handedOut.delete(key);
|
|
2177
|
+
}
|
|
2178
|
+
}
|
|
2179
|
+
}
|
|
2180
|
+
/**
|
|
2181
|
+
* After a claim failed, possibly after the database recorded its batch: gives back the recorded
|
|
2182
|
+
* deliveries that were never handed out, keeping exactly the claims handed out and not settled.
|
|
2183
|
+
* A failure is remembered and retried before the owner's next claim: other deliveries keep
|
|
2184
|
+
* renewing the lane, so an unreturned one would otherwise never be recovered.
|
|
2185
|
+
*/
|
|
2186
|
+
async #reconcile(leaseOwner) {
|
|
2187
|
+
const generation = this.#generation(leaseOwner, true);
|
|
2188
|
+
const handedOut = [...this.#handedOut.values()].filter((entry) => entry.owner === leaseOwner).map(({ deliveryId, lane, epoch, claims }) => ({ id: deliveryId, lane, epoch, claims }));
|
|
2189
|
+
try {
|
|
2190
|
+
await this.#query(this.#statements.reconcileClaims, [
|
|
2191
|
+
leaseOwner,
|
|
2192
|
+
JSON.stringify(handedOut),
|
|
2193
|
+
generation.toString()
|
|
2194
|
+
]);
|
|
2195
|
+
this.#unreconciled.delete(leaseOwner);
|
|
2196
|
+
return true;
|
|
2197
|
+
} catch (error) {
|
|
2198
|
+
this.#unreconciled.add(leaseOwner);
|
|
2199
|
+
this.#logger.error(
|
|
2200
|
+
`[PostgresEventStore](reconcile) Giving back unreturned deliveries failed, it is retried before the next claim | owner=${leaseOwner} | error=${error instanceof Error ? error.message : String(error)}`
|
|
2201
|
+
);
|
|
2202
|
+
return false;
|
|
2203
|
+
}
|
|
2204
|
+
}
|
|
2205
|
+
/** Fast claim, then the slow one when the fast window was full of blocked followers. */
|
|
2206
|
+
async #pickDeliveries(parameters, transaction) {
|
|
2207
|
+
const limit = parameters[2];
|
|
2208
|
+
const fastRows = await this.#query(this.#statements.claim, parameters, transaction);
|
|
2209
|
+
const picked = fastRows.filter((row) => row["delivery_id"] !== null);
|
|
2210
|
+
if (picked.length < limit && fastRows[0]["window_saturated"] === true) {
|
|
2211
|
+
const slowRows = await this.#query(
|
|
2212
|
+
this.#statements.claimBehindBlockedKeys,
|
|
2213
|
+
parameters,
|
|
2214
|
+
transaction
|
|
2215
|
+
);
|
|
2216
|
+
const pickedIds = new Set(picked.map((row) => row["delivery_id"]));
|
|
2217
|
+
picked.push(
|
|
2218
|
+
...slowRows.filter(
|
|
2219
|
+
(row) => row["delivery_id"] !== null && !pickedIds.has(row["delivery_id"])
|
|
2220
|
+
)
|
|
2221
|
+
);
|
|
2222
|
+
const seenKeys = /* @__PURE__ */ new Set();
|
|
2223
|
+
const kept = [...picked].sort((left, right) => compareSequence(left, right)).filter((row) => {
|
|
2224
|
+
if (row["delivery_key"] === null) {
|
|
2225
|
+
return true;
|
|
2226
|
+
}
|
|
2227
|
+
const group = `${row["handler"]}\0${row["delivery_key"]}`;
|
|
2228
|
+
return !seenKeys.has(group) && Boolean(seenKeys.add(group));
|
|
2229
|
+
});
|
|
2230
|
+
kept.sort(compareDue);
|
|
2231
|
+
return kept.slice(0, limit);
|
|
2232
|
+
}
|
|
2233
|
+
return picked;
|
|
745
2234
|
}
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
2235
|
+
/** Writes the picked ids into their lane rows and frees the lanes that picked nothing. */
|
|
2236
|
+
async #recordClaim(claimed, lanes, request, transaction) {
|
|
2237
|
+
const idsByLane = /* @__PURE__ */ new Map();
|
|
2238
|
+
for (const row of claimed) {
|
|
2239
|
+
const lane = readCount(row, "lane");
|
|
2240
|
+
idsByLane.set(lane, [...idsByLane.get(lane) ?? [], row["delivery_id"]]);
|
|
2241
|
+
}
|
|
2242
|
+
const idleLanes = [...lanes].filter((lane) => !idsByLane.has(lane));
|
|
2243
|
+
if (idsByLane.size === 0 && idleLanes.length === 0) {
|
|
2244
|
+
return;
|
|
2245
|
+
}
|
|
2246
|
+
await this.#query(
|
|
2247
|
+
this.#statements.recordClaim,
|
|
2248
|
+
[
|
|
2249
|
+
JSON.stringify([...idsByLane].map(([lane, ids]) => ({ lane, ids }))),
|
|
2250
|
+
JSON.stringify(idleLanes),
|
|
2251
|
+
request.leaseOwner,
|
|
2252
|
+
request.leaseExpiresAt.toISOString()
|
|
2253
|
+
],
|
|
750
2254
|
transaction
|
|
751
2255
|
);
|
|
752
2256
|
}
|
|
2257
|
+
extendLease(lease, leaseExpiresAt) {
|
|
2258
|
+
return this.#whenIssued(
|
|
2259
|
+
lease,
|
|
2260
|
+
(held) => this.#changed(this.#statements.extendLease, [
|
|
2261
|
+
lease.deliveryId,
|
|
2262
|
+
held.lane,
|
|
2263
|
+
held.epoch,
|
|
2264
|
+
held.claims,
|
|
2265
|
+
leaseExpiresAt.toISOString()
|
|
2266
|
+
])
|
|
2267
|
+
);
|
|
2268
|
+
}
|
|
2269
|
+
async complete(lease, now, transaction) {
|
|
2270
|
+
return (await this.#completeLeases([lease], now, transaction))[0];
|
|
2271
|
+
}
|
|
2272
|
+
completeMany(leases, now) {
|
|
2273
|
+
return this.#completeLeases(leases, now, void 0);
|
|
2274
|
+
}
|
|
753
2275
|
reschedule(lease, availableAt, error, now) {
|
|
754
|
-
return this.#
|
|
755
|
-
lease
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
2276
|
+
return this.#settled(
|
|
2277
|
+
lease,
|
|
2278
|
+
this.#whenIssued(
|
|
2279
|
+
lease,
|
|
2280
|
+
(held) => this.#changed(this.#statements.reschedule, [
|
|
2281
|
+
lease.deliveryId,
|
|
2282
|
+
held.lane,
|
|
2283
|
+
held.epoch,
|
|
2284
|
+
held.claims,
|
|
2285
|
+
availableAt.toISOString(),
|
|
2286
|
+
JSON.stringify(error),
|
|
2287
|
+
now.toISOString()
|
|
2288
|
+
])
|
|
2289
|
+
)
|
|
2290
|
+
);
|
|
761
2291
|
}
|
|
762
2292
|
bury(lease, error, now) {
|
|
763
|
-
return this.#
|
|
764
|
-
lease
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
2293
|
+
return this.#settled(
|
|
2294
|
+
lease,
|
|
2295
|
+
this.#whenIssued(
|
|
2296
|
+
lease,
|
|
2297
|
+
(held) => this.#changed(this.#statements.bury, [
|
|
2298
|
+
lease.deliveryId,
|
|
2299
|
+
held.lane,
|
|
2300
|
+
held.epoch,
|
|
2301
|
+
held.claims,
|
|
2302
|
+
JSON.stringify(error),
|
|
2303
|
+
now.toISOString()
|
|
2304
|
+
])
|
|
2305
|
+
)
|
|
2306
|
+
);
|
|
769
2307
|
}
|
|
770
2308
|
/** Returns a leased delivery to pending, due now, without counting the attempt. */
|
|
771
2309
|
release(lease, now) {
|
|
772
|
-
return this.#
|
|
773
|
-
lease
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
2310
|
+
return this.#settled(
|
|
2311
|
+
lease,
|
|
2312
|
+
this.#whenIssued(
|
|
2313
|
+
lease,
|
|
2314
|
+
(held) => this.#changed(this.#statements.release, [
|
|
2315
|
+
lease.deliveryId,
|
|
2316
|
+
held.lane,
|
|
2317
|
+
held.epoch,
|
|
2318
|
+
held.claims,
|
|
2319
|
+
now.toISOString()
|
|
2320
|
+
])
|
|
2321
|
+
)
|
|
2322
|
+
);
|
|
2323
|
+
}
|
|
2324
|
+
/**
|
|
2325
|
+
* Calls `listener` soon after a transaction that published deliveries due at once commits, in any
|
|
2326
|
+
* process: `append` sends one `pg_notify` per call (merged per transaction, none for delayed
|
|
2327
|
+
* deliveries), and this listens with `LISTEN` on a connection of its own, opened with the settings of
|
|
2328
|
+
* the store's pool (one connection more than the pool's `max`). A lost connection is replaced after a
|
|
2329
|
+
* second. Resolves with `undefined` for a store built on an `executor`, which has no connection of its
|
|
2330
|
+
* own to listen on, and with `wakeUp: 'poll'`: polling then finds new deliveries (and the bus still
|
|
2331
|
+
* wakes its own dispatcher).
|
|
2332
|
+
*/
|
|
2333
|
+
async watchNewDeliveries(listener) {
|
|
2334
|
+
if (this.#pool === void 0 || !this.#notifies) {
|
|
2335
|
+
return void 0;
|
|
2336
|
+
}
|
|
2337
|
+
const options = this.#pool.options;
|
|
2338
|
+
const watcher = new NotificationListener(
|
|
2339
|
+
async () => {
|
|
2340
|
+
const client = new pg.Client(options);
|
|
2341
|
+
await client.connect();
|
|
2342
|
+
return client;
|
|
2343
|
+
},
|
|
2344
|
+
this.#notificationChannel,
|
|
2345
|
+
listener,
|
|
2346
|
+
this.#logger
|
|
2347
|
+
);
|
|
2348
|
+
await watcher.start();
|
|
2349
|
+
return () => watcher.close();
|
|
2350
|
+
}
|
|
2351
|
+
/** Frees the lanes `leaseOwner` holds that run no delivery, so other workers need not wait for the lease to expire. */
|
|
2352
|
+
async relinquish(leaseOwner) {
|
|
2353
|
+
await this.#query(this.#statements.relinquish, [leaseOwner]);
|
|
777
2354
|
}
|
|
778
2355
|
async listDeliveries(query) {
|
|
779
2356
|
const statement = this.#statements.listDeliveries(query);
|
|
@@ -781,7 +2358,7 @@ var PostgresEventStore = class {
|
|
|
781
2358
|
return rows.map(mapDeliveryRecord);
|
|
782
2359
|
}
|
|
783
2360
|
async requeue(deliveryIds, now) {
|
|
784
|
-
const rows = await this.#
|
|
2361
|
+
const rows = await this.#query(this.#statements.requeue, [
|
|
785
2362
|
JSON.stringify(deliveryIds),
|
|
786
2363
|
now.toISOString()
|
|
787
2364
|
]);
|
|
@@ -799,16 +2376,31 @@ var PostgresEventStore = class {
|
|
|
799
2376
|
async purge(request) {
|
|
800
2377
|
const total = { deletedDeliveries: 0, deletedEvents: 0 };
|
|
801
2378
|
const batches = [
|
|
802
|
-
[
|
|
803
|
-
|
|
2379
|
+
[
|
|
2380
|
+
this.#statements.purgeCompletedBatch(),
|
|
2381
|
+
request.completedBefore,
|
|
2382
|
+
this.#statements.deleteOrphanEventsWithoutCompletion
|
|
2383
|
+
],
|
|
2384
|
+
[
|
|
2385
|
+
this.#statements.purgeDeadBatch(),
|
|
2386
|
+
request.deadBefore,
|
|
2387
|
+
this.#statements.deleteOrphanEventsWithoutCompletion
|
|
2388
|
+
]
|
|
804
2389
|
];
|
|
805
|
-
|
|
2390
|
+
if (this.#completionHistory) {
|
|
2391
|
+
batches.push([
|
|
2392
|
+
this.#statements.purgeCompletionHistoryBatch(),
|
|
2393
|
+
request.completedBefore,
|
|
2394
|
+
this.#statements.deleteOrphanEventsWithoutCompletion
|
|
2395
|
+
]);
|
|
2396
|
+
}
|
|
2397
|
+
for (const [sql, cutoff, deleteOrphans] of batches) {
|
|
806
2398
|
if (cutoff === void 0) {
|
|
807
2399
|
continue;
|
|
808
2400
|
}
|
|
809
2401
|
let deletedInBatch = this.#purgeBatchSize;
|
|
810
2402
|
while (deletedInBatch === this.#purgeBatchSize) {
|
|
811
|
-
const batch = await this.#purgeBatch(sql, cutoff);
|
|
2403
|
+
const batch = await this.#purgeBatch(sql, cutoff, deleteOrphans);
|
|
812
2404
|
deletedInBatch = batch.deletedDeliveries;
|
|
813
2405
|
total.deletedDeliveries += batch.deletedDeliveries;
|
|
814
2406
|
total.deletedEvents += batch.deletedEvents;
|
|
@@ -816,69 +2408,116 @@ var PostgresEventStore = class {
|
|
|
816
2408
|
}
|
|
817
2409
|
const orphanCutoff = [request.completedBefore, request.deadBefore].filter((cutoff) => cutoff !== void 0).sort((first, second) => first.getTime() - second.getTime())[0];
|
|
818
2410
|
if (orphanCutoff !== void 0) {
|
|
819
|
-
total.deletedEvents += await this.#purgeOrphanEvents(
|
|
2411
|
+
total.deletedEvents += await this.#purgeOrphanEvents(
|
|
2412
|
+
this.#statements.lockOrphanEventsBatch,
|
|
2413
|
+
this.#statements.deleteOrphanEventsWithoutCompletion,
|
|
2414
|
+
orphanCutoff,
|
|
2415
|
+
false
|
|
2416
|
+
);
|
|
2417
|
+
}
|
|
2418
|
+
if (request.completedBefore !== void 0) {
|
|
2419
|
+
total.deletedEvents += await this.#purgeOrphanEvents(
|
|
2420
|
+
this.#statements.lockCompletedOrphanEventsBatch,
|
|
2421
|
+
this.#statements.deleteExpiredCompletedOrphanEvents,
|
|
2422
|
+
request.completedBefore,
|
|
2423
|
+
true
|
|
2424
|
+
);
|
|
820
2425
|
}
|
|
821
2426
|
return total;
|
|
822
2427
|
}
|
|
823
2428
|
async statistics(now) {
|
|
824
|
-
const
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
2429
|
+
const stored = { pending: 0, dead: 0, completed: 0 };
|
|
2430
|
+
for (const row of await this.#query(this.#statements.countByStatus, [])) {
|
|
2431
|
+
stored[row["status"]] = readCount(row, "count");
|
|
2432
|
+
}
|
|
2433
|
+
const [running] = await this.#query(this.#statements.countRunning, []);
|
|
2434
|
+
const processing = readCount(running, "count");
|
|
2435
|
+
let completed = stored["completed"];
|
|
2436
|
+
if (this.#completionHistory) {
|
|
2437
|
+
const [history] = await this.#query(this.#statements.countCompletionHistory, []);
|
|
2438
|
+
completed += readCount(history, "count");
|
|
832
2439
|
}
|
|
833
|
-
const oldestDue = await this.#
|
|
834
|
-
return {
|
|
2440
|
+
const oldestDue = await this.#query(this.#statements.oldestDue, [now.toISOString()]);
|
|
2441
|
+
return {
|
|
2442
|
+
// The counts come from separate statements; a delivery that finishes in between must not make this negative.
|
|
2443
|
+
pending: Math.max(0, stored["pending"] - processing),
|
|
2444
|
+
processing,
|
|
2445
|
+
completed,
|
|
2446
|
+
dead: stored["dead"],
|
|
2447
|
+
oldestDueAt: mapOldestDueAt(oldestDue)
|
|
2448
|
+
};
|
|
835
2449
|
}
|
|
836
2450
|
/** Ends the pool this store created from `connectionString`. Pools and executors you passed in stay open. */
|
|
837
2451
|
async close() {
|
|
2452
|
+
this.#closed = true;
|
|
2453
|
+
clearTimeout(this.#trailingNotification);
|
|
2454
|
+
this.#trailingNotification = void 0;
|
|
838
2455
|
const pool = this.#ownedPool;
|
|
839
2456
|
this.#ownedPool = void 0;
|
|
840
2457
|
await pool?.end();
|
|
841
2458
|
}
|
|
842
|
-
|
|
843
|
-
|
|
2459
|
+
/** Runs a statement of {@link PostgresStatements} whose text never changes, prepared when `preparedStatements` is on. */
|
|
2460
|
+
#query(sql, parameters, transaction) {
|
|
2461
|
+
return this.#executor.query(sql, parameters, transaction, this.#queryOptions);
|
|
2462
|
+
}
|
|
2463
|
+
async #changed(sql, parameters) {
|
|
2464
|
+
const rows = await this.#query(sql, parameters);
|
|
844
2465
|
return readCount(rows[0], "count") > 0;
|
|
845
2466
|
}
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
2467
|
+
/** Runs `operation` for a lease this store issued; any other token cannot be held. */
|
|
2468
|
+
async #whenIssued(lease, operation) {
|
|
2469
|
+
const held = parseLeaseToken(lease.leaseToken);
|
|
2470
|
+
return held !== void 0 && await operation(held);
|
|
2471
|
+
}
|
|
2472
|
+
/**
|
|
2473
|
+
* Deletes the delivery of every lease that is still the current claim of its
|
|
2474
|
+
* delivery. Returns one flag per lease, in order. Results are matched to leases
|
|
2475
|
+
* by position, so two leases of one delivery (a stale and a current one) get
|
|
2476
|
+
* their own answers; a lease listed twice completes once, at its first position.
|
|
2477
|
+
*/
|
|
2478
|
+
async #completeLeases(leases, now, transaction) {
|
|
2479
|
+
const issued = [];
|
|
2480
|
+
const seen = /* @__PURE__ */ new Set();
|
|
2481
|
+
leases.forEach((lease, ord) => {
|
|
2482
|
+
const held = parseLeaseToken(lease.leaseToken);
|
|
2483
|
+
const identity = `${lease.deliveryId}\0${lease.leaseToken}`;
|
|
2484
|
+
if (held !== void 0 && !seen.has(identity)) {
|
|
2485
|
+
seen.add(identity);
|
|
2486
|
+
issued.push({ ord, id: lease.deliveryId, ...held });
|
|
2487
|
+
}
|
|
2488
|
+
});
|
|
2489
|
+
const completed = /* @__PURE__ */ new Set();
|
|
2490
|
+
if (issued.length > 0) {
|
|
2491
|
+
const rows = await this.#query(
|
|
2492
|
+
this.#statements.complete,
|
|
2493
|
+
[JSON.stringify(issued), now.toISOString()],
|
|
2494
|
+
transaction
|
|
2495
|
+
);
|
|
2496
|
+
for (const row of rows) {
|
|
2497
|
+
completed.add(readCount(row, "ord"));
|
|
851
2498
|
}
|
|
852
2499
|
}
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
const requested = [...counts].map(([key, count]) => ({ key, count }));
|
|
858
|
-
const rows = await this.#executor.query(
|
|
859
|
-
this.#statements.reserveSequences,
|
|
860
|
-
[JSON.stringify(requested)],
|
|
861
|
-
transaction
|
|
862
|
-
);
|
|
863
|
-
for (const row of rows) {
|
|
864
|
-
const key = row["key"];
|
|
865
|
-
firstSequences.set(key, BigInt(row["sequence"]) - BigInt(counts.get(key)) + 1n);
|
|
2500
|
+
if (transaction === void 0) {
|
|
2501
|
+
for (const lease of leases) {
|
|
2502
|
+
this.#handedOut.delete(handedOutKey(lease.deliveryId, lease.leaseToken));
|
|
2503
|
+
}
|
|
866
2504
|
}
|
|
867
|
-
return
|
|
2505
|
+
return leases.map((_, ord) => completed.has(ord));
|
|
868
2506
|
}
|
|
869
|
-
async #purgeOrphanEvents(cutoff) {
|
|
2507
|
+
async #purgeOrphanEvents(lockStatement, deleteStatement, cutoff, recheckCutoff) {
|
|
870
2508
|
let deletedEvents = 0;
|
|
871
2509
|
let removedInBatch = this.#purgeBatchSize;
|
|
872
2510
|
while (removedInBatch === this.#purgeBatchSize) {
|
|
873
2511
|
removedInBatch = await this.#executor.transaction(async (transaction) => {
|
|
874
|
-
const locked = await this.#
|
|
875
|
-
|
|
2512
|
+
const locked = await this.#query(
|
|
2513
|
+
lockStatement,
|
|
876
2514
|
[cutoff.toISOString(), this.#purgeBatchSize],
|
|
877
2515
|
transaction
|
|
878
2516
|
);
|
|
879
|
-
const
|
|
880
|
-
|
|
881
|
-
|
|
2517
|
+
const ids = JSON.stringify(locked.map((row) => row["id"]));
|
|
2518
|
+
const removed = await this.#query(
|
|
2519
|
+
deleteStatement,
|
|
2520
|
+
recheckCutoff ? [ids, cutoff.toISOString()] : [ids],
|
|
882
2521
|
transaction
|
|
883
2522
|
);
|
|
884
2523
|
return readCount(removed[0], "count");
|
|
@@ -887,9 +2526,9 @@ var PostgresEventStore = class {
|
|
|
887
2526
|
}
|
|
888
2527
|
return deletedEvents;
|
|
889
2528
|
}
|
|
890
|
-
async #purgeBatch(sql, cutoff) {
|
|
2529
|
+
async #purgeBatch(sql, cutoff, deleteOrphans) {
|
|
891
2530
|
return this.#executor.transaction(async (transaction) => {
|
|
892
|
-
const deleted = await this.#
|
|
2531
|
+
const deleted = await this.#query(
|
|
893
2532
|
sql,
|
|
894
2533
|
[cutoff.toISOString(), this.#purgeBatchSize],
|
|
895
2534
|
transaction
|
|
@@ -900,13 +2539,9 @@ var PostgresEventStore = class {
|
|
|
900
2539
|
const eventIds = JSON.stringify([
|
|
901
2540
|
...new Set(deleted.map((row) => row["event_id"]))
|
|
902
2541
|
]);
|
|
903
|
-
const locked = await this.#
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
transaction
|
|
907
|
-
);
|
|
908
|
-
const removed = await this.#executor.query(
|
|
909
|
-
this.#statements.deleteOrphanEvents,
|
|
2542
|
+
const locked = await this.#query(this.#statements.lockEvents, [eventIds], transaction);
|
|
2543
|
+
const removed = await this.#query(
|
|
2544
|
+
deleteOrphans,
|
|
910
2545
|
[JSON.stringify(locked.map((row) => row["id"]))],
|
|
911
2546
|
transaction
|
|
912
2547
|
);
|
|
@@ -942,28 +2577,31 @@ function planAppend(entries) {
|
|
|
942
2577
|
}
|
|
943
2578
|
return { events, deliveries, outcomes };
|
|
944
2579
|
}
|
|
945
|
-
function
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
2580
|
+
function toEventRow(event, sequence, handlers) {
|
|
2581
|
+
return {
|
|
2582
|
+
id: event.id,
|
|
2583
|
+
type: event.type,
|
|
2584
|
+
source: event.source,
|
|
2585
|
+
key: event.key,
|
|
2586
|
+
sequence,
|
|
2587
|
+
payload: event.payload,
|
|
2588
|
+
headers: event.headers,
|
|
2589
|
+
correlation_id: event.correlationId,
|
|
2590
|
+
causation_id: event.causationId,
|
|
2591
|
+
occurred_at: event.occurredAt.toISOString(),
|
|
2592
|
+
handlers
|
|
2593
|
+
};
|
|
2594
|
+
}
|
|
2595
|
+
function compareSequence(left, right) {
|
|
2596
|
+
const first = BigInt(left["sequence"] ?? 0);
|
|
2597
|
+
const second = BigInt(right["sequence"] ?? 0);
|
|
2598
|
+
return Number(first > second) - Number(first < second);
|
|
2599
|
+
}
|
|
2600
|
+
function compareDue(left, right) {
|
|
2601
|
+
return compareText(left["due_at"], right["due_at"]) || compareText(left["delivery_id"], right["delivery_id"]);
|
|
2602
|
+
}
|
|
2603
|
+
function compareText(left, right) {
|
|
2604
|
+
return Number(left > right) - Number(left < right);
|
|
967
2605
|
}
|
|
968
2606
|
|
|
969
2607
|
exports.PgExecutor = PgExecutor;
|