@relaybus/mongodb 0.0.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +242 -88
- package/dist/index.cjs +1423 -285
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +155 -19
- package/dist/index.d.ts +155 -19
- package/dist/index.js +1424 -286
- package/dist/index.js.map +1 -1
- package/package.json +22 -9
package/dist/index.js
CHANGED
|
@@ -1,50 +1,45 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { createUuidV7, ConfigurationError, EventValidationError } from '@relaybus/core';
|
|
2
2
|
|
|
3
3
|
// src/mongo-event-store.ts
|
|
4
4
|
|
|
5
5
|
// src/claim-scan.ts
|
|
6
6
|
var ClaimScan = class {
|
|
7
7
|
#keys = /* @__PURE__ */ new Map();
|
|
8
|
-
/**
|
|
9
|
-
|
|
8
|
+
/** Whether an unfinished delivery this scan knows of comes before the delivery with `sequence`. */
|
|
9
|
+
waits(orderingId, sequence) {
|
|
10
10
|
const known = this.#keys.get(orderingId);
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
11
|
+
return known?.openSequence !== void 0 && sequence > known.openSequence;
|
|
12
|
+
}
|
|
13
|
+
/** Whether this scan already learned anything about the key. */
|
|
14
|
+
knows(orderingId) {
|
|
15
|
+
return this.#keys.has(orderingId);
|
|
15
16
|
}
|
|
16
17
|
/** Records an unfinished delivery of the key with this sequence. */
|
|
17
18
|
noteOpen(orderingId, sequence) {
|
|
18
19
|
const known = this.#know(orderingId);
|
|
19
20
|
known.openSequence = Math.min(known.openSequence ?? sequence, sequence);
|
|
20
21
|
}
|
|
21
|
-
/**
|
|
22
|
-
|
|
23
|
-
this
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
/** Records that a live lease of the key blocks every delivery of it. */
|
|
27
|
-
noteLeaseHeld(orderingId) {
|
|
28
|
-
this.#know(orderingId).leaseHeld = true;
|
|
29
|
-
}
|
|
30
|
-
/** Remembers a delivery that waited behind an unfinished predecessor. */
|
|
31
|
-
noteWaiting(orderingId, delivery) {
|
|
32
|
-
if (delivery.status === "pending") {
|
|
33
|
-
this.#know(orderingId).waiting.push(delivery._id);
|
|
34
|
-
}
|
|
22
|
+
/** Remembers a pending delivery of lane `laneId` that waited behind an unfinished predecessor. */
|
|
23
|
+
noteWaiting(orderingId, deliveryId, laneId) {
|
|
24
|
+
const known = this.#know(orderingId);
|
|
25
|
+
known.waiting.push(deliveryId);
|
|
26
|
+
known.laneId = laneId;
|
|
35
27
|
}
|
|
36
28
|
/**
|
|
37
29
|
* Ids of the waiting deliveries of every key that accumulated at least
|
|
38
|
-
* `minimum` of them
|
|
39
|
-
*
|
|
30
|
+
* `minimum` of them, or that lives in one of `handedBack`. Shorter queues
|
|
31
|
+
* of a lane that stays held remain in the claim index: moving them out and
|
|
32
|
+
* back would cost more than skipping them. In a lane handed back they are
|
|
33
|
+
* all its due work, and leaving them ready would make every later claim take
|
|
34
|
+
* the lane for nothing.
|
|
40
35
|
*/
|
|
41
|
-
waitingBacklogs(minimum) {
|
|
42
|
-
return [...this.#keys.values()].filter((known) => known.waiting.length >= minimum).flatMap((known) => known.waiting);
|
|
36
|
+
waitingBacklogs(minimum, handedBack) {
|
|
37
|
+
return [...this.#keys.values()].filter((known) => known.waiting.length >= minimum || handedBack.has(known.laneId)).flatMap((known) => known.waiting);
|
|
43
38
|
}
|
|
44
39
|
#know(orderingId) {
|
|
45
40
|
let known = this.#keys.get(orderingId);
|
|
46
41
|
if (known === void 0) {
|
|
47
|
-
known = { openSequence: void 0,
|
|
42
|
+
known = { openSequence: void 0, waiting: [], laneId: void 0 };
|
|
48
43
|
this.#keys.set(orderingId, known);
|
|
49
44
|
}
|
|
50
45
|
return known;
|
|
@@ -53,21 +48,49 @@ var ClaimScan = class {
|
|
|
53
48
|
function orderingIdentity(handler, key) {
|
|
54
49
|
return `${handler}\0${key}`;
|
|
55
50
|
}
|
|
56
|
-
|
|
51
|
+
|
|
52
|
+
// src/lanes.ts
|
|
53
|
+
var LANE_COUNT = 256;
|
|
54
|
+
var FNV_OFFSET_BASIS = 2166136261;
|
|
55
|
+
var FNV_PRIME = 16777619;
|
|
56
|
+
var utf8 = new TextEncoder();
|
|
57
|
+
function laneOf(key, eventId) {
|
|
58
|
+
let hash = FNV_OFFSET_BASIS;
|
|
59
|
+
for (const byte of utf8.encode(key ?? eventId)) {
|
|
60
|
+
hash = Math.imul(hash ^ byte, FNV_PRIME);
|
|
61
|
+
}
|
|
62
|
+
return hash & LANE_COUNT - 1;
|
|
63
|
+
}
|
|
64
|
+
function formatLeaseToken(laneId, epoch, claims) {
|
|
65
|
+
return `${laneId}:${epoch}:${claims}`;
|
|
66
|
+
}
|
|
67
|
+
var LEASE_TOKEN_PATTERN = /^(\d{1,3}):(\d{1,15}):(\d{1,15})$/;
|
|
68
|
+
function parseLeaseToken(token) {
|
|
69
|
+
const match = LEASE_TOKEN_PATTERN.exec(token);
|
|
70
|
+
if (match === null || Number(match[1]) >= LANE_COUNT) {
|
|
71
|
+
return void 0;
|
|
72
|
+
}
|
|
73
|
+
return { laneId: Number(match[1]), epoch: Number(match[2]), claims: Number(match[3]) };
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
// src/documents.ts
|
|
77
|
+
function toEventDocument(event, sequence, handlers, createdAt) {
|
|
57
78
|
const payloadJson = serializeWellFormed(event.payload, "payload");
|
|
58
79
|
const headersJson = serializeWellFormed(event.headers, "headers");
|
|
59
80
|
return {
|
|
60
81
|
_id: event.id,
|
|
61
82
|
type: event.type,
|
|
62
83
|
source: event.source,
|
|
63
|
-
|
|
64
|
-
|
|
84
|
+
// Fields that are null or 0 are left out: every byte of a new document is written to the oplog too.
|
|
85
|
+
...event.key === null ? {} : { key: event.key },
|
|
86
|
+
...sequence === null ? {} : { sequence },
|
|
65
87
|
...payloadJson === void 0 ? {} : { payload: JSON.parse(payloadJson) },
|
|
66
88
|
headers: JSON.parse(headersJson),
|
|
67
|
-
correlationId: event.correlationId,
|
|
68
|
-
causationId: event.causationId,
|
|
89
|
+
...event.correlationId === null ? {} : { correlationId: event.correlationId },
|
|
90
|
+
...event.causationId === null ? {} : { causationId: event.causationId },
|
|
69
91
|
occurredAt: event.occurredAt,
|
|
70
|
-
|
|
92
|
+
handlers,
|
|
93
|
+
createdAt
|
|
71
94
|
};
|
|
72
95
|
}
|
|
73
96
|
var LONE_SURROGATE_ESCAPE = /(?<!\\)(?:\\\\)*\\ud[89a-f][0-9a-f]{2}/i;
|
|
@@ -88,11 +111,11 @@ function toEnvelope(document) {
|
|
|
88
111
|
id: document._id,
|
|
89
112
|
type: document.type,
|
|
90
113
|
source: document.source,
|
|
91
|
-
key: document.key,
|
|
114
|
+
key: document.key ?? null,
|
|
92
115
|
payload: document.payload,
|
|
93
116
|
headers: document.headers,
|
|
94
|
-
correlationId: document.correlationId,
|
|
95
|
-
causationId: document.causationId,
|
|
117
|
+
correlationId: document.correlationId ?? null,
|
|
118
|
+
causationId: document.causationId ?? null,
|
|
96
119
|
occurredAt: document.occurredAt
|
|
97
120
|
};
|
|
98
121
|
}
|
|
@@ -103,31 +126,60 @@ function toDeliveryDocument(delivery, event, ordering, now) {
|
|
|
103
126
|
eventType: event.type,
|
|
104
127
|
handler: delivery.handler,
|
|
105
128
|
...ordering.key === null || ordering.sequence === null ? {} : { key: ordering.key, sequence: ordering.sequence },
|
|
129
|
+
laneId: laneOf(ordering.key, event.id),
|
|
106
130
|
status: "pending",
|
|
107
131
|
lane: "ready",
|
|
108
132
|
attempts: 0,
|
|
133
|
+
// No updatedAt (it equals createdAt): every byte of a new document is written to the oplog too.
|
|
109
134
|
availableAt: delivery.availableAt,
|
|
110
135
|
dueAt: delivery.availableAt,
|
|
111
|
-
createdAt: now
|
|
112
|
-
updatedAt: now
|
|
136
|
+
createdAt: now
|
|
113
137
|
};
|
|
114
138
|
}
|
|
115
|
-
function toDeliveryRecord(document, event) {
|
|
139
|
+
function toDeliveryRecord(document, event, runningIn) {
|
|
116
140
|
return {
|
|
117
141
|
id: document._id,
|
|
118
142
|
handler: document.handler,
|
|
119
|
-
status: document.status,
|
|
120
|
-
attempts: document.attempts,
|
|
143
|
+
status: runningIn === void 0 ? document.status : "processing",
|
|
144
|
+
attempts: runningIn === void 0 ? document.attempts : document.attempts + 1,
|
|
121
145
|
availableAt: document.availableAt,
|
|
122
|
-
leaseOwner:
|
|
123
|
-
leaseExpiresAt:
|
|
146
|
+
leaseOwner: runningIn === void 0 ? null : runningIn.owner,
|
|
147
|
+
leaseExpiresAt: runningIn === void 0 ? null : runningIn.leaseExpiresAt,
|
|
124
148
|
lastError: document.lastError === void 0 ? null : toErrorInfo(document.lastError),
|
|
125
149
|
createdAt: document.createdAt,
|
|
126
|
-
updatedAt: document.updatedAt,
|
|
150
|
+
updatedAt: document.updatedAt ?? document.createdAt,
|
|
127
151
|
completedAt: document.completedAt ?? null,
|
|
128
152
|
event
|
|
129
153
|
};
|
|
130
154
|
}
|
|
155
|
+
function toCompletedDeliveryRecord(document, event) {
|
|
156
|
+
return {
|
|
157
|
+
id: document._id,
|
|
158
|
+
handler: document.handler,
|
|
159
|
+
status: "completed",
|
|
160
|
+
attempts: document.attempts,
|
|
161
|
+
availableAt: document.availableAt,
|
|
162
|
+
leaseOwner: null,
|
|
163
|
+
leaseExpiresAt: null,
|
|
164
|
+
lastError: null,
|
|
165
|
+
createdAt: document.createdAt,
|
|
166
|
+
updatedAt: document.completedAt,
|
|
167
|
+
completedAt: document.completedAt,
|
|
168
|
+
event
|
|
169
|
+
};
|
|
170
|
+
}
|
|
171
|
+
function toCompletedDeliveryDocument(document, completedAt) {
|
|
172
|
+
return {
|
|
173
|
+
_id: document._id,
|
|
174
|
+
eventId: document.eventId,
|
|
175
|
+
eventType: document.eventType,
|
|
176
|
+
handler: document.handler,
|
|
177
|
+
attempts: document.attempts + 1,
|
|
178
|
+
availableAt: document.availableAt,
|
|
179
|
+
createdAt: document.createdAt,
|
|
180
|
+
completedAt
|
|
181
|
+
};
|
|
182
|
+
}
|
|
131
183
|
function toErrorInfo(error) {
|
|
132
184
|
return error.stack === void 0 ? { name: error.name, message: error.message } : { name: error.name, message: error.message, stack: error.stack };
|
|
133
185
|
}
|
|
@@ -165,11 +217,17 @@ var DEFAULT_TRANSACTION_OPTIONS = {
|
|
|
165
217
|
readPreference: "primary"
|
|
166
218
|
};
|
|
167
219
|
var NAMESPACE_EXISTS = 48;
|
|
220
|
+
var INDEX_NOT_FOUND = 27;
|
|
168
221
|
var DEFAULT_PURGE_BATCH_SIZE = 500;
|
|
169
222
|
var DEFAULT_LIST_LIMIT = 100;
|
|
170
223
|
var MINIMUM_CANDIDATE_BATCH_SIZE = 32;
|
|
171
224
|
var PARK_BACKLOG_THRESHOLD = 32;
|
|
172
225
|
var PARK_UPDATE_CHUNK_SIZE = 1e3;
|
|
226
|
+
var UPGRADE_BATCH_SIZE = 1e3;
|
|
227
|
+
var INITIAL_LANES_PER_ROUND = 4;
|
|
228
|
+
var LANES_PER_CANDIDATE_QUERY = 128;
|
|
229
|
+
var CLAIM_INDEX_KEY = { laneId: 1, dueAt: 1, _id: 1, handler: 1 };
|
|
230
|
+
var OBSOLETE_INDEXES = ["running"];
|
|
173
231
|
var LEASE_FIELDS = { leaseOwner: "", leaseToken: "", leaseExpiresAt: "" };
|
|
174
232
|
var FINISHED_FIELDS = { dueAt: "", lane: "" };
|
|
175
233
|
var MongoEventStore = class {
|
|
@@ -178,9 +236,33 @@ var MongoEventStore = class {
|
|
|
178
236
|
#events;
|
|
179
237
|
#deliveries;
|
|
180
238
|
#keySequences;
|
|
239
|
+
#lanes;
|
|
240
|
+
#completions;
|
|
241
|
+
#completionHistory;
|
|
181
242
|
#transactionOptions;
|
|
182
243
|
#purgeBatchSize;
|
|
183
|
-
#
|
|
244
|
+
#logger;
|
|
245
|
+
/**
|
|
246
|
+
* Identifies this store instance in the lanes it holds. A process that restarts
|
|
247
|
+
* with the same `leaseOwner` gets a new instance, so it never mistakes the
|
|
248
|
+
* lanes of its dead predecessor for its own: it waits for their leases to
|
|
249
|
+
* expire and takes them over like any other worker.
|
|
250
|
+
*/
|
|
251
|
+
#instanceId = createUuidV7();
|
|
252
|
+
/** The last claim of each owner still running on this instance; the next one waits for it. */
|
|
253
|
+
#claimsInProgress = /* @__PURE__ */ new Map();
|
|
254
|
+
/**
|
|
255
|
+
* Claims this instance handed out that are not settled yet, keyed by
|
|
256
|
+
* {@link leaseKey} (delivery id and exact lease token), with the owner. A
|
|
257
|
+
* claim is settled when a completion, retry, release or burial of exactly
|
|
258
|
+
* that lease returned, or, for a completion inside the caller's transaction
|
|
259
|
+
* (which may still retry or roll back), when a later claim of the owner finds
|
|
260
|
+
* the delivery gone. After a failed claim, a listed delivery stays listed only
|
|
261
|
+
* if one of these is its current claim (current lane epoch and claim generation).
|
|
262
|
+
*/
|
|
263
|
+
#handedOut = /* @__PURE__ */ new Map();
|
|
264
|
+
/** Owners whose lanes on this instance still need reconciling after a failed claim. */
|
|
265
|
+
#unreconciledOwners = /* @__PURE__ */ new Set();
|
|
184
266
|
constructor(options) {
|
|
185
267
|
const prefix = options.collectionPrefix ?? "relaybus_";
|
|
186
268
|
assertCollectionPrefix(prefix);
|
|
@@ -203,13 +285,19 @@ var MongoEventStore = class {
|
|
|
203
285
|
this.#events = database.collection(`${prefix}events`, collectionOptions);
|
|
204
286
|
this.#deliveries = database.collection(`${prefix}deliveries`, collectionOptions);
|
|
205
287
|
this.#keySequences = database.collection(`${prefix}key_sequences`, collectionOptions);
|
|
288
|
+
this.#lanes = database.collection(`${prefix}lanes`, collectionOptions);
|
|
289
|
+
this.#completions = database.collection(`${prefix}completed_deliveries`, collectionOptions);
|
|
290
|
+
this.#completionHistory = options.completionHistory ?? false;
|
|
206
291
|
this.#transactionOptions = options.transactionOptions ?? DEFAULT_TRANSACTION_OPTIONS;
|
|
207
292
|
this.#purgeBatchSize = purgeBatchSize;
|
|
208
|
-
this.#
|
|
293
|
+
this.#logger = options.logger ?? console;
|
|
209
294
|
}
|
|
210
295
|
/**
|
|
211
|
-
* Verifies the deployment supports transactions
|
|
212
|
-
*
|
|
296
|
+
* Verifies the deployment supports transactions, creates the collections and
|
|
297
|
+
* indexes and seeds the lane documents. Upgrades a 0.1.0 database in place
|
|
298
|
+
* first (see {@link MongoEventStore.#upgradeFromVersion010}); the seeded lanes
|
|
299
|
+
* mark the upgrade as done, so later runs skip it. Safe to run repeatedly and
|
|
300
|
+
* from several processes. Stop 0.1.0 workers before the first run.
|
|
213
301
|
*
|
|
214
302
|
* @throws ConfigurationError when the server is a standalone mongod.
|
|
215
303
|
*/
|
|
@@ -219,18 +307,27 @@ var MongoEventStore = class {
|
|
|
219
307
|
if (!supportsTransactions) {
|
|
220
308
|
throw createTransactionsUnsupportedError(void 0);
|
|
221
309
|
}
|
|
222
|
-
|
|
223
|
-
|
|
310
|
+
const collectionNames = [
|
|
311
|
+
this.#events.collectionName,
|
|
312
|
+
this.#deliveries.collectionName,
|
|
313
|
+
this.#keySequences.collectionName,
|
|
314
|
+
this.#lanes.collectionName,
|
|
315
|
+
...this.#completionHistory ? [this.#completions.collectionName] : []
|
|
316
|
+
];
|
|
317
|
+
for (const name of collectionNames) {
|
|
318
|
+
await this.#createCollectionIfMissing(name);
|
|
224
319
|
}
|
|
320
|
+
if (await this.#lanes.countDocuments() < LANE_COUNT) {
|
|
321
|
+
await this.#upgradeFromVersion010();
|
|
322
|
+
await this.#seedLanes();
|
|
323
|
+
}
|
|
324
|
+
await this.#dropObsoleteIndexes();
|
|
225
325
|
await this.#deliveries.createIndexes([
|
|
226
326
|
// Idempotent appends and lookups by event.
|
|
227
327
|
{ key: { eventId: 1, handler: 1 }, name: "event_handler", unique: true },
|
|
228
|
-
// Claim: due work of a
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
name: "claim",
|
|
232
|
-
partialFilterExpression: { lane: "ready" }
|
|
233
|
-
},
|
|
328
|
+
// Claim: due work of a lane in due order. Holds ready deliveries only. The handler is
|
|
329
|
+
// last, so a query over many lanes and handlers stays a merge of one scan per lane.
|
|
330
|
+
{ key: CLAIM_INDEX_KEY, name: "claim", partialFilterExpression: { lane: "ready" } },
|
|
234
331
|
// Ordering: unfinished predecessors of a keyed delivery, parked or not.
|
|
235
332
|
{
|
|
236
333
|
key: { handler: 1, key: 1, sequence: 1 },
|
|
@@ -243,15 +340,9 @@ var MongoEventStore = class {
|
|
|
243
340
|
name: "parked",
|
|
244
341
|
partialFilterExpression: { lane: "parked" }
|
|
245
342
|
},
|
|
246
|
-
// Live leases: a keyed delivery may not start while another of its key is running.
|
|
247
|
-
{
|
|
248
|
-
key: { handler: 1, key: 1, dueAt: 1 },
|
|
249
|
-
name: "running",
|
|
250
|
-
partialFilterExpression: { status: "processing", key: { $exists: true } }
|
|
251
|
-
},
|
|
252
343
|
// Statistics: oldest due delivery across handlers.
|
|
253
344
|
{ key: { dueAt: 1 }, name: "due", partialFilterExpression: { dueAt: { $exists: true } } },
|
|
254
|
-
// Purge of completed deliveries.
|
|
345
|
+
// Purge of completed deliveries left by 0.1.0.
|
|
255
346
|
{
|
|
256
347
|
key: { completedAt: 1 },
|
|
257
348
|
name: "completed",
|
|
@@ -264,16 +355,146 @@ var MongoEventStore = class {
|
|
|
264
355
|
{ key: { handler: 1, createdAt: 1 }, name: "handler_created" },
|
|
265
356
|
{ key: { eventType: 1, createdAt: 1 }, name: "type_created" }
|
|
266
357
|
]);
|
|
358
|
+
await this.#events.createIndexes([
|
|
359
|
+
{ key: { createdAt: 1, _id: 1 }, name: "created" },
|
|
360
|
+
// Purge of events whose deliveries all completed: they are kept until `completedBefore` passes this.
|
|
361
|
+
{
|
|
362
|
+
key: { completedAt: 1, _id: 1 },
|
|
363
|
+
name: "completed",
|
|
364
|
+
partialFilterExpression: { completedAt: { $exists: true } }
|
|
365
|
+
}
|
|
366
|
+
]);
|
|
367
|
+
if (this.#completionHistory) {
|
|
368
|
+
await this.#completions.createIndexes([
|
|
369
|
+
{ key: { completedAt: 1, _id: 1 }, name: "completed" },
|
|
370
|
+
{ key: { eventId: 1 }, name: "event" },
|
|
371
|
+
{ key: { createdAt: 1, _id: 1 }, name: "created" }
|
|
372
|
+
]);
|
|
373
|
+
}
|
|
267
374
|
}
|
|
268
375
|
async #createCollectionIfMissing(name) {
|
|
269
376
|
try {
|
|
270
377
|
await this.#database.createCollection(name);
|
|
271
378
|
} catch (error) {
|
|
272
|
-
if (!(
|
|
379
|
+
if (!hasErrorCode(error, NAMESPACE_EXISTS)) {
|
|
273
380
|
throw error;
|
|
274
381
|
}
|
|
275
382
|
}
|
|
276
383
|
}
|
|
384
|
+
/**
|
|
385
|
+
* Converts the documents of a 0.1.0 database; every step is idempotent, so a
|
|
386
|
+
* run that crashed is simply repeated. Deliveries that were `processing`
|
|
387
|
+
* become `pending`, due at once (their attempt stays counted, as a lost lease
|
|
388
|
+
* would count it). The per-delivery lease fields go, every delivery gets its
|
|
389
|
+
* lane, and every event learns which handlers it already has deliveries for
|
|
390
|
+
* and when it was created. Deliveries completed under 0.1.0 stay as
|
|
391
|
+
* `completed` documents until a purge removes them.
|
|
392
|
+
*/
|
|
393
|
+
async #upgradeFromVersion010() {
|
|
394
|
+
const legacy = this.#database.collection(this.#deliveries.collectionName);
|
|
395
|
+
await legacy.updateMany({ status: "processing" }, [
|
|
396
|
+
{ $set: { status: "pending", lane: "ready", dueAt: "$availableAt" } }
|
|
397
|
+
]);
|
|
398
|
+
await legacy.updateMany(
|
|
399
|
+
{
|
|
400
|
+
$or: [
|
|
401
|
+
{ leaseOwner: { $exists: true } },
|
|
402
|
+
{ leaseToken: { $exists: true } },
|
|
403
|
+
{ leaseExpiresAt: { $exists: true } }
|
|
404
|
+
]
|
|
405
|
+
},
|
|
406
|
+
{ $unset: LEASE_FIELDS }
|
|
407
|
+
);
|
|
408
|
+
await inBatches(
|
|
409
|
+
this.#deliveries.find(
|
|
410
|
+
{ laneId: { $exists: false } },
|
|
411
|
+
{ projection: { eventId: 1, key: 1 }, batchSize: UPGRADE_BATCH_SIZE }
|
|
412
|
+
),
|
|
413
|
+
async (documents) => {
|
|
414
|
+
await this.#deliveries.bulkWrite(
|
|
415
|
+
documents.map((document) => ({
|
|
416
|
+
updateOne: {
|
|
417
|
+
filter: { _id: document._id },
|
|
418
|
+
update: { $set: { laneId: laneOf(document.key ?? null, document.eventId) } }
|
|
419
|
+
}
|
|
420
|
+
})),
|
|
421
|
+
{ ordered: false }
|
|
422
|
+
);
|
|
423
|
+
}
|
|
424
|
+
);
|
|
425
|
+
await inBatches(
|
|
426
|
+
this.#events.find(
|
|
427
|
+
{ handlers: { $exists: false } },
|
|
428
|
+
{ projection: { occurredAt: 1 }, batchSize: UPGRADE_BATCH_SIZE }
|
|
429
|
+
),
|
|
430
|
+
async (events) => {
|
|
431
|
+
const known = new Map(
|
|
432
|
+
(await this.#deliveries.aggregate([
|
|
433
|
+
{ $match: { eventId: { $in: events.map((event) => event._id) } } },
|
|
434
|
+
{
|
|
435
|
+
$group: {
|
|
436
|
+
_id: "$eventId",
|
|
437
|
+
handlers: { $addToSet: "$handler" },
|
|
438
|
+
createdAt: { $min: "$createdAt" }
|
|
439
|
+
}
|
|
440
|
+
}
|
|
441
|
+
]).toArray()).map((group) => [group._id, group])
|
|
442
|
+
);
|
|
443
|
+
await this.#events.bulkWrite(
|
|
444
|
+
events.map((event) => {
|
|
445
|
+
const group = known.get(event._id);
|
|
446
|
+
return {
|
|
447
|
+
updateOne: {
|
|
448
|
+
// Only an event nobody converted yet: another process may have finished and subscribed a handler.
|
|
449
|
+
filter: { _id: event._id, handlers: { $exists: false } },
|
|
450
|
+
update: {
|
|
451
|
+
$set: group === void 0 ? { handlers: [], createdAt: event.occurredAt } : { handlers: group.handlers, createdAt: group.createdAt }
|
|
452
|
+
}
|
|
453
|
+
}
|
|
454
|
+
};
|
|
455
|
+
}),
|
|
456
|
+
{ ordered: false }
|
|
457
|
+
);
|
|
458
|
+
}
|
|
459
|
+
);
|
|
460
|
+
}
|
|
461
|
+
/** Creates the lane documents that do not exist yet. */
|
|
462
|
+
async #seedLanes() {
|
|
463
|
+
await this.#lanes.bulkWrite(
|
|
464
|
+
Array.from({ length: LANE_COUNT }, (_, laneId) => ({
|
|
465
|
+
updateOne: {
|
|
466
|
+
filter: { _id: laneId },
|
|
467
|
+
update: {
|
|
468
|
+
$setOnInsert: {
|
|
469
|
+
epoch: 0,
|
|
470
|
+
owner: null,
|
|
471
|
+
holder: null,
|
|
472
|
+
leaseExpiresAt: null,
|
|
473
|
+
inFlight: [],
|
|
474
|
+
lost: []
|
|
475
|
+
}
|
|
476
|
+
},
|
|
477
|
+
upsert: true
|
|
478
|
+
}
|
|
479
|
+
})),
|
|
480
|
+
{ ordered: false }
|
|
481
|
+
);
|
|
482
|
+
}
|
|
483
|
+
/** Drops the 0.1.0 indexes that no query uses, and a `claim` index with the 0.1.0 key. */
|
|
484
|
+
async #dropObsoleteIndexes() {
|
|
485
|
+
for (const index of await this.#deliveries.indexes()) {
|
|
486
|
+
const obsolete = OBSOLETE_INDEXES.includes(index.name) || index.name === "claim" && Object.keys(index.key).join() !== Object.keys(CLAIM_INDEX_KEY).join();
|
|
487
|
+
if (obsolete) {
|
|
488
|
+
try {
|
|
489
|
+
await this.#deliveries.dropIndex(index.name);
|
|
490
|
+
} catch (error) {
|
|
491
|
+
if (!hasErrorCode(error, INDEX_NOT_FOUND)) {
|
|
492
|
+
throw error;
|
|
493
|
+
}
|
|
494
|
+
}
|
|
495
|
+
}
|
|
496
|
+
}
|
|
497
|
+
}
|
|
277
498
|
/**
|
|
278
499
|
* Runs `work` in a transaction on a new session and commits when it resolves.
|
|
279
500
|
*
|
|
@@ -302,23 +523,34 @@ var MongoEventStore = class {
|
|
|
302
523
|
throw error;
|
|
303
524
|
}
|
|
304
525
|
}
|
|
526
|
+
/**
|
|
527
|
+
* Writes the entries in `session`. An existing event gets a delivery only for
|
|
528
|
+
* handlers it does not list yet; the added handlers are appended to its list
|
|
529
|
+
* together with a `revision` increment, a real write that makes a concurrent
|
|
530
|
+
* purge of the event conflict with this transaction.
|
|
531
|
+
*/
|
|
305
532
|
async append(session, entries, now) {
|
|
306
533
|
assertInTransaction(session, "append");
|
|
307
534
|
const eventIds = [...new Set(entries.map((entry) => entry.event.id))];
|
|
308
535
|
const storedEvents = new Map(
|
|
309
|
-
(await this.#events.find(
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
536
|
+
(await this.#events.find(
|
|
537
|
+
{ _id: { $in: eventIds } },
|
|
538
|
+
{ session, projection: { key: 1, sequence: 1, handlers: 1 } }
|
|
539
|
+
).toArray()).map((stored) => [stored._id, stored])
|
|
313
540
|
);
|
|
314
541
|
const appendedEventIds = [];
|
|
315
542
|
const duplicateEventIds = [];
|
|
316
543
|
const newEvents = [];
|
|
317
544
|
const newDeliveries = [];
|
|
318
|
-
const
|
|
545
|
+
const knownHandlers = /* @__PURE__ */ new Map();
|
|
546
|
+
const addedHandlers = /* @__PURE__ */ new Map();
|
|
319
547
|
const orderingByEventId = /* @__PURE__ */ new Map();
|
|
320
548
|
for (const [eventId, stored] of storedEvents) {
|
|
321
|
-
orderingByEventId.set(eventId, {
|
|
549
|
+
orderingByEventId.set(eventId, {
|
|
550
|
+
key: stored.key ?? null,
|
|
551
|
+
sequence: stored.sequence ?? null
|
|
552
|
+
});
|
|
553
|
+
knownHandlers.set(eventId, new Set(stored.handlers));
|
|
322
554
|
}
|
|
323
555
|
for (const { event, deliveries } of entries) {
|
|
324
556
|
let ordering = orderingByEventId.get(event.id);
|
|
@@ -326,42 +558,49 @@ var MongoEventStore = class {
|
|
|
326
558
|
const sequence = event.key === null ? null : await this.#nextSequence(event.key, session);
|
|
327
559
|
ordering = { key: event.key, sequence };
|
|
328
560
|
orderingByEventId.set(event.id, ordering);
|
|
329
|
-
|
|
561
|
+
knownHandlers.set(event.id, /* @__PURE__ */ new Set());
|
|
562
|
+
newEvents.push({ event, sequence });
|
|
330
563
|
appendedEventIds.push(event.id);
|
|
331
564
|
} else {
|
|
332
565
|
duplicateEventIds.push(event.id);
|
|
333
566
|
}
|
|
567
|
+
const handlers = knownHandlers.get(event.id);
|
|
334
568
|
for (const delivery of deliveries) {
|
|
335
|
-
|
|
336
|
-
if (storedDeliveryKeys.has(pairKey)) {
|
|
569
|
+
if (handlers.has(delivery.handler)) {
|
|
337
570
|
continue;
|
|
338
571
|
}
|
|
339
|
-
|
|
572
|
+
handlers.add(delivery.handler);
|
|
573
|
+
addedHandlers.set(event.id, [...addedHandlers.get(event.id) ?? [], delivery.handler]);
|
|
340
574
|
newDeliveries.push(toDeliveryDocument(delivery, event, ordering, now));
|
|
341
|
-
if (storedEvents.has(event.id)) {
|
|
342
|
-
touchedEventIds.add(event.id);
|
|
343
|
-
}
|
|
344
575
|
}
|
|
345
576
|
}
|
|
346
577
|
if (newEvents.length > 0) {
|
|
347
578
|
await this.#events.bulkWrite(
|
|
348
|
-
newEvents.map((
|
|
579
|
+
newEvents.map(({ event, sequence }) => {
|
|
580
|
+
const document = toEventDocument(event, sequence, addedHandlers.get(event.id) ?? [], now);
|
|
581
|
+
return {
|
|
582
|
+
updateOne: {
|
|
583
|
+
filter: { _id: document._id },
|
|
584
|
+
update: { $setOnInsert: document },
|
|
585
|
+
upsert: true
|
|
586
|
+
}
|
|
587
|
+
};
|
|
588
|
+
}),
|
|
589
|
+
{ session, ordered: true }
|
|
590
|
+
);
|
|
591
|
+
}
|
|
592
|
+
const extendedEvents = [...addedHandlers].filter(([eventId]) => storedEvents.has(eventId));
|
|
593
|
+
if (extendedEvents.length > 0) {
|
|
594
|
+
await this.#events.bulkWrite(
|
|
595
|
+
extendedEvents.map(([eventId, handlers]) => ({
|
|
349
596
|
updateOne: {
|
|
350
|
-
filter: { _id:
|
|
351
|
-
update: { $
|
|
352
|
-
upsert: true
|
|
597
|
+
filter: { _id: eventId },
|
|
598
|
+
update: { $inc: { revision: 1 }, $push: { handlers: { $each: handlers } } }
|
|
353
599
|
}
|
|
354
600
|
})),
|
|
355
601
|
{ session, ordered: true }
|
|
356
602
|
);
|
|
357
603
|
}
|
|
358
|
-
if (touchedEventIds.size > 0) {
|
|
359
|
-
await this.#events.updateMany(
|
|
360
|
-
{ _id: { $in: [...touchedEventIds] } },
|
|
361
|
-
{ $inc: { revision: 1 } },
|
|
362
|
-
{ session }
|
|
363
|
-
);
|
|
364
|
-
}
|
|
365
604
|
if (newDeliveries.length > 0) {
|
|
366
605
|
await this.#deliveries.bulkWrite(
|
|
367
606
|
newDeliveries.map((document) => ({
|
|
@@ -376,86 +615,371 @@ var MongoEventStore = class {
|
|
|
376
615
|
}
|
|
377
616
|
return { appendedEventIds, duplicateEventIds, appendedDeliveryCount: newDeliveries.length };
|
|
378
617
|
}
|
|
618
|
+
/**
|
|
619
|
+
* Claims up to `limit` due deliveries. It claims from the lanes this owner
|
|
620
|
+
* (and this store instance) holds under a live lease first, then takes free
|
|
621
|
+
* lanes with due work, a few at a time (4, 8, 16, ...) and the lane with the
|
|
622
|
+
* oldest due delivery first, so a worker holds only as many lanes as it
|
|
623
|
+
* needs. It then records the batch with one write per lane and hands back the
|
|
624
|
+
* lanes that yielded nothing and run nothing. No delivery document is
|
|
625
|
+
* written, except for the attempt charged to deliveries of a lane taken over
|
|
626
|
+
* and for parking (see {@link MongoEventStore.#parkDeliveries}).
|
|
627
|
+
*
|
|
628
|
+
* Taking a lane is one conditional `findOneAndUpdate` that also moves the
|
|
629
|
+
* previous owner's in-flight list to the lane's `lost` list; a lane another
|
|
630
|
+
* claimer took first is skipped, never waited for. Lost deliveries are charged
|
|
631
|
+
* one attempt before anything is read from the lane, and `lost` is cleared
|
|
632
|
+
* only after that. Whoever holds the lane next charges a `lost` list it finds
|
|
633
|
+
* first, so a claim that fails anywhere after the take loses no charge; the
|
|
634
|
+
* charge of a lost claim is recorded on the delivery, so charging the list
|
|
635
|
+
* again after a failure before the clear charges nothing twice.
|
|
636
|
+
* Nothing a failed claim did leaves a delivery listed as running.
|
|
637
|
+
*/
|
|
379
638
|
async claim(request) {
|
|
380
639
|
if (request.handlers.length === 0 || request.limit <= 0) {
|
|
381
640
|
return [];
|
|
382
641
|
}
|
|
383
|
-
|
|
384
|
-
const
|
|
385
|
-
const
|
|
386
|
-
const
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
642
|
+
const owner = request.leaseOwner;
|
|
643
|
+
const previous = this.#claimsInProgress.get(owner) ?? Promise.resolve();
|
|
644
|
+
const claimed = previous.then(() => this.#claimReconciled(request));
|
|
645
|
+
const settled = Promise.allSettled([claimed]);
|
|
646
|
+
this.#claimsInProgress.set(owner, settled);
|
|
647
|
+
void settled.then(() => {
|
|
648
|
+
if (this.#claimsInProgress.get(owner) === settled) {
|
|
649
|
+
this.#claimsInProgress.delete(owner);
|
|
391
650
|
}
|
|
392
|
-
);
|
|
651
|
+
});
|
|
652
|
+
return claimed;
|
|
653
|
+
}
|
|
654
|
+
/**
|
|
655
|
+
* Runs one claim and remembers what it hands out. When the claim fails at any
|
|
656
|
+
* point, also after its record reached the database without the reply
|
|
657
|
+
* reaching this process, the lanes this owner holds on this instance are
|
|
658
|
+
* reconciled: every listed delivery this instance has not handed out, or has
|
|
659
|
+
* handed out and seen settled, is taken out of them, so nothing stays listed
|
|
660
|
+
* as running that nobody runs.
|
|
661
|
+
*/
|
|
662
|
+
async #claimReconciled(request) {
|
|
663
|
+
const owner = request.leaseOwner;
|
|
664
|
+
if (this.#unreconciledOwners.has(owner)) {
|
|
665
|
+
await this.#reconcileLanes(owner);
|
|
666
|
+
this.#unreconciledOwners.delete(owner);
|
|
667
|
+
}
|
|
393
668
|
try {
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
669
|
+
const claimed = await this.#claimNow(request);
|
|
670
|
+
for (const delivery of claimed) {
|
|
671
|
+
const { laneId, epoch } = parseLeaseToken(delivery.leaseToken);
|
|
672
|
+
this.#handedOut.set(leaseKey(delivery.deliveryId, delivery.leaseToken), {
|
|
673
|
+
deliveryId: delivery.deliveryId,
|
|
674
|
+
owner: request.leaseOwner,
|
|
675
|
+
laneId,
|
|
676
|
+
epoch
|
|
677
|
+
});
|
|
678
|
+
}
|
|
679
|
+
return claimed;
|
|
680
|
+
} catch (error) {
|
|
681
|
+
const [reconciled] = await Promise.allSettled([this.#reconcileLanes(owner)]);
|
|
682
|
+
if (reconciled.status === "rejected") {
|
|
683
|
+
this.#unreconciledOwners.add(owner);
|
|
684
|
+
}
|
|
685
|
+
throw error;
|
|
686
|
+
}
|
|
687
|
+
}
|
|
688
|
+
/**
|
|
689
|
+
* Takes every delivery this instance does not run out of the lanes `owner`
|
|
690
|
+
* holds on it: every listed delivery whose current claim (lane epoch and
|
|
691
|
+
* claim generation now) was not handed out by this instance, or was settled.
|
|
692
|
+
* One transaction, and the lanes are written first under their epoch and
|
|
693
|
+
* holder: a worker taking a lane over meanwhile conflicts with it, so its
|
|
694
|
+
* claim is never touched. The claim generation of every delivery taken out
|
|
695
|
+
* moves on, so no lease of those claims (a completion that failed and is
|
|
696
|
+
* retried later, say) can ever match a later claim of the same delivery.
|
|
697
|
+
* Registrations of this owner that are not the current claim of a listed
|
|
698
|
+
* delivery are dropped.
|
|
699
|
+
*/
|
|
700
|
+
async #reconcileLanes(owner) {
|
|
701
|
+
await this.transaction(async (session) => {
|
|
702
|
+
const held = await this.#lanes.find({ owner, holder: this.#instanceId }, { session }).toArray();
|
|
703
|
+
const listed = held.flatMap((lane) => lane.inFlight);
|
|
704
|
+
const claimsById = new Map(
|
|
705
|
+
(await this.#deliveries.find(
|
|
706
|
+
{ _id: { $in: listed }, status: "pending" },
|
|
707
|
+
{ session, projection: { claims: 1 } }
|
|
708
|
+
).toArray()).map((document) => [document._id, document.claims ?? 0])
|
|
709
|
+
);
|
|
710
|
+
const current = /* @__PURE__ */ new Set();
|
|
711
|
+
const unclaimed = held.map((lane) => ({
|
|
712
|
+
lane,
|
|
713
|
+
ids: lane.inFlight.filter((id) => {
|
|
714
|
+
const key = leaseKey(id, formatLeaseToken(lane._id, lane.epoch, claimsById.get(id) ?? 0));
|
|
715
|
+
current.add(key);
|
|
716
|
+
return !this.#handedOut.has(key);
|
|
717
|
+
})
|
|
718
|
+
}));
|
|
719
|
+
for (const [key, entry] of this.#handedOut) {
|
|
720
|
+
if (entry.owner === owner && !current.has(key)) {
|
|
721
|
+
this.#handedOut.delete(key);
|
|
722
|
+
}
|
|
723
|
+
}
|
|
724
|
+
if (held.length === 0) {
|
|
725
|
+
return;
|
|
726
|
+
}
|
|
727
|
+
await this.#lanes.bulkWrite(
|
|
728
|
+
unclaimed.map(({ lane, ids }) => ({
|
|
729
|
+
updateOne: {
|
|
730
|
+
filter: { _id: lane._id, epoch: lane.epoch, holder: this.#instanceId },
|
|
731
|
+
update: { $pull: { inFlight: { $in: ids } }, $inc: { revision: 1 } }
|
|
400
732
|
}
|
|
733
|
+
})),
|
|
734
|
+
{ session, ordered: false }
|
|
735
|
+
);
|
|
736
|
+
await this.#deliveries.updateMany(
|
|
737
|
+
{ _id: { $in: unclaimed.flatMap((entry) => entry.ids) }, status: "pending" },
|
|
738
|
+
{ $inc: { claims: 1 } },
|
|
739
|
+
{ session }
|
|
740
|
+
);
|
|
741
|
+
});
|
|
742
|
+
}
|
|
743
|
+
/**
|
|
744
|
+
* Runs `operation`, which ends the claims of `leases` (completion, retry,
|
|
745
|
+
* release, burial), and once it returned forgets exactly those leases as
|
|
746
|
+
* handed out. When it throws, they stay registered: the caller may retry
|
|
747
|
+
* with the same lease, which must still find its claim.
|
|
748
|
+
*/
|
|
749
|
+
async #settle(leases, operation) {
|
|
750
|
+
const result = await operation();
|
|
751
|
+
for (const lease of leases) {
|
|
752
|
+
this.#handedOut.delete(leaseKey(String(lease.deliveryId), String(lease.leaseToken)));
|
|
753
|
+
}
|
|
754
|
+
return result;
|
|
755
|
+
}
|
|
756
|
+
async #claimNow(request) {
|
|
757
|
+
const [held] = await Promise.all([
|
|
758
|
+
this.#lanes.find({
|
|
759
|
+
owner: request.leaseOwner,
|
|
760
|
+
holder: this.#instanceId,
|
|
761
|
+
leaseExpiresAt: { $gt: request.now }
|
|
762
|
+
}).toArray(),
|
|
763
|
+
this.#unparkNextDeliveries(request.handlers)
|
|
764
|
+
]);
|
|
765
|
+
await this.#chargeLost(held);
|
|
766
|
+
const scope = new Map(
|
|
767
|
+
held.map((lane) => [
|
|
768
|
+
lane._id,
|
|
769
|
+
{
|
|
770
|
+
laneId: lane._id,
|
|
771
|
+
epoch: lane.epoch,
|
|
772
|
+
listed: lane.inFlight,
|
|
773
|
+
revision: lane.revision ?? 0
|
|
401
774
|
}
|
|
775
|
+
])
|
|
776
|
+
);
|
|
777
|
+
const running = this.#findRunning(held.flatMap((lane) => lane.inFlight)).then((documents) => ({
|
|
778
|
+
ids: new Set(documents.map((document) => document._id)),
|
|
779
|
+
keys: new Set(
|
|
780
|
+
documents.flatMap(
|
|
781
|
+
(document) => document.key === void 0 ? [] : [orderingIdentity(document.handler, document.key)]
|
|
782
|
+
)
|
|
783
|
+
)
|
|
784
|
+
}));
|
|
785
|
+
const scan = new ClaimScan();
|
|
786
|
+
const picked = [];
|
|
787
|
+
const pickFrom = async (lanes) => {
|
|
788
|
+
for (let offset = 0; offset < lanes.length; offset += LANES_PER_CANDIDATE_QUERY) {
|
|
789
|
+
if (picked.length < request.limit) {
|
|
790
|
+
await this.#pickCandidates(
|
|
791
|
+
lanes.slice(offset, offset + LANES_PER_CANDIDATE_QUERY),
|
|
792
|
+
request,
|
|
793
|
+
scan,
|
|
794
|
+
running,
|
|
795
|
+
picked
|
|
796
|
+
);
|
|
797
|
+
}
|
|
798
|
+
}
|
|
799
|
+
};
|
|
800
|
+
await pickFrom([...scope.values()]);
|
|
801
|
+
const passed = new Set(scope.keys());
|
|
802
|
+
for (let laneCount = INITIAL_LANES_PER_ROUND; picked.length < request.limit; laneCount *= 2) {
|
|
803
|
+
const free = await this.#findFreeLanes(request, laneCount, [...passed]);
|
|
804
|
+
if (free.length === 0) {
|
|
805
|
+
break;
|
|
806
|
+
}
|
|
807
|
+
for (const laneId of free) {
|
|
808
|
+
passed.add(laneId);
|
|
809
|
+
}
|
|
810
|
+
const taken = await this.#takeLanes(free, request);
|
|
811
|
+
for (const lane of taken) {
|
|
812
|
+
scope.set(lane.laneId, lane);
|
|
813
|
+
}
|
|
814
|
+
await pickFrom(taken);
|
|
815
|
+
}
|
|
816
|
+
const events = await this.#readEnvelopes(picked.map((document) => document.eventId));
|
|
817
|
+
const runningIds = (await running).ids;
|
|
818
|
+
const gone = new Set(
|
|
819
|
+
held.flatMap(
|
|
820
|
+
(lane) => lane.inFlight.filter((id) => !runningIds.has(id)).map((id) => `${id}\0${lane._id}:${lane.epoch}`)
|
|
821
|
+
)
|
|
822
|
+
);
|
|
823
|
+
for (const [key, entry] of this.#handedOut) {
|
|
824
|
+
if (entry.owner === request.leaseOwner && gone.has(`${entry.deliveryId}\0${entry.laneId}:${entry.epoch}`)) {
|
|
825
|
+
this.#handedOut.delete(key);
|
|
402
826
|
}
|
|
403
|
-
} finally {
|
|
404
|
-
await candidates.close();
|
|
405
827
|
}
|
|
406
|
-
await this.#
|
|
407
|
-
|
|
828
|
+
const { recorded, handedBack } = await this.#recordClaim(scope, picked, runningIds, request);
|
|
829
|
+
const claimed = await this.#releaseOvertaken(
|
|
830
|
+
picked.filter((document) => recorded.has(document.laneId)),
|
|
831
|
+
scope
|
|
832
|
+
);
|
|
833
|
+
await this.#parkWaiting(scan.waitingBacklogs(PARK_BACKLOG_THRESHOLD, handedBack));
|
|
834
|
+
return claimed.map((document) => ({
|
|
835
|
+
deliveryId: document._id,
|
|
836
|
+
leaseToken: formatLeaseToken(
|
|
837
|
+
document.laneId,
|
|
838
|
+
scope.get(document.laneId).epoch,
|
|
839
|
+
document.claims ?? 0
|
|
840
|
+
),
|
|
841
|
+
handler: document.handler,
|
|
842
|
+
attempt: document.attempts + 1,
|
|
843
|
+
leaseExpiresAt: request.leaseExpiresAt,
|
|
844
|
+
event: events.get(document.eventId)
|
|
845
|
+
}));
|
|
408
846
|
}
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
847
|
+
/**
|
|
848
|
+
* Extends the lease of the lane a running delivery belongs to. The lane lease
|
|
849
|
+
* never shrinks, so a repeated renewal with the same value still holds. An
|
|
850
|
+
* expired lease nobody took over is still renewed, so `_now` is not needed.
|
|
851
|
+
*/
|
|
852
|
+
async extendLease(lease, leaseExpiresAt, _now) {
|
|
853
|
+
const held = parseLeaseToken(lease.leaseToken);
|
|
854
|
+
if (held === void 0) {
|
|
855
|
+
return false;
|
|
856
|
+
}
|
|
857
|
+
const deliveryId = String(lease.deliveryId);
|
|
858
|
+
return this.transaction(async (session) => {
|
|
859
|
+
const running = await this.#deliveries.findOne(ofClaim({ ...held, deliveryId }), {
|
|
860
|
+
session,
|
|
861
|
+
projection: { _id: 1 }
|
|
862
|
+
});
|
|
863
|
+
if (running === null) {
|
|
864
|
+
return false;
|
|
865
|
+
}
|
|
866
|
+
const renewed = await this.#lanes.updateOne(
|
|
867
|
+
{ _id: held.laneId, epoch: held.epoch, inFlight: deliveryId },
|
|
868
|
+
{ $max: { leaseExpiresAt } },
|
|
869
|
+
{ session }
|
|
870
|
+
);
|
|
871
|
+
return renewed.matchedCount === 1;
|
|
412
872
|
});
|
|
413
873
|
}
|
|
874
|
+
/**
|
|
875
|
+
* Deletes the delivery (and records it in the completion history, when kept).
|
|
876
|
+
* With `session` the deletion commits with the caller's transaction; without,
|
|
877
|
+
* it runs in a transaction of its own. Returns false when the lease was lost.
|
|
878
|
+
*/
|
|
414
879
|
async complete(lease, now, session) {
|
|
415
880
|
if (session !== void 0) {
|
|
416
881
|
assertInTransaction(session, "complete");
|
|
417
882
|
}
|
|
418
|
-
return this.#
|
|
419
|
-
$set: { status: "completed", completedAt: now, updatedAt: now },
|
|
420
|
-
$unset: { ...LEASE_FIELDS, ...FINISHED_FIELDS }
|
|
421
|
-
});
|
|
883
|
+
return (await this.#completeLeases([lease], now, session))[0];
|
|
422
884
|
}
|
|
885
|
+
/**
|
|
886
|
+
* Completes many leases with a fixed number of round trips, whatever lanes
|
|
887
|
+
* they belong to: three commands without a completion history, one
|
|
888
|
+
* transaction with it (see {@link MongoEventStore.#deleteFenced}).
|
|
889
|
+
* Returns one flag per lease in input order; a lease listed twice completes
|
|
890
|
+
* once, and a token this store cannot have issued is refused.
|
|
891
|
+
*/
|
|
892
|
+
completeMany(leases, now) {
|
|
893
|
+
return this.#completeLeases(leases, now, void 0);
|
|
894
|
+
}
|
|
895
|
+
/** Returns a running delivery to pending, due at `availableAt`, and counts the attempt that ended. */
|
|
423
896
|
async reschedule(lease, availableAt, error, now) {
|
|
424
|
-
return this.#
|
|
425
|
-
$set: {
|
|
426
|
-
|
|
427
|
-
availableAt,
|
|
428
|
-
dueAt: availableAt,
|
|
429
|
-
lastError: toErrorInfo(error),
|
|
430
|
-
updatedAt: now
|
|
431
|
-
},
|
|
432
|
-
$unset: LEASE_FIELDS
|
|
897
|
+
return this.#leaveFlight(lease, {
|
|
898
|
+
$set: { availableAt, dueAt: availableAt, lastError: toErrorInfo(error), updatedAt: now },
|
|
899
|
+
$inc: { attempts: 1, claims: 1 }
|
|
433
900
|
});
|
|
434
901
|
}
|
|
902
|
+
/** Gives a running delivery back without counting the attempt: pending, due `now`. */
|
|
435
903
|
async release(lease, now) {
|
|
436
|
-
return this.#
|
|
437
|
-
$set: {
|
|
438
|
-
$inc: {
|
|
439
|
-
$unset: LEASE_FIELDS
|
|
904
|
+
return this.#leaveFlight(lease, {
|
|
905
|
+
$set: { availableAt: now, dueAt: now, updatedAt: now },
|
|
906
|
+
$inc: { claims: 1 }
|
|
440
907
|
});
|
|
441
908
|
}
|
|
909
|
+
/** Moves a running delivery to dead and counts the attempt that ended. */
|
|
442
910
|
async bury(lease, error, now) {
|
|
443
|
-
return this.#
|
|
911
|
+
return this.#leaveFlight(lease, {
|
|
444
912
|
$set: { status: "dead", lastError: toErrorInfo(error), updatedAt: now },
|
|
445
|
-
$unset:
|
|
913
|
+
$unset: FINISHED_FIELDS,
|
|
914
|
+
$inc: { attempts: 1, claims: 1 }
|
|
446
915
|
});
|
|
447
916
|
}
|
|
917
|
+
/**
|
|
918
|
+
* Frees every lane `leaseOwner` holds that runs no delivery, so other workers
|
|
919
|
+
* need not wait for the lease to expire. Lanes with running deliveries stay leased.
|
|
920
|
+
*/
|
|
921
|
+
async relinquish(leaseOwner) {
|
|
922
|
+
const held = await this.#lanes.find({ owner: String(leaseOwner), holder: this.#instanceId }).toArray();
|
|
923
|
+
const runningIds = new Set(
|
|
924
|
+
(await this.#findRunning(held.flatMap((lane) => lane.inFlight))).map(
|
|
925
|
+
(document) => document._id
|
|
926
|
+
)
|
|
927
|
+
);
|
|
928
|
+
const idle = held.filter((lane) => lane.inFlight.every((id) => !runningIds.has(id)));
|
|
929
|
+
if (idle.length > 0) {
|
|
930
|
+
await this.#lanes.bulkWrite(
|
|
931
|
+
idle.map((lane) => freeLane(lane._id, lane.epoch, lane.revision ?? 0, lane.inFlight)),
|
|
932
|
+
{ ordered: false }
|
|
933
|
+
);
|
|
934
|
+
}
|
|
935
|
+
}
|
|
936
|
+
/**
|
|
937
|
+
* Lists deliveries in creation order. A delivery is `processing` while its
|
|
938
|
+
* document exists as `pending` and its lane lists it; its attempt count then
|
|
939
|
+
* includes the running attempt, and owner and expiry are those of the lane
|
|
940
|
+
* lease. Completion history records (when kept) are listed as `completed`.
|
|
941
|
+
*/
|
|
448
942
|
async listDeliveries(query) {
|
|
449
|
-
const
|
|
450
|
-
|
|
943
|
+
const status = query.status === void 0 ? void 0 : String(query.status);
|
|
944
|
+
const common = {
|
|
451
945
|
...query.handler === void 0 ? {} : { handler: String(query.handler) },
|
|
452
946
|
...query.eventType === void 0 ? {} : { eventType: String(query.eventType) },
|
|
453
947
|
...query.eventId === void 0 ? {} : { eventId: String(query.eventId) }
|
|
454
948
|
};
|
|
455
|
-
const
|
|
456
|
-
const
|
|
457
|
-
|
|
949
|
+
const limit = query.limit ?? DEFAULT_LIST_LIMIT;
|
|
950
|
+
const lanes = await this.#lanes.find({ "inFlight.0": { $exists: true } }, { projection: { epoch: 0 } }).toArray();
|
|
951
|
+
const listed = lanes.flatMap((lane) => lane.inFlight);
|
|
952
|
+
const derivedStatuses = /* @__PURE__ */ new Map([
|
|
953
|
+
["processing", { status: "pending", _id: { $in: listed } }],
|
|
954
|
+
["pending", { status: "pending", _id: { $nin: listed } }]
|
|
955
|
+
]);
|
|
956
|
+
const filter = {
|
|
957
|
+
...common,
|
|
958
|
+
...status === void 0 ? {} : derivedStatuses.get(status) ?? { status }
|
|
959
|
+
};
|
|
960
|
+
const sort = { createdAt: 1, _id: 1 };
|
|
961
|
+
const documents = await this.#deliveries.find(filter, { sort, limit }).toArray();
|
|
962
|
+
const history = this.#completionHistory && (status === void 0 || status === "completed") ? await this.#completions.find(common, { sort, limit }).toArray() : [];
|
|
963
|
+
const events = await this.#readEnvelopes([
|
|
964
|
+
...documents.map((document) => document.eventId),
|
|
965
|
+
...history.map((document) => document.eventId)
|
|
966
|
+
]);
|
|
967
|
+
const laneById = new Map(lanes.map((lane) => [lane._id, lane]));
|
|
968
|
+
const records = [
|
|
969
|
+
...documents.map((document) => {
|
|
970
|
+
const lane = laneById.get(document.laneId);
|
|
971
|
+
const runs = document.status === "pending" && lane?.inFlight.includes(document._id) === true;
|
|
972
|
+
return toDeliveryRecord(document, events.get(document.eventId), runs ? lane : void 0);
|
|
973
|
+
}),
|
|
974
|
+
...history.map(
|
|
975
|
+
(document) => toCompletedDeliveryRecord(document, events.get(document.eventId))
|
|
976
|
+
)
|
|
977
|
+
];
|
|
978
|
+
return records.sort(
|
|
979
|
+
(left, right) => left.createdAt.getTime() - right.createdAt.getTime() || compareText(left.id, right.id)
|
|
980
|
+
).slice(0, limit);
|
|
458
981
|
}
|
|
982
|
+
/** Moves dead deliveries back to pending with a fresh attempt budget. */
|
|
459
983
|
async requeue(deliveryIds, now) {
|
|
460
984
|
const result = await this.#deliveries.updateMany(
|
|
461
985
|
{ _id: { $in: deliveryIds.map(String) }, status: "dead" },
|
|
@@ -472,38 +996,93 @@ var MongoEventStore = class {
|
|
|
472
996
|
);
|
|
473
997
|
return result.modifiedCount;
|
|
474
998
|
}
|
|
999
|
+
/**
|
|
1000
|
+
* Deletes, in batches of `purgeBatchSize`, dead deliveries last updated before
|
|
1001
|
+
* `deadBefore`, and completion history records (and `completed` documents left
|
|
1002
|
+
* by 0.1.0) completed before `completedBefore`, each batch in one transaction
|
|
1003
|
+
* together with the events it left without any reference. Finally, events
|
|
1004
|
+
* nothing references are deleted: one with a completed delivery once its last
|
|
1005
|
+
* completion is before `completedBefore` (never by `deadBefore` alone), one
|
|
1006
|
+
* that never had a completion (no subscribers, or only dead letters) once it
|
|
1007
|
+
* was created before the earliest given cutoff. The batch step applies the
|
|
1008
|
+
* same rule to events whose last completion is recorded.
|
|
1009
|
+
*/
|
|
475
1010
|
async purge(request) {
|
|
476
|
-
const
|
|
477
|
-
|
|
478
|
-
|
|
1011
|
+
const total = { deletedDeliveries: 0, deletedEvents: 0 };
|
|
1012
|
+
const steps = [];
|
|
1013
|
+
const { completedBefore, deadBefore } = request;
|
|
1014
|
+
if (completedBefore !== void 0) {
|
|
1015
|
+
steps.push(
|
|
1016
|
+
() => this.#purgeMatching(
|
|
1017
|
+
this.#purgeable(this.#deliveries.collectionName),
|
|
1018
|
+
{ status: "completed", completedAt: { $lt: completedBefore } },
|
|
1019
|
+
completedBefore
|
|
1020
|
+
)
|
|
1021
|
+
);
|
|
1022
|
+
if (this.#completionHistory) {
|
|
1023
|
+
steps.push(
|
|
1024
|
+
() => this.#purgeMatching(
|
|
1025
|
+
this.#purgeable(this.#completions.collectionName),
|
|
1026
|
+
{ completedAt: { $lt: completedBefore } },
|
|
1027
|
+
completedBefore
|
|
1028
|
+
)
|
|
1029
|
+
);
|
|
1030
|
+
}
|
|
479
1031
|
}
|
|
480
|
-
if (
|
|
481
|
-
|
|
1032
|
+
if (deadBefore !== void 0) {
|
|
1033
|
+
steps.push(
|
|
1034
|
+
() => this.#purgeMatching(
|
|
1035
|
+
this.#purgeable(this.#deliveries.collectionName),
|
|
1036
|
+
{ status: "dead", updatedAt: { $lt: deadBefore } },
|
|
1037
|
+
completedBefore
|
|
1038
|
+
)
|
|
1039
|
+
);
|
|
482
1040
|
}
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
deletedDeliveries += result.deletedDeliveries;
|
|
488
|
-
deletedEvents += result.deletedEvents;
|
|
1041
|
+
for (const step of steps) {
|
|
1042
|
+
const result = await step();
|
|
1043
|
+
total.deletedDeliveries += result.deletedDeliveries;
|
|
1044
|
+
total.deletedEvents += result.deletedEvents;
|
|
489
1045
|
}
|
|
490
|
-
|
|
1046
|
+
const orphanCutoff = [completedBefore, deadBefore].filter((cutoff) => cutoff !== void 0).sort((first, second) => first.getTime() - second.getTime())[0];
|
|
1047
|
+
if (orphanCutoff !== void 0) {
|
|
1048
|
+
total.deletedEvents += await this.#purgeOrphanEvents(orphanCutoff, completedBefore);
|
|
1049
|
+
}
|
|
1050
|
+
return total;
|
|
491
1051
|
}
|
|
1052
|
+
/**
|
|
1053
|
+
* Counts by derived status. `pending` excludes running deliveries; `completed`
|
|
1054
|
+
* counts history records and 0.1.0 `completed` documents. `oldestDueAt` is the
|
|
1055
|
+
* oldest instant at which a delivery became claimable: the due time of a
|
|
1056
|
+
* pending delivery nobody runs, or the lease expiry of a lane that still runs
|
|
1057
|
+
* deliveries.
|
|
1058
|
+
*/
|
|
492
1059
|
async statistics(now) {
|
|
493
|
-
const
|
|
494
|
-
const [
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
1060
|
+
const lanes = await this.#lanes.find({ "inFlight.0": { $exists: true } }, { projection: { inFlight: 1, leaseExpiresAt: 1 } }).toArray();
|
|
1061
|
+
const [stored, dead, legacyCompleted, history, running] = await Promise.all([
|
|
1062
|
+
this.#deliveries.countDocuments({ status: "pending" }),
|
|
1063
|
+
this.#deliveries.countDocuments({ status: "dead" }),
|
|
1064
|
+
this.#deliveries.countDocuments({ status: "completed" }),
|
|
1065
|
+
this.#completionHistory ? this.#completions.countDocuments() : Promise.resolve(0),
|
|
1066
|
+
this.#findRunning(lanes.flatMap((lane) => lane.inFlight))
|
|
1067
|
+
]);
|
|
1068
|
+
const runningIds = new Set(running.map((document) => document._id));
|
|
1069
|
+
const waiting = await this.#deliveries.findOne(
|
|
1070
|
+
{ dueAt: { $lte: now }, _id: { $nin: [...runningIds] } },
|
|
499
1071
|
{ sort: { dueAt: 1 }, projection: { dueAt: 1 } }
|
|
500
1072
|
);
|
|
1073
|
+
const dueTimes = lanes.filter(
|
|
1074
|
+
(lane) => lane.leaseExpiresAt !== null && lane.leaseExpiresAt <= now && lane.inFlight.some((id) => runningIds.has(id))
|
|
1075
|
+
).map((lane) => lane.leaseExpiresAt.getTime());
|
|
1076
|
+
if (waiting !== null) {
|
|
1077
|
+
dueTimes.push(waiting.dueAt.getTime());
|
|
1078
|
+
}
|
|
501
1079
|
return {
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
1080
|
+
// The counts are separate reads; a completion between them must not make this negative.
|
|
1081
|
+
pending: Math.max(0, stored - running.length),
|
|
1082
|
+
processing: running.length,
|
|
1083
|
+
completed: legacyCompleted + history,
|
|
505
1084
|
dead,
|
|
506
|
-
oldestDueAt:
|
|
1085
|
+
oldestDueAt: dueTimes.length === 0 ? null : new Date(Math.min(...dueTimes))
|
|
507
1086
|
};
|
|
508
1087
|
}
|
|
509
1088
|
/** Atomically increments the counter of `key`; conflicting appends of the key serialize here. */
|
|
@@ -515,92 +1094,376 @@ var MongoEventStore = class {
|
|
|
515
1094
|
);
|
|
516
1095
|
return counter.value;
|
|
517
1096
|
}
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
if (
|
|
521
|
-
return
|
|
1097
|
+
/** The listed deliveries that run: those whose document still exists as pending. */
|
|
1098
|
+
async #findRunning(listed) {
|
|
1099
|
+
if (listed.length === 0) {
|
|
1100
|
+
return [];
|
|
522
1101
|
}
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
1102
|
+
return this.#deliveries.find(
|
|
1103
|
+
{ _id: { $in: [...listed] }, status: "pending" },
|
|
1104
|
+
{ projection: { handler: 1, key: 1, laneId: 1 } }
|
|
1105
|
+
).toArray();
|
|
1106
|
+
}
|
|
1107
|
+
/**
|
|
1108
|
+
* Up to `laneCount` lanes nobody holds under a live lease that have a due,
|
|
1109
|
+
* ready delivery of the requested handlers, the lane with the oldest such
|
|
1110
|
+
* delivery first, leaving out `passed`. One round trip: the oldest delivery
|
|
1111
|
+
* of each lane is one index probe on the server. A lane whose due deliveries
|
|
1112
|
+
* all wait behind an unfinished predecessor is taken at most once: the claim
|
|
1113
|
+
* that hands it back parks them (see {@link ClaimScan.waitingBacklogs}).
|
|
1114
|
+
*/
|
|
1115
|
+
async #findFreeLanes(request, laneCount, passed) {
|
|
1116
|
+
const lanes = await this.#lanes.aggregate([
|
|
1117
|
+
{
|
|
1118
|
+
$match: {
|
|
1119
|
+
_id: { $nin: passed },
|
|
1120
|
+
$or: [{ leaseExpiresAt: null }, { leaseExpiresAt: { $lte: request.now } }]
|
|
1121
|
+
}
|
|
1122
|
+
},
|
|
1123
|
+
{
|
|
1124
|
+
$lookup: {
|
|
1125
|
+
from: this.#deliveries.collectionName,
|
|
1126
|
+
localField: "_id",
|
|
1127
|
+
foreignField: "laneId",
|
|
1128
|
+
pipeline: [
|
|
1129
|
+
{
|
|
1130
|
+
$match: {
|
|
1131
|
+
handler: { $in: [...request.handlers] },
|
|
1132
|
+
lane: "ready",
|
|
1133
|
+
dueAt: { $lte: request.now }
|
|
1134
|
+
}
|
|
1135
|
+
},
|
|
1136
|
+
{ $sort: { dueAt: 1, _id: 1 } },
|
|
1137
|
+
{ $limit: 1 },
|
|
1138
|
+
{ $project: { dueAt: 1 } }
|
|
1139
|
+
],
|
|
1140
|
+
as: "oldest"
|
|
1141
|
+
}
|
|
1142
|
+
},
|
|
1143
|
+
{ $unwind: "$oldest" },
|
|
1144
|
+
{ $sort: { "oldest.dueAt": 1, "oldest._id": 1 } },
|
|
1145
|
+
{ $limit: laneCount },
|
|
1146
|
+
{ $project: { _id: 1 } }
|
|
1147
|
+
]).toArray();
|
|
1148
|
+
return lanes.map((lane) => lane._id);
|
|
1149
|
+
}
|
|
1150
|
+
/**
|
|
1151
|
+
* Takes the lanes with one conditional write each, run in parallel; a lane
|
|
1152
|
+
* taken by someone else meanwhile is skipped. The same write moves the
|
|
1153
|
+
* previous owner's in-flight list to `lost`, which is charged before any
|
|
1154
|
+
* delivery of the lanes is read (see {@link MongoEventStore.#chargeLost}).
|
|
1155
|
+
*/
|
|
1156
|
+
async #takeLanes(laneIds, request) {
|
|
1157
|
+
const after = await Promise.all(
|
|
1158
|
+
laneIds.map(
|
|
1159
|
+
(laneId) => this.#lanes.findOneAndUpdate(
|
|
1160
|
+
{
|
|
1161
|
+
_id: laneId,
|
|
1162
|
+
$or: [{ leaseExpiresAt: null }, { leaseExpiresAt: { $lte: request.now } }]
|
|
1163
|
+
},
|
|
1164
|
+
[
|
|
1165
|
+
{
|
|
1166
|
+
$set: {
|
|
1167
|
+
epoch: { $add: ["$epoch", 1] },
|
|
1168
|
+
owner: { $literal: request.leaseOwner },
|
|
1169
|
+
holder: { $literal: this.#instanceId },
|
|
1170
|
+
leaseExpiresAt: { $literal: request.leaseExpiresAt },
|
|
1171
|
+
// Each lost delivery keeps the epoch it was claimed under: the key of its charge.
|
|
1172
|
+
lost: {
|
|
1173
|
+
$setUnion: [
|
|
1174
|
+
{ $ifNull: ["$lost", []] },
|
|
1175
|
+
{ $map: { input: "$inFlight", in: { id: "$$this", epoch: "$epoch" } } }
|
|
1176
|
+
]
|
|
1177
|
+
},
|
|
1178
|
+
inFlight: { $literal: [] }
|
|
1179
|
+
}
|
|
1180
|
+
}
|
|
1181
|
+
],
|
|
1182
|
+
{ returnDocument: "after" }
|
|
1183
|
+
)
|
|
1184
|
+
)
|
|
1185
|
+
);
|
|
1186
|
+
const taken = after.filter((lane) => lane !== null);
|
|
1187
|
+
await this.#chargeLost(taken);
|
|
1188
|
+
return taken.map((lane) => ({
|
|
1189
|
+
laneId: lane._id,
|
|
1190
|
+
epoch: lane.epoch,
|
|
1191
|
+
listed: lane.inFlight,
|
|
1192
|
+
revision: lane.revision ?? 0
|
|
1193
|
+
}));
|
|
1194
|
+
}
|
|
1195
|
+
/**
|
|
1196
|
+
* Charges one attempt to every pending delivery listed in the `lost` lists of
|
|
1197
|
+
* `lanes` (deliveries whose lane was taken while they ran), then clears those
|
|
1198
|
+
* lists under the lanes' current epochs. The charge is a write on each lost
|
|
1199
|
+
* delivery that comes before any read of the lane's deliveries: a completion
|
|
1200
|
+
* of the previous owner deletes the same documents, so the two conflict and
|
|
1201
|
+
* cannot both succeed. The charge also moves the claim generation on, which
|
|
1202
|
+
* fences every lease of the lost claim, and records the lost claim
|
|
1203
|
+
* (`lostAt` = lane and epoch it was claimed under): a holder that charges the
|
|
1204
|
+
* list again after a failure before the clear skips deliveries already
|
|
1205
|
+
* charged for that claim.
|
|
1206
|
+
*/
|
|
1207
|
+
async #chargeLost(lanes) {
|
|
1208
|
+
const losing = lanes.filter((lane) => lane.lost.length > 0);
|
|
1209
|
+
if (losing.length === 0) {
|
|
1210
|
+
return;
|
|
528
1211
|
}
|
|
529
|
-
|
|
530
|
-
|
|
1212
|
+
const lost = losing.flatMap((lane) => lane.lost).sort((left, right) => left.epoch - right.epoch);
|
|
1213
|
+
await this.#deliveries.bulkWrite(
|
|
1214
|
+
lost.map(({ id, epoch }) => ({
|
|
1215
|
+
updateOne: {
|
|
1216
|
+
filter: { _id: id, status: "pending", lostAt: { $not: { $gte: epoch } } },
|
|
1217
|
+
update: { $inc: { attempts: 1, claims: 1 }, $set: { lostAt: epoch } }
|
|
1218
|
+
}
|
|
1219
|
+
})),
|
|
1220
|
+
{ ordered: true }
|
|
1221
|
+
);
|
|
1222
|
+
await this.#lanes.bulkWrite(
|
|
1223
|
+
losing.map((lane) => ({
|
|
1224
|
+
updateOne: { filter: { _id: lane._id, epoch: lane.epoch }, update: { $set: { lost: [] } } }
|
|
1225
|
+
})),
|
|
1226
|
+
{ ordered: false }
|
|
1227
|
+
);
|
|
1228
|
+
}
|
|
1229
|
+
/**
|
|
1230
|
+
* Adds the claimable candidates of `lanes` to `picked`, in due order, until
|
|
1231
|
+
* `request.limit` is reached. A delivery is a candidate when it is ready, due
|
|
1232
|
+
* and not in flight in its lane. A keyed candidate is claimable only when
|
|
1233
|
+
* (a) no earlier delivery of its handler and key is pending (a running one is
|
|
1234
|
+
* still a pending document) and (b) no other delivery of them runs. Running
|
|
1235
|
+
* deliveries of a key live in the key's lane, which this owner holds, so the
|
|
1236
|
+
* running keys (loaded once from the held lanes) and this claim's own picks
|
|
1237
|
+
* are all of them.
|
|
1238
|
+
*
|
|
1239
|
+
* Candidates are read in pages with a limit, so no cursor is left open, and
|
|
1240
|
+
* the predecessors of the keys of a page are looked up in parallel, one
|
|
1241
|
+
* lookup per key.
|
|
1242
|
+
*/
|
|
1243
|
+
async #pickCandidates(lanes, request, scan, running, picked) {
|
|
1244
|
+
const pageSize = Math.max((request.limit - picked.length) * 2, MINIMUM_CANDIDATE_BATCH_SIZE);
|
|
1245
|
+
const filter = {
|
|
1246
|
+
laneId: { $in: lanes.map((lane) => lane.laneId) },
|
|
1247
|
+
handler: { $in: [...request.handlers] },
|
|
1248
|
+
lane: "ready",
|
|
1249
|
+
dueAt: { $lte: request.now },
|
|
1250
|
+
_id: { $nin: lanes.flatMap((lane) => lane.listed) }
|
|
1251
|
+
};
|
|
1252
|
+
let after;
|
|
1253
|
+
for (; ; ) {
|
|
1254
|
+
const [page, { keys }] = await Promise.all([
|
|
1255
|
+
this.#deliveries.find(
|
|
1256
|
+
after === void 0 ? filter : {
|
|
1257
|
+
$and: [
|
|
1258
|
+
filter,
|
|
1259
|
+
{
|
|
1260
|
+
$or: [
|
|
1261
|
+
{ dueAt: { $gt: after.dueAt } },
|
|
1262
|
+
{ dueAt: after.dueAt, _id: { $gt: after._id } }
|
|
1263
|
+
]
|
|
1264
|
+
}
|
|
1265
|
+
]
|
|
1266
|
+
},
|
|
1267
|
+
{ sort: { dueAt: 1, _id: 1 }, limit: pageSize }
|
|
1268
|
+
).toArray(),
|
|
1269
|
+
running
|
|
1270
|
+
]);
|
|
1271
|
+
for (let offset = 0; offset < page.length; ) {
|
|
1272
|
+
const slice = page.slice(offset, offset + request.limit - picked.length);
|
|
1273
|
+
offset += slice.length;
|
|
1274
|
+
await this.#notePredecessors(slice, scan);
|
|
1275
|
+
for (const candidate of slice) {
|
|
1276
|
+
if (isClaimable(candidate, scan, keys)) {
|
|
1277
|
+
picked.push(candidate);
|
|
1278
|
+
}
|
|
1279
|
+
}
|
|
1280
|
+
if (picked.length >= request.limit) {
|
|
1281
|
+
return;
|
|
1282
|
+
}
|
|
1283
|
+
}
|
|
1284
|
+
if (page.length < pageSize) {
|
|
1285
|
+
return;
|
|
1286
|
+
}
|
|
1287
|
+
after = page.at(-1);
|
|
1288
|
+
}
|
|
1289
|
+
}
|
|
1290
|
+
/**
|
|
1291
|
+
* Looks up, in parallel and once per key, whether an unfinished delivery
|
|
1292
|
+
* precedes the earliest candidate of each keyed (handler, key) of `page` that
|
|
1293
|
+
* this claim knows nothing about yet, and tells `scan`. The earliest candidate
|
|
1294
|
+
* is itself unfinished, so every later candidate of its key waits at least for it.
|
|
1295
|
+
*/
|
|
1296
|
+
async #notePredecessors(page, scan) {
|
|
1297
|
+
const earliest = /* @__PURE__ */ new Map();
|
|
1298
|
+
for (const candidate of page) {
|
|
1299
|
+
if (candidate.key !== void 0) {
|
|
1300
|
+
const orderingId = orderingIdentity(candidate.handler, candidate.key);
|
|
1301
|
+
const known = earliest.get(orderingId);
|
|
1302
|
+
if (!scan.knows(orderingId) && (known?.sequence ?? Infinity) > candidate.sequence) {
|
|
1303
|
+
earliest.set(orderingId, candidate);
|
|
1304
|
+
}
|
|
1305
|
+
}
|
|
531
1306
|
}
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
1307
|
+
await Promise.all(
|
|
1308
|
+
[...earliest].map(async ([orderingId, candidate]) => {
|
|
1309
|
+
const predecessor = await this.#findUnfinishedPredecessor(
|
|
1310
|
+
candidate.handler,
|
|
1311
|
+
candidate.key,
|
|
1312
|
+
candidate.sequence
|
|
1313
|
+
);
|
|
1314
|
+
scan.noteOpen(orderingId, predecessor ?? candidate.sequence);
|
|
1315
|
+
})
|
|
1316
|
+
);
|
|
1317
|
+
}
|
|
1318
|
+
/**
|
|
1319
|
+
* Records the picks in their lanes, one conditional write per lane and one
|
|
1320
|
+
* round trip for all of them, and frees the lanes that yielded nothing and run
|
|
1321
|
+
* nothing. A picked lane lists its running deliveries plus the new ones (ids
|
|
1322
|
+
* whose documents are gone are pruned) and its lease reaches
|
|
1323
|
+
* `request.leaseExpiresAt`. Returns the lanes whose picks were recorded: every
|
|
1324
|
+
* write is conditional on the claim's epoch, owner and store instance, so a
|
|
1325
|
+
* lane that changed hands, or was handed back by `relinquish`, in the meantime
|
|
1326
|
+
* keeps its deliveries for whoever takes it next.
|
|
1327
|
+
*/
|
|
1328
|
+
async #recordClaim(scope, picked, runningIds, request) {
|
|
1329
|
+
const picksByLane = /* @__PURE__ */ new Map();
|
|
1330
|
+
const handedBack = /* @__PURE__ */ new Set();
|
|
1331
|
+
for (const document of picked) {
|
|
1332
|
+
picksByLane.set(document.laneId, [...picksByLane.get(document.laneId) ?? [], document._id]);
|
|
537
1333
|
}
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
1334
|
+
const operations = [];
|
|
1335
|
+
for (const lane of scope.values()) {
|
|
1336
|
+
const gone = lane.listed.filter((id) => !runningIds.has(id));
|
|
1337
|
+
const ids = picksByLane.get(lane.laneId);
|
|
1338
|
+
if (ids !== void 0) {
|
|
1339
|
+
operations.push({
|
|
1340
|
+
updateOne: {
|
|
1341
|
+
filter: {
|
|
1342
|
+
_id: lane.laneId,
|
|
1343
|
+
epoch: lane.epoch,
|
|
1344
|
+
owner: request.leaseOwner,
|
|
1345
|
+
holder: this.#instanceId,
|
|
1346
|
+
// A record that reaches the server late (after its claim failed and reconciled)
|
|
1347
|
+
// finds a newer revision and changes nothing.
|
|
1348
|
+
revision: atCount(lane.revision)
|
|
1349
|
+
},
|
|
1350
|
+
update: [
|
|
1351
|
+
{
|
|
1352
|
+
$set: {
|
|
1353
|
+
inFlight: {
|
|
1354
|
+
$concatArrays: [
|
|
1355
|
+
{
|
|
1356
|
+
$filter: {
|
|
1357
|
+
input: "$inFlight",
|
|
1358
|
+
cond: { $not: [{ $in: ["$$this", { $literal: gone }] }] }
|
|
1359
|
+
}
|
|
1360
|
+
},
|
|
1361
|
+
{ $literal: ids }
|
|
1362
|
+
]
|
|
1363
|
+
},
|
|
1364
|
+
leaseExpiresAt: { $max: ["$leaseExpiresAt", request.leaseExpiresAt] },
|
|
1365
|
+
revision: { $add: [{ $ifNull: ["$revision", 0] }, 1] }
|
|
1366
|
+
}
|
|
1367
|
+
}
|
|
1368
|
+
]
|
|
1369
|
+
}
|
|
1370
|
+
});
|
|
1371
|
+
} else if (gone.length === lane.listed.length) {
|
|
1372
|
+
operations.push(freeLane(lane.laneId, lane.epoch, lane.revision, gone));
|
|
1373
|
+
handedBack.add(lane.laneId);
|
|
1374
|
+
}
|
|
541
1375
|
}
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
return void 0;
|
|
1376
|
+
if (operations.length === 0) {
|
|
1377
|
+
return { recorded: /* @__PURE__ */ new Set(), handedBack };
|
|
545
1378
|
}
|
|
546
|
-
const
|
|
547
|
-
if (
|
|
548
|
-
|
|
549
|
-
scan.noteOpen(orderingId, lateBlocker);
|
|
550
|
-
return void 0;
|
|
1379
|
+
const result = await this.#lanes.bulkWrite(operations, { ordered: false });
|
|
1380
|
+
if (result.matchedCount === operations.length) {
|
|
1381
|
+
return { recorded: new Set(picksByLane.keys()), handedBack };
|
|
551
1382
|
}
|
|
552
|
-
|
|
553
|
-
|
|
1383
|
+
const current = await this.#lanes.find(
|
|
1384
|
+
{ _id: { $in: [...picksByLane.keys()] } },
|
|
1385
|
+
{ projection: { epoch: 1, owner: 1, holder: 1, revision: 1 } }
|
|
1386
|
+
).toArray();
|
|
1387
|
+
const recorded = new Set(
|
|
1388
|
+
current.filter(
|
|
1389
|
+
(lane) => lane.epoch === scope.get(lane._id).epoch && lane.revision === scope.get(lane._id).revision + 1 && lane.owner === request.leaseOwner && lane.holder === this.#instanceId
|
|
1390
|
+
).map((lane) => lane._id)
|
|
1391
|
+
);
|
|
1392
|
+
return { recorded, handedBack };
|
|
554
1393
|
}
|
|
555
1394
|
/**
|
|
556
|
-
*
|
|
557
|
-
*
|
|
558
|
-
*
|
|
1395
|
+
* Gives back the recorded keyed picks that a delivery requeued during the
|
|
1396
|
+
* claim now precedes, without charging an attempt, and returns the rest. The
|
|
1397
|
+
* claim checked their predecessors before recording them; a dead letter
|
|
1398
|
+
* requeued in between would otherwise wait behind its own successor. Once a
|
|
1399
|
+
* pick is recorded, a later requeue waits for it like for any running delivery.
|
|
559
1400
|
*/
|
|
560
|
-
async #
|
|
561
|
-
const
|
|
1401
|
+
async #releaseOvertaken(claimed, scope) {
|
|
1402
|
+
const keyed = claimed.filter((document) => document.key !== void 0);
|
|
1403
|
+
if (keyed.length === 0) {
|
|
1404
|
+
return [...claimed];
|
|
1405
|
+
}
|
|
1406
|
+
const blockers = await this.#deliveries.find(
|
|
562
1407
|
{
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
1408
|
+
$or: keyed.map((document) => ({
|
|
1409
|
+
handler: document.handler,
|
|
1410
|
+
key: document.key,
|
|
1411
|
+
sequence: { $lt: document.sequence },
|
|
1412
|
+
dueAt: { $exists: true }
|
|
1413
|
+
}))
|
|
568
1414
|
},
|
|
569
|
-
{
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
updatedAt: request.now
|
|
577
|
-
},
|
|
578
|
-
$inc: { attempts: 1 }
|
|
579
|
-
},
|
|
580
|
-
{ returnDocument: "after" }
|
|
1415
|
+
{ projection: { handler: 1, key: 1 } }
|
|
1416
|
+
).toArray();
|
|
1417
|
+
const blocked = new Set(
|
|
1418
|
+
blockers.map((blocker) => orderingIdentity(blocker.handler, blocker.key))
|
|
1419
|
+
);
|
|
1420
|
+
const overtaken = new Set(
|
|
1421
|
+
keyed.filter((document) => blocked.has(orderingIdentity(document.handler, document.key))).map((document) => document._id)
|
|
581
1422
|
);
|
|
582
|
-
|
|
1423
|
+
if (overtaken.size === 0) {
|
|
1424
|
+
return [...claimed];
|
|
1425
|
+
}
|
|
1426
|
+
await this.#releasePicks(
|
|
1427
|
+
claimed.filter((document) => overtaken.has(document._id)),
|
|
1428
|
+
scope
|
|
1429
|
+
);
|
|
1430
|
+
return claimed.filter((document) => !overtaken.has(document._id));
|
|
583
1431
|
}
|
|
584
|
-
/**
|
|
585
|
-
async #
|
|
586
|
-
const
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
1432
|
+
/** Takes recorded picks out of their lanes' in-flight lists again, under the claim's epochs and record revisions. */
|
|
1433
|
+
async #releasePicks(documents, scope) {
|
|
1434
|
+
const idsByLane = /* @__PURE__ */ new Map();
|
|
1435
|
+
for (const document of documents) {
|
|
1436
|
+
idsByLane.set(document.laneId, [...idsByLane.get(document.laneId) ?? [], document._id]);
|
|
1437
|
+
}
|
|
1438
|
+
await Promise.all(
|
|
1439
|
+
[...idsByLane].map(
|
|
1440
|
+
([laneId, ids]) => this.#lanes.updateOne(
|
|
1441
|
+
// Under the revision the record left: a pull that reaches the server late, after a
|
|
1442
|
+
// reconciliation and a newer claim of the same delivery, matches nothing.
|
|
1443
|
+
{
|
|
1444
|
+
_id: laneId,
|
|
1445
|
+
epoch: scope.get(laneId).epoch,
|
|
1446
|
+
revision: scope.get(laneId).revision + 1
|
|
1447
|
+
},
|
|
1448
|
+
{ $pull: { inFlight: { $in: ids } } }
|
|
1449
|
+
)
|
|
1450
|
+
)
|
|
602
1451
|
);
|
|
603
1452
|
}
|
|
1453
|
+
/**
|
|
1454
|
+
* Parks the waiting backlogs a claim found. Best effort: parking only saves
|
|
1455
|
+
* later claims work, so a failure is logged and the claim still returns its
|
|
1456
|
+
* batch; the next claim that meets the backlog parks it.
|
|
1457
|
+
*/
|
|
1458
|
+
async #parkWaiting(deliveryIds) {
|
|
1459
|
+
try {
|
|
1460
|
+
await this.#parkDeliveries(deliveryIds);
|
|
1461
|
+
} catch (error) {
|
|
1462
|
+
this.#logger.warn(
|
|
1463
|
+
`[MongoEventStore](parkWaiting) Parking waiting deliveries failed, a later claim parks them | count=${deliveryIds.length} | error=${String(error)}`
|
|
1464
|
+
);
|
|
1465
|
+
}
|
|
1466
|
+
}
|
|
604
1467
|
/** Sequence of the lowest unfinished delivery of the handler and key before `sequence`, if any. */
|
|
605
1468
|
async #findUnfinishedPredecessor(handler, key, sequence) {
|
|
606
1469
|
const predecessor = await this.#deliveries.findOne(
|
|
@@ -609,14 +1472,6 @@ var MongoEventStore = class {
|
|
|
609
1472
|
);
|
|
610
1473
|
return predecessor?.sequence;
|
|
611
1474
|
}
|
|
612
|
-
/** Whether some delivery of the handler and key runs under a lease that has not expired. */
|
|
613
|
-
async #hasLiveLease(handler, key, now) {
|
|
614
|
-
const running = await this.#deliveries.findOne(
|
|
615
|
-
{ handler, key, status: "processing", dueAt: { $gt: now } },
|
|
616
|
-
{ projection: { _id: 1 } }
|
|
617
|
-
);
|
|
618
|
-
return running !== null;
|
|
619
|
-
}
|
|
620
1475
|
/**
|
|
621
1476
|
* Parks pending deliveries that waited behind an unfinished predecessor, which
|
|
622
1477
|
* removes them from the claim index until their turn. Parking is a hint, not a
|
|
@@ -651,82 +1506,220 @@ var MongoEventStore = class {
|
|
|
651
1506
|
if (anyParked === null) {
|
|
652
1507
|
return;
|
|
653
1508
|
}
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
1509
|
+
const lowest = (await Promise.all(
|
|
1510
|
+
[...new Set(handlers)].map(
|
|
1511
|
+
(handler) => this.#deliveries.aggregate([
|
|
1512
|
+
{ $match: { handler, lane: "parked" } },
|
|
1513
|
+
{ $sort: { handler: 1, key: 1, sequence: 1 } },
|
|
658
1514
|
{
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
}
|
|
1515
|
+
$group: { _id: "$key", id: { $first: "$_id" }, sequence: { $first: "$sequence" } }
|
|
1516
|
+
}
|
|
1517
|
+
]).toArray().then((groups) => groups.map((group) => ({ ...group, handler })))
|
|
1518
|
+
)
|
|
1519
|
+
)).flat();
|
|
1520
|
+
const next = await Promise.all(
|
|
1521
|
+
lowest.map(
|
|
1522
|
+
async (parked) => await this.#findUnfinishedPredecessor(parked.handler, parked._id, parked.sequence) === void 0 ? [parked.id] : []
|
|
1523
|
+
)
|
|
1524
|
+
);
|
|
1525
|
+
const ready = next.flat();
|
|
1526
|
+
if (ready.length > 0) {
|
|
1527
|
+
await this.#deliveries.updateMany(
|
|
1528
|
+
{ _id: { $in: ready }, lane: "parked" },
|
|
1529
|
+
{ $set: { lane: "ready" } }
|
|
1530
|
+
);
|
|
676
1531
|
}
|
|
677
1532
|
}
|
|
678
|
-
async #toClaimedDeliveries(leased, leaseExpiresAt) {
|
|
679
|
-
const events = await this.#readEnvelopes(leased.map((document) => document.eventId));
|
|
680
|
-
return leased.map((document) => ({
|
|
681
|
-
deliveryId: document._id,
|
|
682
|
-
leaseToken: document.leaseToken,
|
|
683
|
-
handler: document.handler,
|
|
684
|
-
attempt: document.attempts,
|
|
685
|
-
leaseExpiresAt,
|
|
686
|
-
event: events.get(document.eventId)
|
|
687
|
-
}));
|
|
688
|
-
}
|
|
689
1533
|
async #readEnvelopes(eventIds) {
|
|
690
1534
|
const documents = await this.#events.find({ _id: { $in: [...new Set(eventIds)] } }).toArray();
|
|
691
1535
|
return new Map(documents.map((document) => [document._id, toEnvelope(document)]));
|
|
692
1536
|
}
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
1537
|
+
/**
|
|
1538
|
+
* Deletes the delivery of every lease whose lane still has the lease's epoch
|
|
1539
|
+
* and lists the delivery, in one transaction (the caller's, when given).
|
|
1540
|
+
* Returns one flag per lease, in order; a lease listed twice completes once.
|
|
1541
|
+
*/
|
|
1542
|
+
async #completeLeases(leases, now, session) {
|
|
1543
|
+
return session === void 0 ? this.#settle(leases, () => this.#completeIssued(leases, now, void 0)) : this.#completeIssued(leases, now, session);
|
|
1544
|
+
}
|
|
1545
|
+
async #completeIssued(leases, now, session) {
|
|
1546
|
+
const parsed = leases.map((lease) => parseLeaseToken(lease.leaseToken));
|
|
1547
|
+
const issued = leases.flatMap(
|
|
1548
|
+
(lease, index) => parsed[index] === void 0 ? [] : [{ deliveryId: String(lease.deliveryId), ...parsed[index] }]
|
|
1549
|
+
);
|
|
1550
|
+
let completed = /* @__PURE__ */ new Set();
|
|
1551
|
+
if (issued.length > 0) {
|
|
1552
|
+
if (session !== void 0) {
|
|
1553
|
+
completed = await this.#deleteCompleted(issued, now, session);
|
|
1554
|
+
} else if (this.#completionHistory) {
|
|
1555
|
+
completed = await this.transaction((own) => this.#deleteCompleted(issued, now, own));
|
|
1556
|
+
} else {
|
|
1557
|
+
completed = await this.#deleteFenced(issued, now);
|
|
1558
|
+
}
|
|
1559
|
+
}
|
|
1560
|
+
return leases.map(
|
|
1561
|
+
(lease, index) => parsed[index] !== void 0 && completed.delete(leaseIdentity({ deliveryId: String(lease.deliveryId), ...parsed[index] }))
|
|
1562
|
+
);
|
|
1563
|
+
}
|
|
1564
|
+
/**
|
|
1565
|
+
* Completion without a transaction, used when no history record has to be
|
|
1566
|
+
* written with the deletion. The deliveries' claim generations are read
|
|
1567
|
+
* first, then the lanes, and a delivery is deleted only if it still has the
|
|
1568
|
+
* lease's generation. A takeover moves the generation of every delivery it
|
|
1569
|
+
* takes from the previous owner on before it reads anything, and so do a
|
|
1570
|
+
* retry, a release and a burial, so:
|
|
1571
|
+
* - a takeover before the delivery read shows up in the lane read (new epoch);
|
|
1572
|
+
* - a takeover between the two reads shows up in the lane read as well;
|
|
1573
|
+
* - a takeover, retry, release or burial after the lane read changed the
|
|
1574
|
+
* generation, so the delete matches nothing.
|
|
1575
|
+
* A completion and a takeover (or the end of the claim) therefore never both succeed. Two concurrent
|
|
1576
|
+
* completions of the same lease may both report true (the dispatcher never
|
|
1577
|
+
* issues them); one call with a lease listed twice reports it once.
|
|
1578
|
+
*/
|
|
1579
|
+
async #deleteFenced(issued, now) {
|
|
1580
|
+
const ids = [...new Set(issued.map((lease) => lease.deliveryId))];
|
|
1581
|
+
const documents = await this.#deliveries.find({ _id: { $in: ids }, status: "pending" }, { projection: { claims: 1, eventId: 1 } }).toArray();
|
|
1582
|
+
const lanes = new Map(
|
|
1583
|
+
(await this.#lanes.find(
|
|
1584
|
+
{ _id: { $in: [...new Set(issued.map((lease) => lease.laneId))] } },
|
|
1585
|
+
{ projection: { epoch: 1, inFlight: 1 } }
|
|
1586
|
+
).toArray()).map((lane) => [lane._id, lane])
|
|
1587
|
+
);
|
|
1588
|
+
const fenced = fencedLeases(issued, lanes, documents);
|
|
1589
|
+
if (fenced.length === 0) {
|
|
1590
|
+
return /* @__PURE__ */ new Set();
|
|
1591
|
+
}
|
|
1592
|
+
await this.#markEventsCompleted(fenced, documents, now);
|
|
1593
|
+
const deleted = await this.#deliveries.deleteMany({ $or: fenced.map(ofClaim) });
|
|
1594
|
+
return deleted.deletedCount === fenced.length ? new Set(fenced.map((lease) => leaseIdentity(lease))) : /* @__PURE__ */ new Set();
|
|
1595
|
+
}
|
|
1596
|
+
/**
|
|
1597
|
+
* The transactional part of a completion. The lanes are only read: the
|
|
1598
|
+
* conflict with a takeover comes from the deleted delivery documents, which a
|
|
1599
|
+
* takeover writes before it reads them (see {@link MongoEventStore.#takeLanes}).
|
|
1600
|
+
* So whichever of the two commits second either fails with a write conflict
|
|
1601
|
+
* and is retried against the new state, or waits for the first.
|
|
1602
|
+
*/
|
|
1603
|
+
async #deleteCompleted(issued, now, session) {
|
|
1604
|
+
const lanes = new Map(
|
|
1605
|
+
(await this.#lanes.find(
|
|
1606
|
+
{ _id: { $in: [...new Set(issued.map((lease) => lease.laneId))] } },
|
|
1607
|
+
{ session, projection: { epoch: 1, inFlight: 1 } }
|
|
1608
|
+
).toArray()).map((lane) => [lane._id, lane])
|
|
1609
|
+
);
|
|
1610
|
+
const documents = await this.#deliveries.find(
|
|
1611
|
+
{ _id: { $in: [...new Set(issued.map((lease) => lease.deliveryId))] }, status: "pending" },
|
|
1612
|
+
{ session }
|
|
1613
|
+
).toArray();
|
|
1614
|
+
const fenced = fencedLeases(issued, lanes, documents);
|
|
1615
|
+
if (fenced.length === 0) {
|
|
1616
|
+
return /* @__PURE__ */ new Set();
|
|
1617
|
+
}
|
|
1618
|
+
await this.#markEventsCompleted(fenced, documents, now, session);
|
|
1619
|
+
await this.#deliveries.deleteMany({ $or: fenced.map(ofClaim) }, { session });
|
|
1620
|
+
if (this.#completionHistory) {
|
|
1621
|
+
const completedIds = new Set(fenced.map((lease) => lease.deliveryId));
|
|
1622
|
+
await this.#completions.insertMany(
|
|
1623
|
+
documents.filter((document) => completedIds.has(document._id)).map((document) => toCompletedDeliveryDocument(document, now)),
|
|
1624
|
+
{ session }
|
|
1625
|
+
);
|
|
1626
|
+
}
|
|
1627
|
+
return new Set(fenced.map((lease) => leaseIdentity(lease)));
|
|
1628
|
+
}
|
|
1629
|
+
/**
|
|
1630
|
+
* Records `now` as the latest completion of the events of the fenced
|
|
1631
|
+
* deliveries (`$max`, so an earlier clock never moves it back). Written
|
|
1632
|
+
* before the deletion, it can only be later than the real last completion
|
|
1633
|
+
* (when the deletion then fails), which keeps the event longer, never shorter.
|
|
1634
|
+
* Inside a transaction it is part of it: the transaction may hold the event
|
|
1635
|
+
* already (a transactional handler that appended to its own event), and a
|
|
1636
|
+
* write from outside would wait for it forever. Deliveries of one event that
|
|
1637
|
+
* complete in concurrent transactions (fan-out to transactional handlers)
|
|
1638
|
+
* therefore conflict on the event document, and one of them retries.
|
|
1639
|
+
*/
|
|
1640
|
+
async #markEventsCompleted(fenced, documents, now, session) {
|
|
1641
|
+
const completedIds = new Set(fenced.map((lease) => lease.deliveryId));
|
|
1642
|
+
const eventIds = [
|
|
1643
|
+
...new Set(
|
|
1644
|
+
documents.filter((document) => completedIds.has(document._id)).map((document) => document.eventId)
|
|
1645
|
+
)
|
|
1646
|
+
];
|
|
1647
|
+
await this.#events.updateMany(
|
|
1648
|
+
{ _id: { $in: eventIds } },
|
|
1649
|
+
{ $max: { completedAt: now } },
|
|
697
1650
|
session === void 0 ? {} : { session }
|
|
698
1651
|
);
|
|
699
|
-
return result.matchedCount === 1;
|
|
700
1652
|
}
|
|
701
1653
|
/**
|
|
702
|
-
*
|
|
703
|
-
*
|
|
704
|
-
*
|
|
705
|
-
* same event writes the event document, so the two conflict and one restarts
|
|
706
|
-
* instead of leaving a delivery without event. A delivery requeued since the
|
|
707
|
-
* batch was read conflicts with its deletion and is read again.
|
|
1654
|
+
* Applies `update` to a running delivery and takes it out of its lane's
|
|
1655
|
+
* in-flight list, if the lane still has the lease's epoch. One transaction:
|
|
1656
|
+
* the lane write fences the change against a takeover.
|
|
708
1657
|
*/
|
|
709
|
-
async #
|
|
1658
|
+
async #leaveFlight(lease, update) {
|
|
1659
|
+
const held = parseLeaseToken(lease.leaseToken);
|
|
1660
|
+
if (held === void 0) {
|
|
1661
|
+
return false;
|
|
1662
|
+
}
|
|
1663
|
+
const deliveryId = String(lease.deliveryId);
|
|
1664
|
+
const ofThisClaim = ofClaim({ ...held, deliveryId });
|
|
1665
|
+
return this.#settle([lease], () => this.#leaveFlightNow(held, deliveryId, ofThisClaim, update));
|
|
1666
|
+
}
|
|
1667
|
+
async #leaveFlightNow(held, deliveryId, ofThisClaim, update) {
|
|
1668
|
+
return this.transaction(async (session) => {
|
|
1669
|
+
if (await this.#deliveries.findOne(ofThisClaim, { session, projection: { _id: 1 } }) === null) {
|
|
1670
|
+
return false;
|
|
1671
|
+
}
|
|
1672
|
+
const fence = await this.#lanes.updateOne(
|
|
1673
|
+
{ _id: held.laneId, epoch: held.epoch, inFlight: deliveryId },
|
|
1674
|
+
{ $pull: { inFlight: deliveryId } },
|
|
1675
|
+
{ session }
|
|
1676
|
+
);
|
|
1677
|
+
if (fence.matchedCount === 0) {
|
|
1678
|
+
return false;
|
|
1679
|
+
}
|
|
1680
|
+
const changed = await this.#deliveries.updateOne(ofThisClaim, update, { session });
|
|
1681
|
+
return changed.matchedCount === 1;
|
|
1682
|
+
});
|
|
1683
|
+
}
|
|
1684
|
+
/**
|
|
1685
|
+
* Deletes batches of matching documents (deliveries, or completion history
|
|
1686
|
+
* records) together with the events nothing references any more, each batch
|
|
1687
|
+
* in one transaction, so a crash can never leave an event without deliveries
|
|
1688
|
+
* behind. An append adding a delivery to the same event writes the event
|
|
1689
|
+
* document, so the two conflict and one restarts instead of leaving a delivery
|
|
1690
|
+
* without event. A delivery requeued since the batch was read conflicts with
|
|
1691
|
+
* its deletion and is read again.
|
|
1692
|
+
*/
|
|
1693
|
+
async #purgeMatching(collection, filter, completedBefore) {
|
|
710
1694
|
let deletedDeliveries = 0;
|
|
711
1695
|
let deletedEvents = 0;
|
|
712
1696
|
for (; ; ) {
|
|
713
1697
|
const step = await this.transaction(async (session) => {
|
|
714
|
-
const batch = await
|
|
1698
|
+
const batch = await collection.find(filter, { session, projection: { eventId: 1 }, limit: this.#purgeBatchSize }).toArray();
|
|
1699
|
+
if (batch.length === 0) {
|
|
1700
|
+
return { batchSize: 0, deletedDeliveries: 0, deletedEvents: 0 };
|
|
1701
|
+
}
|
|
715
1702
|
const eventIds = [...new Set(batch.map((document) => document.eventId))];
|
|
716
|
-
const
|
|
1703
|
+
const deleted = await collection.deleteMany(
|
|
717
1704
|
{ _id: { $in: batch.map((document) => document._id) } },
|
|
718
1705
|
{ session }
|
|
719
1706
|
);
|
|
720
|
-
const stillReferenced =
|
|
721
|
-
(await this.#deliveries.find({ eventId: { $in: eventIds } }, { session, projection: { eventId: 1 } }).toArray()).map((document) => document.eventId)
|
|
722
|
-
);
|
|
1707
|
+
const stillReferenced = await this.#referencedEventIds(eventIds, session);
|
|
723
1708
|
const events = await this.#events.deleteMany(
|
|
724
|
-
{
|
|
1709
|
+
{
|
|
1710
|
+
_id: { $in: eventIds.filter((eventId) => !stillReferenced.has(eventId)) },
|
|
1711
|
+
// An event a delivery of which completed stays as deduplication record until
|
|
1712
|
+
// `completedBefore` passes its last completion, whatever this batch purged.
|
|
1713
|
+
$or: [
|
|
1714
|
+
{ completedAt: { $exists: false } },
|
|
1715
|
+
...completedBefore === void 0 ? [] : [{ completedAt: { $lt: completedBefore } }]
|
|
1716
|
+
]
|
|
1717
|
+
},
|
|
725
1718
|
{ session }
|
|
726
1719
|
);
|
|
727
1720
|
return {
|
|
728
1721
|
batchSize: batch.length,
|
|
729
|
-
deletedDeliveries:
|
|
1722
|
+
deletedDeliveries: deleted.deletedCount,
|
|
730
1723
|
deletedEvents: events.deletedCount
|
|
731
1724
|
};
|
|
732
1725
|
});
|
|
@@ -737,9 +1730,154 @@ var MongoEventStore = class {
|
|
|
737
1730
|
}
|
|
738
1731
|
}
|
|
739
1732
|
}
|
|
1733
|
+
/** A view of a collection that purge deletes from: deliveries or completion history records. */
|
|
1734
|
+
#purgeable(name) {
|
|
1735
|
+
return this.#database.collection(name, {
|
|
1736
|
+
readPreference: "primary",
|
|
1737
|
+
writeConcern: { w: "majority" }
|
|
1738
|
+
});
|
|
1739
|
+
}
|
|
1740
|
+
/** Those of `eventIds` that a delivery or a completion history record references. */
|
|
1741
|
+
async #referencedEventIds(eventIds, session) {
|
|
1742
|
+
const filter = { eventId: { $in: [...eventIds] } };
|
|
1743
|
+
const options = { session, projection: { eventId: 1 } };
|
|
1744
|
+
const references = [
|
|
1745
|
+
...await this.#deliveries.find(filter, options).toArray(),
|
|
1746
|
+
...this.#completionHistory ? await this.#completions.find(filter, options).toArray() : []
|
|
1747
|
+
];
|
|
1748
|
+
return new Set(references.map((reference) => reference.eventId));
|
|
1749
|
+
}
|
|
1750
|
+
/**
|
|
1751
|
+
* Deletes, in batches, events created before `cutoff` that no delivery (or
|
|
1752
|
+
* completion history record) references. Each batch is found and deleted in
|
|
1753
|
+
* one transaction: an append adding a delivery to one of them writes the
|
|
1754
|
+
* event, so it conflicts with the deletion, which then restarts and sees the
|
|
1755
|
+
* delivery. Returns how many events were deleted.
|
|
1756
|
+
*/
|
|
1757
|
+
async #purgeOrphanEvents(cutoff, completedBefore) {
|
|
1758
|
+
const references = [
|
|
1759
|
+
this.#deliveries.collectionName,
|
|
1760
|
+
...this.#completionHistory ? [this.#completions.collectionName] : []
|
|
1761
|
+
];
|
|
1762
|
+
let deletedEvents = 0;
|
|
1763
|
+
for (; ; ) {
|
|
1764
|
+
const step = await this.transaction(async (session) => {
|
|
1765
|
+
const orphans = await this.#events.aggregate(
|
|
1766
|
+
[
|
|
1767
|
+
{
|
|
1768
|
+
$match: {
|
|
1769
|
+
$or: [
|
|
1770
|
+
// Never completed (no subscribers, or every delivery dead): by age.
|
|
1771
|
+
{ completedAt: { $exists: false }, createdAt: { $lt: cutoff } },
|
|
1772
|
+
// Completed: by the time of its last completion, and only under the completed cutoff.
|
|
1773
|
+
...completedBefore === void 0 ? [] : [{ completedAt: { $lt: completedBefore } }]
|
|
1774
|
+
]
|
|
1775
|
+
}
|
|
1776
|
+
},
|
|
1777
|
+
{ $sort: { createdAt: 1, _id: 1 } },
|
|
1778
|
+
...references.flatMap((from) => [
|
|
1779
|
+
{
|
|
1780
|
+
$lookup: {
|
|
1781
|
+
from,
|
|
1782
|
+
localField: "_id",
|
|
1783
|
+
foreignField: "eventId",
|
|
1784
|
+
pipeline: [{ $limit: 1 }, { $project: { _id: 1 } }],
|
|
1785
|
+
as: "references"
|
|
1786
|
+
}
|
|
1787
|
+
},
|
|
1788
|
+
{ $match: { references: { $size: 0 } } }
|
|
1789
|
+
]),
|
|
1790
|
+
{ $limit: this.#purgeBatchSize },
|
|
1791
|
+
{ $project: { _id: 1 } }
|
|
1792
|
+
],
|
|
1793
|
+
{ session }
|
|
1794
|
+
).toArray();
|
|
1795
|
+
const deleted = await this.#events.deleteMany(
|
|
1796
|
+
{ _id: { $in: orphans.map((orphan) => orphan._id) } },
|
|
1797
|
+
{ session }
|
|
1798
|
+
);
|
|
1799
|
+
return { found: orphans.length, deleted: deleted.deletedCount };
|
|
1800
|
+
});
|
|
1801
|
+
deletedEvents += step.deleted;
|
|
1802
|
+
if (step.found < this.#purgeBatchSize) {
|
|
1803
|
+
return deletedEvents;
|
|
1804
|
+
}
|
|
1805
|
+
}
|
|
1806
|
+
}
|
|
740
1807
|
};
|
|
741
|
-
function
|
|
742
|
-
|
|
1808
|
+
function isClaimable(candidate, scan, blockedKeys) {
|
|
1809
|
+
const { key, sequence } = candidate;
|
|
1810
|
+
if (key === void 0) {
|
|
1811
|
+
return true;
|
|
1812
|
+
}
|
|
1813
|
+
const orderingId = orderingIdentity(candidate.handler, key);
|
|
1814
|
+
if (scan.waits(orderingId, sequence)) {
|
|
1815
|
+
scan.noteWaiting(orderingId, candidate._id, candidate.laneId);
|
|
1816
|
+
return false;
|
|
1817
|
+
}
|
|
1818
|
+
if (blockedKeys.has(orderingId)) {
|
|
1819
|
+
return false;
|
|
1820
|
+
}
|
|
1821
|
+
blockedKeys.add(orderingId);
|
|
1822
|
+
return true;
|
|
1823
|
+
}
|
|
1824
|
+
function freeLane(laneId, epoch, revision, finished) {
|
|
1825
|
+
return {
|
|
1826
|
+
updateOne: {
|
|
1827
|
+
filter: {
|
|
1828
|
+
_id: laneId,
|
|
1829
|
+
epoch,
|
|
1830
|
+
revision: atCount(revision),
|
|
1831
|
+
inFlight: { $not: { $elemMatch: { $nin: [...finished] } } }
|
|
1832
|
+
},
|
|
1833
|
+
update: {
|
|
1834
|
+
$set: { owner: null, holder: null, leaseExpiresAt: null, inFlight: [] },
|
|
1835
|
+
$inc: { revision: 1 }
|
|
1836
|
+
}
|
|
1837
|
+
}
|
|
1838
|
+
};
|
|
1839
|
+
}
|
|
1840
|
+
async function inBatches(cursor, work) {
|
|
1841
|
+
let batch = [];
|
|
1842
|
+
for await (const document of cursor) {
|
|
1843
|
+
batch.push(document);
|
|
1844
|
+
if (batch.length === UPGRADE_BATCH_SIZE) {
|
|
1845
|
+
await work(batch);
|
|
1846
|
+
batch = [];
|
|
1847
|
+
}
|
|
1848
|
+
}
|
|
1849
|
+
if (batch.length > 0) {
|
|
1850
|
+
await work(batch);
|
|
1851
|
+
}
|
|
1852
|
+
}
|
|
1853
|
+
function atCount(count) {
|
|
1854
|
+
return count === 0 ? { $not: { $gt: 0 } } : count;
|
|
1855
|
+
}
|
|
1856
|
+
function fencedLeases(issued, lanes, documents) {
|
|
1857
|
+
const claimsById = new Map(documents.map((document) => [document._id, document.claims ?? 0]));
|
|
1858
|
+
const fenced = /* @__PURE__ */ new Map();
|
|
1859
|
+
for (const lease of issued) {
|
|
1860
|
+
const lane = lanes.get(lease.laneId);
|
|
1861
|
+
if (claimsById.get(lease.deliveryId) === lease.claims && lane.epoch === lease.epoch && lane.inFlight.includes(lease.deliveryId)) {
|
|
1862
|
+
fenced.set(leaseIdentity(lease), lease);
|
|
1863
|
+
}
|
|
1864
|
+
}
|
|
1865
|
+
return [...fenced.values()];
|
|
1866
|
+
}
|
|
1867
|
+
function ofClaim(lease) {
|
|
1868
|
+
return { _id: lease.deliveryId, status: "pending", claims: atCount(lease.claims) };
|
|
1869
|
+
}
|
|
1870
|
+
function leaseKey(deliveryId, leaseToken) {
|
|
1871
|
+
return `${deliveryId}\0${leaseToken}`;
|
|
1872
|
+
}
|
|
1873
|
+
function leaseIdentity(lease) {
|
|
1874
|
+
return `${lease.deliveryId}\0${formatLeaseToken(lease.laneId, lease.epoch, lease.claims)}`;
|
|
1875
|
+
}
|
|
1876
|
+
function hasErrorCode(error, code) {
|
|
1877
|
+
return typeof error === "object" && error !== null && "code" in error && error.code === code;
|
|
1878
|
+
}
|
|
1879
|
+
function compareText(left, right) {
|
|
1880
|
+
return Number(left > right) - Number(left < right);
|
|
743
1881
|
}
|
|
744
1882
|
function assertInTransaction(session, operation) {
|
|
745
1883
|
if (!session.inTransaction()) {
|