@relaybus/mongodb 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +264 -2
- package/dist/index.cjs +756 -0
- package/dist/index.cjs.map +1 -0
- package/dist/index.d.cts +80 -0
- package/dist/index.d.ts +80 -0
- package/dist/index.js +754 -0
- package/dist/index.js.map +1 -0
- package/package.json +65 -3
package/dist/index.js
ADDED
|
@@ -0,0 +1,754 @@
|
|
|
1
|
+
import { ConfigurationError, createUuidV7, EventValidationError } from '@relaybus/core';
|
|
2
|
+
|
|
3
|
+
// src/mongo-event-store.ts
|
|
4
|
+
|
|
5
|
+
// src/claim-scan.ts
|
|
6
|
+
var ClaimScan = class {
|
|
7
|
+
#keys = /* @__PURE__ */ new Map();
|
|
8
|
+
/** Why the delivery must wait according to what this scan already knows, if it does. */
|
|
9
|
+
verdict(orderingId, sequence) {
|
|
10
|
+
const known = this.#keys.get(orderingId);
|
|
11
|
+
if (known?.openSequence !== void 0 && sequence > known.openSequence) {
|
|
12
|
+
return "open-predecessor";
|
|
13
|
+
}
|
|
14
|
+
return known?.leaseHeld === true ? "live-lease" : void 0;
|
|
15
|
+
}
|
|
16
|
+
/** Records an unfinished delivery of the key with this sequence. */
|
|
17
|
+
noteOpen(orderingId, sequence) {
|
|
18
|
+
const known = this.#know(orderingId);
|
|
19
|
+
known.openSequence = Math.min(known.openSequence ?? sequence, sequence);
|
|
20
|
+
}
|
|
21
|
+
/** Records that this scan leased the delivery, which is now unfinished and holds a live lease. */
|
|
22
|
+
noteClaimed(orderingId, sequence) {
|
|
23
|
+
this.noteOpen(orderingId, sequence);
|
|
24
|
+
this.#know(orderingId).leaseHeld = true;
|
|
25
|
+
}
|
|
26
|
+
/** Records that a live lease of the key blocks every delivery of it. */
|
|
27
|
+
noteLeaseHeld(orderingId) {
|
|
28
|
+
this.#know(orderingId).leaseHeld = true;
|
|
29
|
+
}
|
|
30
|
+
/** Remembers a delivery that waited behind an unfinished predecessor. */
|
|
31
|
+
noteWaiting(orderingId, delivery) {
|
|
32
|
+
if (delivery.status === "pending") {
|
|
33
|
+
this.#know(orderingId).waiting.push(delivery._id);
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* Ids of the waiting deliveries of every key that accumulated at least
|
|
38
|
+
* `minimum` of them. Shorter queues stay in the claim index: moving them out
|
|
39
|
+
* and back would cost more than skipping them.
|
|
40
|
+
*/
|
|
41
|
+
waitingBacklogs(minimum) {
|
|
42
|
+
return [...this.#keys.values()].filter((known) => known.waiting.length >= minimum).flatMap((known) => known.waiting);
|
|
43
|
+
}
|
|
44
|
+
#know(orderingId) {
|
|
45
|
+
let known = this.#keys.get(orderingId);
|
|
46
|
+
if (known === void 0) {
|
|
47
|
+
known = { openSequence: void 0, leaseHeld: false, waiting: [] };
|
|
48
|
+
this.#keys.set(orderingId, known);
|
|
49
|
+
}
|
|
50
|
+
return known;
|
|
51
|
+
}
|
|
52
|
+
};
|
|
53
|
+
function orderingIdentity(handler, key) {
|
|
54
|
+
return `${handler}\0${key}`;
|
|
55
|
+
}
|
|
56
|
+
function toEventDocument(event, sequence) {
|
|
57
|
+
const payloadJson = serializeWellFormed(event.payload, "payload");
|
|
58
|
+
const headersJson = serializeWellFormed(event.headers, "headers");
|
|
59
|
+
return {
|
|
60
|
+
_id: event.id,
|
|
61
|
+
type: event.type,
|
|
62
|
+
source: event.source,
|
|
63
|
+
key: event.key,
|
|
64
|
+
sequence,
|
|
65
|
+
...payloadJson === void 0 ? {} : { payload: JSON.parse(payloadJson) },
|
|
66
|
+
headers: JSON.parse(headersJson),
|
|
67
|
+
correlationId: event.correlationId,
|
|
68
|
+
causationId: event.causationId,
|
|
69
|
+
occurredAt: event.occurredAt,
|
|
70
|
+
revision: 0
|
|
71
|
+
};
|
|
72
|
+
}
|
|
73
|
+
var LONE_SURROGATE_ESCAPE = /(?<!\\)(?:\\\\)*\\ud[89a-f][0-9a-f]{2}/i;
|
|
74
|
+
function serializeWellFormed(value, field) {
|
|
75
|
+
const json = JSON.stringify(value);
|
|
76
|
+
if (json !== void 0 && LONE_SURROGATE_ESCAPE.test(json)) {
|
|
77
|
+
throw new EventValidationError(`The event ${field} contains a lone UTF-16 surrogate`, [
|
|
78
|
+
{
|
|
79
|
+
message: "A lone surrogate cannot be stored as UTF-8 and would be replaced by U+FFFD. Remove it or encode the text first (for example with String.prototype.toWellFormed)",
|
|
80
|
+
path: [field]
|
|
81
|
+
}
|
|
82
|
+
]);
|
|
83
|
+
}
|
|
84
|
+
return json;
|
|
85
|
+
}
|
|
86
|
+
function toEnvelope(document) {
|
|
87
|
+
return {
|
|
88
|
+
id: document._id,
|
|
89
|
+
type: document.type,
|
|
90
|
+
source: document.source,
|
|
91
|
+
key: document.key,
|
|
92
|
+
payload: document.payload,
|
|
93
|
+
headers: document.headers,
|
|
94
|
+
correlationId: document.correlationId,
|
|
95
|
+
causationId: document.causationId,
|
|
96
|
+
occurredAt: document.occurredAt
|
|
97
|
+
};
|
|
98
|
+
}
|
|
99
|
+
function toDeliveryDocument(delivery, event, ordering, now) {
|
|
100
|
+
return {
|
|
101
|
+
_id: delivery.id,
|
|
102
|
+
eventId: event.id,
|
|
103
|
+
eventType: event.type,
|
|
104
|
+
handler: delivery.handler,
|
|
105
|
+
...ordering.key === null || ordering.sequence === null ? {} : { key: ordering.key, sequence: ordering.sequence },
|
|
106
|
+
status: "pending",
|
|
107
|
+
lane: "ready",
|
|
108
|
+
attempts: 0,
|
|
109
|
+
availableAt: delivery.availableAt,
|
|
110
|
+
dueAt: delivery.availableAt,
|
|
111
|
+
createdAt: now,
|
|
112
|
+
updatedAt: now
|
|
113
|
+
};
|
|
114
|
+
}
|
|
115
|
+
function toDeliveryRecord(document, event) {
|
|
116
|
+
return {
|
|
117
|
+
id: document._id,
|
|
118
|
+
handler: document.handler,
|
|
119
|
+
status: document.status,
|
|
120
|
+
attempts: document.attempts,
|
|
121
|
+
availableAt: document.availableAt,
|
|
122
|
+
leaseOwner: document.leaseOwner ?? null,
|
|
123
|
+
leaseExpiresAt: document.leaseExpiresAt ?? null,
|
|
124
|
+
lastError: document.lastError === void 0 ? null : toErrorInfo(document.lastError),
|
|
125
|
+
createdAt: document.createdAt,
|
|
126
|
+
updatedAt: document.updatedAt,
|
|
127
|
+
completedAt: document.completedAt ?? null,
|
|
128
|
+
event
|
|
129
|
+
};
|
|
130
|
+
}
|
|
131
|
+
function toErrorInfo(error) {
|
|
132
|
+
return error.stack === void 0 ? { name: error.name, message: error.message } : { name: error.name, message: error.message, stack: error.stack };
|
|
133
|
+
}
|
|
134
|
+
var ILLEGAL_OPERATION = 20;
|
|
135
|
+
function isTransactionsUnsupportedError(error) {
|
|
136
|
+
return typeof error === "object" && error !== null && "code" in error && error.code === ILLEGAL_OPERATION && "message" in error && typeof error.message === "string" && error.message.includes("replica set member or mongos");
|
|
137
|
+
}
|
|
138
|
+
function createTransactionsUnsupportedError(cause) {
|
|
139
|
+
return new ConfigurationError(
|
|
140
|
+
"MongoEventStore needs multi-document transactions, which MongoDB only offers on a replica set or a sharded cluster. Run a single-node replica set for development (mongod --replSet rs0, then rs.initiate())",
|
|
141
|
+
{ cause }
|
|
142
|
+
);
|
|
143
|
+
}
|
|
144
|
+
var COLLECTION_PREFIX_PATTERN = /^[A-Za-z_][A-Za-z0-9_]{0,39}$/;
|
|
145
|
+
var DATABASE_NAME_PATTERN = /^[A-Za-z0-9_-]{1,63}$/;
|
|
146
|
+
function assertCollectionPrefix(prefix) {
|
|
147
|
+
if (!COLLECTION_PREFIX_PATTERN.test(prefix)) {
|
|
148
|
+
throw new ConfigurationError(
|
|
149
|
+
`collectionPrefix '${prefix}' must be 1-40 characters of letters, digits or '_', starting with a letter or '_'`
|
|
150
|
+
);
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
function assertDatabaseName(name) {
|
|
154
|
+
if (!DATABASE_NAME_PATTERN.test(name)) {
|
|
155
|
+
throw new ConfigurationError(
|
|
156
|
+
`database '${name}' must be 1-63 characters of letters, digits, '_' or '-'`
|
|
157
|
+
);
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
// src/mongo-event-store.ts
|
|
162
|
+
var DEFAULT_TRANSACTION_OPTIONS = {
|
|
163
|
+
readConcern: { level: "snapshot" },
|
|
164
|
+
writeConcern: { w: "majority" },
|
|
165
|
+
readPreference: "primary"
|
|
166
|
+
};
|
|
167
|
+
var NAMESPACE_EXISTS = 48;
|
|
168
|
+
var DEFAULT_PURGE_BATCH_SIZE = 500;
|
|
169
|
+
var DEFAULT_LIST_LIMIT = 100;
|
|
170
|
+
var MINIMUM_CANDIDATE_BATCH_SIZE = 32;
|
|
171
|
+
var PARK_BACKLOG_THRESHOLD = 32;
|
|
172
|
+
var PARK_UPDATE_CHUNK_SIZE = 1e3;
|
|
173
|
+
var LEASE_FIELDS = { leaseOwner: "", leaseToken: "", leaseExpiresAt: "" };
|
|
174
|
+
var FINISHED_FIELDS = { dueAt: "", lane: "" };
|
|
175
|
+
var MongoEventStore = class {
|
|
176
|
+
#client;
|
|
177
|
+
#database;
|
|
178
|
+
#events;
|
|
179
|
+
#deliveries;
|
|
180
|
+
#keySequences;
|
|
181
|
+
#transactionOptions;
|
|
182
|
+
#purgeBatchSize;
|
|
183
|
+
#generateIdentifier;
|
|
184
|
+
constructor(options) {
|
|
185
|
+
const prefix = options.collectionPrefix ?? "relaybus_";
|
|
186
|
+
assertCollectionPrefix(prefix);
|
|
187
|
+
if (options.database !== void 0) {
|
|
188
|
+
assertDatabaseName(options.database);
|
|
189
|
+
}
|
|
190
|
+
const purgeBatchSize = options.purgeBatchSize ?? DEFAULT_PURGE_BATCH_SIZE;
|
|
191
|
+
if (!Number.isInteger(purgeBatchSize) || purgeBatchSize < 1) {
|
|
192
|
+
throw new ConfigurationError(
|
|
193
|
+
`purgeBatchSize must be a positive integer, received ${String(purgeBatchSize)}`
|
|
194
|
+
);
|
|
195
|
+
}
|
|
196
|
+
this.#client = options.client;
|
|
197
|
+
const database = options.client.db(options.database);
|
|
198
|
+
this.#database = database;
|
|
199
|
+
const collectionOptions = {
|
|
200
|
+
readPreference: "primary",
|
|
201
|
+
writeConcern: { w: "majority" }
|
|
202
|
+
};
|
|
203
|
+
this.#events = database.collection(`${prefix}events`, collectionOptions);
|
|
204
|
+
this.#deliveries = database.collection(`${prefix}deliveries`, collectionOptions);
|
|
205
|
+
this.#keySequences = database.collection(`${prefix}key_sequences`, collectionOptions);
|
|
206
|
+
this.#transactionOptions = options.transactionOptions ?? DEFAULT_TRANSACTION_OPTIONS;
|
|
207
|
+
this.#purgeBatchSize = purgeBatchSize;
|
|
208
|
+
this.#generateIdentifier = options.generateIdentifier ?? (() => createUuidV7());
|
|
209
|
+
}
|
|
210
|
+
/**
|
|
211
|
+
* Verifies the deployment supports transactions and creates the indexes.
|
|
212
|
+
* Safe to run repeatedly and from several processes.
|
|
213
|
+
*
|
|
214
|
+
* @throws ConfigurationError when the server is a standalone mongod.
|
|
215
|
+
*/
|
|
216
|
+
async setup() {
|
|
217
|
+
const hello = await this.#client.db("admin").command({ hello: 1 });
|
|
218
|
+
const supportsTransactions = typeof hello["setName"] === "string" || hello["msg"] === "isdbgrid";
|
|
219
|
+
if (!supportsTransactions) {
|
|
220
|
+
throw createTransactionsUnsupportedError(void 0);
|
|
221
|
+
}
|
|
222
|
+
for (const collection of [this.#events, this.#deliveries, this.#keySequences]) {
|
|
223
|
+
await this.#createCollectionIfMissing(collection.collectionName);
|
|
224
|
+
}
|
|
225
|
+
await this.#deliveries.createIndexes([
|
|
226
|
+
// Idempotent appends and lookups by event.
|
|
227
|
+
{ key: { eventId: 1, handler: 1 }, name: "event_handler", unique: true },
|
|
228
|
+
// Claim: due work of a handler in due order. Holds ready deliveries only.
|
|
229
|
+
{
|
|
230
|
+
key: { handler: 1, dueAt: 1, _id: 1 },
|
|
231
|
+
name: "claim",
|
|
232
|
+
partialFilterExpression: { lane: "ready" }
|
|
233
|
+
},
|
|
234
|
+
// Ordering: unfinished predecessors of a keyed delivery, parked or not.
|
|
235
|
+
{
|
|
236
|
+
key: { handler: 1, key: 1, sequence: 1 },
|
|
237
|
+
name: "ordering",
|
|
238
|
+
partialFilterExpression: { dueAt: { $exists: true }, sequence: { $exists: true } }
|
|
239
|
+
},
|
|
240
|
+
// Parked deliveries: the lowest parked delivery of each key, to move it back once it is next.
|
|
241
|
+
{
|
|
242
|
+
key: { handler: 1, key: 1, sequence: 1 },
|
|
243
|
+
name: "parked",
|
|
244
|
+
partialFilterExpression: { lane: "parked" }
|
|
245
|
+
},
|
|
246
|
+
// Live leases: a keyed delivery may not start while another of its key is running.
|
|
247
|
+
{
|
|
248
|
+
key: { handler: 1, key: 1, dueAt: 1 },
|
|
249
|
+
name: "running",
|
|
250
|
+
partialFilterExpression: { status: "processing", key: { $exists: true } }
|
|
251
|
+
},
|
|
252
|
+
// Statistics: oldest due delivery across handlers.
|
|
253
|
+
{ key: { dueAt: 1 }, name: "due", partialFilterExpression: { dueAt: { $exists: true } } },
|
|
254
|
+
// Purge of completed deliveries.
|
|
255
|
+
{
|
|
256
|
+
key: { completedAt: 1 },
|
|
257
|
+
name: "completed",
|
|
258
|
+
partialFilterExpression: { completedAt: { $exists: true } }
|
|
259
|
+
},
|
|
260
|
+
// Listing and status counts.
|
|
261
|
+
{ key: { status: 1, createdAt: 1 }, name: "status_created" },
|
|
262
|
+
// Purge of dead deliveries.
|
|
263
|
+
{ key: { status: 1, updatedAt: 1 }, name: "status_updated" },
|
|
264
|
+
{ key: { handler: 1, createdAt: 1 }, name: "handler_created" },
|
|
265
|
+
{ key: { eventType: 1, createdAt: 1 }, name: "type_created" }
|
|
266
|
+
]);
|
|
267
|
+
}
|
|
268
|
+
async #createCollectionIfMissing(name) {
|
|
269
|
+
try {
|
|
270
|
+
await this.#database.createCollection(name);
|
|
271
|
+
} catch (error) {
|
|
272
|
+
if (!(typeof error === "object" && error !== null && "code" in error && error.code === NAMESPACE_EXISTS)) {
|
|
273
|
+
throw error;
|
|
274
|
+
}
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
/**
|
|
278
|
+
* Runs `work` in a transaction on a new session and commits when it resolves.
|
|
279
|
+
*
|
|
280
|
+
* `work` can run more than once. The driver restarts the whole callback after
|
|
281
|
+
* a transient failure such as a write conflict, and re-sends only the commit
|
|
282
|
+
* after an unknown commit result. A restart is safe for everything written
|
|
283
|
+
* through the session, because the aborted attempt left nothing behind. It is
|
|
284
|
+
* not safe for side effects outside the session (HTTP calls, other databases),
|
|
285
|
+
* so keep those out of `work`; relaybus handlers are already at-least-once.
|
|
286
|
+
*
|
|
287
|
+
* @throws ConfigurationError when the deployment does not support transactions.
|
|
288
|
+
*/
|
|
289
|
+
async transaction(work) {
|
|
290
|
+
try {
|
|
291
|
+
return await this.#client.withSession(async (session) => {
|
|
292
|
+
let result;
|
|
293
|
+
await session.withTransaction(async () => {
|
|
294
|
+
result = await work(session);
|
|
295
|
+
}, this.#transactionOptions);
|
|
296
|
+
return result;
|
|
297
|
+
});
|
|
298
|
+
} catch (error) {
|
|
299
|
+
if (isTransactionsUnsupportedError(error)) {
|
|
300
|
+
throw createTransactionsUnsupportedError(error);
|
|
301
|
+
}
|
|
302
|
+
throw error;
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
async append(session, entries, now) {
|
|
306
|
+
assertInTransaction(session, "append");
|
|
307
|
+
const eventIds = [...new Set(entries.map((entry) => entry.event.id))];
|
|
308
|
+
const storedEvents = new Map(
|
|
309
|
+
(await this.#events.find({ _id: { $in: eventIds } }, { session, projection: { key: 1, sequence: 1 } }).toArray()).map((stored) => [stored._id, stored])
|
|
310
|
+
);
|
|
311
|
+
const storedDeliveryKeys = new Set(
|
|
312
|
+
(await this.#deliveries.find({ eventId: { $in: eventIds } }, { session, projection: { eventId: 1, handler: 1 } }).toArray()).map((stored) => deliveryPairKey(stored.eventId, stored.handler))
|
|
313
|
+
);
|
|
314
|
+
const appendedEventIds = [];
|
|
315
|
+
const duplicateEventIds = [];
|
|
316
|
+
const newEvents = [];
|
|
317
|
+
const newDeliveries = [];
|
|
318
|
+
const touchedEventIds = /* @__PURE__ */ new Set();
|
|
319
|
+
const orderingByEventId = /* @__PURE__ */ new Map();
|
|
320
|
+
for (const [eventId, stored] of storedEvents) {
|
|
321
|
+
orderingByEventId.set(eventId, { key: stored.key, sequence: stored.sequence });
|
|
322
|
+
}
|
|
323
|
+
for (const { event, deliveries } of entries) {
|
|
324
|
+
let ordering = orderingByEventId.get(event.id);
|
|
325
|
+
if (ordering === void 0) {
|
|
326
|
+
const sequence = event.key === null ? null : await this.#nextSequence(event.key, session);
|
|
327
|
+
ordering = { key: event.key, sequence };
|
|
328
|
+
orderingByEventId.set(event.id, ordering);
|
|
329
|
+
newEvents.push(toEventDocument(event, sequence));
|
|
330
|
+
appendedEventIds.push(event.id);
|
|
331
|
+
} else {
|
|
332
|
+
duplicateEventIds.push(event.id);
|
|
333
|
+
}
|
|
334
|
+
for (const delivery of deliveries) {
|
|
335
|
+
const pairKey = deliveryPairKey(event.id, delivery.handler);
|
|
336
|
+
if (storedDeliveryKeys.has(pairKey)) {
|
|
337
|
+
continue;
|
|
338
|
+
}
|
|
339
|
+
storedDeliveryKeys.add(pairKey);
|
|
340
|
+
newDeliveries.push(toDeliveryDocument(delivery, event, ordering, now));
|
|
341
|
+
if (storedEvents.has(event.id)) {
|
|
342
|
+
touchedEventIds.add(event.id);
|
|
343
|
+
}
|
|
344
|
+
}
|
|
345
|
+
}
|
|
346
|
+
if (newEvents.length > 0) {
|
|
347
|
+
await this.#events.bulkWrite(
|
|
348
|
+
newEvents.map((document) => ({
|
|
349
|
+
updateOne: {
|
|
350
|
+
filter: { _id: document._id },
|
|
351
|
+
update: { $setOnInsert: document },
|
|
352
|
+
upsert: true
|
|
353
|
+
}
|
|
354
|
+
})),
|
|
355
|
+
{ session, ordered: true }
|
|
356
|
+
);
|
|
357
|
+
}
|
|
358
|
+
if (touchedEventIds.size > 0) {
|
|
359
|
+
await this.#events.updateMany(
|
|
360
|
+
{ _id: { $in: [...touchedEventIds] } },
|
|
361
|
+
{ $inc: { revision: 1 } },
|
|
362
|
+
{ session }
|
|
363
|
+
);
|
|
364
|
+
}
|
|
365
|
+
if (newDeliveries.length > 0) {
|
|
366
|
+
await this.#deliveries.bulkWrite(
|
|
367
|
+
newDeliveries.map((document) => ({
|
|
368
|
+
updateOne: {
|
|
369
|
+
filter: { eventId: document.eventId, handler: document.handler },
|
|
370
|
+
update: { $setOnInsert: document },
|
|
371
|
+
upsert: true
|
|
372
|
+
}
|
|
373
|
+
})),
|
|
374
|
+
{ session, ordered: true }
|
|
375
|
+
);
|
|
376
|
+
}
|
|
377
|
+
return { appendedEventIds, duplicateEventIds, appendedDeliveryCount: newDeliveries.length };
|
|
378
|
+
}
|
|
379
|
+
async claim(request) {
|
|
380
|
+
if (request.handlers.length === 0 || request.limit <= 0) {
|
|
381
|
+
return [];
|
|
382
|
+
}
|
|
383
|
+
await this.#unparkNextDeliveries(request.handlers);
|
|
384
|
+
const claimed = [];
|
|
385
|
+
const scan = new ClaimScan();
|
|
386
|
+
const candidates = this.#deliveries.find(
|
|
387
|
+
{ handler: { $in: [...request.handlers] }, dueAt: { $lte: request.now }, lane: "ready" },
|
|
388
|
+
{
|
|
389
|
+
sort: { dueAt: 1, _id: 1 },
|
|
390
|
+
batchSize: Math.max(request.limit * 2, MINIMUM_CANDIDATE_BATCH_SIZE)
|
|
391
|
+
}
|
|
392
|
+
);
|
|
393
|
+
try {
|
|
394
|
+
for await (const candidate of candidates) {
|
|
395
|
+
const leased = await this.#claimCandidate(candidate, request, scan);
|
|
396
|
+
if (leased !== void 0) {
|
|
397
|
+
claimed.push(leased);
|
|
398
|
+
if (claimed.length >= request.limit) {
|
|
399
|
+
break;
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
}
|
|
403
|
+
} finally {
|
|
404
|
+
await candidates.close();
|
|
405
|
+
}
|
|
406
|
+
await this.#parkDeliveries(scan.waitingBacklogs(PARK_BACKLOG_THRESHOLD));
|
|
407
|
+
return this.#toClaimedDeliveries(claimed, request.leaseExpiresAt);
|
|
408
|
+
}
|
|
409
|
+
async extendLease(lease, leaseExpiresAt, now) {
|
|
410
|
+
return this.#updateLeased(lease, void 0, {
|
|
411
|
+
$set: { leaseExpiresAt, dueAt: leaseExpiresAt, updatedAt: now }
|
|
412
|
+
});
|
|
413
|
+
}
|
|
414
|
+
async complete(lease, now, session) {
|
|
415
|
+
if (session !== void 0) {
|
|
416
|
+
assertInTransaction(session, "complete");
|
|
417
|
+
}
|
|
418
|
+
return this.#updateLeased(lease, session, {
|
|
419
|
+
$set: { status: "completed", completedAt: now, updatedAt: now },
|
|
420
|
+
$unset: { ...LEASE_FIELDS, ...FINISHED_FIELDS }
|
|
421
|
+
});
|
|
422
|
+
}
|
|
423
|
+
async reschedule(lease, availableAt, error, now) {
|
|
424
|
+
return this.#updateLeased(lease, void 0, {
|
|
425
|
+
$set: {
|
|
426
|
+
status: "pending",
|
|
427
|
+
availableAt,
|
|
428
|
+
dueAt: availableAt,
|
|
429
|
+
lastError: toErrorInfo(error),
|
|
430
|
+
updatedAt: now
|
|
431
|
+
},
|
|
432
|
+
$unset: LEASE_FIELDS
|
|
433
|
+
});
|
|
434
|
+
}
|
|
435
|
+
async release(lease, now) {
|
|
436
|
+
return this.#updateLeased(lease, void 0, {
|
|
437
|
+
$set: { status: "pending", availableAt: now, dueAt: now, updatedAt: now },
|
|
438
|
+
$inc: { attempts: -1 },
|
|
439
|
+
$unset: LEASE_FIELDS
|
|
440
|
+
});
|
|
441
|
+
}
|
|
442
|
+
async bury(lease, error, now) {
|
|
443
|
+
return this.#updateLeased(lease, void 0, {
|
|
444
|
+
$set: { status: "dead", lastError: toErrorInfo(error), updatedAt: now },
|
|
445
|
+
$unset: { ...LEASE_FIELDS, ...FINISHED_FIELDS }
|
|
446
|
+
});
|
|
447
|
+
}
|
|
448
|
+
async listDeliveries(query) {
|
|
449
|
+
const filter = {
|
|
450
|
+
...query.status === void 0 ? {} : { status: String(query.status) },
|
|
451
|
+
...query.handler === void 0 ? {} : { handler: String(query.handler) },
|
|
452
|
+
...query.eventType === void 0 ? {} : { eventType: String(query.eventType) },
|
|
453
|
+
...query.eventId === void 0 ? {} : { eventId: String(query.eventId) }
|
|
454
|
+
};
|
|
455
|
+
const documents = await this.#deliveries.find(filter, { sort: { createdAt: 1, _id: 1 }, limit: query.limit ?? DEFAULT_LIST_LIMIT }).toArray();
|
|
456
|
+
const events = await this.#readEnvelopes(documents.map((document) => document.eventId));
|
|
457
|
+
return documents.map((document) => toDeliveryRecord(document, events.get(document.eventId)));
|
|
458
|
+
}
|
|
459
|
+
async requeue(deliveryIds, now) {
|
|
460
|
+
const result = await this.#deliveries.updateMany(
|
|
461
|
+
{ _id: { $in: deliveryIds.map(String) }, status: "dead" },
|
|
462
|
+
{
|
|
463
|
+
$set: {
|
|
464
|
+
status: "pending",
|
|
465
|
+
lane: "ready",
|
|
466
|
+
attempts: 0,
|
|
467
|
+
availableAt: now,
|
|
468
|
+
dueAt: now,
|
|
469
|
+
updatedAt: now
|
|
470
|
+
}
|
|
471
|
+
}
|
|
472
|
+
);
|
|
473
|
+
return result.modifiedCount;
|
|
474
|
+
}
|
|
475
|
+
async purge(request) {
|
|
476
|
+
const filters = [];
|
|
477
|
+
if (request.completedBefore !== void 0) {
|
|
478
|
+
filters.push({ status: "completed", completedAt: { $lt: request.completedBefore } });
|
|
479
|
+
}
|
|
480
|
+
if (request.deadBefore !== void 0) {
|
|
481
|
+
filters.push({ status: "dead", updatedAt: { $lt: request.deadBefore } });
|
|
482
|
+
}
|
|
483
|
+
let deletedDeliveries = 0;
|
|
484
|
+
let deletedEvents = 0;
|
|
485
|
+
for (const filter of filters) {
|
|
486
|
+
const result = await this.#purgeMatching(filter);
|
|
487
|
+
deletedDeliveries += result.deletedDeliveries;
|
|
488
|
+
deletedEvents += result.deletedEvents;
|
|
489
|
+
}
|
|
490
|
+
return { deletedDeliveries, deletedEvents };
|
|
491
|
+
}
|
|
492
|
+
async statistics(now) {
|
|
493
|
+
const statuses = ["pending", "processing", "completed", "dead"];
|
|
494
|
+
const [pending, processing, completed, dead] = await Promise.all(
|
|
495
|
+
statuses.map((status) => this.#deliveries.countDocuments({ status }))
|
|
496
|
+
);
|
|
497
|
+
const oldestDue = await this.#deliveries.findOne(
|
|
498
|
+
{ dueAt: { $lte: now } },
|
|
499
|
+
{ sort: { dueAt: 1 }, projection: { dueAt: 1 } }
|
|
500
|
+
);
|
|
501
|
+
return {
|
|
502
|
+
pending,
|
|
503
|
+
processing,
|
|
504
|
+
completed,
|
|
505
|
+
dead,
|
|
506
|
+
oldestDueAt: oldestDue?.dueAt ?? null
|
|
507
|
+
};
|
|
508
|
+
}
|
|
509
|
+
/** Atomically increments the counter of `key`; conflicting appends of the key serialize here. */
|
|
510
|
+
async #nextSequence(key, session) {
|
|
511
|
+
const counter = await this.#keySequences.findOneAndUpdate(
|
|
512
|
+
{ _id: key },
|
|
513
|
+
{ $inc: { value: 1 } },
|
|
514
|
+
{ upsert: true, returnDocument: "after", session }
|
|
515
|
+
);
|
|
516
|
+
return counter.value;
|
|
517
|
+
}
|
|
518
|
+
async #claimCandidate(candidate, request, scan) {
|
|
519
|
+
const { key, sequence } = candidate;
|
|
520
|
+
if (key === void 0) {
|
|
521
|
+
return this.#lease(candidate, request);
|
|
522
|
+
}
|
|
523
|
+
const orderingId = orderingIdentity(candidate.handler, key);
|
|
524
|
+
const known = scan.verdict(orderingId, sequence);
|
|
525
|
+
if (known === "open-predecessor") {
|
|
526
|
+
scan.noteWaiting(orderingId, candidate);
|
|
527
|
+
return void 0;
|
|
528
|
+
}
|
|
529
|
+
if (known === "live-lease") {
|
|
530
|
+
return void 0;
|
|
531
|
+
}
|
|
532
|
+
const predecessor = await this.#findUnfinishedPredecessor(candidate.handler, key, sequence);
|
|
533
|
+
if (predecessor !== void 0) {
|
|
534
|
+
scan.noteOpen(orderingId, predecessor);
|
|
535
|
+
scan.noteWaiting(orderingId, candidate);
|
|
536
|
+
return void 0;
|
|
537
|
+
}
|
|
538
|
+
if (await this.#hasLiveLease(candidate.handler, key, request.now)) {
|
|
539
|
+
scan.noteLeaseHeld(orderingId);
|
|
540
|
+
return void 0;
|
|
541
|
+
}
|
|
542
|
+
const leased = await this.#lease(candidate, request);
|
|
543
|
+
if (leased === void 0) {
|
|
544
|
+
return void 0;
|
|
545
|
+
}
|
|
546
|
+
const lateBlocker = await this.#findUnfinishedPredecessor(candidate.handler, key, sequence);
|
|
547
|
+
if (lateBlocker !== void 0) {
|
|
548
|
+
await this.#restore(candidate, leased);
|
|
549
|
+
scan.noteOpen(orderingId, lateBlocker);
|
|
550
|
+
return void 0;
|
|
551
|
+
}
|
|
552
|
+
scan.noteClaimed(orderingId, sequence);
|
|
553
|
+
return leased;
|
|
554
|
+
}
|
|
555
|
+
/**
|
|
556
|
+
* Takes the delivery with one conditional update that pins the exact state
|
|
557
|
+
* that was selected, so a competing claimer, a finished lease, a reschedule
|
|
558
|
+
* or a parking since then makes it match nothing.
|
|
559
|
+
*/
|
|
560
|
+
async #lease(candidate, request) {
|
|
561
|
+
const leased = await this.#deliveries.findOneAndUpdate(
|
|
562
|
+
{
|
|
563
|
+
_id: candidate._id,
|
|
564
|
+
status: candidate.status,
|
|
565
|
+
lane: "ready",
|
|
566
|
+
attempts: candidate.attempts,
|
|
567
|
+
dueAt: candidate.dueAt
|
|
568
|
+
},
|
|
569
|
+
{
|
|
570
|
+
$set: {
|
|
571
|
+
status: "processing",
|
|
572
|
+
leaseOwner: request.leaseOwner,
|
|
573
|
+
leaseToken: this.#generateIdentifier(),
|
|
574
|
+
leaseExpiresAt: request.leaseExpiresAt,
|
|
575
|
+
dueAt: request.leaseExpiresAt,
|
|
576
|
+
updatedAt: request.now
|
|
577
|
+
},
|
|
578
|
+
$inc: { attempts: 1 }
|
|
579
|
+
},
|
|
580
|
+
{ returnDocument: "after" }
|
|
581
|
+
);
|
|
582
|
+
return leased ?? void 0;
|
|
583
|
+
}
|
|
584
|
+
/** Puts a delivery that was leased a moment ago back exactly as it was selected. */
|
|
585
|
+
async #restore(selected, leased) {
|
|
586
|
+
const state = {
|
|
587
|
+
status: selected.status,
|
|
588
|
+
attempts: selected.attempts,
|
|
589
|
+
dueAt: selected.dueAt,
|
|
590
|
+
updatedAt: selected.updatedAt
|
|
591
|
+
};
|
|
592
|
+
await this.#deliveries.updateOne(
|
|
593
|
+
{ _id: leased._id, status: "processing", leaseToken: leased.leaseToken },
|
|
594
|
+
selected.status === "processing" ? {
|
|
595
|
+
$set: {
|
|
596
|
+
...state,
|
|
597
|
+
leaseOwner: selected.leaseOwner,
|
|
598
|
+
leaseToken: selected.leaseToken,
|
|
599
|
+
leaseExpiresAt: selected.leaseExpiresAt
|
|
600
|
+
}
|
|
601
|
+
} : { $set: state, $unset: LEASE_FIELDS }
|
|
602
|
+
);
|
|
603
|
+
}
|
|
604
|
+
/** Sequence of the lowest unfinished delivery of the handler and key before `sequence`, if any. */
|
|
605
|
+
async #findUnfinishedPredecessor(handler, key, sequence) {
|
|
606
|
+
const predecessor = await this.#deliveries.findOne(
|
|
607
|
+
{ handler, key, sequence: { $lt: sequence }, dueAt: { $exists: true } },
|
|
608
|
+
{ sort: { sequence: 1 }, projection: { sequence: 1 } }
|
|
609
|
+
);
|
|
610
|
+
return predecessor?.sequence;
|
|
611
|
+
}
|
|
612
|
+
/** Whether some delivery of the handler and key runs under a lease that has not expired. */
|
|
613
|
+
async #hasLiveLease(handler, key, now) {
|
|
614
|
+
const running = await this.#deliveries.findOne(
|
|
615
|
+
{ handler, key, status: "processing", dueAt: { $gt: now } },
|
|
616
|
+
{ projection: { _id: 1 } }
|
|
617
|
+
);
|
|
618
|
+
return running !== null;
|
|
619
|
+
}
|
|
620
|
+
/**
|
|
621
|
+
* Parks pending deliveries that waited behind an unfinished predecessor, which
|
|
622
|
+
* removes them from the claim index until their turn. Parking is a hint, not a
|
|
623
|
+
* guarantee: a predecessor that finishes meanwhile is found by the next
|
|
624
|
+
* {@link MongoEventStore.claim}, which moves the lowest parked delivery back.
|
|
625
|
+
*/
|
|
626
|
+
async #parkDeliveries(deliveryIds) {
|
|
627
|
+
for (let offset = 0; offset < deliveryIds.length; offset += PARK_UPDATE_CHUNK_SIZE) {
|
|
628
|
+
await this.#deliveries.updateMany(
|
|
629
|
+
{
|
|
630
|
+
_id: { $in: deliveryIds.slice(offset, offset + PARK_UPDATE_CHUNK_SIZE) },
|
|
631
|
+
status: "pending",
|
|
632
|
+
lane: "ready"
|
|
633
|
+
},
|
|
634
|
+
{ $set: { lane: "parked" } }
|
|
635
|
+
);
|
|
636
|
+
}
|
|
637
|
+
}
|
|
638
|
+
/**
|
|
639
|
+
* Moves the lowest parked delivery of every parked key of `handlers` back into
|
|
640
|
+
* the claim index when nothing unfinished precedes it. Level-triggered on
|
|
641
|
+
* purpose: it looks at the current state only, so a crash anywhere, or a
|
|
642
|
+
* predecessor finishing while its followers were being parked, can delay a key
|
|
643
|
+
* until the next claim but never strand it.
|
|
644
|
+
*/
|
|
645
|
+
async #unparkNextDeliveries(handlers) {
|
|
646
|
+
const anyParked = await this.#deliveries.findOne(
|
|
647
|
+
{ handler: { $in: [...handlers] }, lane: "parked" },
|
|
648
|
+
// The sort lets the planner pick the small partial index over the handler's whole history.
|
|
649
|
+
{ sort: { handler: 1, key: 1, sequence: 1 }, projection: { _id: 1 } }
|
|
650
|
+
);
|
|
651
|
+
if (anyParked === null) {
|
|
652
|
+
return;
|
|
653
|
+
}
|
|
654
|
+
for (const handler of new Set(handlers)) {
|
|
655
|
+
let afterKey;
|
|
656
|
+
for (; ; ) {
|
|
657
|
+
const lowest = await this.#deliveries.findOne(
|
|
658
|
+
{
|
|
659
|
+
handler,
|
|
660
|
+
lane: "parked",
|
|
661
|
+
...afterKey === void 0 ? {} : { key: { $gt: afterKey } }
|
|
662
|
+
},
|
|
663
|
+
{ sort: { key: 1, sequence: 1 }, projection: { key: 1, sequence: 1 } }
|
|
664
|
+
);
|
|
665
|
+
if (lowest === null) {
|
|
666
|
+
break;
|
|
667
|
+
}
|
|
668
|
+
afterKey = lowest.key;
|
|
669
|
+
if (await this.#findUnfinishedPredecessor(handler, afterKey, lowest.sequence) === void 0) {
|
|
670
|
+
await this.#deliveries.updateOne(
|
|
671
|
+
{ _id: lowest._id, lane: "parked" },
|
|
672
|
+
{ $set: { lane: "ready" } }
|
|
673
|
+
);
|
|
674
|
+
}
|
|
675
|
+
}
|
|
676
|
+
}
|
|
677
|
+
}
|
|
678
|
+
async #toClaimedDeliveries(leased, leaseExpiresAt) {
|
|
679
|
+
const events = await this.#readEnvelopes(leased.map((document) => document.eventId));
|
|
680
|
+
return leased.map((document) => ({
|
|
681
|
+
deliveryId: document._id,
|
|
682
|
+
leaseToken: document.leaseToken,
|
|
683
|
+
handler: document.handler,
|
|
684
|
+
attempt: document.attempts,
|
|
685
|
+
leaseExpiresAt,
|
|
686
|
+
event: events.get(document.eventId)
|
|
687
|
+
}));
|
|
688
|
+
}
|
|
689
|
+
async #readEnvelopes(eventIds) {
|
|
690
|
+
const documents = await this.#events.find({ _id: { $in: [...new Set(eventIds)] } }).toArray();
|
|
691
|
+
return new Map(documents.map((document) => [document._id, toEnvelope(document)]));
|
|
692
|
+
}
|
|
693
|
+
async #updateLeased(lease, session, update) {
|
|
694
|
+
const result = await this.#deliveries.updateOne(
|
|
695
|
+
{ _id: lease.deliveryId, status: "processing", leaseToken: lease.leaseToken },
|
|
696
|
+
update,
|
|
697
|
+
session === void 0 ? {} : { session }
|
|
698
|
+
);
|
|
699
|
+
return result.matchedCount === 1;
|
|
700
|
+
}
|
|
701
|
+
/**
|
|
702
|
+
* Deletes batches of matching deliveries together with the events no delivery
|
|
703
|
+
* references any more, each batch in one transaction, so a crash can never
|
|
704
|
+
* leave an event without deliveries behind. An append adding a delivery to the
|
|
705
|
+
* same event writes the event document, so the two conflict and one restarts
|
|
706
|
+
* instead of leaving a delivery without event. A delivery requeued since the
|
|
707
|
+
* batch was read conflicts with its deletion and is read again.
|
|
708
|
+
*/
|
|
709
|
+
async #purgeMatching(filter) {
|
|
710
|
+
let deletedDeliveries = 0;
|
|
711
|
+
let deletedEvents = 0;
|
|
712
|
+
for (; ; ) {
|
|
713
|
+
const step = await this.transaction(async (session) => {
|
|
714
|
+
const batch = await this.#deliveries.find(filter, { session, projection: { eventId: 1 }, limit: this.#purgeBatchSize }).toArray();
|
|
715
|
+
const eventIds = [...new Set(batch.map((document) => document.eventId))];
|
|
716
|
+
const deliveries = await this.#deliveries.deleteMany(
|
|
717
|
+
{ _id: { $in: batch.map((document) => document._id) } },
|
|
718
|
+
{ session }
|
|
719
|
+
);
|
|
720
|
+
const stillReferenced = new Set(
|
|
721
|
+
(await this.#deliveries.find({ eventId: { $in: eventIds } }, { session, projection: { eventId: 1 } }).toArray()).map((document) => document.eventId)
|
|
722
|
+
);
|
|
723
|
+
const events = await this.#events.deleteMany(
|
|
724
|
+
{ _id: { $in: eventIds.filter((eventId) => !stillReferenced.has(eventId)) } },
|
|
725
|
+
{ session }
|
|
726
|
+
);
|
|
727
|
+
return {
|
|
728
|
+
batchSize: batch.length,
|
|
729
|
+
deletedDeliveries: deliveries.deletedCount,
|
|
730
|
+
deletedEvents: events.deletedCount
|
|
731
|
+
};
|
|
732
|
+
});
|
|
733
|
+
deletedDeliveries += step.deletedDeliveries;
|
|
734
|
+
deletedEvents += step.deletedEvents;
|
|
735
|
+
if (step.batchSize < this.#purgeBatchSize) {
|
|
736
|
+
return { deletedDeliveries, deletedEvents };
|
|
737
|
+
}
|
|
738
|
+
}
|
|
739
|
+
}
|
|
740
|
+
};
|
|
741
|
+
function deliveryPairKey(eventId, handler) {
|
|
742
|
+
return `${eventId}\0${handler}`;
|
|
743
|
+
}
|
|
744
|
+
function assertInTransaction(session, operation) {
|
|
745
|
+
if (!session.inTransaction()) {
|
|
746
|
+
throw new ConfigurationError(
|
|
747
|
+
`MongoEventStore.${operation} needs a ClientSession inside a started transaction (session.withTransaction or startTransaction). Without one, MongoDB commits each write immediately and the write is no longer atomic with yours`
|
|
748
|
+
);
|
|
749
|
+
}
|
|
750
|
+
}
|
|
751
|
+
|
|
752
|
+
export { MongoEventStore };
|
|
753
|
+
//# sourceMappingURL=index.js.map
|
|
754
|
+
//# sourceMappingURL=index.js.map
|