@relaybus/mongodb 0.2.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -0
- package/dist/index.cjs +1730 -1306
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +8 -5
- package/dist/index.d.ts +8 -5
- package/dist/index.js +1730 -1306
- package/dist/index.js.map +1 -1
- package/package.json +4 -4
package/dist/index.cjs
CHANGED
|
@@ -1,57 +1,9 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
|
|
3
3
|
var core = require('@relaybus/core');
|
|
4
|
+
var internal = require('@relaybus/core/internal');
|
|
4
5
|
|
|
5
|
-
// src/
|
|
6
|
-
|
|
7
|
-
// src/claim-scan.ts
|
|
8
|
-
var ClaimScan = class {
|
|
9
|
-
#keys = /* @__PURE__ */ new Map();
|
|
10
|
-
/** Whether an unfinished delivery this scan knows of comes before the delivery with `sequence`. */
|
|
11
|
-
waits(orderingId, sequence) {
|
|
12
|
-
const known = this.#keys.get(orderingId);
|
|
13
|
-
return known?.openSequence !== void 0 && sequence > known.openSequence;
|
|
14
|
-
}
|
|
15
|
-
/** Whether this scan already learned anything about the key. */
|
|
16
|
-
knows(orderingId) {
|
|
17
|
-
return this.#keys.has(orderingId);
|
|
18
|
-
}
|
|
19
|
-
/** Records an unfinished delivery of the key with this sequence. */
|
|
20
|
-
noteOpen(orderingId, sequence) {
|
|
21
|
-
const known = this.#know(orderingId);
|
|
22
|
-
known.openSequence = Math.min(known.openSequence ?? sequence, sequence);
|
|
23
|
-
}
|
|
24
|
-
/** Remembers a pending delivery of lane `laneId` that waited behind an unfinished predecessor. */
|
|
25
|
-
noteWaiting(orderingId, deliveryId, laneId) {
|
|
26
|
-
const known = this.#know(orderingId);
|
|
27
|
-
known.waiting.push(deliveryId);
|
|
28
|
-
known.laneId = laneId;
|
|
29
|
-
}
|
|
30
|
-
/**
|
|
31
|
-
* Ids of the waiting deliveries of every key that accumulated at least
|
|
32
|
-
* `minimum` of them, or that lives in one of `handedBack`. Shorter queues
|
|
33
|
-
* of a lane that stays held remain in the claim index: moving them out and
|
|
34
|
-
* back would cost more than skipping them. In a lane handed back they are
|
|
35
|
-
* all its due work, and leaving them ready would make every later claim take
|
|
36
|
-
* the lane for nothing.
|
|
37
|
-
*/
|
|
38
|
-
waitingBacklogs(minimum, handedBack) {
|
|
39
|
-
return [...this.#keys.values()].filter((known) => known.waiting.length >= minimum || handedBack.has(known.laneId)).flatMap((known) => known.waiting);
|
|
40
|
-
}
|
|
41
|
-
#know(orderingId) {
|
|
42
|
-
let known = this.#keys.get(orderingId);
|
|
43
|
-
if (known === void 0) {
|
|
44
|
-
known = { openSequence: void 0, waiting: [], laneId: void 0 };
|
|
45
|
-
this.#keys.set(orderingId, known);
|
|
46
|
-
}
|
|
47
|
-
return known;
|
|
48
|
-
}
|
|
49
|
-
};
|
|
50
|
-
function orderingIdentity(handler, key) {
|
|
51
|
-
return `${handler}\0${key}`;
|
|
52
|
-
}
|
|
53
|
-
|
|
54
|
-
// src/lanes.ts
|
|
6
|
+
// src/documents.ts
|
|
55
7
|
var LANE_COUNT = 256;
|
|
56
8
|
var FNV_OFFSET_BASIS = 2166136261;
|
|
57
9
|
var FNV_PRIME = 16777619;
|
|
@@ -63,20 +15,28 @@ function laneOf(key, eventId) {
|
|
|
63
15
|
}
|
|
64
16
|
return hash & LANE_COUNT - 1;
|
|
65
17
|
}
|
|
66
|
-
|
|
67
|
-
|
|
18
|
+
var LEASE_TOKENS = new internal.LaneLeaseTokenFormat({
|
|
19
|
+
laneCount: LANE_COUNT,
|
|
20
|
+
epochDigits: 15,
|
|
21
|
+
claimsDigits: 15
|
|
22
|
+
});
|
|
23
|
+
function formatLeaseToken(laneId, { epoch, claims }) {
|
|
24
|
+
return LEASE_TOKENS.format(laneId, { epoch, claims });
|
|
68
25
|
}
|
|
69
|
-
var LEASE_TOKEN_PATTERN = /^(\d{1,3}):(\d{1,15}):(\d{1,15})$/;
|
|
70
26
|
function parseLeaseToken(token) {
|
|
71
|
-
const
|
|
72
|
-
if (
|
|
27
|
+
const parts = LEASE_TOKENS.parse(token);
|
|
28
|
+
if (parts === void 0) {
|
|
73
29
|
return void 0;
|
|
74
30
|
}
|
|
75
|
-
return { laneId:
|
|
31
|
+
return { laneId: parts.lane, epoch: Number(parts.epoch), claims: parts.claims };
|
|
76
32
|
}
|
|
77
33
|
|
|
78
34
|
// src/documents.ts
|
|
79
|
-
function toEventDocument(event,
|
|
35
|
+
function toEventDocument(event, {
|
|
36
|
+
sequence,
|
|
37
|
+
handlers,
|
|
38
|
+
createdAt
|
|
39
|
+
}) {
|
|
80
40
|
const payloadJson = serializeWellFormed(event.payload, "payload");
|
|
81
41
|
const headersJson = serializeWellFormed(event.headers, "headers");
|
|
82
42
|
return {
|
|
@@ -121,7 +81,11 @@ function toEnvelope(document) {
|
|
|
121
81
|
occurredAt: document.occurredAt
|
|
122
82
|
};
|
|
123
83
|
}
|
|
124
|
-
function toDeliveryDocument(delivery,
|
|
84
|
+
function toDeliveryDocument(delivery, {
|
|
85
|
+
event,
|
|
86
|
+
ordering,
|
|
87
|
+
now
|
|
88
|
+
}) {
|
|
125
89
|
return {
|
|
126
90
|
_id: delivery.id,
|
|
127
91
|
eventId: event.id,
|
|
@@ -138,7 +102,10 @@ function toDeliveryDocument(delivery, event, ordering, now) {
|
|
|
138
102
|
createdAt: now
|
|
139
103
|
};
|
|
140
104
|
}
|
|
141
|
-
function toDeliveryRecord(document,
|
|
105
|
+
function toDeliveryRecord(document, {
|
|
106
|
+
event,
|
|
107
|
+
runningIn
|
|
108
|
+
}) {
|
|
142
109
|
return {
|
|
143
110
|
id: document._id,
|
|
144
111
|
handler: document.handler,
|
|
@@ -183,7 +150,17 @@ function toCompletedDeliveryDocument(document, completedAt) {
|
|
|
183
150
|
};
|
|
184
151
|
}
|
|
185
152
|
function toErrorInfo(error) {
|
|
186
|
-
|
|
153
|
+
const info = {
|
|
154
|
+
name: error.name,
|
|
155
|
+
message: error.message
|
|
156
|
+
};
|
|
157
|
+
if (error.stack !== void 0) {
|
|
158
|
+
info.stack = error.stack;
|
|
159
|
+
}
|
|
160
|
+
if (error.code !== void 0) {
|
|
161
|
+
info.code = error.code;
|
|
162
|
+
}
|
|
163
|
+
return info;
|
|
187
164
|
}
|
|
188
165
|
var ILLEGAL_OPERATION = 20;
|
|
189
166
|
function isTransactionsUnsupportedError(error) {
|
|
@@ -195,463 +172,362 @@ function createTransactionsUnsupportedError(cause) {
|
|
|
195
172
|
{ cause }
|
|
196
173
|
);
|
|
197
174
|
}
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
function
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
175
|
+
|
|
176
|
+
// src/mongo-store-state.ts
|
|
177
|
+
async function runInTransaction(state, work) {
|
|
178
|
+
return runInTransactionWith(state, { work, transactionOptions: state.transactionOptions });
|
|
179
|
+
}
|
|
180
|
+
async function runInBudgetedTransaction(state, {
|
|
181
|
+
work,
|
|
182
|
+
timeoutMs
|
|
183
|
+
}) {
|
|
184
|
+
const budget = Math.ceil(timeoutMs);
|
|
185
|
+
return runInTransactionWith(state, {
|
|
186
|
+
work,
|
|
187
|
+
transactionOptions: { ...state.transactionOptions, maxCommitTimeMS: budget }
|
|
188
|
+
});
|
|
189
|
+
}
|
|
190
|
+
async function runInTransactionWith(state, {
|
|
191
|
+
work,
|
|
192
|
+
transactionOptions
|
|
193
|
+
}) {
|
|
194
|
+
try {
|
|
195
|
+
return await state.client.withSession(async (session) => {
|
|
196
|
+
let result;
|
|
197
|
+
await session.withTransaction(async () => {
|
|
198
|
+
result = await work(session);
|
|
199
|
+
}, transactionOptions);
|
|
200
|
+
return result;
|
|
201
|
+
});
|
|
202
|
+
} catch (error) {
|
|
203
|
+
if (isTransactionsUnsupportedError(error)) {
|
|
204
|
+
throw createTransactionsUnsupportedError(error);
|
|
205
|
+
}
|
|
206
|
+
throw error;
|
|
205
207
|
}
|
|
206
208
|
}
|
|
207
|
-
function
|
|
208
|
-
if (!
|
|
209
|
+
function assertInTransaction(session, operation) {
|
|
210
|
+
if (!session.inTransaction()) {
|
|
209
211
|
throw new core.ConfigurationError(
|
|
210
|
-
`
|
|
212
|
+
`MongoEventStore.${operation} needs a ClientSession inside a started transaction (session.withTransaction or startTransaction). Without one, MongoDB commits each write immediately and the write is no longer atomic with yours`
|
|
211
213
|
);
|
|
212
214
|
}
|
|
213
215
|
}
|
|
216
|
+
function atCount(count) {
|
|
217
|
+
return count === 0 ? { $not: { $gt: 0 } } : count;
|
|
218
|
+
}
|
|
214
219
|
|
|
215
|
-
// src/mongo-
|
|
216
|
-
var
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
}
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
#database;
|
|
238
|
-
#events;
|
|
239
|
-
#deliveries;
|
|
240
|
-
#keySequences;
|
|
241
|
-
#lanes;
|
|
242
|
-
#completions;
|
|
243
|
-
#completionHistory;
|
|
244
|
-
#transactionOptions;
|
|
245
|
-
#purgeBatchSize;
|
|
246
|
-
#logger;
|
|
247
|
-
/**
|
|
248
|
-
* Identifies this store instance in the lanes it holds. A process that restarts
|
|
249
|
-
* with the same `leaseOwner` gets a new instance, so it never mistakes the
|
|
250
|
-
* lanes of its dead predecessor for its own: it waits for their leases to
|
|
251
|
-
* expire and takes them over like any other worker.
|
|
252
|
-
*/
|
|
253
|
-
#instanceId = core.createUuidV7();
|
|
254
|
-
/** The last claim of each owner still running on this instance; the next one waits for it. */
|
|
255
|
-
#claimsInProgress = /* @__PURE__ */ new Map();
|
|
256
|
-
/**
|
|
257
|
-
* Claims this instance handed out that are not settled yet, keyed by
|
|
258
|
-
* {@link leaseKey} (delivery id and exact lease token), with the owner. A
|
|
259
|
-
* claim is settled when a completion, retry, release or burial of exactly
|
|
260
|
-
* that lease returned, or, for a completion inside the caller's transaction
|
|
261
|
-
* (which may still retry or roll back), when a later claim of the owner finds
|
|
262
|
-
* the delivery gone. After a failed claim, a listed delivery stays listed only
|
|
263
|
-
* if one of these is its current claim (current lane epoch and claim generation).
|
|
264
|
-
*/
|
|
265
|
-
#handedOut = /* @__PURE__ */ new Map();
|
|
266
|
-
/** Owners whose lanes on this instance still need reconciling after a failed claim. */
|
|
267
|
-
#unreconciledOwners = /* @__PURE__ */ new Set();
|
|
268
|
-
constructor(options) {
|
|
269
|
-
const prefix = options.collectionPrefix ?? "relaybus_";
|
|
270
|
-
assertCollectionPrefix(prefix);
|
|
271
|
-
if (options.database !== void 0) {
|
|
272
|
-
assertDatabaseName(options.database);
|
|
273
|
-
}
|
|
274
|
-
const purgeBatchSize = options.purgeBatchSize ?? DEFAULT_PURGE_BATCH_SIZE;
|
|
275
|
-
if (!Number.isInteger(purgeBatchSize) || purgeBatchSize < 1) {
|
|
276
|
-
throw new core.ConfigurationError(
|
|
277
|
-
`purgeBatchSize must be a positive integer, received ${String(purgeBatchSize)}`
|
|
278
|
-
);
|
|
279
|
-
}
|
|
280
|
-
this.#client = options.client;
|
|
281
|
-
const database = options.client.db(options.database);
|
|
282
|
-
this.#database = database;
|
|
283
|
-
const collectionOptions = {
|
|
284
|
-
readPreference: "primary",
|
|
285
|
-
writeConcern: { w: "majority" }
|
|
220
|
+
// src/mongo-append.ts
|
|
221
|
+
var MongoAppend = class {
|
|
222
|
+
#state;
|
|
223
|
+
constructor(state) {
|
|
224
|
+
this.#state = state;
|
|
225
|
+
}
|
|
226
|
+
/** Implements {@link MongoEventStore.append}. */
|
|
227
|
+
async append(session, { entries, now }) {
|
|
228
|
+
assertInTransaction(session, "append");
|
|
229
|
+
const eventIds = [...new Set(entries.map((entry) => entry.event.id))];
|
|
230
|
+
const storedEvents = new Map(
|
|
231
|
+
(await this.#state.events.find(
|
|
232
|
+
{ _id: { $in: eventIds } },
|
|
233
|
+
{ session, projection: { key: 1, sequence: 1, handlers: 1 } }
|
|
234
|
+
).toArray()).map((stored) => [stored._id, stored])
|
|
235
|
+
);
|
|
236
|
+
const plan = await this.#planAppend({ session, entries, storedEvents, now });
|
|
237
|
+
await this.#writeAppend({ session, plan, storedEvents, now });
|
|
238
|
+
return {
|
|
239
|
+
appendedEventIds: plan.appendedEventIds,
|
|
240
|
+
duplicateEventIds: plan.duplicateEventIds,
|
|
241
|
+
appendedDeliveryCount: plan.newDeliveries.length
|
|
286
242
|
};
|
|
287
|
-
this.#events = database.collection(`${prefix}events`, collectionOptions);
|
|
288
|
-
this.#deliveries = database.collection(`${prefix}deliveries`, collectionOptions);
|
|
289
|
-
this.#keySequences = database.collection(`${prefix}key_sequences`, collectionOptions);
|
|
290
|
-
this.#lanes = database.collection(`${prefix}lanes`, collectionOptions);
|
|
291
|
-
this.#completions = database.collection(`${prefix}completed_deliveries`, collectionOptions);
|
|
292
|
-
this.#completionHistory = options.completionHistory ?? false;
|
|
293
|
-
this.#transactionOptions = options.transactionOptions ?? DEFAULT_TRANSACTION_OPTIONS;
|
|
294
|
-
this.#purgeBatchSize = purgeBatchSize;
|
|
295
|
-
this.#logger = options.logger ?? console;
|
|
296
243
|
}
|
|
297
244
|
/**
|
|
298
|
-
*
|
|
299
|
-
*
|
|
300
|
-
* first (see {@link MongoEventStore.#upgradeFromVersion010}); the seeded lanes
|
|
301
|
-
* mark the upgrade as done, so later runs skip it. Safe to run repeatedly and
|
|
302
|
-
* from several processes. Stop 0.1.0 workers before the first run.
|
|
303
|
-
*
|
|
304
|
-
* @throws ConfigurationError when the server is a standalone mongod.
|
|
245
|
+
* Decides, in entry order, which events are new (taking their key sequence) and which deliveries
|
|
246
|
+
* each event still lacks.
|
|
305
247
|
*/
|
|
306
|
-
async
|
|
307
|
-
const
|
|
308
|
-
const
|
|
309
|
-
|
|
310
|
-
|
|
248
|
+
async #planAppend(input) {
|
|
249
|
+
const { session, entries, storedEvents, now } = input;
|
|
250
|
+
const appendedEventIds = [];
|
|
251
|
+
const duplicateEventIds = [];
|
|
252
|
+
const newEvents = [];
|
|
253
|
+
const newDeliveries = [];
|
|
254
|
+
const addedHandlers = /* @__PURE__ */ new Map();
|
|
255
|
+
const { orderingByEventId, knownHandlers } = indexStoredEvents(storedEvents);
|
|
256
|
+
for (const { event, deliveries } of entries) {
|
|
257
|
+
let ordering = orderingByEventId.get(event.id);
|
|
258
|
+
if (ordering === void 0) {
|
|
259
|
+
const sequence = event.key === null ? null : await this.#nextSequence(event.key, session);
|
|
260
|
+
ordering = { key: event.key, sequence };
|
|
261
|
+
orderingByEventId.set(event.id, ordering);
|
|
262
|
+
knownHandlers.set(event.id, /* @__PURE__ */ new Set());
|
|
263
|
+
newEvents.push({ event, sequence });
|
|
264
|
+
appendedEventIds.push(event.id);
|
|
265
|
+
} else {
|
|
266
|
+
duplicateEventIds.push(event.id);
|
|
267
|
+
}
|
|
268
|
+
const handlers = knownHandlers.get(event.id);
|
|
269
|
+
for (const delivery of deliveries) {
|
|
270
|
+
if (handlers.has(delivery.handler)) {
|
|
271
|
+
continue;
|
|
272
|
+
}
|
|
273
|
+
handlers.add(delivery.handler);
|
|
274
|
+
addedHandlers.set(event.id, [...addedHandlers.get(event.id) ?? [], delivery.handler]);
|
|
275
|
+
newDeliveries.push(toDeliveryDocument(delivery, { event, ordering, now }));
|
|
276
|
+
}
|
|
311
277
|
}
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
278
|
+
return { appendedEventIds, duplicateEventIds, newEvents, newDeliveries, addedHandlers };
|
|
279
|
+
}
|
|
280
|
+
/** Writes the new events, the handlers added to existing events and the new deliveries, in that order. */
|
|
281
|
+
async #writeAppend(input) {
|
|
282
|
+
const { session, plan, storedEvents, now } = input;
|
|
283
|
+
const { newEvents, newDeliveries, addedHandlers } = plan;
|
|
284
|
+
if (newEvents.length > 0) {
|
|
285
|
+
await this.#state.events.bulkWrite(newEventOperations({ newEvents, addedHandlers, now }), {
|
|
286
|
+
session,
|
|
287
|
+
ordered: true
|
|
288
|
+
});
|
|
321
289
|
}
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
await this.#
|
|
290
|
+
const extendedEvents = [...addedHandlers].filter(([eventId]) => storedEvents.has(eventId));
|
|
291
|
+
if (extendedEvents.length > 0) {
|
|
292
|
+
await this.#state.events.bulkWrite(extendedEventOperations(extendedEvents), {
|
|
293
|
+
session,
|
|
294
|
+
ordered: true
|
|
295
|
+
});
|
|
325
296
|
}
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
// last, so a query over many lanes and handlers stays a merge of one scan per lane.
|
|
332
|
-
{ key: CLAIM_INDEX_KEY, name: "claim", partialFilterExpression: { lane: "ready" } },
|
|
333
|
-
// Ordering: unfinished predecessors of a keyed delivery, parked or not.
|
|
334
|
-
{
|
|
335
|
-
key: { handler: 1, key: 1, sequence: 1 },
|
|
336
|
-
name: "ordering",
|
|
337
|
-
partialFilterExpression: { dueAt: { $exists: true }, sequence: { $exists: true } }
|
|
338
|
-
},
|
|
339
|
-
// Parked deliveries: the lowest parked delivery of each key, to move it back once it is next.
|
|
340
|
-
{
|
|
341
|
-
key: { handler: 1, key: 1, sequence: 1 },
|
|
342
|
-
name: "parked",
|
|
343
|
-
partialFilterExpression: { lane: "parked" }
|
|
344
|
-
},
|
|
345
|
-
// Statistics: oldest due delivery across handlers.
|
|
346
|
-
{ key: { dueAt: 1 }, name: "due", partialFilterExpression: { dueAt: { $exists: true } } },
|
|
347
|
-
// Purge of completed deliveries left by 0.1.0.
|
|
348
|
-
{
|
|
349
|
-
key: { completedAt: 1 },
|
|
350
|
-
name: "completed",
|
|
351
|
-
partialFilterExpression: { completedAt: { $exists: true } }
|
|
352
|
-
},
|
|
353
|
-
// Listing and status counts.
|
|
354
|
-
{ key: { status: 1, createdAt: 1 }, name: "status_created" },
|
|
355
|
-
// Purge of dead deliveries.
|
|
356
|
-
{ key: { status: 1, updatedAt: 1 }, name: "status_updated" },
|
|
357
|
-
{ key: { handler: 1, createdAt: 1 }, name: "handler_created" },
|
|
358
|
-
{ key: { eventType: 1, createdAt: 1 }, name: "type_created" }
|
|
359
|
-
]);
|
|
360
|
-
await this.#events.createIndexes([
|
|
361
|
-
{ key: { createdAt: 1, _id: 1 }, name: "created" },
|
|
362
|
-
// Purge of events whose deliveries all completed: they are kept until `completedBefore` passes this.
|
|
363
|
-
{
|
|
364
|
-
key: { completedAt: 1, _id: 1 },
|
|
365
|
-
name: "completed",
|
|
366
|
-
partialFilterExpression: { completedAt: { $exists: true } }
|
|
367
|
-
}
|
|
368
|
-
]);
|
|
369
|
-
if (this.#completionHistory) {
|
|
370
|
-
await this.#completions.createIndexes([
|
|
371
|
-
{ key: { completedAt: 1, _id: 1 }, name: "completed" },
|
|
372
|
-
{ key: { eventId: 1 }, name: "event" },
|
|
373
|
-
{ key: { createdAt: 1, _id: 1 }, name: "created" }
|
|
374
|
-
]);
|
|
297
|
+
if (newDeliveries.length > 0) {
|
|
298
|
+
await this.#state.deliveries.bulkWrite(newDeliveryOperations(newDeliveries), {
|
|
299
|
+
session,
|
|
300
|
+
ordered: true
|
|
301
|
+
});
|
|
375
302
|
}
|
|
376
303
|
}
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
304
|
+
/** Atomically increments the counter of `key`; conflicting appends of the key serialize here. */
|
|
305
|
+
async #nextSequence(key, session) {
|
|
306
|
+
const counter = await this.#state.keySequences.findOneAndUpdate(
|
|
307
|
+
{ _id: key },
|
|
308
|
+
{ $inc: { value: 1 } },
|
|
309
|
+
{ upsert: true, returnDocument: "after", session }
|
|
310
|
+
);
|
|
311
|
+
return counter.value;
|
|
312
|
+
}
|
|
313
|
+
};
|
|
314
|
+
function indexStoredEvents(storedEvents) {
|
|
315
|
+
const orderingByEventId = /* @__PURE__ */ new Map();
|
|
316
|
+
const knownHandlers = /* @__PURE__ */ new Map();
|
|
317
|
+
for (const [eventId, stored] of storedEvents) {
|
|
318
|
+
orderingByEventId.set(eventId, {
|
|
319
|
+
key: stored.key ?? null,
|
|
320
|
+
sequence: stored.sequence ?? null
|
|
321
|
+
});
|
|
322
|
+
knownHandlers.set(eventId, new Set(stored.handlers));
|
|
323
|
+
}
|
|
324
|
+
return { orderingByEventId, knownHandlers };
|
|
325
|
+
}
|
|
326
|
+
function newEventOperations(input) {
|
|
327
|
+
const { newEvents, addedHandlers, now } = input;
|
|
328
|
+
return newEvents.map(({ event, sequence }) => {
|
|
329
|
+
const document = toEventDocument(event, {
|
|
330
|
+
sequence,
|
|
331
|
+
handlers: addedHandlers.get(event.id) ?? [],
|
|
332
|
+
createdAt: now
|
|
333
|
+
});
|
|
334
|
+
return {
|
|
335
|
+
updateOne: {
|
|
336
|
+
filter: { _id: document._id },
|
|
337
|
+
update: { $setOnInsert: document },
|
|
338
|
+
upsert: true
|
|
383
339
|
}
|
|
340
|
+
};
|
|
341
|
+
});
|
|
342
|
+
}
|
|
343
|
+
function extendedEventOperations(extendedEvents) {
|
|
344
|
+
return extendedEvents.map(([eventId, handlers]) => ({
|
|
345
|
+
updateOne: {
|
|
346
|
+
filter: { _id: eventId },
|
|
347
|
+
update: { $inc: { revision: 1 }, $push: { handlers: { $each: handlers } } }
|
|
384
348
|
}
|
|
349
|
+
}));
|
|
350
|
+
}
|
|
351
|
+
function newDeliveryOperations(newDeliveries) {
|
|
352
|
+
return newDeliveries.map((document) => ({
|
|
353
|
+
updateOne: {
|
|
354
|
+
filter: { eventId: document.eventId, handler: document.handler },
|
|
355
|
+
update: { $setOnInsert: document },
|
|
356
|
+
upsert: true
|
|
357
|
+
}
|
|
358
|
+
}));
|
|
359
|
+
}
|
|
360
|
+
var ClaimScan = class {
|
|
361
|
+
#keys = /* @__PURE__ */ new Map();
|
|
362
|
+
/** Whether an unfinished delivery this scan knows of comes before the delivery with `sequence`. */
|
|
363
|
+
waits(orderingId, sequence) {
|
|
364
|
+
const known = this.#keys.get(orderingId);
|
|
365
|
+
return known?.openSequence !== void 0 && sequence > known.openSequence;
|
|
366
|
+
}
|
|
367
|
+
/** Whether this scan already learned anything about the key. */
|
|
368
|
+
knows(orderingId) {
|
|
369
|
+
return this.#keys.has(orderingId);
|
|
370
|
+
}
|
|
371
|
+
/** Records an unfinished delivery of the key with this sequence. */
|
|
372
|
+
noteOpen(orderingId, sequence) {
|
|
373
|
+
const known = this.#know(orderingId);
|
|
374
|
+
known.openSequence = Math.min(known.openSequence ?? sequence, sequence);
|
|
375
|
+
}
|
|
376
|
+
/** Remembers a pending delivery of lane `laneId` that waited behind an unfinished predecessor. */
|
|
377
|
+
noteWaiting(orderingId, { deliveryId, laneId }) {
|
|
378
|
+
const known = this.#know(orderingId);
|
|
379
|
+
known.waiting.push(deliveryId);
|
|
380
|
+
known.laneId = laneId;
|
|
385
381
|
}
|
|
386
382
|
/**
|
|
387
|
-
*
|
|
388
|
-
*
|
|
389
|
-
*
|
|
390
|
-
* would
|
|
391
|
-
*
|
|
392
|
-
*
|
|
393
|
-
* `completed` documents until a purge removes them.
|
|
383
|
+
* Ids of the waiting deliveries of every key that accumulated at least
|
|
384
|
+
* `minimum` of them, or that lives in one of `handedBack`. Shorter queues
|
|
385
|
+
* of a lane that stays held remain in the claim index: moving them out and
|
|
386
|
+
* back would cost more than skipping them. In a lane handed back they are
|
|
387
|
+
* all its due work, and leaving them ready would make every later claim take
|
|
388
|
+
* the lane for nothing.
|
|
394
389
|
*/
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
await legacy.updateMany({ status: "processing" }, [
|
|
398
|
-
{ $set: { status: "pending", lane: "ready", dueAt: "$availableAt" } }
|
|
399
|
-
]);
|
|
400
|
-
await legacy.updateMany(
|
|
401
|
-
{
|
|
402
|
-
$or: [
|
|
403
|
-
{ leaseOwner: { $exists: true } },
|
|
404
|
-
{ leaseToken: { $exists: true } },
|
|
405
|
-
{ leaseExpiresAt: { $exists: true } }
|
|
406
|
-
]
|
|
407
|
-
},
|
|
408
|
-
{ $unset: LEASE_FIELDS }
|
|
409
|
-
);
|
|
410
|
-
await inBatches(
|
|
411
|
-
this.#deliveries.find(
|
|
412
|
-
{ laneId: { $exists: false } },
|
|
413
|
-
{ projection: { eventId: 1, key: 1 }, batchSize: UPGRADE_BATCH_SIZE }
|
|
414
|
-
),
|
|
415
|
-
async (documents) => {
|
|
416
|
-
await this.#deliveries.bulkWrite(
|
|
417
|
-
documents.map((document) => ({
|
|
418
|
-
updateOne: {
|
|
419
|
-
filter: { _id: document._id },
|
|
420
|
-
update: { $set: { laneId: laneOf(document.key ?? null, document.eventId) } }
|
|
421
|
-
}
|
|
422
|
-
})),
|
|
423
|
-
{ ordered: false }
|
|
424
|
-
);
|
|
425
|
-
}
|
|
426
|
-
);
|
|
427
|
-
await inBatches(
|
|
428
|
-
this.#events.find(
|
|
429
|
-
{ handlers: { $exists: false } },
|
|
430
|
-
{ projection: { occurredAt: 1 }, batchSize: UPGRADE_BATCH_SIZE }
|
|
431
|
-
),
|
|
432
|
-
async (events) => {
|
|
433
|
-
const known = new Map(
|
|
434
|
-
(await this.#deliveries.aggregate([
|
|
435
|
-
{ $match: { eventId: { $in: events.map((event) => event._id) } } },
|
|
436
|
-
{
|
|
437
|
-
$group: {
|
|
438
|
-
_id: "$eventId",
|
|
439
|
-
handlers: { $addToSet: "$handler" },
|
|
440
|
-
createdAt: { $min: "$createdAt" }
|
|
441
|
-
}
|
|
442
|
-
}
|
|
443
|
-
]).toArray()).map((group) => [group._id, group])
|
|
444
|
-
);
|
|
445
|
-
await this.#events.bulkWrite(
|
|
446
|
-
events.map((event) => {
|
|
447
|
-
const group = known.get(event._id);
|
|
448
|
-
return {
|
|
449
|
-
updateOne: {
|
|
450
|
-
// Only an event nobody converted yet: another process may have finished and subscribed a handler.
|
|
451
|
-
filter: { _id: event._id, handlers: { $exists: false } },
|
|
452
|
-
update: {
|
|
453
|
-
$set: group === void 0 ? { handlers: [], createdAt: event.occurredAt } : { handlers: group.handlers, createdAt: group.createdAt }
|
|
454
|
-
}
|
|
455
|
-
}
|
|
456
|
-
};
|
|
457
|
-
}),
|
|
458
|
-
{ ordered: false }
|
|
459
|
-
);
|
|
460
|
-
}
|
|
461
|
-
);
|
|
462
|
-
}
|
|
463
|
-
/** Creates the lane documents that do not exist yet. */
|
|
464
|
-
async #seedLanes() {
|
|
465
|
-
await this.#lanes.bulkWrite(
|
|
466
|
-
Array.from({ length: LANE_COUNT }, (_, laneId) => ({
|
|
467
|
-
updateOne: {
|
|
468
|
-
filter: { _id: laneId },
|
|
469
|
-
update: {
|
|
470
|
-
$setOnInsert: {
|
|
471
|
-
epoch: 0,
|
|
472
|
-
owner: null,
|
|
473
|
-
holder: null,
|
|
474
|
-
leaseExpiresAt: null,
|
|
475
|
-
inFlight: [],
|
|
476
|
-
lost: []
|
|
477
|
-
}
|
|
478
|
-
},
|
|
479
|
-
upsert: true
|
|
480
|
-
}
|
|
481
|
-
})),
|
|
482
|
-
{ ordered: false }
|
|
483
|
-
);
|
|
390
|
+
waitingBacklogs(minimum, handedBack) {
|
|
391
|
+
return [...this.#keys.values()].filter((known) => known.waiting.length >= minimum || handedBack.has(known.laneId)).flatMap((known) => known.waiting);
|
|
484
392
|
}
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
try {
|
|
491
|
-
await this.#deliveries.dropIndex(index.name);
|
|
492
|
-
} catch (error) {
|
|
493
|
-
if (!hasErrorCode(error, INDEX_NOT_FOUND)) {
|
|
494
|
-
throw error;
|
|
495
|
-
}
|
|
496
|
-
}
|
|
497
|
-
}
|
|
393
|
+
#know(orderingId) {
|
|
394
|
+
let known = this.#keys.get(orderingId);
|
|
395
|
+
if (known === void 0) {
|
|
396
|
+
known = { openSequence: void 0, waiting: [], laneId: void 0 };
|
|
397
|
+
this.#keys.set(orderingId, known);
|
|
498
398
|
}
|
|
399
|
+
return known;
|
|
499
400
|
}
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
* so keep those out of `work`; relaybus handlers are already at-least-once.
|
|
509
|
-
*
|
|
510
|
-
* @throws ConfigurationError when the deployment does not support transactions.
|
|
511
|
-
*/
|
|
512
|
-
async transaction(work) {
|
|
513
|
-
try {
|
|
514
|
-
return await this.#client.withSession(async (session) => {
|
|
515
|
-
let result;
|
|
516
|
-
await session.withTransaction(async () => {
|
|
517
|
-
result = await work(session);
|
|
518
|
-
}, this.#transactionOptions);
|
|
519
|
-
return result;
|
|
520
|
-
});
|
|
521
|
-
} catch (error) {
|
|
522
|
-
if (isTransactionsUnsupportedError(error)) {
|
|
523
|
-
throw createTransactionsUnsupportedError(error);
|
|
524
|
-
}
|
|
525
|
-
throw error;
|
|
526
|
-
}
|
|
401
|
+
};
|
|
402
|
+
function orderingIdentity(handler, key) {
|
|
403
|
+
return internal.CompositeKey.of(handler, key);
|
|
404
|
+
}
|
|
405
|
+
var MongoLaneReconciler = class {
|
|
406
|
+
#state;
|
|
407
|
+
constructor(state) {
|
|
408
|
+
this.#state = state;
|
|
527
409
|
}
|
|
528
410
|
/**
|
|
529
|
-
*
|
|
530
|
-
*
|
|
531
|
-
*
|
|
532
|
-
*
|
|
411
|
+
* Takes every delivery this instance does not run out of the lanes `owner`
|
|
412
|
+
* holds on it: every listed delivery whose current claim (lane epoch and
|
|
413
|
+
* claim generation now) was not handed out by this instance, or was settled.
|
|
414
|
+
* One transaction, and the lanes are written first under their epoch and
|
|
415
|
+
* holder: a worker taking a lane over meanwhile conflicts with it, so its
|
|
416
|
+
* claim is never touched. The claim generation of every delivery taken out
|
|
417
|
+
* moves on, so no lease of those claims (a completion that failed and is
|
|
418
|
+
* retried later, say) can ever match a later claim of the same delivery.
|
|
419
|
+
* Registrations of this owner that are not the current claim of a listed
|
|
420
|
+
* delivery are dropped.
|
|
533
421
|
*/
|
|
534
|
-
async
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
{ session, projection: { key: 1, sequence: 1, handlers: 1 } }
|
|
541
|
-
).toArray()).map((stored) => [stored._id, stored])
|
|
542
|
-
);
|
|
543
|
-
const appendedEventIds = [];
|
|
544
|
-
const duplicateEventIds = [];
|
|
545
|
-
const newEvents = [];
|
|
546
|
-
const newDeliveries = [];
|
|
547
|
-
const knownHandlers = /* @__PURE__ */ new Map();
|
|
548
|
-
const addedHandlers = /* @__PURE__ */ new Map();
|
|
549
|
-
const orderingByEventId = /* @__PURE__ */ new Map();
|
|
550
|
-
for (const [eventId, stored] of storedEvents) {
|
|
551
|
-
orderingByEventId.set(eventId, {
|
|
552
|
-
key: stored.key ?? null,
|
|
553
|
-
sequence: stored.sequence ?? null
|
|
554
|
-
});
|
|
555
|
-
knownHandlers.set(eventId, new Set(stored.handlers));
|
|
556
|
-
}
|
|
557
|
-
for (const { event, deliveries } of entries) {
|
|
558
|
-
let ordering = orderingByEventId.get(event.id);
|
|
559
|
-
if (ordering === void 0) {
|
|
560
|
-
const sequence = event.key === null ? null : await this.#nextSequence(event.key, session);
|
|
561
|
-
ordering = { key: event.key, sequence };
|
|
562
|
-
orderingByEventId.set(event.id, ordering);
|
|
563
|
-
knownHandlers.set(event.id, /* @__PURE__ */ new Set());
|
|
564
|
-
newEvents.push({ event, sequence });
|
|
565
|
-
appendedEventIds.push(event.id);
|
|
566
|
-
} else {
|
|
567
|
-
duplicateEventIds.push(event.id);
|
|
568
|
-
}
|
|
569
|
-
const handlers = knownHandlers.get(event.id);
|
|
570
|
-
for (const delivery of deliveries) {
|
|
571
|
-
if (handlers.has(delivery.handler)) {
|
|
572
|
-
continue;
|
|
573
|
-
}
|
|
574
|
-
handlers.add(delivery.handler);
|
|
575
|
-
addedHandlers.set(event.id, [...addedHandlers.get(event.id) ?? [], delivery.handler]);
|
|
576
|
-
newDeliveries.push(toDeliveryDocument(delivery, event, ordering, now));
|
|
577
|
-
}
|
|
578
|
-
}
|
|
579
|
-
if (newEvents.length > 0) {
|
|
580
|
-
await this.#events.bulkWrite(
|
|
581
|
-
newEvents.map(({ event, sequence }) => {
|
|
582
|
-
const document = toEventDocument(event, sequence, addedHandlers.get(event.id) ?? [], now);
|
|
583
|
-
return {
|
|
584
|
-
updateOne: {
|
|
585
|
-
filter: { _id: document._id },
|
|
586
|
-
update: { $setOnInsert: document },
|
|
587
|
-
upsert: true
|
|
588
|
-
}
|
|
589
|
-
};
|
|
590
|
-
}),
|
|
591
|
-
{ session, ordered: true }
|
|
592
|
-
);
|
|
593
|
-
}
|
|
594
|
-
const extendedEvents = [...addedHandlers].filter(([eventId]) => storedEvents.has(eventId));
|
|
595
|
-
if (extendedEvents.length > 0) {
|
|
596
|
-
await this.#events.bulkWrite(
|
|
597
|
-
extendedEvents.map(([eventId, handlers]) => ({
|
|
598
|
-
updateOne: {
|
|
599
|
-
filter: { _id: eventId },
|
|
600
|
-
update: { $inc: { revision: 1 }, $push: { handlers: { $each: handlers } } }
|
|
601
|
-
}
|
|
602
|
-
})),
|
|
603
|
-
{ session, ordered: true }
|
|
422
|
+
async reconcile(owner) {
|
|
423
|
+
await runInTransaction(this.#state, async (session) => {
|
|
424
|
+
const held = await this.#state.lanes.find({ owner, holder: this.#state.instanceId }, { session }).toArray();
|
|
425
|
+
const claimsById = await this.#claimGenerations(
|
|
426
|
+
held.flatMap((lane) => lane.inFlight),
|
|
427
|
+
session
|
|
604
428
|
);
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
newDeliveries.map((document) => ({
|
|
609
|
-
updateOne: {
|
|
610
|
-
filter: { eventId: document.eventId, handler: document.handler },
|
|
611
|
-
update: { $setOnInsert: document },
|
|
612
|
-
upsert: true
|
|
613
|
-
}
|
|
614
|
-
})),
|
|
615
|
-
{ session, ordered: true }
|
|
429
|
+
const { unclaimed, current } = this.#splitUnclaimed(held, claimsById);
|
|
430
|
+
this.#state.handedOut.forgetWhere(
|
|
431
|
+
(entry, identity) => entry.owner === owner && !current.has(identity)
|
|
616
432
|
);
|
|
617
|
-
|
|
618
|
-
|
|
433
|
+
if (held.length === 0) {
|
|
434
|
+
return;
|
|
435
|
+
}
|
|
436
|
+
await this.#takeOut(unclaimed, session);
|
|
437
|
+
});
|
|
438
|
+
}
|
|
439
|
+
/** The claim generation now of every listed delivery that is still pending. */
|
|
440
|
+
async #claimGenerations(listed, session) {
|
|
441
|
+
return new Map(
|
|
442
|
+
(await this.#state.deliveries.find({ _id: { $in: listed }, status: "pending" }, { session, projection: { claims: 1 } }).toArray()).map((document) => [document._id, document.claims ?? 0])
|
|
443
|
+
);
|
|
619
444
|
}
|
|
620
445
|
/**
|
|
621
|
-
*
|
|
622
|
-
*
|
|
623
|
-
* lanes with due work, a few at a time (4, 8, 16, ...) and the lane with the
|
|
624
|
-
* oldest due delivery first, so a worker holds only as many lanes as it
|
|
625
|
-
* needs. It then records the batch with one write per lane and hands back the
|
|
626
|
-
* lanes that yielded nothing and run nothing. No delivery document is
|
|
627
|
-
* written, except for the attempt charged to deliveries of a lane taken over
|
|
628
|
-
* and for parking (see {@link MongoEventStore.#parkDeliveries}).
|
|
629
|
-
*
|
|
630
|
-
* Taking a lane is one conditional `findOneAndUpdate` that also moves the
|
|
631
|
-
* previous owner's in-flight list to the lane's `lost` list; a lane another
|
|
632
|
-
* claimer took first is skipped, never waited for. Lost deliveries are charged
|
|
633
|
-
* one attempt before anything is read from the lane, and `lost` is cleared
|
|
634
|
-
* only after that. Whoever holds the lane next charges a `lost` list it finds
|
|
635
|
-
* first, so a claim that fails anywhere after the take loses no charge; the
|
|
636
|
-
* charge of a lost claim is recorded on the delivery, so charging the list
|
|
637
|
-
* again after a failure before the clear charges nothing twice.
|
|
638
|
-
* Nothing a failed claim did leaves a delivery listed as running.
|
|
446
|
+
* Per held lane, the listed deliveries this instance did not hand out under their current claim,
|
|
447
|
+
* and the identities of all current claims.
|
|
639
448
|
*/
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
449
|
+
#splitUnclaimed(held, claimsById) {
|
|
450
|
+
const current = /* @__PURE__ */ new Set();
|
|
451
|
+
const unclaimed = held.map((lane) => ({
|
|
452
|
+
lane,
|
|
453
|
+
ids: lane.inFlight.filter((id) => {
|
|
454
|
+
const key = internal.DeliveryLeaseIdentity.of(
|
|
455
|
+
id,
|
|
456
|
+
formatLeaseToken(lane._id, { epoch: lane.epoch, claims: claimsById.get(id) ?? 0 })
|
|
457
|
+
);
|
|
458
|
+
current.add(key);
|
|
459
|
+
return !this.#state.handedOut.has(key);
|
|
460
|
+
})
|
|
461
|
+
}));
|
|
462
|
+
return { unclaimed, current };
|
|
463
|
+
}
|
|
464
|
+
/** Takes the unclaimed deliveries out of their lanes, then moves their claim generation on. */
|
|
465
|
+
async #takeOut(unclaimed, session) {
|
|
466
|
+
await this.#state.lanes.bulkWrite(
|
|
467
|
+
unclaimed.map(({ lane, ids }) => ({
|
|
468
|
+
updateOne: {
|
|
469
|
+
filter: { _id: lane._id, epoch: lane.epoch, holder: this.#state.instanceId },
|
|
470
|
+
update: { $pull: { inFlight: { $in: ids } }, $inc: { revision: 1 } }
|
|
471
|
+
}
|
|
472
|
+
})),
|
|
473
|
+
{ session, ordered: false }
|
|
474
|
+
);
|
|
475
|
+
await this.#state.deliveries.updateMany(
|
|
476
|
+
{ _id: { $in: unclaimed.flatMap((entry) => entry.ids) }, status: "pending" },
|
|
477
|
+
{ $inc: { claims: 1 } },
|
|
478
|
+
{ session }
|
|
479
|
+
);
|
|
480
|
+
}
|
|
481
|
+
};
|
|
482
|
+
|
|
483
|
+
// src/mongo-claim.ts
|
|
484
|
+
var PARK_BACKLOG_THRESHOLD = 32;
|
|
485
|
+
var INITIAL_LANES_PER_ROUND = 4;
|
|
486
|
+
var LANES_PER_CANDIDATE_QUERY = 128;
|
|
487
|
+
function toClaimedDeliveries(claimed, context) {
|
|
488
|
+
const { scope, events, request } = context;
|
|
489
|
+
return claimed.map((document) => ({
|
|
490
|
+
deliveryId: document._id,
|
|
491
|
+
leaseToken: formatLeaseToken(document.laneId, {
|
|
492
|
+
epoch: scope.get(document.laneId).epoch,
|
|
493
|
+
claims: document.claims ?? 0
|
|
494
|
+
}),
|
|
495
|
+
handler: document.handler,
|
|
496
|
+
attempt: document.attempts + 1,
|
|
497
|
+
leaseExpiresAt: request.leaseExpiresAt,
|
|
498
|
+
event: events.get(document.eventId)
|
|
499
|
+
}));
|
|
500
|
+
}
|
|
501
|
+
function scopeOf(held) {
|
|
502
|
+
return new Map(
|
|
503
|
+
held.map((lane) => [
|
|
504
|
+
lane._id,
|
|
505
|
+
{
|
|
506
|
+
laneId: lane._id,
|
|
507
|
+
epoch: lane.epoch,
|
|
508
|
+
listed: lane.inFlight,
|
|
509
|
+
revision: lane.revision ?? 0
|
|
652
510
|
}
|
|
653
|
-
|
|
654
|
-
|
|
511
|
+
])
|
|
512
|
+
);
|
|
513
|
+
}
|
|
514
|
+
var MongoClaims = class extends internal.OwnerSerializedClaims {
|
|
515
|
+
#state;
|
|
516
|
+
#laneLeases;
|
|
517
|
+
#ordering;
|
|
518
|
+
#queries;
|
|
519
|
+
#reconciler;
|
|
520
|
+
constructor(state, collaborators) {
|
|
521
|
+
super();
|
|
522
|
+
this.#state = state;
|
|
523
|
+
this.#laneLeases = collaborators.laneLeases;
|
|
524
|
+
this.#ordering = collaborators.ordering;
|
|
525
|
+
this.#queries = collaborators.queries;
|
|
526
|
+
this.#reconciler = new MongoLaneReconciler(state);
|
|
527
|
+
}
|
|
528
|
+
/** Mongo claims nothing for a limit of 0 or below (a limit that is not a number is tried). */
|
|
529
|
+
claimsNothing(request) {
|
|
530
|
+
return request.handlers.length === 0 || request.limit <= 0;
|
|
655
531
|
}
|
|
656
532
|
/**
|
|
657
533
|
* Runs one claim and remembers what it hands out. When the claim fails at any
|
|
@@ -661,17 +537,17 @@ var MongoEventStore = class {
|
|
|
661
537
|
* handed out and seen settled, is taken out of them, so nothing stays listed
|
|
662
538
|
* as running that nobody runs.
|
|
663
539
|
*/
|
|
664
|
-
async
|
|
540
|
+
async claimInTurn(request) {
|
|
665
541
|
const owner = request.leaseOwner;
|
|
666
|
-
if (this.#unreconciledOwners.has(owner)) {
|
|
667
|
-
await this.#
|
|
668
|
-
this.#unreconciledOwners.delete(owner);
|
|
542
|
+
if (this.#state.unreconciledOwners.has(owner)) {
|
|
543
|
+
await this.#reconciler.reconcile(owner);
|
|
544
|
+
this.#state.unreconciledOwners.delete(owner);
|
|
669
545
|
}
|
|
670
546
|
try {
|
|
671
547
|
const claimed = await this.#claimNow(request);
|
|
672
548
|
for (const delivery of claimed) {
|
|
673
549
|
const { laneId, epoch } = parseLeaseToken(delivery.leaseToken);
|
|
674
|
-
this.#handedOut.
|
|
550
|
+
this.#state.handedOut.register(delivery, {
|
|
675
551
|
deliveryId: delivery.deliveryId,
|
|
676
552
|
owner: request.leaseOwner,
|
|
677
553
|
laneId,
|
|
@@ -680,103 +556,51 @@ var MongoEventStore = class {
|
|
|
680
556
|
}
|
|
681
557
|
return claimed;
|
|
682
558
|
} catch (error) {
|
|
683
|
-
const [reconciled] = await Promise.allSettled([this.#
|
|
559
|
+
const [reconciled] = await Promise.allSettled([this.#reconciler.reconcile(owner)]);
|
|
684
560
|
if (reconciled.status === "rejected") {
|
|
685
|
-
this.#unreconciledOwners.add(owner);
|
|
561
|
+
this.#state.unreconciledOwners.add(owner);
|
|
686
562
|
}
|
|
687
563
|
throw error;
|
|
688
564
|
}
|
|
689
565
|
}
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
const listed = held.flatMap((lane) => lane.inFlight);
|
|
706
|
-
const claimsById = new Map(
|
|
707
|
-
(await this.#deliveries.find(
|
|
708
|
-
{ _id: { $in: listed }, status: "pending" },
|
|
709
|
-
{ session, projection: { claims: 1 } }
|
|
710
|
-
).toArray()).map((document) => [document._id, document.claims ?? 0])
|
|
711
|
-
);
|
|
712
|
-
const current = /* @__PURE__ */ new Set();
|
|
713
|
-
const unclaimed = held.map((lane) => ({
|
|
714
|
-
lane,
|
|
715
|
-
ids: lane.inFlight.filter((id) => {
|
|
716
|
-
const key = leaseKey(id, formatLeaseToken(lane._id, lane.epoch, claimsById.get(id) ?? 0));
|
|
717
|
-
current.add(key);
|
|
718
|
-
return !this.#handedOut.has(key);
|
|
719
|
-
})
|
|
720
|
-
}));
|
|
721
|
-
for (const [key, entry] of this.#handedOut) {
|
|
722
|
-
if (entry.owner === owner && !current.has(key)) {
|
|
723
|
-
this.#handedOut.delete(key);
|
|
724
|
-
}
|
|
725
|
-
}
|
|
726
|
-
if (held.length === 0) {
|
|
727
|
-
return;
|
|
728
|
-
}
|
|
729
|
-
await this.#lanes.bulkWrite(
|
|
730
|
-
unclaimed.map(({ lane, ids }) => ({
|
|
731
|
-
updateOne: {
|
|
732
|
-
filter: { _id: lane._id, epoch: lane.epoch, holder: this.#instanceId },
|
|
733
|
-
update: { $pull: { inFlight: { $in: ids } }, $inc: { revision: 1 } }
|
|
734
|
-
}
|
|
735
|
-
})),
|
|
736
|
-
{ session, ordered: false }
|
|
737
|
-
);
|
|
738
|
-
await this.#deliveries.updateMany(
|
|
739
|
-
{ _id: { $in: unclaimed.flatMap((entry) => entry.ids) }, status: "pending" },
|
|
740
|
-
{ $inc: { claims: 1 } },
|
|
741
|
-
{ session }
|
|
742
|
-
);
|
|
566
|
+
async #claimNow(request) {
|
|
567
|
+
const held = await this.#readHeldLanes(request);
|
|
568
|
+
await this.#laneLeases.chargeLost(held);
|
|
569
|
+
const scope = scopeOf(held);
|
|
570
|
+
const running = this.#loadRunning(held);
|
|
571
|
+
const scan = new ClaimScan();
|
|
572
|
+
const picked = [];
|
|
573
|
+
await this.#pickAcrossLanes({ request, scope, scan, running, picked });
|
|
574
|
+
const events = await this.#queries.readEnvelopes(picked.map((document) => document.eventId));
|
|
575
|
+
const runningIds = (await running).ids;
|
|
576
|
+
this.#forgetSettled(held, { runningIds, owner: request.leaseOwner });
|
|
577
|
+
const { recorded, handedBack } = await this.#laneLeases.recordClaim(scope, {
|
|
578
|
+
picked,
|
|
579
|
+
runningIds,
|
|
580
|
+
request
|
|
743
581
|
});
|
|
582
|
+
const claimed = await this.#releaseOvertaken(
|
|
583
|
+
picked.filter((document) => recorded.has(document.laneId)),
|
|
584
|
+
scope
|
|
585
|
+
);
|
|
586
|
+
await this.#ordering.parkWaiting(scan.waitingBacklogs(PARK_BACKLOG_THRESHOLD, handedBack));
|
|
587
|
+
return toClaimedDeliveries(claimed, { scope, events, request });
|
|
744
588
|
}
|
|
745
|
-
/**
|
|
746
|
-
|
|
747
|
-
* release, burial), and once it returned forgets exactly those leases as
|
|
748
|
-
* handed out. When it throws, they stay registered: the caller may retry
|
|
749
|
-
* with the same lease, which must still find its claim.
|
|
750
|
-
*/
|
|
751
|
-
async #settle(leases, operation) {
|
|
752
|
-
const result = await operation();
|
|
753
|
-
for (const lease of leases) {
|
|
754
|
-
this.#handedOut.delete(leaseKey(String(lease.deliveryId), String(lease.leaseToken)));
|
|
755
|
-
}
|
|
756
|
-
return result;
|
|
757
|
-
}
|
|
758
|
-
async #claimNow(request) {
|
|
589
|
+
/** The lanes the owner holds on this instance, read while the next deliveries are unparked. */
|
|
590
|
+
async #readHeldLanes(request) {
|
|
759
591
|
const [held] = await Promise.all([
|
|
760
|
-
this.#lanes.find({
|
|
592
|
+
this.#state.lanes.find({
|
|
761
593
|
owner: request.leaseOwner,
|
|
762
|
-
holder: this.#instanceId,
|
|
594
|
+
holder: this.#state.instanceId,
|
|
763
595
|
leaseExpiresAt: { $gt: request.now }
|
|
764
596
|
}).toArray(),
|
|
765
|
-
this.#unparkNextDeliveries(request.handlers)
|
|
597
|
+
this.#ordering.unparkNextDeliveries(request.handlers)
|
|
766
598
|
]);
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
laneId: lane._id,
|
|
773
|
-
epoch: lane.epoch,
|
|
774
|
-
listed: lane.inFlight,
|
|
775
|
-
revision: lane.revision ?? 0
|
|
776
|
-
}
|
|
777
|
-
])
|
|
778
|
-
);
|
|
779
|
-
const running = this.#findRunning(held.flatMap((lane) => lane.inFlight)).then((documents) => ({
|
|
599
|
+
return held;
|
|
600
|
+
}
|
|
601
|
+
/** The deliveries still running in the held lanes, with their (handler, key) identities. */
|
|
602
|
+
#loadRunning(held) {
|
|
603
|
+
return this.#queries.findRunning(held.flatMap((lane) => lane.inFlight)).then((documents) => ({
|
|
780
604
|
ids: new Set(documents.map((document) => document._id)),
|
|
781
605
|
keys: new Set(
|
|
782
606
|
documents.flatMap(
|
|
@@ -784,88 +608,161 @@ var MongoEventStore = class {
|
|
|
784
608
|
)
|
|
785
609
|
)
|
|
786
610
|
}));
|
|
787
|
-
|
|
788
|
-
|
|
611
|
+
}
|
|
612
|
+
/**
|
|
613
|
+
* Picks from the lanes in `scope`, then takes free lanes with due work, a few more each round
|
|
614
|
+
* (4, 8, 16, ...), and picks from them, until the claim is full or no free lane is left.
|
|
615
|
+
*/
|
|
616
|
+
async #pickAcrossLanes(input) {
|
|
617
|
+
const { request, scope, scan, running, picked } = input;
|
|
789
618
|
const pickFrom = async (lanes) => {
|
|
790
619
|
for (let offset = 0; offset < lanes.length; offset += LANES_PER_CANDIDATE_QUERY) {
|
|
791
620
|
if (picked.length < request.limit) {
|
|
792
|
-
await this.#pickCandidates(
|
|
793
|
-
lanes.slice(offset, offset + LANES_PER_CANDIDATE_QUERY),
|
|
621
|
+
await this.#ordering.pickCandidates({
|
|
622
|
+
lanes: lanes.slice(offset, offset + LANES_PER_CANDIDATE_QUERY),
|
|
794
623
|
request,
|
|
795
624
|
scan,
|
|
796
625
|
running,
|
|
797
626
|
picked
|
|
798
|
-
);
|
|
627
|
+
});
|
|
799
628
|
}
|
|
800
629
|
}
|
|
801
630
|
};
|
|
802
631
|
await pickFrom([...scope.values()]);
|
|
803
632
|
const passed = new Set(scope.keys());
|
|
804
633
|
for (let laneCount = INITIAL_LANES_PER_ROUND; picked.length < request.limit; laneCount *= 2) {
|
|
805
|
-
const free = await this.#findFreeLanes(request,
|
|
634
|
+
const free = await this.#laneLeases.findFreeLanes(request, {
|
|
635
|
+
laneCount,
|
|
636
|
+
passed: [...passed]
|
|
637
|
+
});
|
|
806
638
|
if (free.length === 0) {
|
|
807
639
|
break;
|
|
808
640
|
}
|
|
809
641
|
for (const laneId of free) {
|
|
810
642
|
passed.add(laneId);
|
|
811
643
|
}
|
|
812
|
-
const taken = await this.#takeLanes(free, request);
|
|
644
|
+
const taken = await this.#laneLeases.takeLanes(free, request);
|
|
813
645
|
for (const lane of taken) {
|
|
814
646
|
scope.set(lane.laneId, lane);
|
|
815
647
|
}
|
|
816
648
|
await pickFrom(taken);
|
|
817
649
|
}
|
|
818
|
-
|
|
819
|
-
|
|
650
|
+
}
|
|
651
|
+
/**
|
|
652
|
+
* Forgets the claims of `owner` that are listed in the held lanes but no longer pending:
|
|
653
|
+
* completed (a burial or retry would have unlisted it). Such a claim is settled, also when it was
|
|
654
|
+
* completed inside a caller's transaction that committed. Only this owner's registrations of the
|
|
655
|
+
* lane epochs it read: the same delivery may meanwhile have been requeued and handed out again,
|
|
656
|
+
* under another owner or lane epoch.
|
|
657
|
+
*/
|
|
658
|
+
#forgetSettled(held, { runningIds, owner }) {
|
|
820
659
|
const gone = new Set(
|
|
821
660
|
held.flatMap(
|
|
822
|
-
(lane) => lane.inFlight.filter((id) => !runningIds.has(id)).map((id) => `${
|
|
661
|
+
(lane) => lane.inFlight.filter((id) => !runningIds.has(id)).map((id) => internal.CompositeKey.of(id, `${lane._id}:${lane.epoch}`))
|
|
823
662
|
)
|
|
824
663
|
);
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
this.#handedOut.delete(key);
|
|
828
|
-
}
|
|
829
|
-
}
|
|
830
|
-
const { recorded, handedBack } = await this.#recordClaim(scope, picked, runningIds, request);
|
|
831
|
-
const claimed = await this.#releaseOvertaken(
|
|
832
|
-
picked.filter((document) => recorded.has(document.laneId)),
|
|
833
|
-
scope
|
|
664
|
+
this.#state.handedOut.forgetWhere(
|
|
665
|
+
(entry) => entry.owner === owner && gone.has(internal.CompositeKey.of(entry.deliveryId, `${entry.laneId}:${entry.epoch}`))
|
|
834
666
|
);
|
|
835
|
-
await this.#parkWaiting(scan.waitingBacklogs(PARK_BACKLOG_THRESHOLD, handedBack));
|
|
836
|
-
return claimed.map((document) => ({
|
|
837
|
-
deliveryId: document._id,
|
|
838
|
-
leaseToken: formatLeaseToken(
|
|
839
|
-
document.laneId,
|
|
840
|
-
scope.get(document.laneId).epoch,
|
|
841
|
-
document.claims ?? 0
|
|
842
|
-
),
|
|
843
|
-
handler: document.handler,
|
|
844
|
-
attempt: document.attempts + 1,
|
|
845
|
-
leaseExpiresAt: request.leaseExpiresAt,
|
|
846
|
-
event: events.get(document.eventId)
|
|
847
|
-
}));
|
|
848
667
|
}
|
|
849
668
|
/**
|
|
850
|
-
*
|
|
851
|
-
*
|
|
852
|
-
*
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
669
|
+
* Gives back the recorded keyed picks that a delivery requeued during the
|
|
670
|
+
* claim now precedes, without charging an attempt, and returns the rest. The
|
|
671
|
+
* claim checked their predecessors before recording them; a dead letter
|
|
672
|
+
* requeued in between would otherwise wait behind its own successor. Once a
|
|
673
|
+
* pick is recorded, a later requeue waits for it like for any running delivery.
|
|
674
|
+
*/
|
|
675
|
+
async #releaseOvertaken(claimed, scope) {
|
|
676
|
+
const keyed = claimed.filter((document) => document.key !== void 0);
|
|
677
|
+
if (keyed.length === 0) {
|
|
678
|
+
return [...claimed];
|
|
679
|
+
}
|
|
680
|
+
const blockers = await this.#state.deliveries.find(
|
|
681
|
+
{
|
|
682
|
+
$or: keyed.map((document) => ({
|
|
683
|
+
handler: document.handler,
|
|
684
|
+
key: document.key,
|
|
685
|
+
sequence: { $lt: document.sequence },
|
|
686
|
+
dueAt: { $exists: true }
|
|
687
|
+
}))
|
|
688
|
+
},
|
|
689
|
+
{ projection: { handler: 1, key: 1 } }
|
|
690
|
+
).toArray();
|
|
691
|
+
const blocked = new Set(
|
|
692
|
+
blockers.map((blocker) => orderingIdentity(blocker.handler, blocker.key))
|
|
693
|
+
);
|
|
694
|
+
const overtaken = new Set(
|
|
695
|
+
keyed.filter((document) => blocked.has(orderingIdentity(document.handler, document.key))).map((document) => document._id)
|
|
696
|
+
);
|
|
697
|
+
if (overtaken.size === 0) {
|
|
698
|
+
return [...claimed];
|
|
699
|
+
}
|
|
700
|
+
await this.#releasePicks(
|
|
701
|
+
claimed.filter((document) => overtaken.has(document._id)),
|
|
702
|
+
scope
|
|
703
|
+
);
|
|
704
|
+
return claimed.filter((document) => !overtaken.has(document._id));
|
|
705
|
+
}
|
|
706
|
+
/** Takes recorded picks out of their lanes' in-flight lists again, under the claim's epochs and record revisions. */
|
|
707
|
+
async #releasePicks(documents, scope) {
|
|
708
|
+
const idsByLane = /* @__PURE__ */ new Map();
|
|
709
|
+
for (const document of documents) {
|
|
710
|
+
idsByLane.set(document.laneId, [...idsByLane.get(document.laneId) ?? [], document._id]);
|
|
711
|
+
}
|
|
712
|
+
await Promise.all(
|
|
713
|
+
[...idsByLane].map(
|
|
714
|
+
([laneId, ids]) => this.#state.lanes.updateOne(
|
|
715
|
+
// Under the revision the record left: a pull that reaches the server late, after a
|
|
716
|
+
// reconciliation and a newer claim of the same delivery, matches nothing.
|
|
717
|
+
{
|
|
718
|
+
_id: laneId,
|
|
719
|
+
epoch: scope.get(laneId).epoch,
|
|
720
|
+
revision: scope.get(laneId).revision + 1
|
|
721
|
+
},
|
|
722
|
+
{ $pull: { inFlight: { $in: ids } } }
|
|
723
|
+
)
|
|
724
|
+
)
|
|
725
|
+
);
|
|
726
|
+
}
|
|
727
|
+
};
|
|
728
|
+
var FINISHED_FIELDS = { dueAt: "", lane: "" };
|
|
729
|
+
var MongoCompletion = class extends internal.LeaseSettlement {
|
|
730
|
+
#state;
|
|
731
|
+
constructor(state) {
|
|
732
|
+
super();
|
|
733
|
+
this.#state = state;
|
|
734
|
+
}
|
|
735
|
+
/** Implements {@link MongoEventStore.complete}. */
|
|
736
|
+
async complete(lease, { now, session }) {
|
|
737
|
+
if (session !== void 0) {
|
|
738
|
+
assertInTransaction(session, "complete");
|
|
739
|
+
}
|
|
740
|
+
return (await this.#completeLeases([lease], { now, session }))[0];
|
|
741
|
+
}
|
|
742
|
+
/** Implements {@link MongoEventStore.completeMany}. */
|
|
743
|
+
completeMany(leases, now) {
|
|
744
|
+
return this.#completeLeases(leases, { now, session: void 0 });
|
|
745
|
+
}
|
|
746
|
+
/** A lease this store issued; any other token cannot be held. */
|
|
747
|
+
findHeld(lease) {
|
|
748
|
+
return parseLeaseToken(lease.leaseToken);
|
|
749
|
+
}
|
|
750
|
+
/**
|
|
751
|
+
* Renews the lane lease of a running claim. One transaction: a stale lease must not extend the
|
|
752
|
+
* lane of a newer claim of the delivery. A claim ending or starting meanwhile writes the delivery
|
|
753
|
+
* or the lane, so the two conflict.
|
|
754
|
+
*/
|
|
755
|
+
writeExtension(held, { lease, leaseExpiresAt }) {
|
|
859
756
|
const deliveryId = String(lease.deliveryId);
|
|
860
|
-
return this
|
|
861
|
-
const running = await this.#deliveries.findOne(ofClaim({ ...held, deliveryId }), {
|
|
757
|
+
return runInTransaction(this.#state, async (session) => {
|
|
758
|
+
const running = await this.#state.deliveries.findOne(ofClaim({ ...held, deliveryId }), {
|
|
862
759
|
session,
|
|
863
760
|
projection: { _id: 1 }
|
|
864
761
|
});
|
|
865
762
|
if (running === null) {
|
|
866
763
|
return false;
|
|
867
764
|
}
|
|
868
|
-
const renewed = await this.#lanes.updateOne(
|
|
765
|
+
const renewed = await this.#state.lanes.updateOne(
|
|
869
766
|
{ _id: held.laneId, epoch: held.epoch, inFlight: deliveryId },
|
|
870
767
|
{ $max: { leaseExpiresAt } },
|
|
871
768
|
{ session }
|
|
@@ -874,248 +771,246 @@ var MongoEventStore = class {
|
|
|
874
771
|
});
|
|
875
772
|
}
|
|
876
773
|
/**
|
|
877
|
-
*
|
|
878
|
-
*
|
|
879
|
-
* it runs in a transaction of its own. Returns false when the lease was lost.
|
|
880
|
-
*/
|
|
881
|
-
async complete(lease, now, session) {
|
|
882
|
-
if (session !== void 0) {
|
|
883
|
-
assertInTransaction(session, "complete");
|
|
884
|
-
}
|
|
885
|
-
return (await this.#completeLeases([lease], now, session))[0];
|
|
886
|
-
}
|
|
887
|
-
/**
|
|
888
|
-
* Completes many leases with a fixed number of round trips, whatever lanes
|
|
889
|
-
* they belong to: three commands without a completion history, one
|
|
890
|
-
* transaction with it (see {@link MongoEventStore.#deleteFenced}).
|
|
891
|
-
* Returns one flag per lease in input order; a lease listed twice completes
|
|
892
|
-
* once, and a token this store cannot have issued is refused.
|
|
774
|
+
* Applies the change to the running delivery and takes it out of its lane's in-flight list, if
|
|
775
|
+
* the lane still has the lease's epoch.
|
|
893
776
|
*/
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
$set: { availableAt, dueAt: availableAt, lastError: toErrorInfo(error), updatedAt: now },
|
|
901
|
-
$inc: { attempts: 1, claims: 1 }
|
|
777
|
+
writeChange(held, { lease, change }) {
|
|
778
|
+
const deliveryId = String(lease.deliveryId);
|
|
779
|
+
return this.#leaveFlightNow(held, {
|
|
780
|
+
deliveryId,
|
|
781
|
+
ofThisClaim: ofClaim({ ...held, deliveryId }),
|
|
782
|
+
update: updateFor(change)
|
|
902
783
|
});
|
|
903
784
|
}
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
return this.#leaveFlight(lease, {
|
|
907
|
-
$set: { availableAt: now, dueAt: now, updatedAt: now },
|
|
908
|
-
$inc: { claims: 1 }
|
|
909
|
-
});
|
|
785
|
+
forgetHandedOut(lease) {
|
|
786
|
+
this.#forget(lease);
|
|
910
787
|
}
|
|
911
|
-
/**
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
$inc: { attempts: 1, claims: 1 }
|
|
788
|
+
/** Forgets a lease as handed out: its claim ended. */
|
|
789
|
+
#forget(lease) {
|
|
790
|
+
this.#state.handedOut.forget({
|
|
791
|
+
deliveryId: String(lease.deliveryId),
|
|
792
|
+
leaseToken: String(lease.leaseToken)
|
|
917
793
|
});
|
|
918
794
|
}
|
|
919
795
|
/**
|
|
920
|
-
*
|
|
921
|
-
*
|
|
796
|
+
* Runs `operation`, which ends the claims of `leases` (completion, retry,
|
|
797
|
+
* release, burial), and once it returned forgets exactly those leases as
|
|
798
|
+
* handed out. When it throws, they stay registered: the caller may retry
|
|
799
|
+
* with the same lease, which must still find its claim.
|
|
922
800
|
*/
|
|
923
|
-
async
|
|
924
|
-
const
|
|
925
|
-
const
|
|
926
|
-
|
|
927
|
-
(document) => document._id
|
|
928
|
-
)
|
|
929
|
-
);
|
|
930
|
-
const idle = held.filter((lane) => lane.inFlight.every((id) => !runningIds.has(id)));
|
|
931
|
-
if (idle.length > 0) {
|
|
932
|
-
await this.#lanes.bulkWrite(
|
|
933
|
-
idle.map((lane) => freeLane(lane._id, lane.epoch, lane.revision ?? 0, lane.inFlight)),
|
|
934
|
-
{ ordered: false }
|
|
935
|
-
);
|
|
801
|
+
async #settle(leases, operation) {
|
|
802
|
+
const result = await operation();
|
|
803
|
+
for (const lease of leases) {
|
|
804
|
+
this.#forget(lease);
|
|
936
805
|
}
|
|
806
|
+
return result;
|
|
937
807
|
}
|
|
938
808
|
/**
|
|
939
|
-
*
|
|
940
|
-
*
|
|
941
|
-
*
|
|
942
|
-
* lease. Completion history records (when kept) are listed as `completed`.
|
|
809
|
+
* Deletes the delivery of every lease whose lane still has the lease's epoch
|
|
810
|
+
* and lists the delivery, in one transaction (the caller's, when given).
|
|
811
|
+
* Returns one flag per lease, in order; a lease listed twice completes once.
|
|
943
812
|
*/
|
|
944
|
-
async
|
|
945
|
-
|
|
946
|
-
const common = {
|
|
947
|
-
...query.handler === void 0 ? {} : { handler: String(query.handler) },
|
|
948
|
-
...query.eventType === void 0 ? {} : { eventType: String(query.eventType) },
|
|
949
|
-
...query.eventId === void 0 ? {} : { eventId: String(query.eventId) }
|
|
950
|
-
};
|
|
951
|
-
const limit = query.limit ?? DEFAULT_LIST_LIMIT;
|
|
952
|
-
const lanes = await this.#lanes.find({ "inFlight.0": { $exists: true } }, { projection: { epoch: 0 } }).toArray();
|
|
953
|
-
const listed = lanes.flatMap((lane) => lane.inFlight);
|
|
954
|
-
const derivedStatuses = /* @__PURE__ */ new Map([
|
|
955
|
-
["processing", { status: "pending", _id: { $in: listed } }],
|
|
956
|
-
["pending", { status: "pending", _id: { $nin: listed } }]
|
|
957
|
-
]);
|
|
958
|
-
const filter = {
|
|
959
|
-
...common,
|
|
960
|
-
...status === void 0 ? {} : derivedStatuses.get(status) ?? { status }
|
|
961
|
-
};
|
|
962
|
-
const sort = { createdAt: 1, _id: 1 };
|
|
963
|
-
const documents = await this.#deliveries.find(filter, { sort, limit }).toArray();
|
|
964
|
-
const history = this.#completionHistory && (status === void 0 || status === "completed") ? await this.#completions.find(common, { sort, limit }).toArray() : [];
|
|
965
|
-
const events = await this.#readEnvelopes([
|
|
966
|
-
...documents.map((document) => document.eventId),
|
|
967
|
-
...history.map((document) => document.eventId)
|
|
968
|
-
]);
|
|
969
|
-
const laneById = new Map(lanes.map((lane) => [lane._id, lane]));
|
|
970
|
-
const records = [
|
|
971
|
-
...documents.map((document) => {
|
|
972
|
-
const lane = laneById.get(document.laneId);
|
|
973
|
-
const runs = document.status === "pending" && lane?.inFlight.includes(document._id) === true;
|
|
974
|
-
return toDeliveryRecord(document, events.get(document.eventId), runs ? lane : void 0);
|
|
975
|
-
}),
|
|
976
|
-
...history.map(
|
|
977
|
-
(document) => toCompletedDeliveryRecord(document, events.get(document.eventId))
|
|
978
|
-
)
|
|
979
|
-
];
|
|
980
|
-
return records.sort(
|
|
981
|
-
(left, right) => left.createdAt.getTime() - right.createdAt.getTime() || compareText(left.id, right.id)
|
|
982
|
-
).slice(0, limit);
|
|
813
|
+
async #completeLeases(leases, { now, session }) {
|
|
814
|
+
return session === void 0 ? this.#settle(leases, () => this.#completeIssued(leases, { now, session: void 0 })) : this.#completeIssued(leases, { now, session });
|
|
983
815
|
}
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
const
|
|
987
|
-
|
|
988
|
-
{
|
|
989
|
-
$set: {
|
|
990
|
-
status: "pending",
|
|
991
|
-
lane: "ready",
|
|
992
|
-
attempts: 0,
|
|
993
|
-
availableAt: now,
|
|
994
|
-
dueAt: now,
|
|
995
|
-
updatedAt: now
|
|
996
|
-
}
|
|
997
|
-
}
|
|
816
|
+
async #completeIssued(leases, { now, session }) {
|
|
817
|
+
const parsed = leases.map((lease) => parseLeaseToken(lease.leaseToken));
|
|
818
|
+
const issued = leases.flatMap(
|
|
819
|
+
(lease, index) => parsed[index] === void 0 ? [] : [{ deliveryId: String(lease.deliveryId), ...parsed[index] }]
|
|
998
820
|
);
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
* completion is before `completedBefore` (never by `deadBefore` alone), one
|
|
1008
|
-
* that never had a completion (no subscribers, or only dead letters) once it
|
|
1009
|
-
* was created before the earliest given cutoff. The batch step applies the
|
|
1010
|
-
* same rule to events whose last completion is recorded.
|
|
1011
|
-
*/
|
|
1012
|
-
async purge(request) {
|
|
1013
|
-
const total = { deletedDeliveries: 0, deletedEvents: 0 };
|
|
1014
|
-
const steps = [];
|
|
1015
|
-
const { completedBefore, deadBefore } = request;
|
|
1016
|
-
if (completedBefore !== void 0) {
|
|
1017
|
-
steps.push(
|
|
1018
|
-
() => this.#purgeMatching(
|
|
1019
|
-
this.#purgeable(this.#deliveries.collectionName),
|
|
1020
|
-
{ status: "completed", completedAt: { $lt: completedBefore } },
|
|
1021
|
-
completedBefore
|
|
1022
|
-
)
|
|
1023
|
-
);
|
|
1024
|
-
if (this.#completionHistory) {
|
|
1025
|
-
steps.push(
|
|
1026
|
-
() => this.#purgeMatching(
|
|
1027
|
-
this.#purgeable(this.#completions.collectionName),
|
|
1028
|
-
{ completedAt: { $lt: completedBefore } },
|
|
1029
|
-
completedBefore
|
|
1030
|
-
)
|
|
821
|
+
let completed = /* @__PURE__ */ new Set();
|
|
822
|
+
if (issued.length > 0) {
|
|
823
|
+
if (session !== void 0) {
|
|
824
|
+
completed = await this.#deleteCompleted(issued, { now, session });
|
|
825
|
+
} else if (this.#state.completionHistory) {
|
|
826
|
+
completed = await runInTransaction(
|
|
827
|
+
this.#state,
|
|
828
|
+
(own) => this.#deleteCompleted(issued, { now, session: own })
|
|
1031
829
|
);
|
|
830
|
+
} else {
|
|
831
|
+
completed = await this.#deleteFenced(issued, now);
|
|
1032
832
|
}
|
|
1033
833
|
}
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
this.#purgeable(this.#deliveries.collectionName),
|
|
1038
|
-
{ status: "dead", updatedAt: { $lt: deadBefore } },
|
|
1039
|
-
completedBefore
|
|
1040
|
-
)
|
|
1041
|
-
);
|
|
1042
|
-
}
|
|
1043
|
-
for (const step of steps) {
|
|
1044
|
-
const result = await step();
|
|
1045
|
-
total.deletedDeliveries += result.deletedDeliveries;
|
|
1046
|
-
total.deletedEvents += result.deletedEvents;
|
|
1047
|
-
}
|
|
1048
|
-
const orphanCutoff = [completedBefore, deadBefore].filter((cutoff) => cutoff !== void 0).sort((first, second) => first.getTime() - second.getTime())[0];
|
|
1049
|
-
if (orphanCutoff !== void 0) {
|
|
1050
|
-
total.deletedEvents += await this.#purgeOrphanEvents(orphanCutoff, completedBefore);
|
|
1051
|
-
}
|
|
1052
|
-
return total;
|
|
834
|
+
return leases.map(
|
|
835
|
+
(lease, index) => parsed[index] !== void 0 && completed.delete(leaseIdentity({ deliveryId: String(lease.deliveryId), ...parsed[index] }))
|
|
836
|
+
);
|
|
1053
837
|
}
|
|
1054
838
|
/**
|
|
1055
|
-
*
|
|
1056
|
-
*
|
|
1057
|
-
*
|
|
1058
|
-
*
|
|
1059
|
-
*
|
|
839
|
+
* Completion without a transaction, used when no history record has to be
|
|
840
|
+
* written with the deletion. The deliveries' claim generations are read
|
|
841
|
+
* first, then the lanes, and a delivery is deleted only if it still has the
|
|
842
|
+
* lease's generation. A takeover moves the generation of every delivery it
|
|
843
|
+
* takes from the previous owner on before it reads anything, and so do a
|
|
844
|
+
* retry, a release and a burial, so:
|
|
845
|
+
* - a takeover before the delivery read shows up in the lane read (new epoch);
|
|
846
|
+
* - a takeover between the two reads shows up in the lane read as well;
|
|
847
|
+
* - a takeover, retry, release or burial after the lane read changed the
|
|
848
|
+
* generation, so the delete matches nothing.
|
|
849
|
+
* A completion and a takeover (or the end of the claim) therefore never both succeed. Two concurrent
|
|
850
|
+
* completions of the same lease may both report true (the dispatcher never
|
|
851
|
+
* issues them); one call with a lease listed twice reports it once.
|
|
1060
852
|
*/
|
|
1061
|
-
async
|
|
1062
|
-
const
|
|
1063
|
-
const
|
|
1064
|
-
|
|
1065
|
-
this.#
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
]);
|
|
1070
|
-
const runningIds = new Set(running.map((document) => document._id));
|
|
1071
|
-
const waiting = await this.#deliveries.findOne(
|
|
1072
|
-
{ dueAt: { $lte: now }, _id: { $nin: [...runningIds] } },
|
|
1073
|
-
{ sort: { dueAt: 1 }, projection: { dueAt: 1 } }
|
|
1074
|
-
);
|
|
1075
|
-
const dueTimes = lanes.filter(
|
|
1076
|
-
(lane) => lane.leaseExpiresAt !== null && lane.leaseExpiresAt <= now && lane.inFlight.some((id) => runningIds.has(id))
|
|
1077
|
-
).map((lane) => lane.leaseExpiresAt.getTime());
|
|
1078
|
-
if (waiting !== null) {
|
|
1079
|
-
dueTimes.push(waiting.dueAt.getTime());
|
|
1080
|
-
}
|
|
1081
|
-
return {
|
|
1082
|
-
// The counts are separate reads; a completion between them must not make this negative.
|
|
1083
|
-
pending: Math.max(0, stored - running.length),
|
|
1084
|
-
processing: running.length,
|
|
1085
|
-
completed: legacyCompleted + history,
|
|
1086
|
-
dead,
|
|
1087
|
-
oldestDueAt: dueTimes.length === 0 ? null : new Date(Math.min(...dueTimes))
|
|
1088
|
-
};
|
|
1089
|
-
}
|
|
1090
|
-
/** Atomically increments the counter of `key`; conflicting appends of the key serialize here. */
|
|
1091
|
-
async #nextSequence(key, session) {
|
|
1092
|
-
const counter = await this.#keySequences.findOneAndUpdate(
|
|
1093
|
-
{ _id: key },
|
|
1094
|
-
{ $inc: { value: 1 } },
|
|
1095
|
-
{ upsert: true, returnDocument: "after", session }
|
|
853
|
+
async #deleteFenced(issued, now) {
|
|
854
|
+
const ids = [...new Set(issued.map((lease) => lease.deliveryId))];
|
|
855
|
+
const documents = await this.#state.deliveries.find({ _id: { $in: ids }, status: "pending" }, { projection: { claims: 1, eventId: 1 } }).toArray();
|
|
856
|
+
const lanes = new Map(
|
|
857
|
+
(await this.#state.lanes.find(
|
|
858
|
+
{ _id: { $in: [...new Set(issued.map((lease) => lease.laneId))] } },
|
|
859
|
+
{ projection: { epoch: 1, inFlight: 1 } }
|
|
860
|
+
).toArray()).map((lane) => [lane._id, lane])
|
|
1096
861
|
);
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
async #findRunning(listed) {
|
|
1101
|
-
if (listed.length === 0) {
|
|
1102
|
-
return [];
|
|
862
|
+
const fenced = fencedLeases(issued, { lanes, documents });
|
|
863
|
+
if (fenced.length === 0) {
|
|
864
|
+
return /* @__PURE__ */ new Set();
|
|
1103
865
|
}
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
).toArray();
|
|
866
|
+
await this.#markEventsCompleted(fenced, { documents, now });
|
|
867
|
+
const deleted = await this.#state.deliveries.deleteMany({ $or: fenced.map(ofClaim) });
|
|
868
|
+
return deleted.deletedCount === fenced.length ? new Set(fenced.map((lease) => leaseIdentity(lease))) : /* @__PURE__ */ new Set();
|
|
1108
869
|
}
|
|
1109
870
|
/**
|
|
1110
|
-
*
|
|
1111
|
-
*
|
|
871
|
+
* The transactional part of a completion. The lanes are only read: the
|
|
872
|
+
* conflict with a takeover comes from the deleted delivery documents, which a
|
|
873
|
+
* takeover writes before it reads them (see `MongoLaneLeases.takeLanes`).
|
|
874
|
+
* So whichever of the two commits second either fails with a write conflict
|
|
875
|
+
* and is retried against the new state, or waits for the first.
|
|
876
|
+
*/
|
|
877
|
+
async #deleteCompleted(issued, { now, session }) {
|
|
878
|
+
const lanes = new Map(
|
|
879
|
+
(await this.#state.lanes.find(
|
|
880
|
+
{ _id: { $in: [...new Set(issued.map((lease) => lease.laneId))] } },
|
|
881
|
+
{ session, projection: { epoch: 1, inFlight: 1 } }
|
|
882
|
+
).toArray()).map((lane) => [lane._id, lane])
|
|
883
|
+
);
|
|
884
|
+
const documents = await this.#state.deliveries.find(
|
|
885
|
+
{ _id: { $in: [...new Set(issued.map((lease) => lease.deliveryId))] }, status: "pending" },
|
|
886
|
+
{ session }
|
|
887
|
+
).toArray();
|
|
888
|
+
const fenced = fencedLeases(issued, { lanes, documents });
|
|
889
|
+
if (fenced.length === 0) {
|
|
890
|
+
return /* @__PURE__ */ new Set();
|
|
891
|
+
}
|
|
892
|
+
await this.#markEventsCompleted(fenced, { documents, now, session });
|
|
893
|
+
await this.#state.deliveries.deleteMany({ $or: fenced.map(ofClaim) }, { session });
|
|
894
|
+
if (this.#state.completionHistory) {
|
|
895
|
+
const completedIds = new Set(fenced.map((lease) => lease.deliveryId));
|
|
896
|
+
await this.#state.completions.insertMany(
|
|
897
|
+
documents.filter((document) => completedIds.has(document._id)).map((document) => toCompletedDeliveryDocument(document, now)),
|
|
898
|
+
{ session }
|
|
899
|
+
);
|
|
900
|
+
}
|
|
901
|
+
return new Set(fenced.map((lease) => leaseIdentity(lease)));
|
|
902
|
+
}
|
|
903
|
+
/**
|
|
904
|
+
* Records `now` as the latest completion of the events of the fenced
|
|
905
|
+
* deliveries (`$max`, so an earlier clock never moves it back). Written
|
|
906
|
+
* before the deletion, it can only be later than the real last completion
|
|
907
|
+
* (when the deletion then fails), which keeps the event longer, never shorter.
|
|
908
|
+
* Inside a transaction it is part of it: the transaction may hold the event
|
|
909
|
+
* already (a transactional handler that appended to its own event), and a
|
|
910
|
+
* write from outside would wait for it forever. Deliveries of one event that
|
|
911
|
+
* complete in concurrent transactions (fan-out to transactional handlers)
|
|
912
|
+
* therefore conflict on the event document, and one of them retries.
|
|
913
|
+
*/
|
|
914
|
+
async #markEventsCompleted(fenced, {
|
|
915
|
+
documents,
|
|
916
|
+
now,
|
|
917
|
+
session
|
|
918
|
+
}) {
|
|
919
|
+
const completedIds = new Set(fenced.map((lease) => lease.deliveryId));
|
|
920
|
+
const eventIds = [
|
|
921
|
+
...new Set(
|
|
922
|
+
documents.filter((document) => completedIds.has(document._id)).map((document) => document.eventId)
|
|
923
|
+
)
|
|
924
|
+
];
|
|
925
|
+
await this.#state.events.updateMany(
|
|
926
|
+
{ _id: { $in: eventIds } },
|
|
927
|
+
{ $max: { completedAt: now } },
|
|
928
|
+
session === void 0 ? {} : { session }
|
|
929
|
+
);
|
|
930
|
+
}
|
|
931
|
+
/** One transaction: the lane write fences the change against a takeover. */
|
|
932
|
+
async #leaveFlightNow(held, {
|
|
933
|
+
deliveryId,
|
|
934
|
+
ofThisClaim,
|
|
935
|
+
update
|
|
936
|
+
}) {
|
|
937
|
+
return runInTransaction(this.#state, async (session) => {
|
|
938
|
+
if (await this.#state.deliveries.findOne(ofThisClaim, { session, projection: { _id: 1 } }) === null) {
|
|
939
|
+
return false;
|
|
940
|
+
}
|
|
941
|
+
const fence = await this.#state.lanes.updateOne(
|
|
942
|
+
{ _id: held.laneId, epoch: held.epoch, inFlight: deliveryId },
|
|
943
|
+
{ $pull: { inFlight: deliveryId } },
|
|
944
|
+
{ session }
|
|
945
|
+
);
|
|
946
|
+
if (fence.matchedCount === 0) {
|
|
947
|
+
return false;
|
|
948
|
+
}
|
|
949
|
+
const changed = await this.#state.deliveries.updateOne(ofThisClaim, update, { session });
|
|
950
|
+
return changed.matchedCount === 1;
|
|
951
|
+
});
|
|
952
|
+
}
|
|
953
|
+
};
|
|
954
|
+
function fencedLeases(issued, {
|
|
955
|
+
lanes,
|
|
956
|
+
documents
|
|
957
|
+
}) {
|
|
958
|
+
const claimsById = new Map(documents.map((document) => [document._id, document.claims ?? 0]));
|
|
959
|
+
const fenced = /* @__PURE__ */ new Map();
|
|
960
|
+
for (const lease of issued) {
|
|
961
|
+
const lane = lanes.get(lease.laneId);
|
|
962
|
+
if (claimsById.get(lease.deliveryId) === lease.claims && lane.epoch === lease.epoch && lane.inFlight.includes(lease.deliveryId)) {
|
|
963
|
+
fenced.set(leaseIdentity(lease), lease);
|
|
964
|
+
}
|
|
965
|
+
}
|
|
966
|
+
return [...fenced.values()];
|
|
967
|
+
}
|
|
968
|
+
function ofClaim(lease) {
|
|
969
|
+
return { _id: lease.deliveryId, status: "pending", claims: atCount(lease.claims) };
|
|
970
|
+
}
|
|
971
|
+
function leaseIdentity(lease) {
|
|
972
|
+
return internal.DeliveryLeaseIdentity.of(
|
|
973
|
+
lease.deliveryId,
|
|
974
|
+
formatLeaseToken(lease.laneId, { epoch: lease.epoch, claims: lease.claims })
|
|
975
|
+
);
|
|
976
|
+
}
|
|
977
|
+
function updateFor(change) {
|
|
978
|
+
if (change.kind === "reschedule") {
|
|
979
|
+
const { availableAt, error, now } = change;
|
|
980
|
+
return {
|
|
981
|
+
$set: { availableAt, dueAt: availableAt, lastError: toErrorInfo(error), updatedAt: now },
|
|
982
|
+
$inc: { attempts: 1, claims: 1 }
|
|
983
|
+
};
|
|
984
|
+
}
|
|
985
|
+
if (change.kind === "release") {
|
|
986
|
+
const { now } = change;
|
|
987
|
+
return { $set: { availableAt: now, dueAt: now, updatedAt: now }, $inc: { claims: 1 } };
|
|
988
|
+
}
|
|
989
|
+
return {
|
|
990
|
+
$set: { status: "dead", lastError: toErrorInfo(change.error), updatedAt: change.now },
|
|
991
|
+
$unset: FINISHED_FIELDS,
|
|
992
|
+
$inc: { attempts: 1, claims: 1 }
|
|
993
|
+
};
|
|
994
|
+
}
|
|
995
|
+
|
|
996
|
+
// src/mongo-lane-leases.ts
|
|
997
|
+
var MongoLaneLeases = class {
|
|
998
|
+
#state;
|
|
999
|
+
#queries;
|
|
1000
|
+
constructor(state, collaborators) {
|
|
1001
|
+
this.#state = state;
|
|
1002
|
+
this.#queries = collaborators.queries;
|
|
1003
|
+
}
|
|
1004
|
+
/**
|
|
1005
|
+
* Up to `laneCount` lanes nobody holds under a live lease that have a due,
|
|
1006
|
+
* ready delivery of the requested handlers, the lane with the oldest such
|
|
1112
1007
|
* delivery first, leaving out `passed`. One round trip: the oldest delivery
|
|
1113
1008
|
* of each lane is one index probe on the server. A lane whose due deliveries
|
|
1114
1009
|
* all wait behind an unfinished predecessor is taken at most once: the claim
|
|
1115
|
-
* that hands it back parks them (see
|
|
1010
|
+
* that hands it back parks them (see `ClaimScan.waitingBacklogs`).
|
|
1116
1011
|
*/
|
|
1117
|
-
async
|
|
1118
|
-
const lanes = await this.#lanes.aggregate([
|
|
1012
|
+
async findFreeLanes(request, { laneCount, passed }) {
|
|
1013
|
+
const lanes = await this.#state.lanes.aggregate([
|
|
1119
1014
|
{
|
|
1120
1015
|
$match: {
|
|
1121
1016
|
_id: { $nin: passed },
|
|
@@ -1124,7 +1019,7 @@ var MongoEventStore = class {
|
|
|
1124
1019
|
},
|
|
1125
1020
|
{
|
|
1126
1021
|
$lookup: {
|
|
1127
|
-
from: this.#deliveries.collectionName,
|
|
1022
|
+
from: this.#state.deliveries.collectionName,
|
|
1128
1023
|
localField: "_id",
|
|
1129
1024
|
foreignField: "laneId",
|
|
1130
1025
|
pipeline: [
|
|
@@ -1153,12 +1048,12 @@ var MongoEventStore = class {
|
|
|
1153
1048
|
* Takes the lanes with one conditional write each, run in parallel; a lane
|
|
1154
1049
|
* taken by someone else meanwhile is skipped. The same write moves the
|
|
1155
1050
|
* previous owner's in-flight list to `lost`, which is charged before any
|
|
1156
|
-
* delivery of the lanes is read (see {@link
|
|
1051
|
+
* delivery of the lanes is read (see {@link MongoLaneLeases.chargeLost}).
|
|
1157
1052
|
*/
|
|
1158
|
-
async
|
|
1053
|
+
async takeLanes(laneIds, request) {
|
|
1159
1054
|
const after = await Promise.all(
|
|
1160
1055
|
laneIds.map(
|
|
1161
|
-
(laneId) => this.#lanes.findOneAndUpdate(
|
|
1056
|
+
(laneId) => this.#state.lanes.findOneAndUpdate(
|
|
1162
1057
|
{
|
|
1163
1058
|
_id: laneId,
|
|
1164
1059
|
$or: [{ leaseExpiresAt: null }, { leaseExpiresAt: { $lte: request.now } }]
|
|
@@ -1168,7 +1063,7 @@ var MongoEventStore = class {
|
|
|
1168
1063
|
$set: {
|
|
1169
1064
|
epoch: { $add: ["$epoch", 1] },
|
|
1170
1065
|
owner: { $literal: request.leaseOwner },
|
|
1171
|
-
holder: { $literal: this.#instanceId },
|
|
1066
|
+
holder: { $literal: this.#state.instanceId },
|
|
1172
1067
|
leaseExpiresAt: { $literal: request.leaseExpiresAt },
|
|
1173
1068
|
// Each lost delivery keeps the epoch it was claimed under: the key of its charge.
|
|
1174
1069
|
lost: {
|
|
@@ -1186,7 +1081,7 @@ var MongoEventStore = class {
|
|
|
1186
1081
|
)
|
|
1187
1082
|
);
|
|
1188
1083
|
const taken = after.filter((lane) => lane !== null);
|
|
1189
|
-
await this
|
|
1084
|
+
await this.chargeLost(taken);
|
|
1190
1085
|
return taken.map((lane) => ({
|
|
1191
1086
|
laneId: lane._id,
|
|
1192
1087
|
epoch: lane.epoch,
|
|
@@ -1206,13 +1101,13 @@ var MongoEventStore = class {
|
|
|
1206
1101
|
* list again after a failure before the clear skips deliveries already
|
|
1207
1102
|
* charged for that claim.
|
|
1208
1103
|
*/
|
|
1209
|
-
async
|
|
1104
|
+
async chargeLost(lanes) {
|
|
1210
1105
|
const losing = lanes.filter((lane) => lane.lost.length > 0);
|
|
1211
1106
|
if (losing.length === 0) {
|
|
1212
1107
|
return;
|
|
1213
1108
|
}
|
|
1214
1109
|
const lost = losing.flatMap((lane) => lane.lost).sort((left, right) => left.epoch - right.epoch);
|
|
1215
|
-
await this.#deliveries.bulkWrite(
|
|
1110
|
+
await this.#state.deliveries.bulkWrite(
|
|
1216
1111
|
lost.map(({ id, epoch }) => ({
|
|
1217
1112
|
updateOne: {
|
|
1218
1113
|
filter: { _id: id, status: "pending", lostAt: { $not: { $gte: epoch } } },
|
|
@@ -1221,13 +1116,175 @@ var MongoEventStore = class {
|
|
|
1221
1116
|
})),
|
|
1222
1117
|
{ ordered: true }
|
|
1223
1118
|
);
|
|
1224
|
-
await this.#lanes.bulkWrite(
|
|
1119
|
+
await this.#state.lanes.bulkWrite(
|
|
1225
1120
|
losing.map((lane) => ({
|
|
1226
1121
|
updateOne: { filter: { _id: lane._id, epoch: lane.epoch }, update: { $set: { lost: [] } } }
|
|
1227
1122
|
})),
|
|
1228
1123
|
{ ordered: false }
|
|
1229
1124
|
);
|
|
1230
1125
|
}
|
|
1126
|
+
/**
|
|
1127
|
+
* Records the picks in their lanes, one conditional write per lane and one
|
|
1128
|
+
* round trip for all of them, and frees the lanes that yielded nothing and run
|
|
1129
|
+
* nothing. A picked lane lists its running deliveries plus the new ones (ids
|
|
1130
|
+
* whose documents are gone are pruned) and its lease reaches
|
|
1131
|
+
* `request.leaseExpiresAt`. Returns the lanes whose picks were recorded: every
|
|
1132
|
+
* write is conditional on the claim's epoch, owner and store instance, so a
|
|
1133
|
+
* lane that changed hands, or was handed back by `relinquish`, in the meantime
|
|
1134
|
+
* keeps its deliveries for whoever takes it next.
|
|
1135
|
+
*/
|
|
1136
|
+
async recordClaim(scope, {
|
|
1137
|
+
picked,
|
|
1138
|
+
runningIds,
|
|
1139
|
+
request
|
|
1140
|
+
}) {
|
|
1141
|
+
const picksByLane = /* @__PURE__ */ new Map();
|
|
1142
|
+
for (const document of picked) {
|
|
1143
|
+
picksByLane.set(document.laneId, [...picksByLane.get(document.laneId) ?? [], document._id]);
|
|
1144
|
+
}
|
|
1145
|
+
const { operations, handedBack } = this.#planLaneRecords(scope, {
|
|
1146
|
+
picksByLane,
|
|
1147
|
+
runningIds,
|
|
1148
|
+
request
|
|
1149
|
+
});
|
|
1150
|
+
if (operations.length === 0) {
|
|
1151
|
+
return { recorded: /* @__PURE__ */ new Set(), handedBack };
|
|
1152
|
+
}
|
|
1153
|
+
const result = await this.#state.lanes.bulkWrite(operations, { ordered: false });
|
|
1154
|
+
if (result.matchedCount === operations.length) {
|
|
1155
|
+
return { recorded: new Set(picksByLane.keys()), handedBack };
|
|
1156
|
+
}
|
|
1157
|
+
return {
|
|
1158
|
+
recorded: await this.#readRecorded(scope, { laneIds: [...picksByLane.keys()], request }),
|
|
1159
|
+
handedBack
|
|
1160
|
+
};
|
|
1161
|
+
}
|
|
1162
|
+
/**
|
|
1163
|
+
* One conditional write per lane of the claim: a picked lane records its picks, a lane that
|
|
1164
|
+
* yielded nothing and runs nothing is freed (and reported as handed back).
|
|
1165
|
+
*/
|
|
1166
|
+
#planLaneRecords(scope, {
|
|
1167
|
+
picksByLane,
|
|
1168
|
+
runningIds,
|
|
1169
|
+
request
|
|
1170
|
+
}) {
|
|
1171
|
+
const handedBack = /* @__PURE__ */ new Set();
|
|
1172
|
+
const operations = [];
|
|
1173
|
+
for (const lane of scope.values()) {
|
|
1174
|
+
const gone = lane.listed.filter((id) => !runningIds.has(id));
|
|
1175
|
+
const ids = picksByLane.get(lane.laneId);
|
|
1176
|
+
if (ids !== void 0) {
|
|
1177
|
+
operations.push(
|
|
1178
|
+
recordInFlight({ lane, ids, gone, request, instanceId: this.#state.instanceId })
|
|
1179
|
+
);
|
|
1180
|
+
} else if (gone.length === lane.listed.length) {
|
|
1181
|
+
operations.push(
|
|
1182
|
+
freeLane(lane.laneId, { epoch: lane.epoch, revision: lane.revision, finished: gone })
|
|
1183
|
+
);
|
|
1184
|
+
handedBack.add(lane.laneId);
|
|
1185
|
+
}
|
|
1186
|
+
}
|
|
1187
|
+
return { operations, handedBack };
|
|
1188
|
+
}
|
|
1189
|
+
/** The lanes of `laneIds` whose record of this claim applied, read back after a partial bulk write. */
|
|
1190
|
+
async #readRecorded(scope, { laneIds, request }) {
|
|
1191
|
+
const current = await this.#state.lanes.find(
|
|
1192
|
+
{ _id: { $in: [...laneIds] } },
|
|
1193
|
+
{ projection: { epoch: 1, owner: 1, holder: 1, revision: 1 } }
|
|
1194
|
+
).toArray();
|
|
1195
|
+
return new Set(
|
|
1196
|
+
current.filter(
|
|
1197
|
+
(lane) => lane.epoch === scope.get(lane._id).epoch && lane.revision === scope.get(lane._id).revision + 1 && lane.owner === request.leaseOwner && lane.holder === this.#state.instanceId
|
|
1198
|
+
).map((lane) => lane._id)
|
|
1199
|
+
);
|
|
1200
|
+
}
|
|
1201
|
+
/** Implements {@link MongoEventStore.relinquish}. */
|
|
1202
|
+
async relinquish(leaseOwner) {
|
|
1203
|
+
const held = await this.#state.lanes.find({ owner: String(leaseOwner), holder: this.#state.instanceId }).toArray();
|
|
1204
|
+
const runningIds = new Set(
|
|
1205
|
+
(await this.#queries.findRunning(held.flatMap((lane) => lane.inFlight))).map(
|
|
1206
|
+
(document) => document._id
|
|
1207
|
+
)
|
|
1208
|
+
);
|
|
1209
|
+
const idle = held.filter((lane) => lane.inFlight.every((id) => !runningIds.has(id)));
|
|
1210
|
+
if (idle.length > 0) {
|
|
1211
|
+
await this.#state.lanes.bulkWrite(
|
|
1212
|
+
idle.map(
|
|
1213
|
+
(lane) => freeLane(lane._id, {
|
|
1214
|
+
epoch: lane.epoch,
|
|
1215
|
+
revision: lane.revision ?? 0,
|
|
1216
|
+
finished: lane.inFlight
|
|
1217
|
+
})
|
|
1218
|
+
),
|
|
1219
|
+
{ ordered: false }
|
|
1220
|
+
);
|
|
1221
|
+
}
|
|
1222
|
+
}
|
|
1223
|
+
};
|
|
1224
|
+
function recordInFlight(input) {
|
|
1225
|
+
const { lane, ids, gone, request, instanceId } = input;
|
|
1226
|
+
return {
|
|
1227
|
+
updateOne: {
|
|
1228
|
+
filter: {
|
|
1229
|
+
_id: lane.laneId,
|
|
1230
|
+
epoch: lane.epoch,
|
|
1231
|
+
owner: request.leaseOwner,
|
|
1232
|
+
holder: instanceId,
|
|
1233
|
+
// A record that reaches the server late (after its claim failed and reconciled)
|
|
1234
|
+
// finds a newer revision and changes nothing.
|
|
1235
|
+
revision: atCount(lane.revision)
|
|
1236
|
+
},
|
|
1237
|
+
update: [
|
|
1238
|
+
{
|
|
1239
|
+
$set: {
|
|
1240
|
+
inFlight: {
|
|
1241
|
+
$concatArrays: [
|
|
1242
|
+
{
|
|
1243
|
+
$filter: {
|
|
1244
|
+
input: "$inFlight",
|
|
1245
|
+
cond: { $not: [{ $in: ["$$this", { $literal: gone }] }] }
|
|
1246
|
+
}
|
|
1247
|
+
},
|
|
1248
|
+
{ $literal: ids }
|
|
1249
|
+
]
|
|
1250
|
+
},
|
|
1251
|
+
leaseExpiresAt: { $max: ["$leaseExpiresAt", request.leaseExpiresAt] },
|
|
1252
|
+
revision: { $add: [{ $ifNull: ["$revision", 0] }, 1] }
|
|
1253
|
+
}
|
|
1254
|
+
}
|
|
1255
|
+
]
|
|
1256
|
+
}
|
|
1257
|
+
};
|
|
1258
|
+
}
|
|
1259
|
+
function freeLane(laneId, {
|
|
1260
|
+
epoch,
|
|
1261
|
+
revision,
|
|
1262
|
+
finished
|
|
1263
|
+
}) {
|
|
1264
|
+
return {
|
|
1265
|
+
updateOne: {
|
|
1266
|
+
filter: {
|
|
1267
|
+
_id: laneId,
|
|
1268
|
+
epoch,
|
|
1269
|
+
revision: atCount(revision),
|
|
1270
|
+
inFlight: { $not: { $elemMatch: { $nin: [...finished] } } }
|
|
1271
|
+
},
|
|
1272
|
+
update: {
|
|
1273
|
+
$set: { owner: null, holder: null, leaseExpiresAt: null, inFlight: [] },
|
|
1274
|
+
$inc: { revision: 1 }
|
|
1275
|
+
}
|
|
1276
|
+
}
|
|
1277
|
+
};
|
|
1278
|
+
}
|
|
1279
|
+
|
|
1280
|
+
// src/mongo-ordering.ts
|
|
1281
|
+
var MINIMUM_CANDIDATE_BATCH_SIZE = 32;
|
|
1282
|
+
var PARK_UPDATE_CHUNK_SIZE = 1e3;
|
|
1283
|
+
var MongoOrdering = class {
|
|
1284
|
+
#state;
|
|
1285
|
+
constructor(state) {
|
|
1286
|
+
this.#state = state;
|
|
1287
|
+
}
|
|
1231
1288
|
/**
|
|
1232
1289
|
* Adds the claimable candidates of `lanes` to `picked`, in due order, until
|
|
1233
1290
|
* `request.limit` is reached. A delivery is a candidate when it is ready, due
|
|
@@ -1242,46 +1299,18 @@ var MongoEventStore = class {
|
|
|
1242
1299
|
* the predecessors of the keys of a page are looked up in parallel, one
|
|
1243
1300
|
* lookup per key.
|
|
1244
1301
|
*/
|
|
1245
|
-
async
|
|
1302
|
+
async pickCandidates(input) {
|
|
1303
|
+
const { lanes, request, scan, running, picked } = input;
|
|
1246
1304
|
const pageSize = Math.max((request.limit - picked.length) * 2, MINIMUM_CANDIDATE_BATCH_SIZE);
|
|
1247
|
-
const filter =
|
|
1248
|
-
laneId: { $in: lanes.map((lane) => lane.laneId) },
|
|
1249
|
-
handler: { $in: [...request.handlers] },
|
|
1250
|
-
lane: "ready",
|
|
1251
|
-
dueAt: { $lte: request.now },
|
|
1252
|
-
_id: { $nin: lanes.flatMap((lane) => lane.listed) }
|
|
1253
|
-
};
|
|
1305
|
+
const filter = candidateFilter(lanes, request);
|
|
1254
1306
|
let after;
|
|
1255
1307
|
for (; ; ) {
|
|
1256
1308
|
const [page, { keys }] = await Promise.all([
|
|
1257
|
-
this.#deliveries.find(
|
|
1258
|
-
after === void 0 ? filter : {
|
|
1259
|
-
$and: [
|
|
1260
|
-
filter,
|
|
1261
|
-
{
|
|
1262
|
-
$or: [
|
|
1263
|
-
{ dueAt: { $gt: after.dueAt } },
|
|
1264
|
-
{ dueAt: after.dueAt, _id: { $gt: after._id } }
|
|
1265
|
-
]
|
|
1266
|
-
}
|
|
1267
|
-
]
|
|
1268
|
-
},
|
|
1269
|
-
{ sort: { dueAt: 1, _id: 1 }, limit: pageSize }
|
|
1270
|
-
).toArray(),
|
|
1309
|
+
this.#state.deliveries.find(pageAfter(filter, after), { sort: { dueAt: 1, _id: 1 }, limit: pageSize }).toArray(),
|
|
1271
1310
|
running
|
|
1272
1311
|
]);
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
offset += slice.length;
|
|
1276
|
-
await this.#notePredecessors(slice, scan);
|
|
1277
|
-
for (const candidate of slice) {
|
|
1278
|
-
if (isClaimable(candidate, scan, keys)) {
|
|
1279
|
-
picked.push(candidate);
|
|
1280
|
-
}
|
|
1281
|
-
}
|
|
1282
|
-
if (picked.length >= request.limit) {
|
|
1283
|
-
return;
|
|
1284
|
-
}
|
|
1312
|
+
if (await this.#pickFromPage(page, { request, scan, picked, keys })) {
|
|
1313
|
+
return;
|
|
1285
1314
|
}
|
|
1286
1315
|
if (page.length < pageSize) {
|
|
1287
1316
|
return;
|
|
@@ -1289,6 +1318,27 @@ var MongoEventStore = class {
|
|
|
1289
1318
|
after = page.at(-1);
|
|
1290
1319
|
}
|
|
1291
1320
|
}
|
|
1321
|
+
/**
|
|
1322
|
+
* Adds the claimable candidates of `page` to `picked`. Looks up predecessors only for as many
|
|
1323
|
+
* candidates as the claim still needs, then more. True once the claim has `request.limit` picks.
|
|
1324
|
+
*/
|
|
1325
|
+
async #pickFromPage(page, claim) {
|
|
1326
|
+
const { request, scan, picked, keys } = claim;
|
|
1327
|
+
for (let offset = 0; offset < page.length; ) {
|
|
1328
|
+
const slice = page.slice(offset, offset + request.limit - picked.length);
|
|
1329
|
+
offset += slice.length;
|
|
1330
|
+
await this.#notePredecessors(slice, scan);
|
|
1331
|
+
for (const candidate of slice) {
|
|
1332
|
+
if (isClaimable(candidate, { scan, blockedKeys: keys })) {
|
|
1333
|
+
picked.push(candidate);
|
|
1334
|
+
}
|
|
1335
|
+
}
|
|
1336
|
+
if (picked.length >= request.limit) {
|
|
1337
|
+
return true;
|
|
1338
|
+
}
|
|
1339
|
+
}
|
|
1340
|
+
return false;
|
|
1341
|
+
}
|
|
1292
1342
|
/**
|
|
1293
1343
|
* Looks up, in parallel and once per key, whether an unfinished delivery
|
|
1294
1344
|
* precedes the earliest candidate of each keyed (handler, key) of `page` that
|
|
@@ -1308,181 +1358,45 @@ var MongoEventStore = class {
|
|
|
1308
1358
|
}
|
|
1309
1359
|
await Promise.all(
|
|
1310
1360
|
[...earliest].map(async ([orderingId, candidate]) => {
|
|
1311
|
-
const predecessor = await this.#findUnfinishedPredecessor(
|
|
1312
|
-
candidate.
|
|
1313
|
-
candidate.
|
|
1314
|
-
|
|
1315
|
-
);
|
|
1361
|
+
const predecessor = await this.#findUnfinishedPredecessor(candidate.handler, {
|
|
1362
|
+
key: candidate.key,
|
|
1363
|
+
sequence: candidate.sequence
|
|
1364
|
+
});
|
|
1316
1365
|
scan.noteOpen(orderingId, predecessor ?? candidate.sequence);
|
|
1317
1366
|
})
|
|
1318
1367
|
);
|
|
1319
1368
|
}
|
|
1320
1369
|
/**
|
|
1321
|
-
*
|
|
1322
|
-
*
|
|
1323
|
-
*
|
|
1324
|
-
* whose documents are gone are pruned) and its lease reaches
|
|
1325
|
-
* `request.leaseExpiresAt`. Returns the lanes whose picks were recorded: every
|
|
1326
|
-
* write is conditional on the claim's epoch, owner and store instance, so a
|
|
1327
|
-
* lane that changed hands, or was handed back by `relinquish`, in the meantime
|
|
1328
|
-
* keeps its deliveries for whoever takes it next.
|
|
1370
|
+
* Parks the waiting backlogs a claim found. Best effort: parking only saves
|
|
1371
|
+
* later claims work, so a failure is logged and the claim still returns its
|
|
1372
|
+
* batch; the next claim that meets the backlog parks it.
|
|
1329
1373
|
*/
|
|
1330
|
-
async
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
|
|
1337
|
-
for (const lane of scope.values()) {
|
|
1338
|
-
const gone = lane.listed.filter((id) => !runningIds.has(id));
|
|
1339
|
-
const ids = picksByLane.get(lane.laneId);
|
|
1340
|
-
if (ids !== void 0) {
|
|
1341
|
-
operations.push({
|
|
1342
|
-
updateOne: {
|
|
1343
|
-
filter: {
|
|
1344
|
-
_id: lane.laneId,
|
|
1345
|
-
epoch: lane.epoch,
|
|
1346
|
-
owner: request.leaseOwner,
|
|
1347
|
-
holder: this.#instanceId,
|
|
1348
|
-
// A record that reaches the server late (after its claim failed and reconciled)
|
|
1349
|
-
// finds a newer revision and changes nothing.
|
|
1350
|
-
revision: atCount(lane.revision)
|
|
1351
|
-
},
|
|
1352
|
-
update: [
|
|
1353
|
-
{
|
|
1354
|
-
$set: {
|
|
1355
|
-
inFlight: {
|
|
1356
|
-
$concatArrays: [
|
|
1357
|
-
{
|
|
1358
|
-
$filter: {
|
|
1359
|
-
input: "$inFlight",
|
|
1360
|
-
cond: { $not: [{ $in: ["$$this", { $literal: gone }] }] }
|
|
1361
|
-
}
|
|
1362
|
-
},
|
|
1363
|
-
{ $literal: ids }
|
|
1364
|
-
]
|
|
1365
|
-
},
|
|
1366
|
-
leaseExpiresAt: { $max: ["$leaseExpiresAt", request.leaseExpiresAt] },
|
|
1367
|
-
revision: { $add: [{ $ifNull: ["$revision", 0] }, 1] }
|
|
1368
|
-
}
|
|
1369
|
-
}
|
|
1370
|
-
]
|
|
1371
|
-
}
|
|
1372
|
-
});
|
|
1373
|
-
} else if (gone.length === lane.listed.length) {
|
|
1374
|
-
operations.push(freeLane(lane.laneId, lane.epoch, lane.revision, gone));
|
|
1375
|
-
handedBack.add(lane.laneId);
|
|
1376
|
-
}
|
|
1377
|
-
}
|
|
1378
|
-
if (operations.length === 0) {
|
|
1379
|
-
return { recorded: /* @__PURE__ */ new Set(), handedBack };
|
|
1380
|
-
}
|
|
1381
|
-
const result = await this.#lanes.bulkWrite(operations, { ordered: false });
|
|
1382
|
-
if (result.matchedCount === operations.length) {
|
|
1383
|
-
return { recorded: new Set(picksByLane.keys()), handedBack };
|
|
1374
|
+
async parkWaiting(deliveryIds) {
|
|
1375
|
+
try {
|
|
1376
|
+
await this.#parkDeliveries(deliveryIds);
|
|
1377
|
+
} catch (error) {
|
|
1378
|
+
this.#state.logger.warn(
|
|
1379
|
+
`[MongoOrdering](parkWaiting) Parking waiting deliveries failed, a later claim parks them | count=${deliveryIds.length} | error=${String(error)}`
|
|
1380
|
+
);
|
|
1384
1381
|
}
|
|
1385
|
-
|
|
1386
|
-
|
|
1387
|
-
|
|
1388
|
-
|
|
1389
|
-
|
|
1390
|
-
|
|
1391
|
-
(lane) => lane.epoch === scope.get(lane._id).epoch && lane.revision === scope.get(lane._id).revision + 1 && lane.owner === request.leaseOwner && lane.holder === this.#instanceId
|
|
1392
|
-
).map((lane) => lane._id)
|
|
1382
|
+
}
|
|
1383
|
+
/** Sequence of the lowest unfinished delivery of the handler and key before `sequence`, if any. */
|
|
1384
|
+
async #findUnfinishedPredecessor(handler, { key, sequence }) {
|
|
1385
|
+
const predecessor = await this.#state.deliveries.findOne(
|
|
1386
|
+
{ handler, key, sequence: { $lt: sequence }, dueAt: { $exists: true } },
|
|
1387
|
+
{ sort: { sequence: 1 }, projection: { sequence: 1 } }
|
|
1393
1388
|
);
|
|
1394
|
-
return
|
|
1389
|
+
return predecessor?.sequence;
|
|
1395
1390
|
}
|
|
1396
1391
|
/**
|
|
1397
|
-
*
|
|
1398
|
-
* claim
|
|
1399
|
-
*
|
|
1400
|
-
*
|
|
1401
|
-
* pick is recorded, a later requeue waits for it like for any running delivery.
|
|
1402
|
-
*/
|
|
1403
|
-
async #releaseOvertaken(claimed, scope) {
|
|
1404
|
-
const keyed = claimed.filter((document) => document.key !== void 0);
|
|
1405
|
-
if (keyed.length === 0) {
|
|
1406
|
-
return [...claimed];
|
|
1407
|
-
}
|
|
1408
|
-
const blockers = await this.#deliveries.find(
|
|
1409
|
-
{
|
|
1410
|
-
$or: keyed.map((document) => ({
|
|
1411
|
-
handler: document.handler,
|
|
1412
|
-
key: document.key,
|
|
1413
|
-
sequence: { $lt: document.sequence },
|
|
1414
|
-
dueAt: { $exists: true }
|
|
1415
|
-
}))
|
|
1416
|
-
},
|
|
1417
|
-
{ projection: { handler: 1, key: 1 } }
|
|
1418
|
-
).toArray();
|
|
1419
|
-
const blocked = new Set(
|
|
1420
|
-
blockers.map((blocker) => orderingIdentity(blocker.handler, blocker.key))
|
|
1421
|
-
);
|
|
1422
|
-
const overtaken = new Set(
|
|
1423
|
-
keyed.filter((document) => blocked.has(orderingIdentity(document.handler, document.key))).map((document) => document._id)
|
|
1424
|
-
);
|
|
1425
|
-
if (overtaken.size === 0) {
|
|
1426
|
-
return [...claimed];
|
|
1427
|
-
}
|
|
1428
|
-
await this.#releasePicks(
|
|
1429
|
-
claimed.filter((document) => overtaken.has(document._id)),
|
|
1430
|
-
scope
|
|
1431
|
-
);
|
|
1432
|
-
return claimed.filter((document) => !overtaken.has(document._id));
|
|
1433
|
-
}
|
|
1434
|
-
/** Takes recorded picks out of their lanes' in-flight lists again, under the claim's epochs and record revisions. */
|
|
1435
|
-
async #releasePicks(documents, scope) {
|
|
1436
|
-
const idsByLane = /* @__PURE__ */ new Map();
|
|
1437
|
-
for (const document of documents) {
|
|
1438
|
-
idsByLane.set(document.laneId, [...idsByLane.get(document.laneId) ?? [], document._id]);
|
|
1439
|
-
}
|
|
1440
|
-
await Promise.all(
|
|
1441
|
-
[...idsByLane].map(
|
|
1442
|
-
([laneId, ids]) => this.#lanes.updateOne(
|
|
1443
|
-
// Under the revision the record left: a pull that reaches the server late, after a
|
|
1444
|
-
// reconciliation and a newer claim of the same delivery, matches nothing.
|
|
1445
|
-
{
|
|
1446
|
-
_id: laneId,
|
|
1447
|
-
epoch: scope.get(laneId).epoch,
|
|
1448
|
-
revision: scope.get(laneId).revision + 1
|
|
1449
|
-
},
|
|
1450
|
-
{ $pull: { inFlight: { $in: ids } } }
|
|
1451
|
-
)
|
|
1452
|
-
)
|
|
1453
|
-
);
|
|
1454
|
-
}
|
|
1455
|
-
/**
|
|
1456
|
-
* Parks the waiting backlogs a claim found. Best effort: parking only saves
|
|
1457
|
-
* later claims work, so a failure is logged and the claim still returns its
|
|
1458
|
-
* batch; the next claim that meets the backlog parks it.
|
|
1459
|
-
*/
|
|
1460
|
-
async #parkWaiting(deliveryIds) {
|
|
1461
|
-
try {
|
|
1462
|
-
await this.#parkDeliveries(deliveryIds);
|
|
1463
|
-
} catch (error) {
|
|
1464
|
-
this.#logger.warn(
|
|
1465
|
-
`[MongoEventStore](parkWaiting) Parking waiting deliveries failed, a later claim parks them | count=${deliveryIds.length} | error=${String(error)}`
|
|
1466
|
-
);
|
|
1467
|
-
}
|
|
1468
|
-
}
|
|
1469
|
-
/** Sequence of the lowest unfinished delivery of the handler and key before `sequence`, if any. */
|
|
1470
|
-
async #findUnfinishedPredecessor(handler, key, sequence) {
|
|
1471
|
-
const predecessor = await this.#deliveries.findOne(
|
|
1472
|
-
{ handler, key, sequence: { $lt: sequence }, dueAt: { $exists: true } },
|
|
1473
|
-
{ sort: { sequence: 1 }, projection: { sequence: 1 } }
|
|
1474
|
-
);
|
|
1475
|
-
return predecessor?.sequence;
|
|
1476
|
-
}
|
|
1477
|
-
/**
|
|
1478
|
-
* Parks pending deliveries that waited behind an unfinished predecessor, which
|
|
1479
|
-
* removes them from the claim index until their turn. Parking is a hint, not a
|
|
1480
|
-
* guarantee: a predecessor that finishes meanwhile is found by the next
|
|
1481
|
-
* {@link MongoEventStore.claim}, which moves the lowest parked delivery back.
|
|
1392
|
+
* Parks pending deliveries that waited behind an unfinished predecessor, which
|
|
1393
|
+
* removes them from the claim index until their turn. Parking is a hint, not a
|
|
1394
|
+
* guarantee: a predecessor that finishes meanwhile is found by the next
|
|
1395
|
+
* {@link MongoEventStore.claim}, which moves the lowest parked delivery back.
|
|
1482
1396
|
*/
|
|
1483
1397
|
async #parkDeliveries(deliveryIds) {
|
|
1484
1398
|
for (let offset = 0; offset < deliveryIds.length; offset += PARK_UPDATE_CHUNK_SIZE) {
|
|
1485
|
-
await this.#deliveries.updateMany(
|
|
1399
|
+
await this.#state.deliveries.updateMany(
|
|
1486
1400
|
{
|
|
1487
1401
|
_id: { $in: deliveryIds.slice(offset, offset + PARK_UPDATE_CHUNK_SIZE) },
|
|
1488
1402
|
status: "pending",
|
|
@@ -1499,8 +1413,8 @@ var MongoEventStore = class {
|
|
|
1499
1413
|
* predecessor finishing while its followers were being parked, can delay a key
|
|
1500
1414
|
* until the next claim but never strand it.
|
|
1501
1415
|
*/
|
|
1502
|
-
async
|
|
1503
|
-
const anyParked = await this.#deliveries.findOne(
|
|
1416
|
+
async unparkNextDeliveries(handlers) {
|
|
1417
|
+
const anyParked = await this.#state.deliveries.findOne(
|
|
1504
1418
|
{ handler: { $in: [...handlers] }, lane: "parked" },
|
|
1505
1419
|
// The sort lets the planner pick the small partial index over the handler's whole history.
|
|
1506
1420
|
{ sort: { handler: 1, key: 1, sequence: 1 }, projection: { _id: 1 } }
|
|
@@ -1508,180 +1422,131 @@ var MongoEventStore = class {
|
|
|
1508
1422
|
if (anyParked === null) {
|
|
1509
1423
|
return;
|
|
1510
1424
|
}
|
|
1511
|
-
const lowest =
|
|
1512
|
-
[...new Set(handlers)].map(
|
|
1513
|
-
(handler) => this.#deliveries.aggregate([
|
|
1514
|
-
{ $match: { handler, lane: "parked" } },
|
|
1515
|
-
{ $sort: { handler: 1, key: 1, sequence: 1 } },
|
|
1516
|
-
{
|
|
1517
|
-
$group: { _id: "$key", id: { $first: "$_id" }, sequence: { $first: "$sequence" } }
|
|
1518
|
-
}
|
|
1519
|
-
]).toArray().then((groups) => groups.map((group) => ({ ...group, handler })))
|
|
1520
|
-
)
|
|
1521
|
-
)).flat();
|
|
1425
|
+
const lowest = await this.#lowestParkedPerKey(handlers);
|
|
1522
1426
|
const next = await Promise.all(
|
|
1523
1427
|
lowest.map(
|
|
1524
|
-
async (parked) => await this.#findUnfinishedPredecessor(parked.handler,
|
|
1428
|
+
async (parked) => await this.#findUnfinishedPredecessor(parked.handler, {
|
|
1429
|
+
key: parked._id,
|
|
1430
|
+
sequence: parked.sequence
|
|
1431
|
+
}) === void 0 ? [parked.id] : []
|
|
1525
1432
|
)
|
|
1526
1433
|
);
|
|
1527
1434
|
const ready = next.flat();
|
|
1528
1435
|
if (ready.length > 0) {
|
|
1529
|
-
await this.#deliveries.updateMany(
|
|
1436
|
+
await this.#state.deliveries.updateMany(
|
|
1530
1437
|
{ _id: { $in: ready }, lane: "parked" },
|
|
1531
1438
|
{ $set: { lane: "ready" } }
|
|
1532
1439
|
);
|
|
1533
1440
|
}
|
|
1534
1441
|
}
|
|
1535
|
-
async #readEnvelopes(eventIds) {
|
|
1536
|
-
const documents = await this.#events.find({ _id: { $in: [...new Set(eventIds)] } }).toArray();
|
|
1537
|
-
return new Map(documents.map((document) => [document._id, toEnvelope(document)]));
|
|
1538
|
-
}
|
|
1539
1442
|
/**
|
|
1540
|
-
*
|
|
1541
|
-
*
|
|
1542
|
-
* Returns one flag per lease, in order; a lease listed twice completes once.
|
|
1443
|
+
* The lowest parked delivery of every parked key of `handlers`, one aggregation per handler in
|
|
1444
|
+
* parallel.
|
|
1543
1445
|
*/
|
|
1544
|
-
async #
|
|
1545
|
-
return
|
|
1446
|
+
async #lowestParkedPerKey(handlers) {
|
|
1447
|
+
return (await Promise.all(
|
|
1448
|
+
[...new Set(handlers)].map(
|
|
1449
|
+
(handler) => this.#state.deliveries.aggregate([
|
|
1450
|
+
{ $match: { handler, lane: "parked" } },
|
|
1451
|
+
{ $sort: { handler: 1, key: 1, sequence: 1 } },
|
|
1452
|
+
{
|
|
1453
|
+
$group: { _id: "$key", id: { $first: "$_id" }, sequence: { $first: "$sequence" } }
|
|
1454
|
+
}
|
|
1455
|
+
]).toArray().then((groups) => groups.map((group) => ({ ...group, handler })))
|
|
1456
|
+
)
|
|
1457
|
+
)).flat();
|
|
1546
1458
|
}
|
|
1547
|
-
|
|
1548
|
-
|
|
1549
|
-
|
|
1550
|
-
|
|
1551
|
-
|
|
1552
|
-
|
|
1553
|
-
|
|
1554
|
-
|
|
1555
|
-
|
|
1556
|
-
|
|
1557
|
-
|
|
1558
|
-
|
|
1559
|
-
|
|
1459
|
+
};
|
|
1460
|
+
function candidateFilter(lanes, request) {
|
|
1461
|
+
return {
|
|
1462
|
+
laneId: { $in: lanes.map((lane) => lane.laneId) },
|
|
1463
|
+
handler: { $in: [...request.handlers] },
|
|
1464
|
+
lane: "ready",
|
|
1465
|
+
dueAt: { $lte: request.now },
|
|
1466
|
+
_id: { $nin: lanes.flatMap((lane) => lane.listed) }
|
|
1467
|
+
};
|
|
1468
|
+
}
|
|
1469
|
+
function pageAfter(filter, after) {
|
|
1470
|
+
return after === void 0 ? filter : {
|
|
1471
|
+
$and: [
|
|
1472
|
+
filter,
|
|
1473
|
+
{
|
|
1474
|
+
$or: [
|
|
1475
|
+
{ dueAt: { $gt: after.dueAt } },
|
|
1476
|
+
{ dueAt: after.dueAt, _id: { $gt: after._id } }
|
|
1477
|
+
]
|
|
1560
1478
|
}
|
|
1561
|
-
|
|
1562
|
-
|
|
1563
|
-
|
|
1564
|
-
|
|
1479
|
+
]
|
|
1480
|
+
};
|
|
1481
|
+
}
|
|
1482
|
+
function isClaimable(candidate, { scan, blockedKeys }) {
|
|
1483
|
+
const { key, sequence } = candidate;
|
|
1484
|
+
if (key === void 0) {
|
|
1485
|
+
return true;
|
|
1565
1486
|
}
|
|
1566
|
-
|
|
1567
|
-
|
|
1568
|
-
|
|
1569
|
-
|
|
1570
|
-
* lease's generation. A takeover moves the generation of every delivery it
|
|
1571
|
-
* takes from the previous owner on before it reads anything, and so do a
|
|
1572
|
-
* retry, a release and a burial, so:
|
|
1573
|
-
* - a takeover before the delivery read shows up in the lane read (new epoch);
|
|
1574
|
-
* - a takeover between the two reads shows up in the lane read as well;
|
|
1575
|
-
* - a takeover, retry, release or burial after the lane read changed the
|
|
1576
|
-
* generation, so the delete matches nothing.
|
|
1577
|
-
* A completion and a takeover (or the end of the claim) therefore never both succeed. Two concurrent
|
|
1578
|
-
* completions of the same lease may both report true (the dispatcher never
|
|
1579
|
-
* issues them); one call with a lease listed twice reports it once.
|
|
1580
|
-
*/
|
|
1581
|
-
async #deleteFenced(issued, now) {
|
|
1582
|
-
const ids = [...new Set(issued.map((lease) => lease.deliveryId))];
|
|
1583
|
-
const documents = await this.#deliveries.find({ _id: { $in: ids }, status: "pending" }, { projection: { claims: 1, eventId: 1 } }).toArray();
|
|
1584
|
-
const lanes = new Map(
|
|
1585
|
-
(await this.#lanes.find(
|
|
1586
|
-
{ _id: { $in: [...new Set(issued.map((lease) => lease.laneId))] } },
|
|
1587
|
-
{ projection: { epoch: 1, inFlight: 1 } }
|
|
1588
|
-
).toArray()).map((lane) => [lane._id, lane])
|
|
1589
|
-
);
|
|
1590
|
-
const fenced = fencedLeases(issued, lanes, documents);
|
|
1591
|
-
if (fenced.length === 0) {
|
|
1592
|
-
return /* @__PURE__ */ new Set();
|
|
1593
|
-
}
|
|
1594
|
-
await this.#markEventsCompleted(fenced, documents, now);
|
|
1595
|
-
const deleted = await this.#deliveries.deleteMany({ $or: fenced.map(ofClaim) });
|
|
1596
|
-
return deleted.deletedCount === fenced.length ? new Set(fenced.map((lease) => leaseIdentity(lease))) : /* @__PURE__ */ new Set();
|
|
1487
|
+
const orderingId = orderingIdentity(candidate.handler, key);
|
|
1488
|
+
if (scan.waits(orderingId, sequence)) {
|
|
1489
|
+
scan.noteWaiting(orderingId, { deliveryId: candidate._id, laneId: candidate.laneId });
|
|
1490
|
+
return false;
|
|
1597
1491
|
}
|
|
1598
|
-
|
|
1599
|
-
|
|
1600
|
-
* conflict with a takeover comes from the deleted delivery documents, which a
|
|
1601
|
-
* takeover writes before it reads them (see {@link MongoEventStore.#takeLanes}).
|
|
1602
|
-
* So whichever of the two commits second either fails with a write conflict
|
|
1603
|
-
* and is retried against the new state, or waits for the first.
|
|
1604
|
-
*/
|
|
1605
|
-
async #deleteCompleted(issued, now, session) {
|
|
1606
|
-
const lanes = new Map(
|
|
1607
|
-
(await this.#lanes.find(
|
|
1608
|
-
{ _id: { $in: [...new Set(issued.map((lease) => lease.laneId))] } },
|
|
1609
|
-
{ session, projection: { epoch: 1, inFlight: 1 } }
|
|
1610
|
-
).toArray()).map((lane) => [lane._id, lane])
|
|
1611
|
-
);
|
|
1612
|
-
const documents = await this.#deliveries.find(
|
|
1613
|
-
{ _id: { $in: [...new Set(issued.map((lease) => lease.deliveryId))] }, status: "pending" },
|
|
1614
|
-
{ session }
|
|
1615
|
-
).toArray();
|
|
1616
|
-
const fenced = fencedLeases(issued, lanes, documents);
|
|
1617
|
-
if (fenced.length === 0) {
|
|
1618
|
-
return /* @__PURE__ */ new Set();
|
|
1619
|
-
}
|
|
1620
|
-
await this.#markEventsCompleted(fenced, documents, now, session);
|
|
1621
|
-
await this.#deliveries.deleteMany({ $or: fenced.map(ofClaim) }, { session });
|
|
1622
|
-
if (this.#completionHistory) {
|
|
1623
|
-
const completedIds = new Set(fenced.map((lease) => lease.deliveryId));
|
|
1624
|
-
await this.#completions.insertMany(
|
|
1625
|
-
documents.filter((document) => completedIds.has(document._id)).map((document) => toCompletedDeliveryDocument(document, now)),
|
|
1626
|
-
{ session }
|
|
1627
|
-
);
|
|
1628
|
-
}
|
|
1629
|
-
return new Set(fenced.map((lease) => leaseIdentity(lease)));
|
|
1492
|
+
if (blockedKeys.has(orderingId)) {
|
|
1493
|
+
return false;
|
|
1630
1494
|
}
|
|
1631
|
-
|
|
1632
|
-
|
|
1633
|
-
|
|
1634
|
-
|
|
1635
|
-
|
|
1636
|
-
|
|
1637
|
-
|
|
1638
|
-
|
|
1639
|
-
|
|
1640
|
-
* therefore conflict on the event document, and one of them retries.
|
|
1641
|
-
*/
|
|
1642
|
-
async #markEventsCompleted(fenced, documents, now, session) {
|
|
1643
|
-
const completedIds = new Set(fenced.map((lease) => lease.deliveryId));
|
|
1644
|
-
const eventIds = [
|
|
1645
|
-
...new Set(
|
|
1646
|
-
documents.filter((document) => completedIds.has(document._id)).map((document) => document.eventId)
|
|
1647
|
-
)
|
|
1648
|
-
];
|
|
1649
|
-
await this.#events.updateMany(
|
|
1650
|
-
{ _id: { $in: eventIds } },
|
|
1651
|
-
{ $max: { completedAt: now } },
|
|
1652
|
-
session === void 0 ? {} : { session }
|
|
1653
|
-
);
|
|
1495
|
+
blockedKeys.add(orderingId);
|
|
1496
|
+
return true;
|
|
1497
|
+
}
|
|
1498
|
+
|
|
1499
|
+
// src/mongo-purge.ts
|
|
1500
|
+
var MongoPurge = class {
|
|
1501
|
+
#state;
|
|
1502
|
+
constructor(state) {
|
|
1503
|
+
this.#state = state;
|
|
1654
1504
|
}
|
|
1655
|
-
/**
|
|
1656
|
-
|
|
1657
|
-
|
|
1658
|
-
|
|
1659
|
-
|
|
1660
|
-
|
|
1661
|
-
|
|
1662
|
-
|
|
1663
|
-
|
|
1505
|
+
/** Implements {@link MongoEventStore.purge}. */
|
|
1506
|
+
async purge(request) {
|
|
1507
|
+
const total = { deletedDeliveries: 0, deletedEvents: 0 };
|
|
1508
|
+
const steps = this.#purgeSteps(request);
|
|
1509
|
+
const { completedBefore, deadBefore } = request;
|
|
1510
|
+
for (const step of steps) {
|
|
1511
|
+
const result = await step();
|
|
1512
|
+
total.deletedDeliveries += result.deletedDeliveries;
|
|
1513
|
+
total.deletedEvents += result.deletedEvents;
|
|
1664
1514
|
}
|
|
1665
|
-
const
|
|
1666
|
-
|
|
1667
|
-
|
|
1515
|
+
const orphanCutoff = [completedBefore, deadBefore].filter((cutoff) => cutoff !== void 0).sort((first, second) => first.getTime() - second.getTime())[0];
|
|
1516
|
+
if (orphanCutoff !== void 0) {
|
|
1517
|
+
total.deletedEvents += await this.#purgeOrphanEvents(orphanCutoff, completedBefore);
|
|
1518
|
+
}
|
|
1519
|
+
return total;
|
|
1668
1520
|
}
|
|
1669
|
-
|
|
1670
|
-
|
|
1671
|
-
|
|
1672
|
-
|
|
1673
|
-
|
|
1674
|
-
|
|
1675
|
-
|
|
1676
|
-
|
|
1677
|
-
|
|
1521
|
+
/** The purges the request asks for, in the order they run. */
|
|
1522
|
+
#purgeSteps(request) {
|
|
1523
|
+
const steps = [];
|
|
1524
|
+
const { completedBefore, deadBefore } = request;
|
|
1525
|
+
if (completedBefore !== void 0) {
|
|
1526
|
+
steps.push(
|
|
1527
|
+
() => this.#purgeMatching(this.#purgeable(this.#state.deliveries.collectionName), {
|
|
1528
|
+
filter: { status: "completed", completedAt: { $lt: completedBefore } },
|
|
1529
|
+
completedBefore
|
|
1530
|
+
})
|
|
1678
1531
|
);
|
|
1679
|
-
if (
|
|
1680
|
-
|
|
1532
|
+
if (this.#state.completionHistory) {
|
|
1533
|
+
steps.push(
|
|
1534
|
+
() => this.#purgeMatching(this.#purgeable(this.#state.completions.collectionName), {
|
|
1535
|
+
filter: { completedAt: { $lt: completedBefore } },
|
|
1536
|
+
completedBefore
|
|
1537
|
+
})
|
|
1538
|
+
);
|
|
1681
1539
|
}
|
|
1682
|
-
|
|
1683
|
-
|
|
1684
|
-
|
|
1540
|
+
}
|
|
1541
|
+
if (deadBefore !== void 0) {
|
|
1542
|
+
steps.push(
|
|
1543
|
+
() => this.#purgeMatching(this.#purgeable(this.#state.deliveries.collectionName), {
|
|
1544
|
+
filter: { status: "dead", updatedAt: { $lt: deadBefore } },
|
|
1545
|
+
completedBefore
|
|
1546
|
+
})
|
|
1547
|
+
);
|
|
1548
|
+
}
|
|
1549
|
+
return steps;
|
|
1685
1550
|
}
|
|
1686
1551
|
/**
|
|
1687
1552
|
* Deletes batches of matching documents (deliveries, or completion history
|
|
@@ -1692,49 +1557,61 @@ var MongoEventStore = class {
|
|
|
1692
1557
|
* without event. A delivery requeued since the batch was read conflicts with
|
|
1693
1558
|
* its deletion and is read again.
|
|
1694
1559
|
*/
|
|
1695
|
-
async #purgeMatching(collection,
|
|
1560
|
+
async #purgeMatching(collection, {
|
|
1561
|
+
filter,
|
|
1562
|
+
completedBefore
|
|
1563
|
+
}) {
|
|
1696
1564
|
let deletedDeliveries = 0;
|
|
1697
1565
|
let deletedEvents = 0;
|
|
1698
1566
|
for (; ; ) {
|
|
1699
|
-
const step = await
|
|
1700
|
-
|
|
1701
|
-
|
|
1702
|
-
|
|
1703
|
-
}
|
|
1704
|
-
const eventIds = [...new Set(batch.map((document) => document.eventId))];
|
|
1705
|
-
const deleted = await collection.deleteMany(
|
|
1706
|
-
{ _id: { $in: batch.map((document) => document._id) } },
|
|
1707
|
-
{ session }
|
|
1708
|
-
);
|
|
1709
|
-
const stillReferenced = await this.#referencedEventIds(eventIds, session);
|
|
1710
|
-
const events = await this.#events.deleteMany(
|
|
1711
|
-
{
|
|
1712
|
-
_id: { $in: eventIds.filter((eventId) => !stillReferenced.has(eventId)) },
|
|
1713
|
-
// An event a delivery of which completed stays as deduplication record until
|
|
1714
|
-
// `completedBefore` passes its last completion, whatever this batch purged.
|
|
1715
|
-
$or: [
|
|
1716
|
-
{ completedAt: { $exists: false } },
|
|
1717
|
-
...completedBefore === void 0 ? [] : [{ completedAt: { $lt: completedBefore } }]
|
|
1718
|
-
]
|
|
1719
|
-
},
|
|
1720
|
-
{ session }
|
|
1721
|
-
);
|
|
1722
|
-
return {
|
|
1723
|
-
batchSize: batch.length,
|
|
1724
|
-
deletedDeliveries: deleted.deletedCount,
|
|
1725
|
-
deletedEvents: events.deletedCount
|
|
1726
|
-
};
|
|
1727
|
-
});
|
|
1567
|
+
const step = await runInTransaction(
|
|
1568
|
+
this.#state,
|
|
1569
|
+
(session) => this.#purgeBatch(collection, { filter, completedBefore, session })
|
|
1570
|
+
);
|
|
1728
1571
|
deletedDeliveries += step.deletedDeliveries;
|
|
1729
1572
|
deletedEvents += step.deletedEvents;
|
|
1730
|
-
if (step.batchSize < this.#purgeBatchSize) {
|
|
1573
|
+
if (step.batchSize < this.#state.purgeBatchSize) {
|
|
1731
1574
|
return { deletedDeliveries, deletedEvents };
|
|
1732
1575
|
}
|
|
1733
1576
|
}
|
|
1734
1577
|
}
|
|
1578
|
+
/** One batch of {@link MongoPurge.#purgeMatching}, inside the caller's transaction. */
|
|
1579
|
+
async #purgeBatch(collection, {
|
|
1580
|
+
filter,
|
|
1581
|
+
completedBefore,
|
|
1582
|
+
session
|
|
1583
|
+
}) {
|
|
1584
|
+
const batch = await collection.find(filter, { session, projection: { eventId: 1 }, limit: this.#state.purgeBatchSize }).toArray();
|
|
1585
|
+
if (batch.length === 0) {
|
|
1586
|
+
return { batchSize: 0, deletedDeliveries: 0, deletedEvents: 0 };
|
|
1587
|
+
}
|
|
1588
|
+
const eventIds = [...new Set(batch.map((document) => document.eventId))];
|
|
1589
|
+
const deleted = await collection.deleteMany(
|
|
1590
|
+
{ _id: { $in: batch.map((document) => document._id) } },
|
|
1591
|
+
{ session }
|
|
1592
|
+
);
|
|
1593
|
+
const stillReferenced = await this.#referencedEventIds(eventIds, session);
|
|
1594
|
+
const events = await this.#state.events.deleteMany(
|
|
1595
|
+
{
|
|
1596
|
+
_id: { $in: eventIds.filter((eventId) => !stillReferenced.has(eventId)) },
|
|
1597
|
+
// An event a delivery of which completed stays as deduplication record until
|
|
1598
|
+
// `completedBefore` passes its last completion, whatever this batch purged.
|
|
1599
|
+
$or: [
|
|
1600
|
+
{ completedAt: { $exists: false } },
|
|
1601
|
+
...completedBefore === void 0 ? [] : [{ completedAt: { $lt: completedBefore } }]
|
|
1602
|
+
]
|
|
1603
|
+
},
|
|
1604
|
+
{ session }
|
|
1605
|
+
);
|
|
1606
|
+
return {
|
|
1607
|
+
batchSize: batch.length,
|
|
1608
|
+
deletedDeliveries: deleted.deletedCount,
|
|
1609
|
+
deletedEvents: events.deletedCount
|
|
1610
|
+
};
|
|
1611
|
+
}
|
|
1735
1612
|
/** A view of a collection that purge deletes from: deliveries or completion history records. */
|
|
1736
1613
|
#purgeable(name) {
|
|
1737
|
-
return this.#database.collection(name, {
|
|
1614
|
+
return this.#state.database.collection(name, {
|
|
1738
1615
|
readPreference: "primary",
|
|
1739
1616
|
writeConcern: { w: "majority" }
|
|
1740
1617
|
});
|
|
@@ -1744,8 +1621,8 @@ var MongoEventStore = class {
|
|
|
1744
1621
|
const filter = { eventId: { $in: [...eventIds] } };
|
|
1745
1622
|
const options = { session, projection: { eventId: 1 } };
|
|
1746
1623
|
const references = [
|
|
1747
|
-
...await this.#deliveries.find(filter, options).toArray(),
|
|
1748
|
-
...this.#completionHistory ? await this.#completions.find(filter, options).toArray() : []
|
|
1624
|
+
...await this.#state.deliveries.find(filter, options).toArray(),
|
|
1625
|
+
...this.#state.completionHistory ? await this.#state.completions.find(filter, options).toArray() : []
|
|
1749
1626
|
];
|
|
1750
1627
|
return new Set(references.map((reference) => reference.eventId));
|
|
1751
1628
|
}
|
|
@@ -1758,87 +1635,426 @@ var MongoEventStore = class {
|
|
|
1758
1635
|
*/
|
|
1759
1636
|
async #purgeOrphanEvents(cutoff, completedBefore) {
|
|
1760
1637
|
const references = [
|
|
1761
|
-
this.#deliveries.collectionName,
|
|
1762
|
-
...this.#completionHistory ? [this.#completions.collectionName] : []
|
|
1638
|
+
this.#state.deliveries.collectionName,
|
|
1639
|
+
...this.#state.completionHistory ? [this.#state.completions.collectionName] : []
|
|
1763
1640
|
];
|
|
1764
1641
|
let deletedEvents = 0;
|
|
1765
1642
|
for (; ; ) {
|
|
1766
|
-
const step = await this
|
|
1767
|
-
const orphans = await this.#events.aggregate(
|
|
1768
|
-
|
|
1769
|
-
|
|
1770
|
-
|
|
1771
|
-
|
|
1772
|
-
|
|
1773
|
-
|
|
1774
|
-
// Completed: by the time of its last completion, and only under the completed cutoff.
|
|
1775
|
-
...completedBefore === void 0 ? [] : [{ completedAt: { $lt: completedBefore } }]
|
|
1776
|
-
]
|
|
1777
|
-
}
|
|
1778
|
-
},
|
|
1779
|
-
{ $sort: { createdAt: 1, _id: 1 } },
|
|
1780
|
-
...references.flatMap((from) => [
|
|
1781
|
-
{
|
|
1782
|
-
$lookup: {
|
|
1783
|
-
from,
|
|
1784
|
-
localField: "_id",
|
|
1785
|
-
foreignField: "eventId",
|
|
1786
|
-
pipeline: [{ $limit: 1 }, { $project: { _id: 1 } }],
|
|
1787
|
-
as: "references"
|
|
1788
|
-
}
|
|
1789
|
-
},
|
|
1790
|
-
{ $match: { references: { $size: 0 } } }
|
|
1791
|
-
]),
|
|
1792
|
-
{ $limit: this.#purgeBatchSize },
|
|
1793
|
-
{ $project: { _id: 1 } }
|
|
1794
|
-
],
|
|
1643
|
+
const step = await runInTransaction(this.#state, async (session) => {
|
|
1644
|
+
const orphans = await this.#state.events.aggregate(
|
|
1645
|
+
orphanPipeline({
|
|
1646
|
+
cutoff,
|
|
1647
|
+
completedBefore,
|
|
1648
|
+
references,
|
|
1649
|
+
limit: this.#state.purgeBatchSize
|
|
1650
|
+
}),
|
|
1795
1651
|
{ session }
|
|
1796
1652
|
).toArray();
|
|
1797
|
-
const deleted = await this.#events.deleteMany(
|
|
1653
|
+
const deleted = await this.#state.events.deleteMany(
|
|
1798
1654
|
{ _id: { $in: orphans.map((orphan) => orphan._id) } },
|
|
1799
1655
|
{ session }
|
|
1800
1656
|
);
|
|
1801
1657
|
return { found: orphans.length, deleted: deleted.deletedCount };
|
|
1802
1658
|
});
|
|
1803
1659
|
deletedEvents += step.deleted;
|
|
1804
|
-
if (step.found < this.#purgeBatchSize) {
|
|
1660
|
+
if (step.found < this.#state.purgeBatchSize) {
|
|
1805
1661
|
return deletedEvents;
|
|
1806
1662
|
}
|
|
1807
1663
|
}
|
|
1808
1664
|
}
|
|
1809
1665
|
};
|
|
1810
|
-
function
|
|
1811
|
-
const {
|
|
1812
|
-
|
|
1813
|
-
|
|
1666
|
+
function orphanPipeline(input) {
|
|
1667
|
+
const { cutoff, completedBefore, references, limit } = input;
|
|
1668
|
+
return [
|
|
1669
|
+
{
|
|
1670
|
+
$match: {
|
|
1671
|
+
$or: [
|
|
1672
|
+
// Never completed (no subscribers, or every delivery dead): by age.
|
|
1673
|
+
{ completedAt: { $exists: false }, createdAt: { $lt: cutoff } },
|
|
1674
|
+
// Completed: by the time of its last completion, and only under the completed cutoff.
|
|
1675
|
+
...completedBefore === void 0 ? [] : [{ completedAt: { $lt: completedBefore } }]
|
|
1676
|
+
]
|
|
1677
|
+
}
|
|
1678
|
+
},
|
|
1679
|
+
{ $sort: { createdAt: 1, _id: 1 } },
|
|
1680
|
+
...references.flatMap((from) => [
|
|
1681
|
+
{
|
|
1682
|
+
$lookup: {
|
|
1683
|
+
from,
|
|
1684
|
+
localField: "_id",
|
|
1685
|
+
foreignField: "eventId",
|
|
1686
|
+
pipeline: [{ $limit: 1 }, { $project: { _id: 1 } }],
|
|
1687
|
+
as: "references"
|
|
1688
|
+
}
|
|
1689
|
+
},
|
|
1690
|
+
{ $match: { references: { $size: 0 } } }
|
|
1691
|
+
]),
|
|
1692
|
+
{ $limit: limit },
|
|
1693
|
+
{ $project: { _id: 1 } }
|
|
1694
|
+
];
|
|
1695
|
+
}
|
|
1696
|
+
|
|
1697
|
+
// src/mongo-queries.ts
|
|
1698
|
+
var DEFAULT_LIST_LIMIT = 100;
|
|
1699
|
+
var MongoQueries = class {
|
|
1700
|
+
#state;
|
|
1701
|
+
constructor(state) {
|
|
1702
|
+
this.#state = state;
|
|
1814
1703
|
}
|
|
1815
|
-
|
|
1816
|
-
|
|
1817
|
-
|
|
1818
|
-
|
|
1704
|
+
/** Implements {@link MongoEventStore.listDeliveries}. */
|
|
1705
|
+
async listDeliveries(query) {
|
|
1706
|
+
const status = query.status === void 0 ? void 0 : String(query.status);
|
|
1707
|
+
const common = commonFilter(query);
|
|
1708
|
+
const limit = query.limit ?? DEFAULT_LIST_LIMIT;
|
|
1709
|
+
const lanes = await this.#state.lanes.find({ "inFlight.0": { $exists: true } }, { projection: { epoch: 0 } }).toArray();
|
|
1710
|
+
const listed = lanes.flatMap((lane) => lane.inFlight);
|
|
1711
|
+
const filter = { ...common, ...statusFilter(status, listed) };
|
|
1712
|
+
const sort = { createdAt: 1, _id: 1 };
|
|
1713
|
+
const documents = await this.#state.deliveries.find(filter, { sort, limit }).toArray();
|
|
1714
|
+
const history = this.#state.completionHistory && (status === void 0 || status === "completed") ? await this.#state.completions.find(common, { sort, limit }).toArray() : [];
|
|
1715
|
+
const events = await this.readEnvelopes([
|
|
1716
|
+
...documents.map((document) => document.eventId),
|
|
1717
|
+
...history.map((document) => document.eventId)
|
|
1718
|
+
]);
|
|
1719
|
+
return toSortedRecords({ documents, history, events, lanes }).slice(0, limit);
|
|
1819
1720
|
}
|
|
1820
|
-
|
|
1821
|
-
|
|
1721
|
+
/** Implements {@link MongoEventStore.requeue}. */
|
|
1722
|
+
async requeue(deliveryIds, now) {
|
|
1723
|
+
const result = await this.#state.deliveries.updateMany(
|
|
1724
|
+
{ _id: { $in: deliveryIds.map(String) }, status: "dead" },
|
|
1725
|
+
{
|
|
1726
|
+
$set: {
|
|
1727
|
+
status: "pending",
|
|
1728
|
+
lane: "ready",
|
|
1729
|
+
attempts: 0,
|
|
1730
|
+
availableAt: now,
|
|
1731
|
+
dueAt: now,
|
|
1732
|
+
updatedAt: now
|
|
1733
|
+
}
|
|
1734
|
+
}
|
|
1735
|
+
);
|
|
1736
|
+
return result.modifiedCount;
|
|
1822
1737
|
}
|
|
1823
|
-
|
|
1824
|
-
|
|
1738
|
+
/** Implements {@link MongoEventStore.statistics}. */
|
|
1739
|
+
async statistics(now) {
|
|
1740
|
+
const lanes = await this.#state.lanes.find({ "inFlight.0": { $exists: true } }, { projection: { inFlight: 1, leaseExpiresAt: 1 } }).toArray();
|
|
1741
|
+
const [stored, dead, legacyCompleted, history, running] = await Promise.all([
|
|
1742
|
+
this.#state.deliveries.countDocuments({ status: "pending" }),
|
|
1743
|
+
this.#state.deliveries.countDocuments({ status: "dead" }),
|
|
1744
|
+
this.#state.deliveries.countDocuments({ status: "completed" }),
|
|
1745
|
+
this.#state.completionHistory ? this.#state.completions.countDocuments() : Promise.resolve(0),
|
|
1746
|
+
this.findRunning(lanes.flatMap((lane) => lane.inFlight))
|
|
1747
|
+
]);
|
|
1748
|
+
const runningIds = new Set(running.map((document) => document._id));
|
|
1749
|
+
const waiting = await this.#state.deliveries.findOne(
|
|
1750
|
+
{ dueAt: { $lte: now }, _id: { $nin: [...runningIds] } },
|
|
1751
|
+
{ sort: { dueAt: 1 }, projection: { dueAt: 1 } }
|
|
1752
|
+
);
|
|
1753
|
+
const dueTimes = lanes.filter(
|
|
1754
|
+
(lane) => lane.leaseExpiresAt !== null && lane.leaseExpiresAt <= now && lane.inFlight.some((id) => runningIds.has(id))
|
|
1755
|
+
).map((lane) => lane.leaseExpiresAt.getTime());
|
|
1756
|
+
if (waiting !== null) {
|
|
1757
|
+
dueTimes.push(waiting.dueAt.getTime());
|
|
1758
|
+
}
|
|
1759
|
+
return {
|
|
1760
|
+
// The counts are separate reads; a completion between them must not make this negative.
|
|
1761
|
+
pending: Math.max(0, stored - running.length),
|
|
1762
|
+
processing: running.length,
|
|
1763
|
+
completed: legacyCompleted + history,
|
|
1764
|
+
dead,
|
|
1765
|
+
oldestDueAt: dueTimes.length === 0 ? null : new Date(Math.min(...dueTimes))
|
|
1766
|
+
};
|
|
1767
|
+
}
|
|
1768
|
+
/** The listed deliveries that run: those whose document still exists as pending. */
|
|
1769
|
+
async findRunning(listed) {
|
|
1770
|
+
if (listed.length === 0) {
|
|
1771
|
+
return [];
|
|
1772
|
+
}
|
|
1773
|
+
return this.#state.deliveries.find(
|
|
1774
|
+
{ _id: { $in: listed }, status: "pending" },
|
|
1775
|
+
{ projection: { handler: 1, key: 1, laneId: 1 } }
|
|
1776
|
+
).toArray();
|
|
1777
|
+
}
|
|
1778
|
+
async readEnvelopes(eventIds) {
|
|
1779
|
+
const documents = await this.#state.events.find({ _id: { $in: [...new Set(eventIds)] } }).toArray();
|
|
1780
|
+
return new Map(documents.map((document) => [document._id, toEnvelope(document)]));
|
|
1781
|
+
}
|
|
1782
|
+
};
|
|
1783
|
+
function statusFilter(status, listed) {
|
|
1784
|
+
if (status === void 0) {
|
|
1785
|
+
return {};
|
|
1786
|
+
}
|
|
1787
|
+
const derivedStatuses = /* @__PURE__ */ new Map([
|
|
1788
|
+
["processing", { status: "pending", _id: { $in: listed } }],
|
|
1789
|
+
["pending", { status: "pending", _id: { $nin: listed } }]
|
|
1790
|
+
]);
|
|
1791
|
+
return derivedStatuses.get(status) ?? { status };
|
|
1792
|
+
}
|
|
1793
|
+
function toSortedRecords(input) {
|
|
1794
|
+
const { documents, history, events, lanes } = input;
|
|
1795
|
+
const laneById = new Map(lanes.map((lane) => [lane._id, lane]));
|
|
1796
|
+
const records = [
|
|
1797
|
+
...documents.map((document) => {
|
|
1798
|
+
const lane = laneById.get(document.laneId);
|
|
1799
|
+
const runs = document.status === "pending" && lane?.inFlight.includes(document._id) === true;
|
|
1800
|
+
return toDeliveryRecord(document, {
|
|
1801
|
+
event: events.get(document.eventId),
|
|
1802
|
+
runningIn: runs ? lane : void 0
|
|
1803
|
+
});
|
|
1804
|
+
}),
|
|
1805
|
+
...history.map(
|
|
1806
|
+
(document) => toCompletedDeliveryRecord(document, events.get(document.eventId))
|
|
1807
|
+
)
|
|
1808
|
+
];
|
|
1809
|
+
return records.sort(
|
|
1810
|
+
(left, right) => left.createdAt.getTime() - right.createdAt.getTime() || compareText(left.id, right.id)
|
|
1811
|
+
);
|
|
1812
|
+
}
|
|
1813
|
+
function compareText(left, right) {
|
|
1814
|
+
return Number(left > right) - Number(left < right);
|
|
1825
1815
|
}
|
|
1826
|
-
function
|
|
1816
|
+
function commonFilter(query) {
|
|
1827
1817
|
return {
|
|
1828
|
-
|
|
1829
|
-
|
|
1830
|
-
|
|
1831
|
-
|
|
1832
|
-
|
|
1833
|
-
|
|
1818
|
+
...query.handler === void 0 ? {} : { handler: String(query.handler) },
|
|
1819
|
+
...query.eventType === void 0 ? {} : { eventType: String(query.eventType) },
|
|
1820
|
+
...query.eventId === void 0 ? {} : { eventId: String(query.eventId) }
|
|
1821
|
+
};
|
|
1822
|
+
}
|
|
1823
|
+
|
|
1824
|
+
// src/mongo-schema.ts
|
|
1825
|
+
var NAMESPACE_EXISTS = 48;
|
|
1826
|
+
var INDEX_NOT_FOUND = 27;
|
|
1827
|
+
var UPGRADE_BATCH_SIZE = 1e3;
|
|
1828
|
+
var CLAIM_INDEX_KEY = { laneId: 1, dueAt: 1, _id: 1, handler: 1 };
|
|
1829
|
+
var OBSOLETE_INDEXES = ["running"];
|
|
1830
|
+
var LEASE_FIELDS = { leaseOwner: "", leaseToken: "", leaseExpiresAt: "" };
|
|
1831
|
+
var MongoSchema = class {
|
|
1832
|
+
#state;
|
|
1833
|
+
constructor(state) {
|
|
1834
|
+
this.#state = state;
|
|
1835
|
+
}
|
|
1836
|
+
/** Implements {@link MongoEventStore.setup}. */
|
|
1837
|
+
async setup() {
|
|
1838
|
+
const hello = await this.#state.client.db("admin").command({ hello: 1 });
|
|
1839
|
+
const supportsTransactions = typeof hello["setName"] === "string" || hello["msg"] === "isdbgrid";
|
|
1840
|
+
if (!supportsTransactions) {
|
|
1841
|
+
throw createTransactionsUnsupportedError(void 0);
|
|
1842
|
+
}
|
|
1843
|
+
const collectionNames = [
|
|
1844
|
+
this.#state.events.collectionName,
|
|
1845
|
+
this.#state.deliveries.collectionName,
|
|
1846
|
+
this.#state.keySequences.collectionName,
|
|
1847
|
+
this.#state.lanes.collectionName,
|
|
1848
|
+
...this.#state.completionHistory ? [this.#state.completions.collectionName] : []
|
|
1849
|
+
];
|
|
1850
|
+
for (const name of collectionNames) {
|
|
1851
|
+
await this.#createCollectionIfMissing(name);
|
|
1852
|
+
}
|
|
1853
|
+
if (await this.#state.lanes.countDocuments() < LANE_COUNT) {
|
|
1854
|
+
await this.#upgradeFromVersion010();
|
|
1855
|
+
await this.#seedLanes();
|
|
1856
|
+
}
|
|
1857
|
+
await this.#dropObsoleteIndexes();
|
|
1858
|
+
await this.#createIndexes();
|
|
1859
|
+
}
|
|
1860
|
+
/** Creates the indexes of every collection; existing indexes are left as they are. */
|
|
1861
|
+
async #createIndexes() {
|
|
1862
|
+
await this.#state.deliveries.createIndexes(deliveryIndexes());
|
|
1863
|
+
await this.#state.events.createIndexes(eventIndexes());
|
|
1864
|
+
if (this.#state.completionHistory) {
|
|
1865
|
+
await this.#state.completions.createIndexes(completionIndexes());
|
|
1866
|
+
}
|
|
1867
|
+
}
|
|
1868
|
+
async #createCollectionIfMissing(name) {
|
|
1869
|
+
try {
|
|
1870
|
+
await this.#state.database.createCollection(name);
|
|
1871
|
+
} catch (error) {
|
|
1872
|
+
if (!hasErrorCode(error, NAMESPACE_EXISTS)) {
|
|
1873
|
+
throw error;
|
|
1874
|
+
}
|
|
1875
|
+
}
|
|
1876
|
+
}
|
|
1877
|
+
/**
|
|
1878
|
+
* Converts the documents of a 0.1.0 database; every step is idempotent, so a
|
|
1879
|
+
* run that crashed is simply repeated. Deliveries that were `processing`
|
|
1880
|
+
* become `pending`, due at once (their attempt stays counted, as a lost lease
|
|
1881
|
+
* would count it). The per-delivery lease fields go, every delivery gets its
|
|
1882
|
+
* lane, and every event learns which handlers it already has deliveries for
|
|
1883
|
+
* and when it was created. Deliveries completed under 0.1.0 stay as
|
|
1884
|
+
* `completed` documents until a purge removes them.
|
|
1885
|
+
*/
|
|
1886
|
+
async #upgradeFromVersion010() {
|
|
1887
|
+
const legacy = this.#state.database.collection(this.#state.deliveries.collectionName);
|
|
1888
|
+
await legacy.updateMany({ status: "processing" }, [
|
|
1889
|
+
{ $set: { status: "pending", lane: "ready", dueAt: "$availableAt" } }
|
|
1890
|
+
]);
|
|
1891
|
+
await legacy.updateMany(
|
|
1892
|
+
{
|
|
1893
|
+
$or: [
|
|
1894
|
+
{ leaseOwner: { $exists: true } },
|
|
1895
|
+
{ leaseToken: { $exists: true } },
|
|
1896
|
+
{ leaseExpiresAt: { $exists: true } }
|
|
1897
|
+
]
|
|
1834
1898
|
},
|
|
1899
|
+
{ $unset: LEASE_FIELDS }
|
|
1900
|
+
);
|
|
1901
|
+
await inBatches(
|
|
1902
|
+
this.#state.deliveries.find(
|
|
1903
|
+
{ laneId: { $exists: false } },
|
|
1904
|
+
{ projection: { eventId: 1, key: 1 }, batchSize: UPGRADE_BATCH_SIZE }
|
|
1905
|
+
),
|
|
1906
|
+
async (documents) => {
|
|
1907
|
+
await this.#state.deliveries.bulkWrite(
|
|
1908
|
+
documents.map((document) => ({
|
|
1909
|
+
updateOne: {
|
|
1910
|
+
filter: { _id: document._id },
|
|
1911
|
+
update: { $set: { laneId: laneOf(document.key ?? null, document.eventId) } }
|
|
1912
|
+
}
|
|
1913
|
+
})),
|
|
1914
|
+
{ ordered: false }
|
|
1915
|
+
);
|
|
1916
|
+
}
|
|
1917
|
+
);
|
|
1918
|
+
await this.#upgradeEventsFromVersion010();
|
|
1919
|
+
}
|
|
1920
|
+
/** Tells every 0.1.0 event which handlers it already has deliveries for and when it was created. */
|
|
1921
|
+
async #upgradeEventsFromVersion010() {
|
|
1922
|
+
await inBatches(
|
|
1923
|
+
this.#state.events.find(
|
|
1924
|
+
{ handlers: { $exists: false } },
|
|
1925
|
+
{ projection: { occurredAt: 1 }, batchSize: UPGRADE_BATCH_SIZE }
|
|
1926
|
+
),
|
|
1927
|
+
async (events) => {
|
|
1928
|
+
const known = await this.#legacyEventGroups(events.map((event) => event._id));
|
|
1929
|
+
await this.#state.events.bulkWrite(
|
|
1930
|
+
events.map((event) => convertEvent(event, known.get(event._id))),
|
|
1931
|
+
{ ordered: false }
|
|
1932
|
+
);
|
|
1933
|
+
}
|
|
1934
|
+
);
|
|
1935
|
+
}
|
|
1936
|
+
/** The handlers and first creation time of the deliveries of each of `eventIds`, by event id. */
|
|
1937
|
+
async #legacyEventGroups(eventIds) {
|
|
1938
|
+
return new Map(
|
|
1939
|
+
(await this.#state.deliveries.aggregate([
|
|
1940
|
+
{ $match: { eventId: { $in: [...eventIds] } } },
|
|
1941
|
+
{
|
|
1942
|
+
$group: {
|
|
1943
|
+
_id: "$eventId",
|
|
1944
|
+
handlers: { $addToSet: "$handler" },
|
|
1945
|
+
createdAt: { $min: "$createdAt" }
|
|
1946
|
+
}
|
|
1947
|
+
}
|
|
1948
|
+
]).toArray()).map((group) => [group._id, group])
|
|
1949
|
+
);
|
|
1950
|
+
}
|
|
1951
|
+
/** Creates the lane documents that do not exist yet. */
|
|
1952
|
+
async #seedLanes() {
|
|
1953
|
+
await this.#state.lanes.bulkWrite(
|
|
1954
|
+
Array.from({ length: LANE_COUNT }, (_, laneId) => ({
|
|
1955
|
+
updateOne: {
|
|
1956
|
+
filter: { _id: laneId },
|
|
1957
|
+
update: {
|
|
1958
|
+
$setOnInsert: {
|
|
1959
|
+
epoch: 0,
|
|
1960
|
+
owner: null,
|
|
1961
|
+
holder: null,
|
|
1962
|
+
leaseExpiresAt: null,
|
|
1963
|
+
inFlight: [],
|
|
1964
|
+
lost: []
|
|
1965
|
+
}
|
|
1966
|
+
},
|
|
1967
|
+
upsert: true
|
|
1968
|
+
}
|
|
1969
|
+
})),
|
|
1970
|
+
{ ordered: false }
|
|
1971
|
+
);
|
|
1972
|
+
}
|
|
1973
|
+
/** Drops the 0.1.0 indexes that no query uses, and a `claim` index with the 0.1.0 key. */
|
|
1974
|
+
async #dropObsoleteIndexes() {
|
|
1975
|
+
for (const index of await this.#state.deliveries.indexes()) {
|
|
1976
|
+
if (isObsoleteIndex(index)) {
|
|
1977
|
+
await this.#dropIndexIfPresent(index.name);
|
|
1978
|
+
}
|
|
1979
|
+
}
|
|
1980
|
+
}
|
|
1981
|
+
async #dropIndexIfPresent(name) {
|
|
1982
|
+
try {
|
|
1983
|
+
await this.#state.deliveries.dropIndex(name);
|
|
1984
|
+
} catch (error) {
|
|
1985
|
+
if (!hasErrorCode(error, INDEX_NOT_FOUND)) {
|
|
1986
|
+
throw error;
|
|
1987
|
+
}
|
|
1988
|
+
}
|
|
1989
|
+
}
|
|
1990
|
+
};
|
|
1991
|
+
function deliveryIndexes() {
|
|
1992
|
+
return [
|
|
1993
|
+
// Idempotent appends and lookups by event.
|
|
1994
|
+
{ key: { eventId: 1, handler: 1 }, name: "event_handler", unique: true },
|
|
1995
|
+
// Claim: due work of a lane in due order. Holds ready deliveries only. The handler is
|
|
1996
|
+
// last, so a query over many lanes and handlers stays a merge of one scan per lane.
|
|
1997
|
+
{ key: CLAIM_INDEX_KEY, name: "claim", partialFilterExpression: { lane: "ready" } },
|
|
1998
|
+
// Ordering: unfinished predecessors of a keyed delivery, parked or not.
|
|
1999
|
+
{
|
|
2000
|
+
key: { handler: 1, key: 1, sequence: 1 },
|
|
2001
|
+
name: "ordering",
|
|
2002
|
+
partialFilterExpression: { dueAt: { $exists: true }, sequence: { $exists: true } }
|
|
2003
|
+
},
|
|
2004
|
+
// Parked deliveries: the lowest parked delivery of each key, to move it back once it is next.
|
|
2005
|
+
{
|
|
2006
|
+
key: { handler: 1, key: 1, sequence: 1 },
|
|
2007
|
+
name: "parked",
|
|
2008
|
+
partialFilterExpression: { lane: "parked" }
|
|
2009
|
+
},
|
|
2010
|
+
// Statistics: oldest due delivery across handlers.
|
|
2011
|
+
{ key: { dueAt: 1 }, name: "due", partialFilterExpression: { dueAt: { $exists: true } } },
|
|
2012
|
+
// Purge of completed deliveries left by 0.1.0.
|
|
2013
|
+
{
|
|
2014
|
+
key: { completedAt: 1 },
|
|
2015
|
+
name: "completed",
|
|
2016
|
+
partialFilterExpression: { completedAt: { $exists: true } }
|
|
2017
|
+
},
|
|
2018
|
+
// Listing and status counts.
|
|
2019
|
+
{ key: { status: 1, createdAt: 1 }, name: "status_created" },
|
|
2020
|
+
// Purge of dead deliveries.
|
|
2021
|
+
{ key: { status: 1, updatedAt: 1 }, name: "status_updated" },
|
|
2022
|
+
{ key: { handler: 1, createdAt: 1 }, name: "handler_created" },
|
|
2023
|
+
{ key: { eventType: 1, createdAt: 1 }, name: "type_created" }
|
|
2024
|
+
];
|
|
2025
|
+
}
|
|
2026
|
+
function eventIndexes() {
|
|
2027
|
+
return [
|
|
2028
|
+
{ key: { createdAt: 1, _id: 1 }, name: "created" },
|
|
2029
|
+
// Purge of events whose deliveries all completed: they are kept until `completedBefore` passes this.
|
|
2030
|
+
{
|
|
2031
|
+
key: { completedAt: 1, _id: 1 },
|
|
2032
|
+
name: "completed",
|
|
2033
|
+
partialFilterExpression: { completedAt: { $exists: true } }
|
|
2034
|
+
}
|
|
2035
|
+
];
|
|
2036
|
+
}
|
|
2037
|
+
function completionIndexes() {
|
|
2038
|
+
return [
|
|
2039
|
+
{ key: { completedAt: 1, _id: 1 }, name: "completed" },
|
|
2040
|
+
{ key: { eventId: 1 }, name: "event" },
|
|
2041
|
+
{ key: { createdAt: 1, _id: 1 }, name: "created" }
|
|
2042
|
+
];
|
|
2043
|
+
}
|
|
2044
|
+
function convertEvent(event, group) {
|
|
2045
|
+
return {
|
|
2046
|
+
updateOne: {
|
|
2047
|
+
// Only an event nobody converted yet: another process may have finished and subscribed a handler.
|
|
2048
|
+
filter: { _id: event._id, handlers: { $exists: false } },
|
|
1835
2049
|
update: {
|
|
1836
|
-
$set: {
|
|
1837
|
-
$inc: { revision: 1 }
|
|
2050
|
+
$set: group === void 0 ? { handlers: [], createdAt: event.occurredAt } : { handlers: group.handlers, createdAt: group.createdAt }
|
|
1838
2051
|
}
|
|
1839
2052
|
}
|
|
1840
2053
|
};
|
|
1841
2054
|
}
|
|
2055
|
+
function isObsoleteIndex(index) {
|
|
2056
|
+
return OBSOLETE_INDEXES.includes(index.name) || index.name === "claim" && Object.keys(index.key).join() !== Object.keys(CLAIM_INDEX_KEY).join();
|
|
2057
|
+
}
|
|
1842
2058
|
async function inBatches(cursor, work) {
|
|
1843
2059
|
let batch = [];
|
|
1844
2060
|
for await (const document of cursor) {
|
|
@@ -1852,43 +2068,251 @@ async function inBatches(cursor, work) {
|
|
|
1852
2068
|
await work(batch);
|
|
1853
2069
|
}
|
|
1854
2070
|
}
|
|
1855
|
-
function atCount(count) {
|
|
1856
|
-
return count === 0 ? { $not: { $gt: 0 } } : count;
|
|
1857
|
-
}
|
|
1858
|
-
function fencedLeases(issued, lanes, documents) {
|
|
1859
|
-
const claimsById = new Map(documents.map((document) => [document._id, document.claims ?? 0]));
|
|
1860
|
-
const fenced = /* @__PURE__ */ new Map();
|
|
1861
|
-
for (const lease of issued) {
|
|
1862
|
-
const lane = lanes.get(lease.laneId);
|
|
1863
|
-
if (claimsById.get(lease.deliveryId) === lease.claims && lane.epoch === lease.epoch && lane.inFlight.includes(lease.deliveryId)) {
|
|
1864
|
-
fenced.set(leaseIdentity(lease), lease);
|
|
1865
|
-
}
|
|
1866
|
-
}
|
|
1867
|
-
return [...fenced.values()];
|
|
1868
|
-
}
|
|
1869
|
-
function ofClaim(lease) {
|
|
1870
|
-
return { _id: lease.deliveryId, status: "pending", claims: atCount(lease.claims) };
|
|
1871
|
-
}
|
|
1872
|
-
function leaseKey(deliveryId, leaseToken) {
|
|
1873
|
-
return `${deliveryId}\0${leaseToken}`;
|
|
1874
|
-
}
|
|
1875
|
-
function leaseIdentity(lease) {
|
|
1876
|
-
return `${lease.deliveryId}\0${formatLeaseToken(lease.laneId, lease.epoch, lease.claims)}`;
|
|
1877
|
-
}
|
|
1878
2071
|
function hasErrorCode(error, code) {
|
|
1879
2072
|
return typeof error === "object" && error !== null && "code" in error && error.code === code;
|
|
1880
2073
|
}
|
|
1881
|
-
|
|
1882
|
-
|
|
2074
|
+
var COLLECTION_PREFIX_PATTERN = /^[A-Za-z_][A-Za-z0-9_]{0,39}$/;
|
|
2075
|
+
var DATABASE_NAME_PATTERN = /^[A-Za-z0-9_-]{1,63}$/;
|
|
2076
|
+
function assertCollectionPrefix(prefix) {
|
|
2077
|
+
if (!COLLECTION_PREFIX_PATTERN.test(prefix)) {
|
|
2078
|
+
throw new core.ConfigurationError(
|
|
2079
|
+
`collectionPrefix '${prefix}' must be 1-40 characters of letters, digits or '_', starting with a letter or '_'`
|
|
2080
|
+
);
|
|
2081
|
+
}
|
|
1883
2082
|
}
|
|
1884
|
-
function
|
|
1885
|
-
if (!
|
|
2083
|
+
function assertDatabaseName(name) {
|
|
2084
|
+
if (!DATABASE_NAME_PATTERN.test(name)) {
|
|
1886
2085
|
throw new core.ConfigurationError(
|
|
1887
|
-
`
|
|
2086
|
+
`database '${name}' must be 1-63 characters of letters, digits, '_' or '-'`
|
|
1888
2087
|
);
|
|
1889
2088
|
}
|
|
1890
2089
|
}
|
|
1891
2090
|
|
|
2091
|
+
// src/mongo-store-factory.ts
|
|
2092
|
+
var DEFAULT_TRANSACTION_OPTIONS = {
|
|
2093
|
+
readConcern: { level: "snapshot" },
|
|
2094
|
+
writeConcern: { w: "majority" },
|
|
2095
|
+
readPreference: "primary"
|
|
2096
|
+
};
|
|
2097
|
+
var DEFAULT_PURGE_BATCH_SIZE = 500;
|
|
2098
|
+
function createMongoStoreState(options) {
|
|
2099
|
+
const prefix = options.collectionPrefix ?? "relaybus_";
|
|
2100
|
+
assertCollectionPrefix(prefix);
|
|
2101
|
+
if (options.database !== void 0) {
|
|
2102
|
+
assertDatabaseName(options.database);
|
|
2103
|
+
}
|
|
2104
|
+
const purgeBatchSize = internal.PositiveInteger.of(
|
|
2105
|
+
options.purgeBatchSize ?? DEFAULT_PURGE_BATCH_SIZE,
|
|
2106
|
+
"purgeBatchSize"
|
|
2107
|
+
);
|
|
2108
|
+
const database = options.client.db(options.database);
|
|
2109
|
+
const collectionOptions = {
|
|
2110
|
+
readPreference: "primary",
|
|
2111
|
+
writeConcern: { w: "majority" }
|
|
2112
|
+
};
|
|
2113
|
+
const state = {
|
|
2114
|
+
client: options.client,
|
|
2115
|
+
database,
|
|
2116
|
+
events: database.collection(`${prefix}events`, collectionOptions),
|
|
2117
|
+
deliveries: database.collection(`${prefix}deliveries`, collectionOptions),
|
|
2118
|
+
keySequences: database.collection(`${prefix}key_sequences`, collectionOptions),
|
|
2119
|
+
lanes: database.collection(`${prefix}lanes`, collectionOptions),
|
|
2120
|
+
completions: database.collection(`${prefix}completed_deliveries`, collectionOptions),
|
|
2121
|
+
completionHistory: options.completionHistory ?? false,
|
|
2122
|
+
transactionOptions: options.transactionOptions ?? DEFAULT_TRANSACTION_OPTIONS,
|
|
2123
|
+
purgeBatchSize,
|
|
2124
|
+
logger: options.logger ?? console,
|
|
2125
|
+
instanceId: core.createUuidV7(),
|
|
2126
|
+
handedOut: new internal.HandedOutLedger(),
|
|
2127
|
+
unreconciledOwners: /* @__PURE__ */ new Set()
|
|
2128
|
+
};
|
|
2129
|
+
return state;
|
|
2130
|
+
}
|
|
2131
|
+
|
|
2132
|
+
// src/mongo-event-store.ts
|
|
2133
|
+
var MongoEventStore = class {
|
|
2134
|
+
#state;
|
|
2135
|
+
#schema;
|
|
2136
|
+
#appender;
|
|
2137
|
+
#claims;
|
|
2138
|
+
#laneLeases;
|
|
2139
|
+
#completion;
|
|
2140
|
+
#queries;
|
|
2141
|
+
#purger;
|
|
2142
|
+
constructor(options) {
|
|
2143
|
+
const state = createMongoStoreState(options);
|
|
2144
|
+
this.#state = state;
|
|
2145
|
+
const queries = new MongoQueries(state);
|
|
2146
|
+
const ordering = new MongoOrdering(state);
|
|
2147
|
+
const laneLeases = new MongoLaneLeases(state, { queries });
|
|
2148
|
+
this.#schema = new MongoSchema(state);
|
|
2149
|
+
this.#appender = new MongoAppend(state);
|
|
2150
|
+
this.#claims = new MongoClaims(state, { laneLeases, ordering, queries });
|
|
2151
|
+
this.#laneLeases = laneLeases;
|
|
2152
|
+
this.#completion = new MongoCompletion(state);
|
|
2153
|
+
this.#queries = queries;
|
|
2154
|
+
this.#purger = new MongoPurge(state);
|
|
2155
|
+
}
|
|
2156
|
+
/**
|
|
2157
|
+
* Verifies the deployment supports transactions, creates the collections and
|
|
2158
|
+
* indexes and seeds the lane documents. Upgrades a 0.1.0 database in place
|
|
2159
|
+
* first (see `MongoSchema.#upgradeFromVersion010`); the seeded lanes
|
|
2160
|
+
* mark the upgrade as done, so later runs skip it. Safe to run repeatedly and
|
|
2161
|
+
* from several processes. Stop 0.1.0 workers before the first run.
|
|
2162
|
+
*
|
|
2163
|
+
* @throws ConfigurationError when the server is a standalone mongod.
|
|
2164
|
+
*/
|
|
2165
|
+
setup() {
|
|
2166
|
+
return this.#schema.setup();
|
|
2167
|
+
}
|
|
2168
|
+
/**
|
|
2169
|
+
* Runs `work` in a transaction on a new session and commits when it resolves.
|
|
2170
|
+
*
|
|
2171
|
+
* `work` can run more than once. The driver restarts the whole callback after
|
|
2172
|
+
* a transient failure such as a write conflict, and re-sends only the commit
|
|
2173
|
+
* after an unknown commit result. A restart is safe for everything written
|
|
2174
|
+
* through the session, because the aborted attempt left nothing behind. It is
|
|
2175
|
+
* not safe for side effects outside the session (HTTP calls, other databases),
|
|
2176
|
+
* so keep those out of `work`; relaybus handlers are already at-least-once.
|
|
2177
|
+
*
|
|
2178
|
+
* With `options.timeoutMs` (from `bus.transaction`) the server bounds the commit with
|
|
2179
|
+
* `maxCommitTimeMS`; the bus aborts a transaction that ran past the budget before its commit.
|
|
2180
|
+
*
|
|
2181
|
+
* @throws ConfigurationError when the deployment does not support transactions.
|
|
2182
|
+
*/
|
|
2183
|
+
transaction(work, options) {
|
|
2184
|
+
return options === void 0 ? runInTransaction(this.#state, work) : runInBudgetedTransaction(this.#state, { work, timeoutMs: options.timeoutMs });
|
|
2185
|
+
}
|
|
2186
|
+
/**
|
|
2187
|
+
* Writes the entries in `session`. An existing event gets a delivery only for
|
|
2188
|
+
* handlers it does not list yet; the added handlers are appended to its list
|
|
2189
|
+
* together with a `revision` increment, a real write that makes a concurrent
|
|
2190
|
+
* purge of the event conflict with this transaction.
|
|
2191
|
+
*/
|
|
2192
|
+
// eslint-disable-next-line relaybus/max-two-params -- EventStore port signature
|
|
2193
|
+
append(session, entries, now) {
|
|
2194
|
+
return this.#appender.append(session, { entries, now });
|
|
2195
|
+
}
|
|
2196
|
+
/**
|
|
2197
|
+
* Claims up to `limit` due deliveries. It claims from the lanes this owner
|
|
2198
|
+
* (and this store instance) holds under a live lease first, then takes free
|
|
2199
|
+
* lanes with due work, a few at a time (4, 8, 16, ...) and the lane with the
|
|
2200
|
+
* oldest due delivery first, so a worker holds only as many lanes as it
|
|
2201
|
+
* needs. It then records the batch with one write per lane and hands back the
|
|
2202
|
+
* lanes that yielded nothing and run nothing. No delivery document is
|
|
2203
|
+
* written, except for the attempt charged to deliveries of a lane taken over
|
|
2204
|
+
* and for parking (see `MongoOrdering.#parkDeliveries`).
|
|
2205
|
+
*
|
|
2206
|
+
* Taking a lane is one conditional `findOneAndUpdate` that also moves the
|
|
2207
|
+
* previous owner's in-flight list to the lane's `lost` list; a lane another
|
|
2208
|
+
* claimer took first is skipped, never waited for. Lost deliveries are charged
|
|
2209
|
+
* one attempt before anything is read from the lane, and `lost` is cleared
|
|
2210
|
+
* only after that. Whoever holds the lane next charges a `lost` list it finds
|
|
2211
|
+
* first, so a claim that fails anywhere after the take loses no charge; the
|
|
2212
|
+
* charge of a lost claim is recorded on the delivery, so charging the list
|
|
2213
|
+
* again after a failure before the clear charges nothing twice.
|
|
2214
|
+
* Nothing a failed claim did leaves a delivery listed as running.
|
|
2215
|
+
*/
|
|
2216
|
+
claim(request) {
|
|
2217
|
+
return this.#claims.claim(request);
|
|
2218
|
+
}
|
|
2219
|
+
/**
|
|
2220
|
+
* Extends the lease of the lane a running delivery belongs to. The lane lease
|
|
2221
|
+
* never shrinks, so a repeated renewal with the same value still holds. An
|
|
2222
|
+
* expired lease nobody took over is still renewed, so `_now` is not needed.
|
|
2223
|
+
*/
|
|
2224
|
+
// eslint-disable-next-line relaybus/max-two-params -- EventStore port signature
|
|
2225
|
+
extendLease(lease, leaseExpiresAt, _now) {
|
|
2226
|
+
return Promise.resolve(this.#completion.extendLease({ lease, leaseExpiresAt }));
|
|
2227
|
+
}
|
|
2228
|
+
/**
|
|
2229
|
+
* Deletes the delivery (and records it in the completion history, when kept).
|
|
2230
|
+
* With `session` the deletion commits with the caller's transaction; without,
|
|
2231
|
+
* it runs in a transaction of its own. Returns false when the lease was lost.
|
|
2232
|
+
*/
|
|
2233
|
+
// eslint-disable-next-line relaybus/max-two-params -- EventStore port signature
|
|
2234
|
+
complete(lease, now, session) {
|
|
2235
|
+
return this.#completion.complete(lease, { now, session });
|
|
2236
|
+
}
|
|
2237
|
+
/**
|
|
2238
|
+
* Completes many leases with a fixed number of round trips, whatever lanes
|
|
2239
|
+
* they belong to: three commands without a completion history, one
|
|
2240
|
+
* transaction with it (see `MongoCompletion.#deleteFenced`).
|
|
2241
|
+
* Returns one flag per lease in input order; a lease listed twice completes
|
|
2242
|
+
* once, and a token this store cannot have issued is refused.
|
|
2243
|
+
*/
|
|
2244
|
+
completeMany(leases, now) {
|
|
2245
|
+
return this.#completion.completeMany(leases, now);
|
|
2246
|
+
}
|
|
2247
|
+
/** Returns a running delivery to pending, due at `availableAt`, and counts the attempt that ended. */
|
|
2248
|
+
// eslint-disable-next-line relaybus/max-two-params -- EventStore port signature
|
|
2249
|
+
reschedule(lease, availableAt, error, now) {
|
|
2250
|
+
return Promise.resolve(
|
|
2251
|
+
this.#completion.changeClaim({
|
|
2252
|
+
lease,
|
|
2253
|
+
change: { kind: "reschedule", availableAt, error, now }
|
|
2254
|
+
})
|
|
2255
|
+
);
|
|
2256
|
+
}
|
|
2257
|
+
/** Gives a running delivery back without counting the attempt: pending, due `now`. */
|
|
2258
|
+
release(lease, now) {
|
|
2259
|
+
return Promise.resolve(
|
|
2260
|
+
this.#completion.changeClaim({ lease, change: { kind: "release", now } })
|
|
2261
|
+
);
|
|
2262
|
+
}
|
|
2263
|
+
/** Moves a running delivery to dead and counts the attempt that ended. */
|
|
2264
|
+
// eslint-disable-next-line relaybus/max-two-params -- EventStore port signature
|
|
2265
|
+
bury(lease, error, now) {
|
|
2266
|
+
return Promise.resolve(
|
|
2267
|
+
this.#completion.changeClaim({ lease, change: { kind: "bury", error, now } })
|
|
2268
|
+
);
|
|
2269
|
+
}
|
|
2270
|
+
/**
|
|
2271
|
+
* Frees every lane `leaseOwner` holds that runs no delivery, so other workers
|
|
2272
|
+
* need not wait for the lease to expire. Lanes with running deliveries stay leased.
|
|
2273
|
+
*/
|
|
2274
|
+
relinquish(leaseOwner) {
|
|
2275
|
+
return this.#laneLeases.relinquish(leaseOwner);
|
|
2276
|
+
}
|
|
2277
|
+
/**
|
|
2278
|
+
* Lists deliveries in creation order. A delivery is `processing` while its
|
|
2279
|
+
* document exists as `pending` and its lane lists it; its attempt count then
|
|
2280
|
+
* includes the running attempt, and owner and expiry are those of the lane
|
|
2281
|
+
* lease. Completion history records (when kept) are listed as `completed`.
|
|
2282
|
+
*/
|
|
2283
|
+
listDeliveries(query) {
|
|
2284
|
+
return this.#queries.listDeliveries(query);
|
|
2285
|
+
}
|
|
2286
|
+
/** Moves dead deliveries back to pending with a fresh attempt budget. */
|
|
2287
|
+
requeue(deliveryIds, now) {
|
|
2288
|
+
return this.#queries.requeue(deliveryIds, now);
|
|
2289
|
+
}
|
|
2290
|
+
/**
|
|
2291
|
+
* Deletes, in batches of `purgeBatchSize`, dead deliveries last updated before
|
|
2292
|
+
* `deadBefore`, and completion history records (and `completed` documents left
|
|
2293
|
+
* by 0.1.0) completed before `completedBefore`, each batch in one transaction
|
|
2294
|
+
* together with the events it left without any reference. Finally, events
|
|
2295
|
+
* nothing references are deleted: one with a completed delivery once its last
|
|
2296
|
+
* completion is before `completedBefore` (never by `deadBefore` alone), one
|
|
2297
|
+
* that never had a completion (no subscribers, or only dead letters) once it
|
|
2298
|
+
* was created before the earliest given cutoff. The batch step applies the
|
|
2299
|
+
* same rule to events whose last completion is recorded.
|
|
2300
|
+
*/
|
|
2301
|
+
purge(request) {
|
|
2302
|
+
return this.#purger.purge(request);
|
|
2303
|
+
}
|
|
2304
|
+
/**
|
|
2305
|
+
* Counts by derived status. `pending` excludes running deliveries; `completed`
|
|
2306
|
+
* counts history records and 0.1.0 `completed` documents. `oldestDueAt` is the
|
|
2307
|
+
* oldest instant at which a delivery became claimable: the due time of a
|
|
2308
|
+
* pending delivery nobody runs, or the lease expiry of a lane that still runs
|
|
2309
|
+
* deliveries.
|
|
2310
|
+
*/
|
|
2311
|
+
statistics(now) {
|
|
2312
|
+
return this.#queries.statistics(now);
|
|
2313
|
+
}
|
|
2314
|
+
};
|
|
2315
|
+
|
|
1892
2316
|
exports.MongoEventStore = MongoEventStore;
|
|
1893
2317
|
//# sourceMappingURL=index.cjs.map
|
|
1894
2318
|
//# sourceMappingURL=index.cjs.map
|