@procedures/mongo 0.48.2 → 0.49.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +35 -0
- package/dist/.tsbuildinfo +1 -1
- package/dist/storage/database-changes.d.ts +15 -2
- package/dist/storage/database-changes.d.ts.map +1 -1
- package/dist/storage/database-changes.js +30 -5
- package/dist/storage/database-changes.js.map +1 -1
- package/dist/storage/document-storage.d.ts +2 -1
- package/dist/storage/document-storage.d.ts.map +1 -1
- package/dist/storage/document-storage.js +66 -18
- package/dist/storage/document-storage.js.map +1 -1
- package/dist/storage/materializer.d.ts.map +1 -1
- package/dist/storage/materializer.js +6 -5
- package/dist/storage/materializer.js.map +1 -1
- package/dist/storage/pool-share.d.ts +24 -0
- package/dist/storage/pool-share.d.ts.map +1 -0
- package/dist/storage/pool-share.js +27 -0
- package/dist/storage/pool-share.js.map +1 -0
- package/dist/storage/split-events.d.ts +57 -0
- package/dist/storage/split-events.d.ts.map +1 -0
- package/dist/storage/split-events.js +94 -0
- package/dist/storage/split-events.js.map +1 -0
- package/dist/storage/storage-hints.d.ts +7 -0
- package/dist/storage/storage-hints.d.ts.map +1 -1
- package/dist/storage/storage-hints.js +41 -7
- package/dist/storage/storage-hints.js.map +1 -1
- package/package.json +6 -3
- package/src/storage/database-changes.ts +44 -7
- package/src/storage/document-storage.ts +102 -35
- package/src/storage/materializer.ts +6 -5
- package/src/storage/pool-share.ts +30 -0
- package/src/storage/split-events.ts +135 -0
- package/src/storage/storage-hints.ts +44 -7
|
@@ -29,6 +29,12 @@
|
|
|
29
29
|
* of the backoff. An `onChange` that runs finishes, and no change comes after it.
|
|
30
30
|
* - Change streams need a replica set or a sharded cluster. The watcher asks the server (`hello`)
|
|
31
31
|
* before the first open: on a single server, it stops with the code 40573.
|
|
32
|
+
* - An event over the BSON limit of 16 MiB comes in fragments. The pipeline ends with the stage
|
|
33
|
+
* `$changeStreamSplitLargeEvent`, and the watcher joins the fragments into one event. A server
|
|
34
|
+
* before 6.0.9 does not know the stage. The watcher then opens the stream without it.
|
|
35
|
+
* - An error with one code at one token, 5 times in a row, does not go away: the watcher stops. An
|
|
36
|
+
* example is an event over the BSON limit (code 10334) with no split stage. A network error and an
|
|
37
|
+
* error that the driver resumes do not count.
|
|
32
38
|
*/
|
|
33
39
|
import type { ChangeStreamDocument, ChangeStreamOptions, Db, Document, ResumeToken, Timestamp } from "mongodb";
|
|
34
40
|
import { type Clock, type Logger, type ResubscribeBackoff } from "@procedures/client";
|
|
@@ -41,8 +47,9 @@ export interface WatchDatabaseChangesOptions<TChange extends Document = ChangeSt
|
|
|
41
47
|
onChange: (change: TChange) => void | Promise<void>;
|
|
42
48
|
/**
|
|
43
49
|
* The function that gets the error that stops the watcher. The error is an `invalidate` event, a
|
|
44
|
-
* lost resume token, or an error of the first open that does not go away. An
|
|
45
|
-
* gives an `Error` with the event in its field
|
|
50
|
+
* lost resume token, or an error of the first open that does not go away. An error with one code at
|
|
51
|
+
* one token 5 times in a row also stops the watcher. An `invalidate` event gives an `Error` with the event in its field
|
|
52
|
+
* `change`. The error of a fault at one token has the fields `code`, `resumeToken` and `cause`.
|
|
46
53
|
*/
|
|
47
54
|
onError?: ((error: unknown) => void) | undefined;
|
|
48
55
|
/** The resume token after which the first stream starts. Give only one of `resumeAfter`, `startAfter` and `startAtOperationTime`. */
|
|
@@ -57,6 +64,12 @@ export interface WatchDatabaseChangesOptions<TChange extends Document = ChangeSt
|
|
|
57
64
|
fullDocumentBeforeChange?: "whenAvailable" | "required" | "off" | undefined;
|
|
58
65
|
/** The pipeline of the stream, for example a `$match` of the collections. @default [] */
|
|
59
66
|
pipeline?: Document[] | undefined;
|
|
67
|
+
/**
|
|
68
|
+
* True to add the stage `$changeStreamSplitLargeEvent` after the pipeline. The watcher then joins
|
|
69
|
+
* the fragments of an event over 16 MiB. With false, such an event stops the watcher.
|
|
70
|
+
* @default true
|
|
71
|
+
*/
|
|
72
|
+
splitLargeEvents?: boolean | undefined;
|
|
60
73
|
/** More options of the stream, for example `maxAwaitTimeMS`, `batchSize` or `showExpandedEvents`. */
|
|
61
74
|
streamOptions?: Omit<ChangeStreamOptions, "resumeAfter" | "startAfter" | "startAtOperationTime" | "fullDocument" | "fullDocumentBeforeChange"> | undefined;
|
|
62
75
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"database-changes.d.ts","sourceRoot":"","sources":["../../src/storage/database-changes.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"database-changes.d.ts","sourceRoot":"","sources":["../../src/storage/database-changes.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqCG;AACH,OAAO,KAAK,EAAgB,oBAAoB,EAAE,mBAAmB,EAAE,EAAE,EAAE,QAAQ,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAC7H,OAAO,EAAoC,KAAK,KAAK,EAAkB,KAAK,MAAM,EAAE,KAAK,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAIxI,+CAA+C;AAC/C,MAAM,WAAW,2BAA2B,CAAC,OAAO,SAAS,QAAQ,GAAG,oBAAoB;IAC1F;;;OAGG;IACH,QAAQ,EAAE,CAAC,MAAM,EAAE,OAAO,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACpD;;;;;OAKG;IACH,OAAO,CAAC,EAAE,CAAC,CAAC,KAAK,EAAE,OAAO,KAAK,IAAI,CAAC,GAAG,SAAS,CAAC;IACjD,qIAAqI;IACrI,WAAW,CAAC,EAAE,WAAW,GAAG,SAAS,CAAC;IACtC,qGAAqG;IACrG,UAAU,CAAC,EAAE,WAAW,GAAG,SAAS,CAAC;IACrC,gEAAgE;IAChE,oBAAoB,CAAC,EAAE,SAAS,GAAG,SAAS,CAAC;IAC7C,8FAA8F;IAC9F,YAAY,CAAC,EAAE,cAAc,GAAG,eAAe,GAAG,UAAU,GAAG,SAAS,CAAC;IACzE,iGAAiG;IACjG,wBAAwB,CAAC,EAAE,eAAe,GAAG,UAAU,GAAG,KAAK,GAAG,SAAS,CAAC;IAC5E,yFAAyF;IACzF,QAAQ,CAAC,EAAE,QAAQ,EAAE,GAAG,SAAS,CAAC;IAClC;;;;OAIG;IACH,gBAAgB,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IACvC,qGAAqG;IACrG,aAAa,CAAC,EAAE,IAAI,CAAC,mBAAmB,EAAE,aAAa,GAAG,YAAY,GAAG,sBAAsB,GAAG,cAAc,GAAG,0BAA0B,CAAC,GAAG,SAAS,CAAC;IAC3J;;;OAGG;IACH,OAAO,CAAC,EAAE,kBAAkB,GAAG,SAAS,CAAC;IACzC;;;;OAIG;IACH,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,yCAAyC;IACzC,MAAM,CAAC,EAAE,WAAW,GAAG,SAAS,CAAC;IACjC,2GAA2G;IAC3G,KAAK,CAAC,EAAE,KAAK,GAAG,SAAS,CAAC;CAC3B;AAED,8CAA8C;AAC9C,MAAM,WAAW,oBAAoB;IACnC;;;OAGG;IACH,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IAC9B,kKAAkK;IAClK,QAAQ,CAAC,WAAW,EAAE,WAAW,GAAG,SAAS,CAAC;IAC9C,qFAAqF;IACrF,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,8FAA8F;IAC9F,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACvB;AAuBD;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,wBAAgB,oBAAoB,CAAC,OAAO,SAAS,QAAQ,GAAG,oBAAoB,EAClF,EAAE,EAAE,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,OAAO,CAAC,EAAE,CAAC,CAAC,EACjC,OAAO,EAAE,2BAA2B,CAAC,OAAO,CAAC,GAC5C,oBAAoB,CA8OtB"}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { realClock, writeLog } from "@procedures/client";
|
|
2
2
|
import { COMMAND_NOT_FOUND, LOST_TOKEN_CODES, backoffOf, codeOf, isPermanent, messageOf } from "./storage-hints.js";
|
|
3
|
+
import { SAME_TOKEN_LIMIT, SPLIT_LARGE_EVENT_STAGE, UNKNOWN_STAGE_CODE, eventJoiner, sameTokenFaults } from "./split-events.js";
|
|
3
4
|
/** An error of `onChange`: the watcher waits, and gives the same change again. */
|
|
4
5
|
class HandlerError extends Error {
|
|
5
6
|
error;
|
|
@@ -63,6 +64,9 @@ export function watchDatabaseChanges(db, options) {
|
|
|
63
64
|
let attempt = 0;
|
|
64
65
|
let database;
|
|
65
66
|
let stream;
|
|
67
|
+
/** True while the pipeline ends with the split stage. A server that does not know the stage sets it to false. */
|
|
68
|
+
let split = options.splitLargeEvents !== false;
|
|
69
|
+
const faults = sameTokenFaults();
|
|
66
70
|
/** The timer of a backoff, and the end of its wait. */
|
|
67
71
|
let timer;
|
|
68
72
|
let wake;
|
|
@@ -130,19 +134,27 @@ export function watchDatabaseChanges(db, options) {
|
|
|
130
134
|
};
|
|
131
135
|
/** One stream: it reads the events until an error, an `invalidate` event or the stop. */
|
|
132
136
|
const read = async (next) => {
|
|
137
|
+
const joiner = eventJoiner();
|
|
133
138
|
for (;;) {
|
|
134
|
-
const
|
|
139
|
+
const raw = await next.tryNext();
|
|
135
140
|
if (stopped)
|
|
136
141
|
return;
|
|
137
142
|
markOpen();
|
|
138
|
-
if (
|
|
139
|
-
// A batch without events: its token is after each event that `onChange` took
|
|
143
|
+
if (raw === null) {
|
|
144
|
+
// A batch without events: its token is after each event that `onChange` took. In the middle of a split event, the token stays before it
|
|
140
145
|
const batchToken = next.resumeToken;
|
|
141
|
-
if (batchToken !== undefined && batchToken !== null)
|
|
146
|
+
if (!joiner.partial && batchToken !== undefined && batchToken !== null)
|
|
142
147
|
token = batchToken;
|
|
143
148
|
succeeded();
|
|
144
149
|
continue;
|
|
145
150
|
}
|
|
151
|
+
const result = joiner.push(raw);
|
|
152
|
+
if (result.kind === "part")
|
|
153
|
+
continue;
|
|
154
|
+
if (result.kind === "missing") {
|
|
155
|
+
log("warn", "A split event comes without its first fragments: the stream started after one of them", { missing: result.missing, collection: collectionOf(result.change) });
|
|
156
|
+
}
|
|
157
|
+
const change = result.change;
|
|
146
158
|
if (change["operationType"] === "invalidate") {
|
|
147
159
|
token = change["_id"];
|
|
148
160
|
return fail(Object.assign(new Error(`the change stream of the database ${database ?? ""} was invalidated`), { change }), { operationType: "invalidate" });
|
|
@@ -154,6 +166,7 @@ export function watchDatabaseChanges(db, options) {
|
|
|
154
166
|
throw new HandlerError(error, change);
|
|
155
167
|
}
|
|
156
168
|
token = change["_id"];
|
|
169
|
+
faults.reset();
|
|
157
170
|
succeeded();
|
|
158
171
|
}
|
|
159
172
|
};
|
|
@@ -194,7 +207,8 @@ export function watchDatabaseChanges(db, options) {
|
|
|
194
207
|
if (token === undefined && opened)
|
|
195
208
|
log("warn", "The change stream opens again without a resume token: it can miss changes", {});
|
|
196
209
|
try {
|
|
197
|
-
const
|
|
210
|
+
const pipeline = [...(options.pipeline ?? []), ...(split ? [SPLIT_LARGE_EVENT_STAGE] : [])];
|
|
211
|
+
const next = handle.watch(pipeline, {
|
|
198
212
|
...options.streamOptions,
|
|
199
213
|
...(options.fullDocument !== undefined && { fullDocument: options.fullDocument }),
|
|
200
214
|
...(options.fullDocumentBeforeChange !== undefined && { fullDocumentBeforeChange: options.fullDocumentBeforeChange }),
|
|
@@ -226,8 +240,19 @@ export function watchDatabaseChanges(db, options) {
|
|
|
226
240
|
const code = codeOf(error);
|
|
227
241
|
if (code !== undefined && LOST_TOKEN_CODES.has(code))
|
|
228
242
|
return fail(error, { code });
|
|
243
|
+
if (code === UNKNOWN_STAGE_CODE && split) {
|
|
244
|
+
// A server before 6.0.9: the stream opens again without the split stage
|
|
245
|
+
split = false;
|
|
246
|
+
log("warn", "The server does not know the stage $changeStreamSplitLargeEvent: an event over 16 MiB stops the watcher", { error, code });
|
|
247
|
+
continue;
|
|
248
|
+
}
|
|
229
249
|
if (!opened && isPermanent(error))
|
|
230
250
|
return fail(error, { code });
|
|
251
|
+
const repeats = faults.count(error, token);
|
|
252
|
+
if (repeats >= SAME_TOKEN_LIMIT) {
|
|
253
|
+
const stuck = Object.assign(new Error(`the change stream failed ${repeats} times at one token with the code ${String(code)}: ${messageOf(error)}`), { code, resumeToken: token, cause: error });
|
|
254
|
+
return fail(stuck, { code, repeats });
|
|
255
|
+
}
|
|
231
256
|
await backoff("The change stream failed: it opens again after a backoff", { error, code });
|
|
232
257
|
}
|
|
233
258
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"database-changes.js","sourceRoot":"","sources":["../../src/storage/database-changes.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"database-changes.js","sourceRoot":"","sources":["../../src/storage/database-changes.ts"],"names":[],"mappings":"AAuCA,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAiF,MAAM,oBAAoB,CAAC;AACxI,OAAO,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,SAAS,EAAE,MAAM,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AACpH,OAAO,EAAE,gBAAgB,EAAE,uBAAuB,EAAE,kBAAkB,EAAE,WAAW,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAoEhI,kFAAkF;AAClF,MAAM,YAAa,SAAQ,KAAK;IACrB,KAAK,CAAU;IACf,MAAM,CAAW;IAE1B,YACE,KAAc,EACd,MAAgB;QAEhB,KAAK,CAAC,oBAAoB,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAC9C,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;CACF;AAED,uDAAuD;AACvD,SAAS,YAAY,CAAC,MAAgB;IACpC,MAAM,EAAE,GAAY,MAAM,CAAC,IAAI,CAAC,CAAC;IACjC,OAAO,OAAO,EAAE,KAAK,QAAQ,IAAI,EAAE,KAAK,IAAI,CAAC,CAAC,CAAE,EAAe,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AACtF,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,MAAM,UAAU,oBAAoB,CAClC,EAAiC,EACjC,OAA6C;IAE7C,IAAI,OAAO,OAAO,EAAE,QAAQ,KAAK,UAAU;QAAE,MAAM,IAAI,SAAS,CAAC,6BAA6B,CAAC,CAAC;IAChG,MAAM,MAAM,GAAG,CAAC,OAAO,CAAC,WAAW,EAAE,OAAO,CAAC,UAAU,EAAE,OAAO,CAAC,oBAAoB,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC;IAC9H,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,mEAAmE,CAAC,CAAC;IAChH,MAAM,OAAO,GAAG,SAAS,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAC3C,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,SAAS,CAAC;IACzC,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAC9B,MAAM,IAAI,GAAG,OAAO,EAAE,KAAK,UAAU,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAO,EAAE,CAAC,EAAE,CAAC;IAE1D,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB,8FAA8F;IAC9F,IAAI,KAAK,GAA4B,OAAO,CAAC,UAAU,IAAI,OAAO,CAAC,WAAW,CAAC;IAC/E,oCAAoC;IACpC,IAAI,MAAM,GAAG,KAAK,CAAC;IACnB,6DAA6D;IAC7D,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB,6FAA6F;IAC7F,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,QAA4B,CAAC;IACjC,IAAI,MAAmD,CAAC;IACxD,iHAAiH;IACjH,IAAI,KAAK,GAAG,OAAO,CAAC,gBAAgB,KAAK,KAAK,CAAC;IAC/C,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;IACjC,uDAAuD;IACvD,IAAI,KAAyB,CAAC;IAC9B,IAAI,IAA8B,CAAC;IACnC,IAAI,SAAS,GAAe,GAAG,EAAE,CAAC,SAAS,CAAC;IAC5C,IAAI,SAAS,GAA6B,GAAG,EAAE,CAAC,SAAS,CAAC;IAC1D,MAAM,KAAK,GAAG,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QAClD,SAAS,GAAG,OAAO,CAAC;QACpB,SAAS,GAAG,MAAM,CAAC;IACrB,CAAC,CAAC,CAAC;IACH,kGAAkG;IAClG,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;IAE7B,MAAM,GAAG,GAAG,CAAC,KAAgC,EAAE,OAAe,EAAE,MAAiB,EAAQ,EAAE;QACzF,MAAM,GAAG,GAAG,EAAE,SAAS,EAAE,kBAAkB,EAAE,QAAQ,EAAE,WAAW,EAAE,KAAK,KAAK,SAAS,EAAE,GAAG,MAAM,EAAE,CAAC;QACrG,QAAQ,CAAC,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,EAAE,sBAAsB,OAAO,EAAE,EAAE,GAAG,CAAC,CAAC;IAC9E,CAAC,CAAC;IAEF,mEAAmE;IACnE,MAAM,IAAI,GAAG,CAAC,KAAc,EAAE,MAAiB,EAAQ,EAAE;QACvD,IAAI,OAAO;YAAE,OAAO;QACpB,GAAG,CAAC,OAAO,EAAE,uDAAuD,EAAE,EAAE,KAAK,EAAE,GAAG,MAAM,EAAE,CAAC,CAAC;QAC5F,IAAI,EAAE,CAAC;QACP,SAAS,CAAC,KAAK,CAAC,CAAC;QACjB,IAAI,CAAC;YACH,OAAO,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC;QAC3B,CAAC;QAAC,OAAO,MAAM,EAAE,CAAC;YAChB,GAAG,CAAC,OAAO,EAAE,gBAAgB,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;QACpD,CAAC;IACH,CAAC,CAAC;IAEF,iDAAiD;IACjD,MAAM,OAAO,GAAG,CAAC,OAAe,EAAE,MAAiB,EAAiB,EAAE;QACpE,MAAM,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;QAC/B,OAAO,EAAE,CAAC;QACV,GAAG,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE,GAAG,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACzE,OAAO,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;YACnC,IAAI,GAAG,OAAO,CAAC;YACf,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,EAAE;gBAC9B,KAAK,GAAG,SAAS,CAAC;gBAClB,IAAI,GAAG,SAAS,CAAC;gBACjB,OAAO,EAAE,CAAC;YACZ,CAAC,CAAC,CAAC;YACH,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC;QAClB,CAAC,CAAC,CAAC;IACL,CAAC,CAAC;IAEF,oFAAoF;IACpF,MAAM,KAAK,GAAG,GAAqF,EAAE;QACnG,IAAI,CAAC,MAAM,IAAI,OAAO,CAAC,WAAW,KAAK,SAAS;YAAE,OAAO,EAAE,WAAW,EAAE,OAAO,CAAC,WAAW,EAAE,CAAC;QAC9F,IAAI,KAAK,KAAK,SAAS;YAAE,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC;QACtD,OAAO,OAAO,CAAC,oBAAoB,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,oBAAoB,EAAE,OAAO,CAAC,oBAAoB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IAClH,CAAC,CAAC;IAEF,8DAA8D;IAC9D,MAAM,QAAQ,GAAG,GAAS,EAAE;QAC1B,IAAI,MAAM,IAAI,OAAO;YAAE,OAAO;QAC9B,MAAM,GAAG,IAAI,CAAC;QACd,SAAS,EAAE,CAAC;IACd,CAAC,CAAC;IAEF,qGAAqG;IACrG,MAAM,SAAS,GAAG,GAAS,EAAE;QAC3B,IAAI,OAAO,GAAG,CAAC;YAAE,GAAG,CAAC,MAAM,EAAE,gCAAgC,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC;QACtF,OAAO,GAAG,CAAC,CAAC;IACd,CAAC,CAAC;IAEF,yFAAyF;IACzF,MAAM,IAAI,GAAG,KAAK,EAAE,IAAqC,EAAiB,EAAE;QAC1E,MAAM,MAAM,GAAG,WAAW,EAAE,CAAC;QAC7B,SAAS,CAAC;YACR,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;YACjC,IAAI,OAAO;gBAAE,OAAO;YACpB,QAAQ,EAAE,CAAC;YACX,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;gBACjB,wIAAwI;gBACxI,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,CAAC;gBACpC,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,UAAU,KAAK,SAAS,IAAI,UAAU,KAAK,IAAI;oBAAE,KAAK,GAAG,UAAU,CAAC;gBAC3F,SAAS,EAAE,CAAC;gBACZ,SAAS;YACX,CAAC;YACD,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAChC,IAAI,MAAM,CAAC,IAAI,KAAK,MAAM;gBAAE,SAAS;YACrC,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;gBAC9B,GAAG,CAAC,MAAM,EAAE,uFAAuF,EAAE,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,UAAU,EAAE,YAAY,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YAC7K,CAAC;YACD,MAAM,MAAM,GAAG,MAAM,CAAC,MAAiB,CAAC;YACxC,IAAI,MAAM,CAAC,eAAe,CAAC,KAAK,YAAY,EAAE,CAAC;gBAC7C,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;gBACtB,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,qCAAqC,QAAQ,IAAI,EAAE,kBAAkB,CAAC,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE,EAAE,aAAa,EAAE,YAAY,EAAE,CAAC,CAAC;YAC5J,CAAC;YACD,IAAI,CAAC;gBACH,MAAM,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YACjC,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,IAAI,YAAY,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;YACxC,CAAC;YACD,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;YACtB,MAAM,CAAC,KAAK,EAAE,CAAC;YACf,SAAS,EAAE,CAAC;QACd,CAAC;IACH,CAAC,CAAC;IAEF,MAAM,GAAG,GAAG,KAAK,IAAmB,EAAE;QACpC,OAAO,CAAC,OAAO,EAAE,CAAC;YAChB,IAAI,MAAU,CAAC;YACf,IAAI,CAAC;gBACH,MAAM,GAAG,MAAM,IAAI,EAAE,CAAC;gBACtB,QAAQ,GAAG,MAAM,CAAC,YAAY,CAAC;gBAC/B,IAAI,CAAC,OAAO,EAAE,CAAC;oBACb,8FAA8F;oBAC9F,IAAI,KAA2B,CAAC;oBAChC,IAAI,CAAC;wBACH,KAAK,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;oBAC7C,CAAC;oBAAC,OAAO,KAAK,EAAE,CAAC;wBACf,0DAA0D;wBAC1D,IAAI,MAAM,CAAC,KAAK,CAAC,KAAK,iBAAiB;4BAAE,MAAM,KAAK,CAAC;oBACvD,CAAC;oBACD,IAAI,KAAK,IAAI,OAAO,KAAK,CAAC,SAAS,CAAC,KAAK,QAAQ,IAAI,KAAK,CAAC,KAAK,CAAC,KAAK,UAAU,EAAE,CAAC;wBACjF,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,2FAA2F,CAAC,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;wBACtJ,OAAO,IAAI,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;oBACvC,CAAC;oBACD,OAAO,GAAG,IAAI,CAAC;gBACjB,CAAC;YACH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,OAAO;oBAAE,OAAO;gBACpB,IAAI,CAAC,MAAM,IAAI,WAAW,CAAC,KAAK,CAAC;oBAAE,OAAO,IAAI,CAAC,KAAK,EAAE,EAAE,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;gBAC/E,MAAM,OAAO,CAAC,8EAA8E,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;gBAC9H,SAAS;YACX,CAAC;YACD,IAAI,OAAO;gBAAE,OAAO;YACpB,IAAI,KAAK,KAAK,SAAS,IAAI,MAAM;gBAAE,GAAG,CAAC,MAAM,EAAE,2EAA2E,EAAE,EAAE,CAAC,CAAC;YAChI,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,uBAAuB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;gBAC5F,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAoB,QAAQ,EAAE;oBACrD,GAAG,OAAO,CAAC,aAAa;oBACxB,GAAG,CAAC,OAAO,CAAC,YAAY,KAAK,SAAS,IAAI,EAAE,YAAY,EAAE,OAAO,CAAC,YAAY,EAAE,CAAC;oBACjF,GAAG,CAAC,OAAO,CAAC,wBAAwB,KAAK,SAAS,IAAI,EAAE,wBAAwB,EAAE,OAAO,CAAC,wBAAwB,EAAE,CAAC;oBACrH,GAAG,KAAK,EAAE;iBACX,CAAC,CAAC;gBACH,MAAM,GAAG,IAAI,CAAC;gBACd,kHAAkH;gBAClH,IAAI,CAAC,EAAE,CAAC,oBAAoB,EAAE,QAAQ,CAAC,CAAC;gBACxC,IAAI,CAAC;oBACH,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC;gBACnB,CAAC;wBAAS,CAAC;oBACT,IAAI,MAAM,KAAK,IAAI;wBAAE,MAAM,GAAG,SAAS,CAAC;oBACxC,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;gBAC5C,CAAC;YACH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,OAAO;oBAAE,OAAO;gBACpB,IAAI,KAAK,YAAY,YAAY,EAAE,CAAC;oBAClC,MAAM,OAAO,CAAC,8DAA8D,EAAE;wBAC5E,KAAK,EAAE,KAAK,CAAC,KAAK;wBAClB,UAAU,EAAE,YAAY,CAAC,KAAK,CAAC,MAAM,CAAC;wBACtC,aAAa,EAAE,KAAK,CAAC,MAAM,CAAC,eAAe,CAAC;qBAC7C,CAAC,CAAC;oBACH,SAAS;gBACX,CAAC;gBACD,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;gBAC3B,IAAI,IAAI,KAAK,SAAS,IAAI,gBAAgB,CAAC,GAAG,CAAC,IAAI,CAAC;oBAAE,OAAO,IAAI,CAAC,KAAK,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;gBACnF,IAAI,IAAI,KAAK,kBAAkB,IAAI,KAAK,EAAE,CAAC;oBACzC,wEAAwE;oBACxE,KAAK,GAAG,KAAK,CAAC;oBACd,GAAG,CAAC,MAAM,EAAE,yGAAyG,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;oBACxI,SAAS;gBACX,CAAC;gBACD,IAAI,CAAC,MAAM,IAAI,WAAW,CAAC,KAAK,CAAC;oBAAE,OAAO,IAAI,CAAC,KAAK,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;gBAChE,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;gBAC3C,IAAI,OAAO,IAAI,gBAAgB,EAAE,CAAC;oBAChC,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,4BAA4B,OAAO,qCAAqC,MAAM,CAAC,IAAI,CAAC,KAAK,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;oBAChM,OAAO,IAAI,CAAC,KAAK,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;gBACxC,CAAC;gBACD,MAAM,OAAO,CAAC,0DAA0D,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YAC7F,CAAC;QACH,CAAC;IACH,CAAC,CAAC;IAEF,uDAAuD;IACvD,MAAM,IAAI,GAAG,GAAS,EAAE;QACtB,OAAO,GAAG,IAAI,CAAC;QACf,KAAK,EAAE,EAAE,CAAC;QACV,KAAK,GAAG,SAAS,CAAC;QAClB,IAAI,EAAE,EAAE,CAAC;QACT,IAAI,GAAG,SAAS,CAAC;QACjB,OAAO,CAAC,MAAM,EAAE,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IACxD,CAAC,CAAC;IAEF,MAAM,IAAI,GAAG,KAAK,IAAmB,EAAE;QACrC,IAAI,OAAO;YAAE,OAAO;QACpB,IAAI,EAAE,CAAC;QACP,SAAS,EAAE,CAAC;QACZ,MAAM,OAAO,GAAG,MAAM,CAAC;QACvB,MAAM,GAAG,SAAS,CAAC;QACnB,MAAM,OAAO,EAAE,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;IAChD,CAAC,CAAC;IAEF,MAAM,OAAO,GAAG,GAAS,EAAE,CAAC,KAAK,IAAI,EAAE,CAAC;IAExC,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;QAC5B,OAAO,GAAG,IAAI,CAAC;QACf,SAAS,EAAE,CAAC;IACd,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,MAAM,EAAE,gBAAgB,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QACnE,KAAK,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC,KAAc,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC;IACxD,CAAC;IAED,OAAO;QACL,KAAK;QACL,IAAI,WAAW;YACb,OAAO,KAAK,CAAC;QACf,CAAC;QACD,IAAI,OAAO;YACT,OAAO,OAAO,CAAC;QACjB,CAAC;QACD,IAAI;KACL,CAAC;AACJ,CAAC"}
|
|
@@ -479,7 +479,8 @@ export declare class MongoDocumentStorage<T> implements CollectionStorage<T> {
|
|
|
479
479
|
private withPendingNumbers;
|
|
480
480
|
/**
|
|
481
481
|
* The watermark of the collection: the highest number below each pending number of a write whose
|
|
482
|
-
* lease has not ended, and the
|
|
482
|
+
* lease has not ended, the epoch, and the floor of the partition. Each number up to it is written
|
|
483
|
+
* or abandoned. One command reads them. Only the first read of a collection writes: it makes the counter.
|
|
483
484
|
*/
|
|
484
485
|
private watermark;
|
|
485
486
|
/** The id of the document of the oldest kept change of the partition. */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"document-storage.d.ts","sourceRoot":"","sources":["../../src/storage/document-storage.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiGG;AAIH,OAAO,EAAyE,KAAK,EAAE,EAAE,KAAK,QAAQ,EAAe,KAAK,WAAW,EAAE,MAAM,SAAS,CAAC;AACvJ,OAAO,EAML,KAAK,iBAAiB,EAEtB,KAAK,wBAAwB,EAC7B,KAAK,cAAc,EACnB,KAAK,uBAAuB,EAC5B,KAAK,eAAe,EACpB,KAAK,YAAY,EACjB,KAAK,mBAAmB,EACxB,KAAK,kBAAkB,EACvB,KAAK,eAAe,EACpB,KAAK,mBAAmB,EAEzB,MAAM,yBAAyB,CAAC;AACjC,OAAO,
|
|
1
|
+
{"version":3,"file":"document-storage.d.ts","sourceRoot":"","sources":["../../src/storage/document-storage.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiGG;AAIH,OAAO,EAAyE,KAAK,EAAE,EAAE,KAAK,QAAQ,EAAe,KAAK,WAAW,EAAE,MAAM,SAAS,CAAC;AACvJ,OAAO,EAML,KAAK,iBAAiB,EAEtB,KAAK,wBAAwB,EAC7B,KAAK,cAAc,EACnB,KAAK,uBAAuB,EAC5B,KAAK,eAAe,EACpB,KAAK,YAAY,EACjB,KAAK,mBAAmB,EACxB,KAAK,kBAAkB,EACvB,KAAK,eAAe,EACpB,KAAK,mBAAmB,EAEzB,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAA+B,KAAK,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAGhF,OAAO,EAAqE,KAAK,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAgB7H,mEAAmE;AACnE,eAAO,MAAM,SAAS,WAAW,CAAC;AAElC;;;;GAIG;AACH,eAAO,MAAM,uBAAuB,EAAE,SAAS,MAAM,EAA+E,CAAC;AAErI,8HAA8H;AAC9H,eAAO,MAAM,eAAe,EAAE,OAAc,CAAC;AAE7C;;;;GAIG;AACH,MAAM,WAAW,UAAU;IACzB;;;OAGG;IACH,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,SAAS,EAAE,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC;IAC7D,kHAAkH;IAClH,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,GAAG,SAAS,CAAC;IAC3E;;;;;;;OAOG;IACH,eAAe,CAAC,SAAS,EAAE,MAAM,GAAG,SAAS,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;IAClF;;;;;;;OAOG;IACH,WAAW,CAAC,GAAG,EAAE,QAAQ,GAAG,MAAM,GAAG,SAAS,CAAC;CAChD;AAED;;;;GAIG;AACH,eAAO,MAAM,mBAAmB,EAAE,UAmBhC,CAAC;AAEH,+CAA+C;AAC/C,MAAM,WAAW,2BAA2B,CAAC,CAAC,GAAG,OAAO;IACtD,8BAA8B;IAC9B,UAAU,EAAE,MAAM,CAAC;IACnB;;;;OAIG;IACH,EAAE,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,OAAO,CAAC,EAAE,CAAC,CAAC,GAAG,SAAS,CAAC;IAC1C,gHAAgH;IAChH,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC/B,0DAA0D;IAC1D,aAAa,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACnC,kEAAkE;IAClE,WAAW,CAAC,EAAE,kBAAkB,GAAG,SAAS,CAAC;IAC7C,8EAA8E;IAC9E,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B;;;OAGG;IACH,UAAU,CAAC,EAAE,QAAQ,GAAG,SAAS,CAAC;IAClC;;;;OAIG;IACH,MAAM,CAAC,EAAE,CAAC,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC,GAAG,SAAS,CAAC;IAChE;;;OAGG;IACH,YAAY,CAAC,EAAE,MAAM,GAAG,OAAO,GAAG,SAAS,CAAC;IAC5C;;;;;;OAMG;IACH,SAAS,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,MAAM,GAAG,SAAS,CAAC,GAAG,SAAS,CAAC;IAC1D;;;OAGG;IACH,IAAI,CAAC,EAAE,UAAU,GAAG,SAAS,CAAC;CAC/B;AAwLD,6CAA6C;AAC7C,MAAM,WAAW,2BAA2B;IAC1C,kGAAkG;IAClG,IAAI,CAAC,EAAE,UAAU,GAAG,SAAS,CAAC;IAC9B,6EAA6E;IAC7E,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC/B;;;OAGG;IACH,KAAK,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;CAC7B;AAED,iEAAiE;AACjE,MAAM,WAAW,mBAAmB,CAAC,CAAC;IACpC,0BAA0B;IAC1B,EAAE,EAAE,MAAM,CAAC;IACX,qCAAqC;IACrC,SAAS,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B,gBAAgB;IAChB,IAAI,EAAE,CAAC,CAAC;CACT;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,oBAAoB,CAAC,CAAC,GAAG,QAAQ,EAAE,GAAG,EAAE,QAAQ,EAAE,OAAO,GAAE,2BAAgC,GAAG,mBAAmB,CAAC,CAAC,CAAC,GAAG,SAAS,CAS/I;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAgB,eAAe,CAAC,GAAG,EAAE,QAAQ,EAAE,IAAI,GAAE,UAAgC,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAK7H;AAQD;;;GAGG;AACH,MAAM,WAAW,mBAAmB;IAClC,mCAAmC;IACnC,EAAE,IAAI,OAAO,CAAC,EAAE,CAAC,CAAC;IAClB,2CAA2C;IAC3C,UAAU,EAAE,MAAM,CAAC;IACnB,+CAA+C;IAC/C,QAAQ,EAAE,MAAM,CAAC;IACjB,uBAAuB;IACvB,IAAI,EAAE,UAAU,CAAC;IACjB,0FAA0F;IAC1F,SAAS,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B,mEAAmE;IACnE,KAAK,CAAC,GAAG,EAAE,eAAe,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;IAClD;;;OAGG;IACH,QAAQ,CAAC,IAAI,EAAE,SAAS,eAAe,EAAE,GAAG,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAAC;IACzE;;;OAGG;IACH,YAAY,CAAC,KAAK,EAAE;QAAE,QAAQ,CAAC,EAAE,EAAE,OAAO,CAAA;KAAE,GAAG,SAAS,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;IAC9F,yHAAyH;IACzH,eAAe,CAAC,GAAG,EAAE,SAAS,OAAO,EAAE,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;IAC9D;;;;OAIG;IACH,UAAU,IAAI,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC;CAClD;AAED;;;;;;;;;;;;;;;GAeG;AACH,MAAM,WAAW,eAAe;IAC9B,4DAA4D;IAC5D,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC;IAC3B,8FAA8F;IAC9F,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,CAAC;IACjC,oEAAoE;IACpE,QAAQ,CAAC,KAAK,CAAC,EAAE,QAAQ,GAAG,SAAS,CAAC;IACtC,kHAAkH;IAClH,QAAQ,CAAC,MAAM,CAAC,EAAE,QAAQ,GAAG,SAAS,CAAC;CACxC;AAED,4LAA4L;AAC5L,MAAM,MAAM,WAAW,GAAG,SAAS,GAAG,MAAM,GAAG,SAAS,CAAC;AAWzD,gHAAgH;AAChH,MAAM,MAAM,iBAAiB,GAAG,WAAW,GAAG;IAAE,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAA;CAAE,CAAC;AAY1E,mHAAmH;AACnH,wBAAgB,mBAAmB,CAAC,CAAC,EAAE,OAAO,EAAE,oBAAoB,CAAC,CAAC,CAAC,GAAG,mBAAmB,CAE5F;AAED;;;;;GAKG;AACH,MAAM,WAAW,kBAAkB;IACjC,2CAA2C;IAC3C,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,+CAA+C;IAC/C,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,uBAAuB;IACvB,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;IAC1B,0FAA0F;IAC1F,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,SAAS,CAAC;IACvC,iEAAiE;IACjE,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,4CAA4C;IAC5C,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,sFAAsF;IACtF,QAAQ,CAAC,eAAe,EAAE,WAAW,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACvD,wEAAwE;IACxE,eAAe,IAAI,QAAQ,CAAC;IAC5B,+EAA+E;IAC/E,QAAQ,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC;IAC9B,iFAAiF;IACjF,OAAO,CAAC,GAAG,EAAE,QAAQ,GAAG,MAAM,GAAG,SAAS,CAAC;IAC3C,2IAA2I;IAC3I,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,QAAQ,CAAC;IACvE,mEAAmE;IACnE,SAAS,CAAC,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,QAAQ,CAAC;IAC7C,sDAAsD;IACtD,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACvB;;;;OAIG;IACH,kBAAkB,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;IAC/F,iFAAiF;IACjF,SAAS,CAAC,EAAE,EAAE,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACjC,sGAAsG;IACtG,KAAK,CAAC,EAAE,EAAE,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC9B;AAKD,kIAAkI;AAClI,wBAAgB,kBAAkB,CAAC,CAAC,EAAE,OAAO,EAAE,oBAAoB,CAAC,CAAC,CAAC,GAAG,kBAAkB,CAE1F;AAmBD,0CAA0C;AAC1C,MAAM,WAAW,wBAAwB;IACvC;;;OAGG;IACH,QAAQ,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,GAAG,SAAS,CAAC;IACjD,wIAAwI;IACxI,aAAa,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,GAAG,SAAS,CAAC;CAC9C;AAiDD;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,eAAe,EAAE,OAAO,GAAE,wBAA6B,GAAG,QAAQ,GAAG,SAAS,CAEzH;AAsCD;;;;;;;;;GASG;AACH,qBAAa,oBAAoB,CAAC,CAAC,CAAE,YAAW,iBAAiB,CAAC,CAAC,CAAC;IAClE,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAiC;IACzD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;IACxC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAqB;IAC/C,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAS;IACvC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IACtC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IACtC,mGAAmG;IACnG,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAuC;IAClE,qHAAqH;IACrH,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;IAC1B,mGAAmG;IACnG,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAA4C;IAC7E,sFAAsF;IACtF,OAAO,CAAC,QAAQ,CAAC,eAAe,CAA+B;IAC/D,oFAAoF;IACpF,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAsB;IAC/C,OAAO,CAAC,OAAO,CAA4B;IAC3C,kIAAkI;IAClI,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,MAAM,CAAS;IAqCvB,YAAY,OAAO,EAAE,2BAA2B,CAAC,CAAC,CAAC,EAAE,UAAU,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,oBAAoB,CAAC,CAAC,CAAC,CAAC,EA4BrG;IAED,2GAA2G;IAC3G,YAAY,CAAC,SAAS,EAAE,MAAM,GAAG,oBAAoB,CAAC,CAAC,CAAC,CAOvD;YAMa,QAAQ;YAKR,UAAU;IAexB,oDAAoD;IACpD,OAAO,CAAC,eAAe;IAMvB,gHAAgH;IAChH,OAAO,CAAC,YAAY;IAIpB,qEAAqE;IACrE,OAAO,CAAC,UAAU;IAIlB,4GAA4G;IAC5G,OAAO,CAAC,QAAQ;IAOhB,qHAAqH;IACrH,OAAO,CAAC,OAAO;IAOf,yGAAyG;IACzG,OAAO,CAAC,OAAO;IAKf,iFAAiF;IACjF,OAAO,CAAC,WAAW;IAInB,iGAAiG;IACjG,OAAO,CAAC,MAAM;IAcd;;;;;;OAMG;IACH,OAAO,CAAC,MAAM;IAqBd,qGAAqG;IACrG,OAAO,CAAC,MAAM;IAMd,yGAAyG;IACzG,OAAO,CAAC,SAAS;IASjB;;;OAGG;YACW,QAAQ;IAmBtB;;;;;OAKG;YACW,OAAO;IAwBrB,uFAAuF;YACzE,IAAI;IAalB,qFAAqF;YACvE,iBAAiB;IAM/B;;;;;OAKG;YACW,WAAW;IA8BzB;;;;;OAKG;YACW,kBAAkB;IA2BhC;;;;OAIG;YACW,SAAS;IAyCvB,yEAAyE;YAC3D,OAAO;IAKrB,OAAO,CAAC,OAAO;IAIf;;;;;;;;;;;;;;OAcG;YACW,SAAS;IAUvB,+GAA+G;YACjG,eAAe;IAkB7B,qGAAqG;IACrG,OAAO,CAAC,KAAK;IASb,+DAA+D;YACjD,QAAQ;IAKtB,2EAA2E;YAC7D,QAAQ;IAgBhB,GAAG,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,GAAG,SAAS,CAAC,CAI5C;IAEK,MAAM,IAAI,OAAO,CAAC,CAAC,EAAE,CAAC,CAE3B;IAEK,UAAU,IAAI,OAAO,CAAC,KAAK,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,CAE9C;IAEK,MAAM,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC,CAIhC;IAEK,IAAI,CAAC,SAAS,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,OAAO,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC,CAExD;IAEK,KAAK,CAAC,QAAQ,EAAE,eAAe,EAAE,OAAO,CAAC,EAAE,mBAAmB,EAAE,MAAM,CAAC,EAAE,kBAAkB,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,CAO9H;IAEK,GAAG,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAGtC;IAEK,IAAI,IAAI,OAAO,CAAC,MAAM,CAAC,CAQ5B;IAEK,QAAQ,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAYrD;IAED;;;;;;OAMG;IACG,SAAS,CAAC,QAAQ,EAAE,SAAS,QAAQ,EAAE,EAAE,MAAM,GAAE,cAAmB,GAAG,OAAO,CAAC;QAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;QAAC,SAAS,EAAE,OAAO,CAAA;KAAE,CAAC,CAW7H;IAED;;;;OAIG;IACG,SAAS,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CAKvD;IAEK,eAAe,CAAC,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAoChE;YAMa,KAAK;IASb,GAAG,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,CAAC,EAAE,uBAAuB,GAAG,OAAO,CAAC,IAAI,CAAC,CAEhF;IAED;;;OAGG;IACG,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,EAAE,wBAAwB,GAAG,OAAO,CAAC,IAAI,CAAC,CAgD3F;IAEK,KAAK,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,QAAQ,CAAC,EAAE,mBAAmB,GAAG,OAAO,CAAC,CAAC,GAAG,SAAS,CAAC,CAoBpG;IAED;;;OAGG;IACG,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,uBAAuB,GAAG,OAAO,CAAC,OAAO,CAAC,CAa5E;IAEK,WAAW,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,QAAQ,CAAC,EAAE,mBAAmB,GAAG,OAAO,CAAC,MAAM,CAAC,CAgBhF;IAEK,KAAK,CAAC,QAAQ,CAAC,EAAE,mBAAmB,GAAG,OAAO,CAAC,IAAI,CAAC,CAOzD;IAED,6FAA6F;IAC7F,OAAO,CAAC,WAAW;IAMnB;;;OAGG;YACW,SAAS;IAiBvB;;;;OAIG;YACW,QAAQ;IAMtB;;;OAGG;YACW,WAAW;IAqBzB,+EAA+E;YACjE,UAAU;IAqBxB,mGAAmG;YACrF,WAAW;IAezB,8FAA8F;YAChF,UAAU;IASxB,8GAA8G;YAChG,YAAY;IAY1B,8GAA8G;YAChG,eAAe;IAM7B;;;;;;OAMG;IACH,OAAO,CAAC,cAAc;IAuBtB,8HAA8H;IAC9H,OAAO,CAAC,KAAK;IAQb;;;;;;;;;;;;OAYG;IACH,OAAO,CAAC,cAAc;IAsBtB,gKAAgK;YAClJ,eAAe;IAgB7B;;;OAGG;IACH,OAAO,CAAC,aAAa;IAOrB,qHAAqH;YACvG,WAAW;IAYzB,4FAA4F;IAC5F,OAAO,CAAC,SAAS;IAKjB;;;;OAIG;YACW,aAAa;IAoB3B;;;OAGG;YACW,YAAY;IAkB1B;;;OAGG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAE3B;IAED,OAAO,CAAC,SAAS;IAIX,WAAW,IAAI,OAAO,CAAC,eAAe,CAAC,CAM5C;CACF"}
|
|
@@ -100,10 +100,11 @@ import { randomUUID } from "node:crypto";
|
|
|
100
100
|
import { isDeepStrictEqual } from "node:util";
|
|
101
101
|
import { BSON } from "mongodb";
|
|
102
102
|
import { VersionConflictError, applyPatch, preconditionNotSupported, queryEntries, versionConflict, } from "@procedures/collections";
|
|
103
|
-
import { optionalMs, toMs } from "@procedures/client";
|
|
103
|
+
import { optionalMs, Semaphore, toMs } from "@procedures/client";
|
|
104
104
|
import { ensureConnection } from "../connection.js";
|
|
105
105
|
import { noteOwnNumbers, notePrunedIds, ownerOf } from "./own-writes.js";
|
|
106
106
|
import { boundPipeline, cutResults, DEFAULT_PIPELINE_LIMITS, scopePipeline } from "./pipeline-scope.js";
|
|
107
|
+
import { CHANGE_READ_DIVISOR, poolShare } from "./pool-share.js";
|
|
107
108
|
/** The fields of the storage in a stored document. */
|
|
108
109
|
const KEY = "_key";
|
|
109
110
|
const PART = "_part";
|
|
@@ -185,24 +186,55 @@ function hasTransactions(db) {
|
|
|
185
186
|
/** The most moves of a late write to new numbers. */
|
|
186
187
|
const MAX_MOVES = 10;
|
|
187
188
|
/**
|
|
188
|
-
* The writes and the change reads of each collection
|
|
189
|
-
*
|
|
189
|
+
* The writes and the change reads of each collection: by the client, then by the names of the database and the
|
|
190
|
+
* collection. A new `Db` object of the same client has the same queue (roadmap CLI-93). A write goes alone, in the
|
|
191
|
+
* order of the calls. A change read goes after each earlier write, next to the other change reads. So a read and a
|
|
192
|
+
* write keep the order of their calls, and a read does not wait for a read (roadmap CLI-91).
|
|
190
193
|
*/
|
|
191
194
|
const QUEUES = new WeakMap();
|
|
192
|
-
|
|
195
|
+
/** The change reads of each client at one time. A burst of reads leaves the pool to the other operations. */
|
|
196
|
+
const READS = new WeakMap();
|
|
197
|
+
function queueOf(db, name) {
|
|
193
198
|
const owner = ownerOf(db);
|
|
194
199
|
let queues = QUEUES.get(owner);
|
|
195
200
|
if (!queues)
|
|
196
201
|
QUEUES.set(owner, (queues = new Map()));
|
|
197
202
|
const collection = `${db.databaseName}\u0000${name}`;
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
queues
|
|
202
|
-
|
|
203
|
-
|
|
203
|
+
let queue = queues.get(collection);
|
|
204
|
+
if (!queue)
|
|
205
|
+
queues.set(collection, (queue = { all: Promise.resolve(), writes: Promise.resolve() }));
|
|
206
|
+
return { queues, collection, queue };
|
|
207
|
+
}
|
|
208
|
+
/** This function drops the queue of a collection when this end is the last of its work. */
|
|
209
|
+
function dropWhenLast(queues, collection, queue, end) {
|
|
210
|
+
void end.then(() => {
|
|
211
|
+
if (queue.all === end && queues.get(collection) === queue)
|
|
204
212
|
queues.delete(collection);
|
|
205
213
|
});
|
|
214
|
+
}
|
|
215
|
+
const settledOf = (run) => run.then(() => undefined, () => undefined);
|
|
216
|
+
/** This function starts a write after each earlier write and change read of the collection. */
|
|
217
|
+
function enqueueWrite(db, name, task) {
|
|
218
|
+
const { queues, collection, queue } = queueOf(db, name);
|
|
219
|
+
const run = queue.all.then(task);
|
|
220
|
+
const settled = settledOf(run);
|
|
221
|
+
queue.all = settled;
|
|
222
|
+
queue.writes = settled;
|
|
223
|
+
dropWhenLast(queues, collection, queue, settled);
|
|
224
|
+
return run;
|
|
225
|
+
}
|
|
226
|
+
/** This function starts a change read after each earlier write of the collection, next to the other change reads. */
|
|
227
|
+
function enqueueRead(db, name, task) {
|
|
228
|
+
const { queues, collection, queue } = queueOf(db, name);
|
|
229
|
+
const owner = ownerOf(db);
|
|
230
|
+
let reads = READS.get(owner);
|
|
231
|
+
if (!reads)
|
|
232
|
+
READS.set(owner, (reads = new Semaphore(poolShare(db, CHANGE_READ_DIVISOR))));
|
|
233
|
+
const permits = reads;
|
|
234
|
+
const run = queue.writes.then(() => permits.withPermit(task));
|
|
235
|
+
const end = Promise.all([queue.all, settledOf(run)]);
|
|
236
|
+
queue.all = end;
|
|
237
|
+
dropWhenLast(queues, collection, queue, end);
|
|
206
238
|
return run;
|
|
207
239
|
}
|
|
208
240
|
/**
|
|
@@ -817,18 +849,20 @@ export class MongoDocumentStorage {
|
|
|
817
849
|
}
|
|
818
850
|
/**
|
|
819
851
|
* The watermark of the collection: the highest number below each pending number of a write whose
|
|
820
|
-
* lease has not ended, and the
|
|
852
|
+
* lease has not ended, the epoch, and the floor of the partition. Each number up to it is written
|
|
853
|
+
* or abandoned. One command reads them. Only the first read of a collection writes: it makes the counter.
|
|
821
854
|
*/
|
|
822
855
|
async watermark(db) {
|
|
823
856
|
const counters = db.collection(this.countersName);
|
|
824
|
-
|
|
825
|
-
const
|
|
857
|
+
const floorId = this.floorId();
|
|
858
|
+
const read = () => counters
|
|
826
859
|
.aggregate([
|
|
827
|
-
{ $match: { _id: this.collectionName } },
|
|
860
|
+
{ $match: { _id: { $in: [this.collectionName, floorId] } } },
|
|
828
861
|
{
|
|
829
862
|
$project: {
|
|
830
863
|
seq: 1,
|
|
831
864
|
epoch: 1,
|
|
865
|
+
floor: 1,
|
|
832
866
|
low: {
|
|
833
867
|
$min: {
|
|
834
868
|
$map: {
|
|
@@ -843,10 +877,21 @@ export class MongoDocumentStorage {
|
|
|
843
877
|
},
|
|
844
878
|
])
|
|
845
879
|
.toArray();
|
|
880
|
+
let docs = await read();
|
|
881
|
+
if (!docs.some((doc) => doc._id === this.collectionName)) {
|
|
882
|
+
// A duplicate key: another read made the counter at the same time
|
|
883
|
+
await counters.updateOne({ _id: this.collectionName }, { $setOnInsert: { seq: 0, epoch: randomUUID() } }, { upsert: true }).catch((error) => {
|
|
884
|
+
if (!isDuplicateKey(error))
|
|
885
|
+
throw error;
|
|
886
|
+
});
|
|
887
|
+
docs = await read();
|
|
888
|
+
}
|
|
889
|
+
const mark = docs.find((doc) => doc._id === this.collectionName);
|
|
846
890
|
if (!mark)
|
|
847
891
|
throw new Error(`The sequence of ${this.collectionName} is not available`);
|
|
848
892
|
const seq = typeof mark.low === "number" ? Math.min(mark.seq, mark.low - 1) : mark.seq;
|
|
849
|
-
|
|
893
|
+
const floor = docs.find((doc) => doc._id === floorId)?.floor;
|
|
894
|
+
return { seq, epoch: mark.epoch, floor: typeof floor === "number" ? floor : 0, held: seq < mark.seq };
|
|
850
895
|
}
|
|
851
896
|
/** The id of the document of the oldest kept change of the partition. */
|
|
852
897
|
async floorOf(db) {
|
|
@@ -1023,7 +1068,7 @@ export class MongoDocumentStorage {
|
|
|
1023
1068
|
}
|
|
1024
1069
|
async getChangesSince(since) {
|
|
1025
1070
|
const { db, items } = await this.collection();
|
|
1026
|
-
return
|
|
1071
|
+
return enqueueRead(db, this.collectionName, async () => {
|
|
1027
1072
|
const counter = await this.watermark(db);
|
|
1028
1073
|
const version = `${counter.epoch}.${counter.seq}`;
|
|
1029
1074
|
const held = counter.held ? { held: true } : {};
|
|
@@ -1038,8 +1083,11 @@ export class MongoDocumentStorage {
|
|
|
1038
1083
|
if (!match || match[1] !== counter.epoch)
|
|
1039
1084
|
return full();
|
|
1040
1085
|
const from = Number(match[2]);
|
|
1041
|
-
if (from <
|
|
1086
|
+
if (from < counter.floor || from > counter.seq)
|
|
1042
1087
|
return full();
|
|
1088
|
+
// No number is above this version: the read needs no query of the items
|
|
1089
|
+
if (from === counter.seq)
|
|
1090
|
+
return { version, full: false, changed: [], deleted: [], ...held };
|
|
1043
1091
|
// A number above the watermark can belong to a write that is not visible yet: the next read gives it
|
|
1044
1092
|
const docs = await items
|
|
1045
1093
|
.find({ ...this.partitionFilter(), [SEQ]: { $gt: from, $lte: counter.seq } })
|
|
@@ -1067,7 +1115,7 @@ export class MongoDocumentStorage {
|
|
|
1067
1115
|
// ===========================================================================
|
|
1068
1116
|
async write(task) {
|
|
1069
1117
|
const { db, items } = await this.collection();
|
|
1070
|
-
return
|
|
1118
|
+
return enqueueWrite(db, this.collectionName, async () => {
|
|
1071
1119
|
// The floor document of the partition, outside each transaction of the task (roadmap O66)
|
|
1072
1120
|
await this.noteFloor(db);
|
|
1073
1121
|
return task(db, items);
|