@procedures/mongo 0.48.1 → 0.49.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +39 -0
- package/dist/.tsbuildinfo +1 -1
- package/dist/storage/database-changes.d.ts +15 -2
- package/dist/storage/database-changes.d.ts.map +1 -1
- package/dist/storage/database-changes.js +30 -5
- package/dist/storage/database-changes.js.map +1 -1
- package/dist/storage/document-storage.d.ts +8 -1
- package/dist/storage/document-storage.d.ts.map +1 -1
- package/dist/storage/document-storage.js +128 -39
- package/dist/storage/document-storage.js.map +1 -1
- package/dist/storage/materializer.d.ts.map +1 -1
- package/dist/storage/materializer.js +6 -5
- package/dist/storage/materializer.js.map +1 -1
- package/dist/storage/pool-share.d.ts +24 -0
- package/dist/storage/pool-share.d.ts.map +1 -0
- package/dist/storage/pool-share.js +27 -0
- package/dist/storage/pool-share.js.map +1 -0
- package/dist/storage/split-events.d.ts +57 -0
- package/dist/storage/split-events.d.ts.map +1 -0
- package/dist/storage/split-events.js +94 -0
- package/dist/storage/split-events.js.map +1 -0
- package/dist/storage/storage-hints.d.ts +7 -0
- package/dist/storage/storage-hints.d.ts.map +1 -1
- package/dist/storage/storage-hints.js +41 -7
- package/dist/storage/storage-hints.js.map +1 -1
- package/package.json +7 -4
- package/src/storage/database-changes.ts +44 -7
- package/src/storage/document-storage.ts +159 -55
- package/src/storage/materializer.ts +6 -5
- package/src/storage/pool-share.ts +30 -0
- package/src/storage/split-events.ts +135 -0
- package/src/storage/storage-hints.ts +44 -7
|
@@ -29,6 +29,12 @@
|
|
|
29
29
|
* of the backoff. An `onChange` that runs finishes, and no change comes after it.
|
|
30
30
|
* - Change streams need a replica set or a sharded cluster. The watcher asks the server (`hello`)
|
|
31
31
|
* before the first open: on a single server, it stops with the code 40573.
|
|
32
|
+
* - An event over the BSON limit of 16 MiB comes in fragments. The pipeline ends with the stage
|
|
33
|
+
* `$changeStreamSplitLargeEvent`, and the watcher joins the fragments into one event. A server
|
|
34
|
+
* before 6.0.9 does not know the stage. The watcher then opens the stream without it.
|
|
35
|
+
* - An error with one code at one token, 5 times in a row, does not go away: the watcher stops. An
|
|
36
|
+
* example is an event over the BSON limit (code 10334) with no split stage. A network error and an
|
|
37
|
+
* error that the driver resumes do not count.
|
|
32
38
|
*/
|
|
33
39
|
import type { ChangeStreamDocument, ChangeStreamOptions, Db, Document, ResumeToken, Timestamp } from "mongodb";
|
|
34
40
|
import { type Clock, type Logger, type ResubscribeBackoff } from "@procedures/client";
|
|
@@ -41,8 +47,9 @@ export interface WatchDatabaseChangesOptions<TChange extends Document = ChangeSt
|
|
|
41
47
|
onChange: (change: TChange) => void | Promise<void>;
|
|
42
48
|
/**
|
|
43
49
|
* The function that gets the error that stops the watcher. The error is an `invalidate` event, a
|
|
44
|
-
* lost resume token, or an error of the first open that does not go away. An
|
|
45
|
-
* gives an `Error` with the event in its field
|
|
50
|
+
* lost resume token, or an error of the first open that does not go away. An error with one code at
|
|
51
|
+
* one token 5 times in a row also stops the watcher. An `invalidate` event gives an `Error` with the event in its field
|
|
52
|
+
* `change`. The error of a fault at one token has the fields `code`, `resumeToken` and `cause`.
|
|
46
53
|
*/
|
|
47
54
|
onError?: ((error: unknown) => void) | undefined;
|
|
48
55
|
/** The resume token after which the first stream starts. Give only one of `resumeAfter`, `startAfter` and `startAtOperationTime`. */
|
|
@@ -57,6 +64,12 @@ export interface WatchDatabaseChangesOptions<TChange extends Document = ChangeSt
|
|
|
57
64
|
fullDocumentBeforeChange?: "whenAvailable" | "required" | "off" | undefined;
|
|
58
65
|
/** The pipeline of the stream, for example a `$match` of the collections. @default [] */
|
|
59
66
|
pipeline?: Document[] | undefined;
|
|
67
|
+
/**
|
|
68
|
+
* True to add the stage `$changeStreamSplitLargeEvent` after the pipeline. The watcher then joins
|
|
69
|
+
* the fragments of an event over 16 MiB. With false, such an event stops the watcher.
|
|
70
|
+
* @default true
|
|
71
|
+
*/
|
|
72
|
+
splitLargeEvents?: boolean | undefined;
|
|
60
73
|
/** More options of the stream, for example `maxAwaitTimeMS`, `batchSize` or `showExpandedEvents`. */
|
|
61
74
|
streamOptions?: Omit<ChangeStreamOptions, "resumeAfter" | "startAfter" | "startAtOperationTime" | "fullDocument" | "fullDocumentBeforeChange"> | undefined;
|
|
62
75
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"database-changes.d.ts","sourceRoot":"","sources":["../../src/storage/database-changes.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"database-changes.d.ts","sourceRoot":"","sources":["../../src/storage/database-changes.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqCG;AACH,OAAO,KAAK,EAAgB,oBAAoB,EAAE,mBAAmB,EAAE,EAAE,EAAE,QAAQ,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAC7H,OAAO,EAAoC,KAAK,KAAK,EAAkB,KAAK,MAAM,EAAE,KAAK,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAIxI,+CAA+C;AAC/C,MAAM,WAAW,2BAA2B,CAAC,OAAO,SAAS,QAAQ,GAAG,oBAAoB;IAC1F;;;OAGG;IACH,QAAQ,EAAE,CAAC,MAAM,EAAE,OAAO,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACpD;;;;;OAKG;IACH,OAAO,CAAC,EAAE,CAAC,CAAC,KAAK,EAAE,OAAO,KAAK,IAAI,CAAC,GAAG,SAAS,CAAC;IACjD,qIAAqI;IACrI,WAAW,CAAC,EAAE,WAAW,GAAG,SAAS,CAAC;IACtC,qGAAqG;IACrG,UAAU,CAAC,EAAE,WAAW,GAAG,SAAS,CAAC;IACrC,gEAAgE;IAChE,oBAAoB,CAAC,EAAE,SAAS,GAAG,SAAS,CAAC;IAC7C,8FAA8F;IAC9F,YAAY,CAAC,EAAE,cAAc,GAAG,eAAe,GAAG,UAAU,GAAG,SAAS,CAAC;IACzE,iGAAiG;IACjG,wBAAwB,CAAC,EAAE,eAAe,GAAG,UAAU,GAAG,KAAK,GAAG,SAAS,CAAC;IAC5E,yFAAyF;IACzF,QAAQ,CAAC,EAAE,QAAQ,EAAE,GAAG,SAAS,CAAC;IAClC;;;;OAIG;IACH,gBAAgB,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IACvC,qGAAqG;IACrG,aAAa,CAAC,EAAE,IAAI,CAAC,mBAAmB,EAAE,aAAa,GAAG,YAAY,GAAG,sBAAsB,GAAG,cAAc,GAAG,0BAA0B,CAAC,GAAG,SAAS,CAAC;IAC3J;;;OAGG;IACH,OAAO,CAAC,EAAE,kBAAkB,GAAG,SAAS,CAAC;IACzC;;;;OAIG;IACH,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,yCAAyC;IACzC,MAAM,CAAC,EAAE,WAAW,GAAG,SAAS,CAAC;IACjC,2GAA2G;IAC3G,KAAK,CAAC,EAAE,KAAK,GAAG,SAAS,CAAC;CAC3B;AAED,8CAA8C;AAC9C,MAAM,WAAW,oBAAoB;IACnC;;;OAGG;IACH,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IAC9B,kKAAkK;IAClK,QAAQ,CAAC,WAAW,EAAE,WAAW,GAAG,SAAS,CAAC;IAC9C,qFAAqF;IACrF,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,8FAA8F;IAC9F,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACvB;AAuBD;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,wBAAgB,oBAAoB,CAAC,OAAO,SAAS,QAAQ,GAAG,oBAAoB,EAClF,EAAE,EAAE,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,OAAO,CAAC,EAAE,CAAC,CAAC,EACjC,OAAO,EAAE,2BAA2B,CAAC,OAAO,CAAC,GAC5C,oBAAoB,CA8OtB"}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { realClock, writeLog } from "@procedures/client";
|
|
2
2
|
import { COMMAND_NOT_FOUND, LOST_TOKEN_CODES, backoffOf, codeOf, isPermanent, messageOf } from "./storage-hints.js";
|
|
3
|
+
import { SAME_TOKEN_LIMIT, SPLIT_LARGE_EVENT_STAGE, UNKNOWN_STAGE_CODE, eventJoiner, sameTokenFaults } from "./split-events.js";
|
|
3
4
|
/** An error of `onChange`: the watcher waits, and gives the same change again. */
|
|
4
5
|
class HandlerError extends Error {
|
|
5
6
|
error;
|
|
@@ -63,6 +64,9 @@ export function watchDatabaseChanges(db, options) {
|
|
|
63
64
|
let attempt = 0;
|
|
64
65
|
let database;
|
|
65
66
|
let stream;
|
|
67
|
+
/** True while the pipeline ends with the split stage. A server that does not know the stage sets it to false. */
|
|
68
|
+
let split = options.splitLargeEvents !== false;
|
|
69
|
+
const faults = sameTokenFaults();
|
|
66
70
|
/** The timer of a backoff, and the end of its wait. */
|
|
67
71
|
let timer;
|
|
68
72
|
let wake;
|
|
@@ -130,19 +134,27 @@ export function watchDatabaseChanges(db, options) {
|
|
|
130
134
|
};
|
|
131
135
|
/** One stream: it reads the events until an error, an `invalidate` event or the stop. */
|
|
132
136
|
const read = async (next) => {
|
|
137
|
+
const joiner = eventJoiner();
|
|
133
138
|
for (;;) {
|
|
134
|
-
const
|
|
139
|
+
const raw = await next.tryNext();
|
|
135
140
|
if (stopped)
|
|
136
141
|
return;
|
|
137
142
|
markOpen();
|
|
138
|
-
if (
|
|
139
|
-
// A batch without events: its token is after each event that `onChange` took
|
|
143
|
+
if (raw === null) {
|
|
144
|
+
// A batch without events: its token is after each event that `onChange` took. In the middle of a split event, the token stays before it
|
|
140
145
|
const batchToken = next.resumeToken;
|
|
141
|
-
if (batchToken !== undefined && batchToken !== null)
|
|
146
|
+
if (!joiner.partial && batchToken !== undefined && batchToken !== null)
|
|
142
147
|
token = batchToken;
|
|
143
148
|
succeeded();
|
|
144
149
|
continue;
|
|
145
150
|
}
|
|
151
|
+
const result = joiner.push(raw);
|
|
152
|
+
if (result.kind === "part")
|
|
153
|
+
continue;
|
|
154
|
+
if (result.kind === "missing") {
|
|
155
|
+
log("warn", "A split event comes without its first fragments: the stream started after one of them", { missing: result.missing, collection: collectionOf(result.change) });
|
|
156
|
+
}
|
|
157
|
+
const change = result.change;
|
|
146
158
|
if (change["operationType"] === "invalidate") {
|
|
147
159
|
token = change["_id"];
|
|
148
160
|
return fail(Object.assign(new Error(`the change stream of the database ${database ?? ""} was invalidated`), { change }), { operationType: "invalidate" });
|
|
@@ -154,6 +166,7 @@ export function watchDatabaseChanges(db, options) {
|
|
|
154
166
|
throw new HandlerError(error, change);
|
|
155
167
|
}
|
|
156
168
|
token = change["_id"];
|
|
169
|
+
faults.reset();
|
|
157
170
|
succeeded();
|
|
158
171
|
}
|
|
159
172
|
};
|
|
@@ -194,7 +207,8 @@ export function watchDatabaseChanges(db, options) {
|
|
|
194
207
|
if (token === undefined && opened)
|
|
195
208
|
log("warn", "The change stream opens again without a resume token: it can miss changes", {});
|
|
196
209
|
try {
|
|
197
|
-
const
|
|
210
|
+
const pipeline = [...(options.pipeline ?? []), ...(split ? [SPLIT_LARGE_EVENT_STAGE] : [])];
|
|
211
|
+
const next = handle.watch(pipeline, {
|
|
198
212
|
...options.streamOptions,
|
|
199
213
|
...(options.fullDocument !== undefined && { fullDocument: options.fullDocument }),
|
|
200
214
|
...(options.fullDocumentBeforeChange !== undefined && { fullDocumentBeforeChange: options.fullDocumentBeforeChange }),
|
|
@@ -226,8 +240,19 @@ export function watchDatabaseChanges(db, options) {
|
|
|
226
240
|
const code = codeOf(error);
|
|
227
241
|
if (code !== undefined && LOST_TOKEN_CODES.has(code))
|
|
228
242
|
return fail(error, { code });
|
|
243
|
+
if (code === UNKNOWN_STAGE_CODE && split) {
|
|
244
|
+
// A server before 6.0.9: the stream opens again without the split stage
|
|
245
|
+
split = false;
|
|
246
|
+
log("warn", "The server does not know the stage $changeStreamSplitLargeEvent: an event over 16 MiB stops the watcher", { error, code });
|
|
247
|
+
continue;
|
|
248
|
+
}
|
|
229
249
|
if (!opened && isPermanent(error))
|
|
230
250
|
return fail(error, { code });
|
|
251
|
+
const repeats = faults.count(error, token);
|
|
252
|
+
if (repeats >= SAME_TOKEN_LIMIT) {
|
|
253
|
+
const stuck = Object.assign(new Error(`the change stream failed ${repeats} times at one token with the code ${String(code)}: ${messageOf(error)}`), { code, resumeToken: token, cause: error });
|
|
254
|
+
return fail(stuck, { code, repeats });
|
|
255
|
+
}
|
|
231
256
|
await backoff("The change stream failed: it opens again after a backoff", { error, code });
|
|
232
257
|
}
|
|
233
258
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"database-changes.js","sourceRoot":"","sources":["../../src/storage/database-changes.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"database-changes.js","sourceRoot":"","sources":["../../src/storage/database-changes.ts"],"names":[],"mappings":"AAuCA,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAiF,MAAM,oBAAoB,CAAC;AACxI,OAAO,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,SAAS,EAAE,MAAM,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AACpH,OAAO,EAAE,gBAAgB,EAAE,uBAAuB,EAAE,kBAAkB,EAAE,WAAW,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAoEhI,kFAAkF;AAClF,MAAM,YAAa,SAAQ,KAAK;IACrB,KAAK,CAAU;IACf,MAAM,CAAW;IAE1B,YACE,KAAc,EACd,MAAgB;QAEhB,KAAK,CAAC,oBAAoB,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAC9C,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;CACF;AAED,uDAAuD;AACvD,SAAS,YAAY,CAAC,MAAgB;IACpC,MAAM,EAAE,GAAY,MAAM,CAAC,IAAI,CAAC,CAAC;IACjC,OAAO,OAAO,EAAE,KAAK,QAAQ,IAAI,EAAE,KAAK,IAAI,CAAC,CAAC,CAAE,EAAe,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AACtF,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,MAAM,UAAU,oBAAoB,CAClC,EAAiC,EACjC,OAA6C;IAE7C,IAAI,OAAO,OAAO,EAAE,QAAQ,KAAK,UAAU;QAAE,MAAM,IAAI,SAAS,CAAC,6BAA6B,CAAC,CAAC;IAChG,MAAM,MAAM,GAAG,CAAC,OAAO,CAAC,WAAW,EAAE,OAAO,CAAC,UAAU,EAAE,OAAO,CAAC,oBAAoB,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC;IAC9H,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,mEAAmE,CAAC,CAAC;IAChH,MAAM,OAAO,GAAG,SAAS,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAC3C,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,SAAS,CAAC;IACzC,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAC9B,MAAM,IAAI,GAAG,OAAO,EAAE,KAAK,UAAU,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAO,EAAE,CAAC,EAAE,CAAC;IAE1D,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB,8FAA8F;IAC9F,IAAI,KAAK,GAA4B,OAAO,CAAC,UAAU,IAAI,OAAO,CAAC,WAAW,CAAC;IAC/E,oCAAoC;IACpC,IAAI,MAAM,GAAG,KAAK,CAAC;IACnB,6DAA6D;IAC7D,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB,6FAA6F;IAC7F,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,QAA4B,CAAC;IACjC,IAAI,MAAmD,CAAC;IACxD,iHAAiH;IACjH,IAAI,KAAK,GAAG,OAAO,CAAC,gBAAgB,KAAK,KAAK,CAAC;IAC/C,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;IACjC,uDAAuD;IACvD,IAAI,KAAyB,CAAC;IAC9B,IAAI,IAA8B,CAAC;IACnC,IAAI,SAAS,GAAe,GAAG,EAAE,CAAC,SAAS,CAAC;IAC5C,IAAI,SAAS,GAA6B,GAAG,EAAE,CAAC,SAAS,CAAC;IAC1D,MAAM,KAAK,GAAG,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QAClD,SAAS,GAAG,OAAO,CAAC;QACpB,SAAS,GAAG,MAAM,CAAC;IACrB,CAAC,CAAC,CAAC;IACH,kGAAkG;IAClG,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;IAE7B,MAAM,GAAG,GAAG,CAAC,KAAgC,EAAE,OAAe,EAAE,MAAiB,EAAQ,EAAE;QACzF,MAAM,GAAG,GAAG,EAAE,SAAS,EAAE,kBAAkB,EAAE,QAAQ,EAAE,WAAW,EAAE,KAAK,KAAK,SAAS,EAAE,GAAG,MAAM,EAAE,CAAC;QACrG,QAAQ,CAAC,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,EAAE,sBAAsB,OAAO,EAAE,EAAE,GAAG,CAAC,CAAC;IAC9E,CAAC,CAAC;IAEF,mEAAmE;IACnE,MAAM,IAAI,GAAG,CAAC,KAAc,EAAE,MAAiB,EAAQ,EAAE;QACvD,IAAI,OAAO;YAAE,OAAO;QACpB,GAAG,CAAC,OAAO,EAAE,uDAAuD,EAAE,EAAE,KAAK,EAAE,GAAG,MAAM,EAAE,CAAC,CAAC;QAC5F,IAAI,EAAE,CAAC;QACP,SAAS,CAAC,KAAK,CAAC,CAAC;QACjB,IAAI,CAAC;YACH,OAAO,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC;QAC3B,CAAC;QAAC,OAAO,MAAM,EAAE,CAAC;YAChB,GAAG,CAAC,OAAO,EAAE,gBAAgB,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;QACpD,CAAC;IACH,CAAC,CAAC;IAEF,iDAAiD;IACjD,MAAM,OAAO,GAAG,CAAC,OAAe,EAAE,MAAiB,EAAiB,EAAE;QACpE,MAAM,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;QAC/B,OAAO,EAAE,CAAC;QACV,GAAG,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE,GAAG,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACzE,OAAO,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;YACnC,IAAI,GAAG,OAAO,CAAC;YACf,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,EAAE;gBAC9B,KAAK,GAAG,SAAS,CAAC;gBAClB,IAAI,GAAG,SAAS,CAAC;gBACjB,OAAO,EAAE,CAAC;YACZ,CAAC,CAAC,CAAC;YACH,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC;QAClB,CAAC,CAAC,CAAC;IACL,CAAC,CAAC;IAEF,oFAAoF;IACpF,MAAM,KAAK,GAAG,GAAqF,EAAE;QACnG,IAAI,CAAC,MAAM,IAAI,OAAO,CAAC,WAAW,KAAK,SAAS;YAAE,OAAO,EAAE,WAAW,EAAE,OAAO,CAAC,WAAW,EAAE,CAAC;QAC9F,IAAI,KAAK,KAAK,SAAS;YAAE,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC;QACtD,OAAO,OAAO,CAAC,oBAAoB,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,oBAAoB,EAAE,OAAO,CAAC,oBAAoB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IAClH,CAAC,CAAC;IAEF,8DAA8D;IAC9D,MAAM,QAAQ,GAAG,GAAS,EAAE;QAC1B,IAAI,MAAM,IAAI,OAAO;YAAE,OAAO;QAC9B,MAAM,GAAG,IAAI,CAAC;QACd,SAAS,EAAE,CAAC;IACd,CAAC,CAAC;IAEF,qGAAqG;IACrG,MAAM,SAAS,GAAG,GAAS,EAAE;QAC3B,IAAI,OAAO,GAAG,CAAC;YAAE,GAAG,CAAC,MAAM,EAAE,gCAAgC,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC;QACtF,OAAO,GAAG,CAAC,CAAC;IACd,CAAC,CAAC;IAEF,yFAAyF;IACzF,MAAM,IAAI,GAAG,KAAK,EAAE,IAAqC,EAAiB,EAAE;QAC1E,MAAM,MAAM,GAAG,WAAW,EAAE,CAAC;QAC7B,SAAS,CAAC;YACR,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;YACjC,IAAI,OAAO;gBAAE,OAAO;YACpB,QAAQ,EAAE,CAAC;YACX,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;gBACjB,wIAAwI;gBACxI,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,CAAC;gBACpC,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,UAAU,KAAK,SAAS,IAAI,UAAU,KAAK,IAAI;oBAAE,KAAK,GAAG,UAAU,CAAC;gBAC3F,SAAS,EAAE,CAAC;gBACZ,SAAS;YACX,CAAC;YACD,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAChC,IAAI,MAAM,CAAC,IAAI,KAAK,MAAM;gBAAE,SAAS;YACrC,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;gBAC9B,GAAG,CAAC,MAAM,EAAE,uFAAuF,EAAE,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,UAAU,EAAE,YAAY,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YAC7K,CAAC;YACD,MAAM,MAAM,GAAG,MAAM,CAAC,MAAiB,CAAC;YACxC,IAAI,MAAM,CAAC,eAAe,CAAC,KAAK,YAAY,EAAE,CAAC;gBAC7C,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;gBACtB,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,qCAAqC,QAAQ,IAAI,EAAE,kBAAkB,CAAC,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE,EAAE,aAAa,EAAE,YAAY,EAAE,CAAC,CAAC;YAC5J,CAAC;YACD,IAAI,CAAC;gBACH,MAAM,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YACjC,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,IAAI,YAAY,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;YACxC,CAAC;YACD,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;YACtB,MAAM,CAAC,KAAK,EAAE,CAAC;YACf,SAAS,EAAE,CAAC;QACd,CAAC;IACH,CAAC,CAAC;IAEF,MAAM,GAAG,GAAG,KAAK,IAAmB,EAAE;QACpC,OAAO,CAAC,OAAO,EAAE,CAAC;YAChB,IAAI,MAAU,CAAC;YACf,IAAI,CAAC;gBACH,MAAM,GAAG,MAAM,IAAI,EAAE,CAAC;gBACtB,QAAQ,GAAG,MAAM,CAAC,YAAY,CAAC;gBAC/B,IAAI,CAAC,OAAO,EAAE,CAAC;oBACb,8FAA8F;oBAC9F,IAAI,KAA2B,CAAC;oBAChC,IAAI,CAAC;wBACH,KAAK,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;oBAC7C,CAAC;oBAAC,OAAO,KAAK,EAAE,CAAC;wBACf,0DAA0D;wBAC1D,IAAI,MAAM,CAAC,KAAK,CAAC,KAAK,iBAAiB;4BAAE,MAAM,KAAK,CAAC;oBACvD,CAAC;oBACD,IAAI,KAAK,IAAI,OAAO,KAAK,CAAC,SAAS,CAAC,KAAK,QAAQ,IAAI,KAAK,CAAC,KAAK,CAAC,KAAK,UAAU,EAAE,CAAC;wBACjF,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,2FAA2F,CAAC,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;wBACtJ,OAAO,IAAI,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;oBACvC,CAAC;oBACD,OAAO,GAAG,IAAI,CAAC;gBACjB,CAAC;YACH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,OAAO;oBAAE,OAAO;gBACpB,IAAI,CAAC,MAAM,IAAI,WAAW,CAAC,KAAK,CAAC;oBAAE,OAAO,IAAI,CAAC,KAAK,EAAE,EAAE,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;gBAC/E,MAAM,OAAO,CAAC,8EAA8E,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;gBAC9H,SAAS;YACX,CAAC;YACD,IAAI,OAAO;gBAAE,OAAO;YACpB,IAAI,KAAK,KAAK,SAAS,IAAI,MAAM;gBAAE,GAAG,CAAC,MAAM,EAAE,2EAA2E,EAAE,EAAE,CAAC,CAAC;YAChI,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,uBAAuB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;gBAC5F,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAoB,QAAQ,EAAE;oBACrD,GAAG,OAAO,CAAC,aAAa;oBACxB,GAAG,CAAC,OAAO,CAAC,YAAY,KAAK,SAAS,IAAI,EAAE,YAAY,EAAE,OAAO,CAAC,YAAY,EAAE,CAAC;oBACjF,GAAG,CAAC,OAAO,CAAC,wBAAwB,KAAK,SAAS,IAAI,EAAE,wBAAwB,EAAE,OAAO,CAAC,wBAAwB,EAAE,CAAC;oBACrH,GAAG,KAAK,EAAE;iBACX,CAAC,CAAC;gBACH,MAAM,GAAG,IAAI,CAAC;gBACd,kHAAkH;gBAClH,IAAI,CAAC,EAAE,CAAC,oBAAoB,EAAE,QAAQ,CAAC,CAAC;gBACxC,IAAI,CAAC;oBACH,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC;gBACnB,CAAC;wBAAS,CAAC;oBACT,IAAI,MAAM,KAAK,IAAI;wBAAE,MAAM,GAAG,SAAS,CAAC;oBACxC,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;gBAC5C,CAAC;YACH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,OAAO;oBAAE,OAAO;gBACpB,IAAI,KAAK,YAAY,YAAY,EAAE,CAAC;oBAClC,MAAM,OAAO,CAAC,8DAA8D,EAAE;wBAC5E,KAAK,EAAE,KAAK,CAAC,KAAK;wBAClB,UAAU,EAAE,YAAY,CAAC,KAAK,CAAC,MAAM,CAAC;wBACtC,aAAa,EAAE,KAAK,CAAC,MAAM,CAAC,eAAe,CAAC;qBAC7C,CAAC,CAAC;oBACH,SAAS;gBACX,CAAC;gBACD,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;gBAC3B,IAAI,IAAI,KAAK,SAAS,IAAI,gBAAgB,CAAC,GAAG,CAAC,IAAI,CAAC;oBAAE,OAAO,IAAI,CAAC,KAAK,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;gBACnF,IAAI,IAAI,KAAK,kBAAkB,IAAI,KAAK,EAAE,CAAC;oBACzC,wEAAwE;oBACxE,KAAK,GAAG,KAAK,CAAC;oBACd,GAAG,CAAC,MAAM,EAAE,yGAAyG,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;oBACxI,SAAS;gBACX,CAAC;gBACD,IAAI,CAAC,MAAM,IAAI,WAAW,CAAC,KAAK,CAAC;oBAAE,OAAO,IAAI,CAAC,KAAK,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;gBAChE,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;gBAC3C,IAAI,OAAO,IAAI,gBAAgB,EAAE,CAAC;oBAChC,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,4BAA4B,OAAO,qCAAqC,MAAM,CAAC,IAAI,CAAC,KAAK,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;oBAChM,OAAO,IAAI,CAAC,KAAK,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;gBACxC,CAAC;gBACD,MAAM,OAAO,CAAC,0DAA0D,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YAC7F,CAAC;QACH,CAAC;IACH,CAAC,CAAC;IAEF,uDAAuD;IACvD,MAAM,IAAI,GAAG,GAAS,EAAE;QACtB,OAAO,GAAG,IAAI,CAAC;QACf,KAAK,EAAE,EAAE,CAAC;QACV,KAAK,GAAG,SAAS,CAAC;QAClB,IAAI,EAAE,EAAE,CAAC;QACT,IAAI,GAAG,SAAS,CAAC;QACjB,OAAO,CAAC,MAAM,EAAE,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IACxD,CAAC,CAAC;IAEF,MAAM,IAAI,GAAG,KAAK,IAAmB,EAAE;QACrC,IAAI,OAAO;YAAE,OAAO;QACpB,IAAI,EAAE,CAAC;QACP,SAAS,EAAE,CAAC;QACZ,MAAM,OAAO,GAAG,MAAM,CAAC;QACvB,MAAM,GAAG,SAAS,CAAC;QACnB,MAAM,OAAO,EAAE,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;IAChD,CAAC,CAAC;IAEF,MAAM,OAAO,GAAG,GAAS,EAAE,CAAC,KAAK,IAAI,EAAE,CAAC;IAExC,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;QAC5B,OAAO,GAAG,IAAI,CAAC;QACf,SAAS,EAAE,CAAC;IACd,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,MAAM,EAAE,gBAAgB,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QACnE,KAAK,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC,KAAc,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC;IACxD,CAAC;IAED,OAAO;QACL,KAAK;QACL,IAAI,WAAW;YACb,OAAO,KAAK,CAAC;QACf,CAAC;QACD,IAAI,OAAO;YACT,OAAO,OAAO,CAAC;QACjB,CAAC;QACD,IAAI;KACL,CAAC;AACJ,CAAC"}
|
|
@@ -68,6 +68,12 @@
|
|
|
68
68
|
* its lease moves its documents to new numbers, so a reader that passed the old numbers gets
|
|
69
69
|
* it. One case stays on a single server: a process that stops between such a late write and the
|
|
70
70
|
* move of its documents. Then a reader that passed the old numbers does not get that change.
|
|
71
|
+
* - A reader can get a write two times, with the same value or a later value. A full answer takes
|
|
72
|
+
* its mark before it lists the items, so it can hold a write above its mark, and the next delta
|
|
73
|
+
* gives the write again. A reader can also read a late write at its old numbers, and then the move
|
|
74
|
+
* gives the write new numbers and a new version. The second case occurs only for a write that ends
|
|
75
|
+
* after `writeLease`.
|
|
76
|
+
* - A version that stops below a pending number has `held`, so a reader knows to read again.
|
|
71
77
|
* - The storage uses a transaction only with a client: the option `client`, or the default
|
|
72
78
|
* connection when `db` is absent. With `db` and no `client`, each write uses the watermark, also
|
|
73
79
|
* on a replica set.
|
|
@@ -473,7 +479,8 @@ export declare class MongoDocumentStorage<T> implements CollectionStorage<T> {
|
|
|
473
479
|
private withPendingNumbers;
|
|
474
480
|
/**
|
|
475
481
|
* The watermark of the collection: the highest number below each pending number of a write whose
|
|
476
|
-
* lease has not ended, and the
|
|
482
|
+
* lease has not ended, the epoch, and the floor of the partition. Each number up to it is written
|
|
483
|
+
* or abandoned. One command reads them. Only the first read of a collection writes: it makes the counter.
|
|
477
484
|
*/
|
|
478
485
|
private watermark;
|
|
479
486
|
/** The id of the document of the oldest kept change of the partition. */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"document-storage.d.ts","sourceRoot":"","sources":["../../src/storage/document-storage.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"document-storage.d.ts","sourceRoot":"","sources":["../../src/storage/document-storage.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiGG;AAIH,OAAO,EAAyE,KAAK,EAAE,EAAE,KAAK,QAAQ,EAAe,KAAK,WAAW,EAAE,MAAM,SAAS,CAAC;AACvJ,OAAO,EAML,KAAK,iBAAiB,EAEtB,KAAK,wBAAwB,EAC7B,KAAK,cAAc,EACnB,KAAK,uBAAuB,EAC5B,KAAK,eAAe,EACpB,KAAK,YAAY,EACjB,KAAK,mBAAmB,EACxB,KAAK,kBAAkB,EACvB,KAAK,eAAe,EACpB,KAAK,mBAAmB,EAEzB,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAA+B,KAAK,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAGhF,OAAO,EAAqE,KAAK,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAgB7H,mEAAmE;AACnE,eAAO,MAAM,SAAS,WAAW,CAAC;AAElC;;;;GAIG;AACH,eAAO,MAAM,uBAAuB,EAAE,SAAS,MAAM,EAA+E,CAAC;AAErI,8HAA8H;AAC9H,eAAO,MAAM,eAAe,EAAE,OAAc,CAAC;AAE7C;;;;GAIG;AACH,MAAM,WAAW,UAAU;IACzB;;;OAGG;IACH,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,SAAS,EAAE,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC;IAC7D,kHAAkH;IAClH,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,GAAG,SAAS,CAAC;IAC3E;;;;;;;OAOG;IACH,eAAe,CAAC,SAAS,EAAE,MAAM,GAAG,SAAS,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;IAClF;;;;;;;OAOG;IACH,WAAW,CAAC,GAAG,EAAE,QAAQ,GAAG,MAAM,GAAG,SAAS,CAAC;CAChD;AAED;;;;GAIG;AACH,eAAO,MAAM,mBAAmB,EAAE,UAmBhC,CAAC;AAEH,+CAA+C;AAC/C,MAAM,WAAW,2BAA2B,CAAC,CAAC,GAAG,OAAO;IACtD,8BAA8B;IAC9B,UAAU,EAAE,MAAM,CAAC;IACnB;;;;OAIG;IACH,EAAE,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,OAAO,CAAC,EAAE,CAAC,CAAC,GAAG,SAAS,CAAC;IAC1C,gHAAgH;IAChH,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC/B,0DAA0D;IAC1D,aAAa,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACnC,kEAAkE;IAClE,WAAW,CAAC,EAAE,kBAAkB,GAAG,SAAS,CAAC;IAC7C,8EAA8E;IAC9E,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B;;;OAGG;IACH,UAAU,CAAC,EAAE,QAAQ,GAAG,SAAS,CAAC;IAClC;;;;OAIG;IACH,MAAM,CAAC,EAAE,CAAC,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC,GAAG,SAAS,CAAC;IAChE;;;OAGG;IACH,YAAY,CAAC,EAAE,MAAM,GAAG,OAAO,GAAG,SAAS,CAAC;IAC5C;;;;;;OAMG;IACH,SAAS,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,MAAM,GAAG,SAAS,CAAC,GAAG,SAAS,CAAC;IAC1D;;;OAGG;IACH,IAAI,CAAC,EAAE,UAAU,GAAG,SAAS,CAAC;CAC/B;AAwLD,6CAA6C;AAC7C,MAAM,WAAW,2BAA2B;IAC1C,kGAAkG;IAClG,IAAI,CAAC,EAAE,UAAU,GAAG,SAAS,CAAC;IAC9B,6EAA6E;IAC7E,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC/B;;;OAGG;IACH,KAAK,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;CAC7B;AAED,iEAAiE;AACjE,MAAM,WAAW,mBAAmB,CAAC,CAAC;IACpC,0BAA0B;IAC1B,EAAE,EAAE,MAAM,CAAC;IACX,qCAAqC;IACrC,SAAS,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B,gBAAgB;IAChB,IAAI,EAAE,CAAC,CAAC;CACT;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,oBAAoB,CAAC,CAAC,GAAG,QAAQ,EAAE,GAAG,EAAE,QAAQ,EAAE,OAAO,GAAE,2BAAgC,GAAG,mBAAmB,CAAC,CAAC,CAAC,GAAG,SAAS,CAS/I;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAgB,eAAe,CAAC,GAAG,EAAE,QAAQ,EAAE,IAAI,GAAE,UAAgC,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAK7H;AAQD;;;GAGG;AACH,MAAM,WAAW,mBAAmB;IAClC,mCAAmC;IACnC,EAAE,IAAI,OAAO,CAAC,EAAE,CAAC,CAAC;IAClB,2CAA2C;IAC3C,UAAU,EAAE,MAAM,CAAC;IACnB,+CAA+C;IAC/C,QAAQ,EAAE,MAAM,CAAC;IACjB,uBAAuB;IACvB,IAAI,EAAE,UAAU,CAAC;IACjB,0FAA0F;IAC1F,SAAS,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B,mEAAmE;IACnE,KAAK,CAAC,GAAG,EAAE,eAAe,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;IAClD;;;OAGG;IACH,QAAQ,CAAC,IAAI,EAAE,SAAS,eAAe,EAAE,GAAG,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAAC;IACzE;;;OAGG;IACH,YAAY,CAAC,KAAK,EAAE;QAAE,QAAQ,CAAC,EAAE,EAAE,OAAO,CAAA;KAAE,GAAG,SAAS,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;IAC9F,yHAAyH;IACzH,eAAe,CAAC,GAAG,EAAE,SAAS,OAAO,EAAE,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;IAC9D;;;;OAIG;IACH,UAAU,IAAI,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC;CAClD;AAED;;;;;;;;;;;;;;;GAeG;AACH,MAAM,WAAW,eAAe;IAC9B,4DAA4D;IAC5D,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC;IAC3B,8FAA8F;IAC9F,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,CAAC;IACjC,oEAAoE;IACpE,QAAQ,CAAC,KAAK,CAAC,EAAE,QAAQ,GAAG,SAAS,CAAC;IACtC,kHAAkH;IAClH,QAAQ,CAAC,MAAM,CAAC,EAAE,QAAQ,GAAG,SAAS,CAAC;CACxC;AAED,4LAA4L;AAC5L,MAAM,MAAM,WAAW,GAAG,SAAS,GAAG,MAAM,GAAG,SAAS,CAAC;AAWzD,gHAAgH;AAChH,MAAM,MAAM,iBAAiB,GAAG,WAAW,GAAG;IAAE,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAA;CAAE,CAAC;AAY1E,mHAAmH;AACnH,wBAAgB,mBAAmB,CAAC,CAAC,EAAE,OAAO,EAAE,oBAAoB,CAAC,CAAC,CAAC,GAAG,mBAAmB,CAE5F;AAED;;;;;GAKG;AACH,MAAM,WAAW,kBAAkB;IACjC,2CAA2C;IAC3C,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,+CAA+C;IAC/C,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,uBAAuB;IACvB,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;IAC1B,0FAA0F;IAC1F,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,SAAS,CAAC;IACvC,iEAAiE;IACjE,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,4CAA4C;IAC5C,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,sFAAsF;IACtF,QAAQ,CAAC,eAAe,EAAE,WAAW,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACvD,wEAAwE;IACxE,eAAe,IAAI,QAAQ,CAAC;IAC5B,+EAA+E;IAC/E,QAAQ,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC;IAC9B,iFAAiF;IACjF,OAAO,CAAC,GAAG,EAAE,QAAQ,GAAG,MAAM,GAAG,SAAS,CAAC;IAC3C,2IAA2I;IAC3I,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,QAAQ,CAAC;IACvE,mEAAmE;IACnE,SAAS,CAAC,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,QAAQ,CAAC;IAC7C,sDAAsD;IACtD,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACvB;;;;OAIG;IACH,kBAAkB,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;IAC/F,iFAAiF;IACjF,SAAS,CAAC,EAAE,EAAE,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACjC,sGAAsG;IACtG,KAAK,CAAC,EAAE,EAAE,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC9B;AAKD,kIAAkI;AAClI,wBAAgB,kBAAkB,CAAC,CAAC,EAAE,OAAO,EAAE,oBAAoB,CAAC,CAAC,CAAC,GAAG,kBAAkB,CAE1F;AAmBD,0CAA0C;AAC1C,MAAM,WAAW,wBAAwB;IACvC;;;OAGG;IACH,QAAQ,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,GAAG,SAAS,CAAC;IACjD,wIAAwI;IACxI,aAAa,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,GAAG,SAAS,CAAC;CAC9C;AAiDD;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,eAAe,EAAE,OAAO,GAAE,wBAA6B,GAAG,QAAQ,GAAG,SAAS,CAEzH;AAsCD;;;;;;;;;GASG;AACH,qBAAa,oBAAoB,CAAC,CAAC,CAAE,YAAW,iBAAiB,CAAC,CAAC,CAAC;IAClE,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAiC;IACzD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;IACxC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAqB;IAC/C,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAS;IACvC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IACtC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IACtC,mGAAmG;IACnG,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAuC;IAClE,qHAAqH;IACrH,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;IAC1B,mGAAmG;IACnG,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAA4C;IAC7E,sFAAsF;IACtF,OAAO,CAAC,QAAQ,CAAC,eAAe,CAA+B;IAC/D,oFAAoF;IACpF,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAsB;IAC/C,OAAO,CAAC,OAAO,CAA4B;IAC3C,kIAAkI;IAClI,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,MAAM,CAAS;IAqCvB,YAAY,OAAO,EAAE,2BAA2B,CAAC,CAAC,CAAC,EAAE,UAAU,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,oBAAoB,CAAC,CAAC,CAAC,CAAC,EA4BrG;IAED,2GAA2G;IAC3G,YAAY,CAAC,SAAS,EAAE,MAAM,GAAG,oBAAoB,CAAC,CAAC,CAAC,CAOvD;YAMa,QAAQ;YAKR,UAAU;IAexB,oDAAoD;IACpD,OAAO,CAAC,eAAe;IAMvB,gHAAgH;IAChH,OAAO,CAAC,YAAY;IAIpB,qEAAqE;IACrE,OAAO,CAAC,UAAU;IAIlB,4GAA4G;IAC5G,OAAO,CAAC,QAAQ;IAOhB,qHAAqH;IACrH,OAAO,CAAC,OAAO;IAOf,yGAAyG;IACzG,OAAO,CAAC,OAAO;IAKf,iFAAiF;IACjF,OAAO,CAAC,WAAW;IAInB,iGAAiG;IACjG,OAAO,CAAC,MAAM;IAcd;;;;;;OAMG;IACH,OAAO,CAAC,MAAM;IAqBd,qGAAqG;IACrG,OAAO,CAAC,MAAM;IAMd,yGAAyG;IACzG,OAAO,CAAC,SAAS;IASjB;;;OAGG;YACW,QAAQ;IAmBtB;;;;;OAKG;YACW,OAAO;IAwBrB,uFAAuF;YACzE,IAAI;IAalB,qFAAqF;YACvE,iBAAiB;IAM/B;;;;;OAKG;YACW,WAAW;IA8BzB;;;;;OAKG;YACW,kBAAkB;IA2BhC;;;;OAIG;YACW,SAAS;IAyCvB,yEAAyE;YAC3D,OAAO;IAKrB,OAAO,CAAC,OAAO;IAIf;;;;;;;;;;;;;;OAcG;YACW,SAAS;IAUvB,+GAA+G;YACjG,eAAe;IAkB7B,qGAAqG;IACrG,OAAO,CAAC,KAAK;IASb,+DAA+D;YACjD,QAAQ;IAKtB,2EAA2E;YAC7D,QAAQ;IAgBhB,GAAG,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,GAAG,SAAS,CAAC,CAI5C;IAEK,MAAM,IAAI,OAAO,CAAC,CAAC,EAAE,CAAC,CAE3B;IAEK,UAAU,IAAI,OAAO,CAAC,KAAK,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,CAE9C;IAEK,MAAM,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC,CAIhC;IAEK,IAAI,CAAC,SAAS,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,OAAO,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC,CAExD;IAEK,KAAK,CAAC,QAAQ,EAAE,eAAe,EAAE,OAAO,CAAC,EAAE,mBAAmB,EAAE,MAAM,CAAC,EAAE,kBAAkB,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,CAO9H;IAEK,GAAG,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAGtC;IAEK,IAAI,IAAI,OAAO,CAAC,MAAM,CAAC,CAQ5B;IAEK,QAAQ,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAYrD;IAED;;;;;;OAMG;IACG,SAAS,CAAC,QAAQ,EAAE,SAAS,QAAQ,EAAE,EAAE,MAAM,GAAE,cAAmB,GAAG,OAAO,CAAC;QAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;QAAC,SAAS,EAAE,OAAO,CAAA;KAAE,CAAC,CAW7H;IAED;;;;OAIG;IACG,SAAS,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CAKvD;IAEK,eAAe,CAAC,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAoChE;YAMa,KAAK;IASb,GAAG,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,CAAC,EAAE,uBAAuB,GAAG,OAAO,CAAC,IAAI,CAAC,CAEhF;IAED;;;OAGG;IACG,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,EAAE,wBAAwB,GAAG,OAAO,CAAC,IAAI,CAAC,CAgD3F;IAEK,KAAK,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,QAAQ,CAAC,EAAE,mBAAmB,GAAG,OAAO,CAAC,CAAC,GAAG,SAAS,CAAC,CAoBpG;IAED;;;OAGG;IACG,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,uBAAuB,GAAG,OAAO,CAAC,OAAO,CAAC,CAa5E;IAEK,WAAW,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,QAAQ,CAAC,EAAE,mBAAmB,GAAG,OAAO,CAAC,MAAM,CAAC,CAgBhF;IAEK,KAAK,CAAC,QAAQ,CAAC,EAAE,mBAAmB,GAAG,OAAO,CAAC,IAAI,CAAC,CAOzD;IAED,6FAA6F;IAC7F,OAAO,CAAC,WAAW;IAMnB;;;OAGG;YACW,SAAS;IAiBvB;;;;OAIG;YACW,QAAQ;IAMtB;;;OAGG;YACW,WAAW;IAqBzB,+EAA+E;YACjE,UAAU;IAqBxB,mGAAmG;YACrF,WAAW;IAezB,8FAA8F;YAChF,UAAU;IASxB,8GAA8G;YAChG,YAAY;IAY1B,8GAA8G;YAChG,eAAe;IAM7B;;;;;;OAMG;IACH,OAAO,CAAC,cAAc;IAuBtB,8HAA8H;IAC9H,OAAO,CAAC,KAAK;IAQb;;;;;;;;;;;;OAYG;IACH,OAAO,CAAC,cAAc;IAsBtB,gKAAgK;YAClJ,eAAe;IAgB7B;;;OAGG;IACH,OAAO,CAAC,aAAa;IAOrB,qHAAqH;YACvG,WAAW;IAYzB,4FAA4F;IAC5F,OAAO,CAAC,SAAS;IAKjB;;;;OAIG;YACW,aAAa;IAoB3B;;;OAGG;YACW,YAAY;IAkB1B;;;OAGG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAE3B;IAED,OAAO,CAAC,SAAS;IAIX,WAAW,IAAI,OAAO,CAAC,eAAe,CAAC,CAM5C;CACF"}
|
|
@@ -68,6 +68,12 @@
|
|
|
68
68
|
* its lease moves its documents to new numbers, so a reader that passed the old numbers gets
|
|
69
69
|
* it. One case stays on a single server: a process that stops between such a late write and the
|
|
70
70
|
* move of its documents. Then a reader that passed the old numbers does not get that change.
|
|
71
|
+
* - A reader can get a write two times, with the same value or a later value. A full answer takes
|
|
72
|
+
* its mark before it lists the items, so it can hold a write above its mark, and the next delta
|
|
73
|
+
* gives the write again. A reader can also read a late write at its old numbers, and then the move
|
|
74
|
+
* gives the write new numbers and a new version. The second case occurs only for a write that ends
|
|
75
|
+
* after `writeLease`.
|
|
76
|
+
* - A version that stops below a pending number has `held`, so a reader knows to read again.
|
|
71
77
|
* - The storage uses a transaction only with a client: the option `client`, or the default
|
|
72
78
|
* connection when `db` is absent. With `db` and no `client`, each write uses the watermark, also
|
|
73
79
|
* on a replica set.
|
|
@@ -94,10 +100,11 @@ import { randomUUID } from "node:crypto";
|
|
|
94
100
|
import { isDeepStrictEqual } from "node:util";
|
|
95
101
|
import { BSON } from "mongodb";
|
|
96
102
|
import { VersionConflictError, applyPatch, preconditionNotSupported, queryEntries, versionConflict, } from "@procedures/collections";
|
|
97
|
-
import { optionalMs, toMs } from "@procedures/client";
|
|
103
|
+
import { optionalMs, Semaphore, toMs } from "@procedures/client";
|
|
98
104
|
import { ensureConnection } from "../connection.js";
|
|
99
105
|
import { noteOwnNumbers, notePrunedIds, ownerOf } from "./own-writes.js";
|
|
100
106
|
import { boundPipeline, cutResults, DEFAULT_PIPELINE_LIMITS, scopePipeline } from "./pipeline-scope.js";
|
|
107
|
+
import { CHANGE_READ_DIVISOR, poolShare } from "./pool-share.js";
|
|
101
108
|
/** The fields of the storage in a stored document. */
|
|
102
109
|
const KEY = "_key";
|
|
103
110
|
const PART = "_part";
|
|
@@ -155,6 +162,12 @@ function isDuplicateKey(error) {
|
|
|
155
162
|
const { code, writeErrors } = (error ?? {});
|
|
156
163
|
return code === 11000 || [writeErrors].flat().some((entry) => entry?.code === 11000);
|
|
157
164
|
}
|
|
165
|
+
/** The place in an ordered bulk write of the write that a duplicate key stopped, or `undefined` when the error does not give one. */
|
|
166
|
+
function duplicateIndex(error) {
|
|
167
|
+
const { writeErrors } = (error ?? {});
|
|
168
|
+
const stopped = [writeErrors].flat().find((entry) => entry?.code === 11000);
|
|
169
|
+
return typeof stopped?.index === "number" ? stopped.index : undefined;
|
|
170
|
+
}
|
|
158
171
|
/** True when a client is on a replica set or a sharded cluster: it has transactions. */
|
|
159
172
|
const TRANSACTIONAL = new WeakMap();
|
|
160
173
|
function hasTransactions(db) {
|
|
@@ -173,24 +186,55 @@ function hasTransactions(db) {
|
|
|
173
186
|
/** The most moves of a late write to new numbers. */
|
|
174
187
|
const MAX_MOVES = 10;
|
|
175
188
|
/**
|
|
176
|
-
* The writes and the change reads of each collection
|
|
177
|
-
*
|
|
189
|
+
* The writes and the change reads of each collection: by the client, then by the names of the database and the
|
|
190
|
+
* collection. A new `Db` object of the same client has the same queue (roadmap CLI-93). A write goes alone, in the
|
|
191
|
+
* order of the calls. A change read goes after each earlier write, next to the other change reads. So a read and a
|
|
192
|
+
* write keep the order of their calls, and a read does not wait for a read (roadmap CLI-91).
|
|
178
193
|
*/
|
|
179
194
|
const QUEUES = new WeakMap();
|
|
180
|
-
|
|
195
|
+
/** The change reads of each client at one time. A burst of reads leaves the pool to the other operations. */
|
|
196
|
+
const READS = new WeakMap();
|
|
197
|
+
function queueOf(db, name) {
|
|
181
198
|
const owner = ownerOf(db);
|
|
182
199
|
let queues = QUEUES.get(owner);
|
|
183
200
|
if (!queues)
|
|
184
201
|
QUEUES.set(owner, (queues = new Map()));
|
|
185
202
|
const collection = `${db.databaseName}\u0000${name}`;
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
queues
|
|
190
|
-
|
|
191
|
-
|
|
203
|
+
let queue = queues.get(collection);
|
|
204
|
+
if (!queue)
|
|
205
|
+
queues.set(collection, (queue = { all: Promise.resolve(), writes: Promise.resolve() }));
|
|
206
|
+
return { queues, collection, queue };
|
|
207
|
+
}
|
|
208
|
+
/** This function drops the queue of a collection when this end is the last of its work. */
|
|
209
|
+
function dropWhenLast(queues, collection, queue, end) {
|
|
210
|
+
void end.then(() => {
|
|
211
|
+
if (queue.all === end && queues.get(collection) === queue)
|
|
192
212
|
queues.delete(collection);
|
|
193
213
|
});
|
|
214
|
+
}
|
|
215
|
+
const settledOf = (run) => run.then(() => undefined, () => undefined);
|
|
216
|
+
/** This function starts a write after each earlier write and change read of the collection. */
|
|
217
|
+
function enqueueWrite(db, name, task) {
|
|
218
|
+
const { queues, collection, queue } = queueOf(db, name);
|
|
219
|
+
const run = queue.all.then(task);
|
|
220
|
+
const settled = settledOf(run);
|
|
221
|
+
queue.all = settled;
|
|
222
|
+
queue.writes = settled;
|
|
223
|
+
dropWhenLast(queues, collection, queue, settled);
|
|
224
|
+
return run;
|
|
225
|
+
}
|
|
226
|
+
/** This function starts a change read after each earlier write of the collection, next to the other change reads. */
|
|
227
|
+
function enqueueRead(db, name, task) {
|
|
228
|
+
const { queues, collection, queue } = queueOf(db, name);
|
|
229
|
+
const owner = ownerOf(db);
|
|
230
|
+
let reads = READS.get(owner);
|
|
231
|
+
if (!reads)
|
|
232
|
+
READS.set(owner, (reads = new Semaphore(poolShare(db, CHANGE_READ_DIVISOR))));
|
|
233
|
+
const permits = reads;
|
|
234
|
+
const run = queue.writes.then(() => permits.withPermit(task));
|
|
235
|
+
const end = Promise.all([queue.all, settledOf(run)]);
|
|
236
|
+
queue.all = end;
|
|
237
|
+
dropWhenLast(queues, collection, queue, end);
|
|
194
238
|
return run;
|
|
195
239
|
}
|
|
196
240
|
/**
|
|
@@ -805,18 +849,20 @@ export class MongoDocumentStorage {
|
|
|
805
849
|
}
|
|
806
850
|
/**
|
|
807
851
|
* The watermark of the collection: the highest number below each pending number of a write whose
|
|
808
|
-
* lease has not ended, and the
|
|
852
|
+
* lease has not ended, the epoch, and the floor of the partition. Each number up to it is written
|
|
853
|
+
* or abandoned. One command reads them. Only the first read of a collection writes: it makes the counter.
|
|
809
854
|
*/
|
|
810
855
|
async watermark(db) {
|
|
811
856
|
const counters = db.collection(this.countersName);
|
|
812
|
-
|
|
813
|
-
const
|
|
857
|
+
const floorId = this.floorId();
|
|
858
|
+
const read = () => counters
|
|
814
859
|
.aggregate([
|
|
815
|
-
{ $match: { _id: this.collectionName } },
|
|
860
|
+
{ $match: { _id: { $in: [this.collectionName, floorId] } } },
|
|
816
861
|
{
|
|
817
862
|
$project: {
|
|
818
863
|
seq: 1,
|
|
819
864
|
epoch: 1,
|
|
865
|
+
floor: 1,
|
|
820
866
|
low: {
|
|
821
867
|
$min: {
|
|
822
868
|
$map: {
|
|
@@ -831,10 +877,21 @@ export class MongoDocumentStorage {
|
|
|
831
877
|
},
|
|
832
878
|
])
|
|
833
879
|
.toArray();
|
|
880
|
+
let docs = await read();
|
|
881
|
+
if (!docs.some((doc) => doc._id === this.collectionName)) {
|
|
882
|
+
// A duplicate key: another read made the counter at the same time
|
|
883
|
+
await counters.updateOne({ _id: this.collectionName }, { $setOnInsert: { seq: 0, epoch: randomUUID() } }, { upsert: true }).catch((error) => {
|
|
884
|
+
if (!isDuplicateKey(error))
|
|
885
|
+
throw error;
|
|
886
|
+
});
|
|
887
|
+
docs = await read();
|
|
888
|
+
}
|
|
889
|
+
const mark = docs.find((doc) => doc._id === this.collectionName);
|
|
834
890
|
if (!mark)
|
|
835
891
|
throw new Error(`The sequence of ${this.collectionName} is not available`);
|
|
836
892
|
const seq = typeof mark.low === "number" ? Math.min(mark.seq, mark.low - 1) : mark.seq;
|
|
837
|
-
|
|
893
|
+
const floor = docs.find((doc) => doc._id === floorId)?.floor;
|
|
894
|
+
return { seq, epoch: mark.epoch, floor: typeof floor === "number" ? floor : 0, held: seq < mark.seq };
|
|
838
895
|
}
|
|
839
896
|
/** The id of the document of the oldest kept change of the partition. */
|
|
840
897
|
async floorOf(db) {
|
|
@@ -1011,21 +1068,26 @@ export class MongoDocumentStorage {
|
|
|
1011
1068
|
}
|
|
1012
1069
|
async getChangesSince(since) {
|
|
1013
1070
|
const { db, items } = await this.collection();
|
|
1014
|
-
return
|
|
1071
|
+
return enqueueRead(db, this.collectionName, async () => {
|
|
1015
1072
|
const counter = await this.watermark(db);
|
|
1016
1073
|
const version = `${counter.epoch}.${counter.seq}`;
|
|
1074
|
+
const held = counter.held ? { held: true } : {};
|
|
1017
1075
|
const full = async () => ({
|
|
1018
1076
|
version,
|
|
1019
1077
|
full: true,
|
|
1020
1078
|
changed: this.decodeAll(await items.find(this.liveFilter()).sort({ [ORD]: 1, _id: 1 }).toArray()),
|
|
1021
1079
|
deleted: [],
|
|
1080
|
+
...held,
|
|
1022
1081
|
});
|
|
1023
1082
|
const match = since === undefined ? null : /^([^.]+)\.(\d+)$/.exec(since);
|
|
1024
1083
|
if (!match || match[1] !== counter.epoch)
|
|
1025
1084
|
return full();
|
|
1026
1085
|
const from = Number(match[2]);
|
|
1027
|
-
if (from <
|
|
1086
|
+
if (from < counter.floor || from > counter.seq)
|
|
1028
1087
|
return full();
|
|
1088
|
+
// No number is above this version: the read needs no query of the items
|
|
1089
|
+
if (from === counter.seq)
|
|
1090
|
+
return { version, full: false, changed: [], deleted: [], ...held };
|
|
1029
1091
|
// A number above the watermark can belong to a write that is not visible yet: the next read gives it
|
|
1030
1092
|
const docs = await items
|
|
1031
1093
|
.find({ ...this.partitionFilter(), [SEQ]: { $gt: from, $lte: counter.seq } })
|
|
@@ -1045,7 +1107,7 @@ export class MongoDocumentStorage {
|
|
|
1045
1107
|
else
|
|
1046
1108
|
changed.push(this.decode(doc, id));
|
|
1047
1109
|
}
|
|
1048
|
-
return { version, full: false, changed, deleted };
|
|
1110
|
+
return { version, full: false, changed, deleted, ...held };
|
|
1049
1111
|
});
|
|
1050
1112
|
}
|
|
1051
1113
|
// ===========================================================================
|
|
@@ -1053,7 +1115,7 @@ export class MongoDocumentStorage {
|
|
|
1053
1115
|
// ===========================================================================
|
|
1054
1116
|
async write(task) {
|
|
1055
1117
|
const { db, items } = await this.collection();
|
|
1056
|
-
return
|
|
1118
|
+
return enqueueWrite(db, this.collectionName, async () => {
|
|
1057
1119
|
// The floor document of the partition, outside each transaction of the task (roadmap O66)
|
|
1058
1120
|
await this.noteFloor(db);
|
|
1059
1121
|
return task(db, items);
|
|
@@ -1079,27 +1141,43 @@ export class MongoDocumentStorage {
|
|
|
1079
1141
|
JSON.stringify(value);
|
|
1080
1142
|
}
|
|
1081
1143
|
await this.write(async (db, collection) => {
|
|
1082
|
-
const
|
|
1144
|
+
const ids = items.map(([id]) => id);
|
|
1145
|
+
const places = await this.placesOf(collection, ids);
|
|
1083
1146
|
await this.withNumbers(db, collection, items.length, async (first, session) => {
|
|
1084
1147
|
const guards = await this.checkVersions(collection, expected, session);
|
|
1085
|
-
const
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
placed.
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1148
|
+
const operationsAt = (known) => {
|
|
1149
|
+
const placed = new Map(known);
|
|
1150
|
+
return items.map(([id, value], index) => {
|
|
1151
|
+
const seq = first + index;
|
|
1152
|
+
// A new id takes its place at its first write; a later write of the batch keeps it
|
|
1153
|
+
if (!placed.has(id))
|
|
1154
|
+
placed.set(id, seq);
|
|
1155
|
+
// The guard of an id with a precondition: the write fails when another writer came first
|
|
1156
|
+
const guard = guards.get(id);
|
|
1157
|
+
guards.delete(id);
|
|
1158
|
+
return { replaceOne: { filter: { ...this.writeFilter(id), ...guard }, replacement: this.encode(id, value, seq, placed.get(id)), upsert: true } };
|
|
1159
|
+
});
|
|
1160
|
+
};
|
|
1161
|
+
// The race of a new id: another process made it after the query of this upsert, so a new try from the
|
|
1162
|
+
// stopped write is a rewrite, with the same number. These numbers stay pending, so no reader passes them.
|
|
1163
|
+
// A real duplicate stops one write two times, and it reaches the caller. So a batch has at most one try more than it has writes.
|
|
1164
|
+
let known = places;
|
|
1165
|
+
for (let from = 0, tries = 0;; tries++) {
|
|
1166
|
+
try {
|
|
1167
|
+
await collection.bulkWrite(operationsAt(known).slice(from), { ordered: true, ...(session ? { session } : {}) });
|
|
1168
|
+
return;
|
|
1169
|
+
}
|
|
1170
|
+
catch (error) {
|
|
1171
|
+
if (session || !isDuplicateKey(error))
|
|
1172
|
+
throw error;
|
|
1173
|
+
if (expected.length > 0)
|
|
1174
|
+
throw await this.lateConflict(collection, expected);
|
|
1175
|
+
const at = duplicateIndex(error);
|
|
1176
|
+
if (at === undefined || (tries > 0 && at === 0))
|
|
1177
|
+
throw error;
|
|
1178
|
+
from += at;
|
|
1179
|
+
known = await this.placesOf(collection, ids);
|
|
1180
|
+
}
|
|
1103
1181
|
}
|
|
1104
1182
|
});
|
|
1105
1183
|
});
|
|
@@ -1112,7 +1190,18 @@ export class MongoDocumentStorage {
|
|
|
1112
1190
|
const next = applyPatch(this.decode(doc, id)[1], fields);
|
|
1113
1191
|
// The encoding is checked before a number of the sequence goes
|
|
1114
1192
|
this.flatOf(id, next);
|
|
1115
|
-
await this.withNumbers(db, collection, 1, (first, session) =>
|
|
1193
|
+
await this.withNumbers(db, collection, 1, async (first, session) => {
|
|
1194
|
+
const write = () => collection.replaceOne(this.writeFilter(id), this.encode(id, next, first, this.placeOf(doc)), { upsert: true, ...(session ? { session } : {}) });
|
|
1195
|
+
try {
|
|
1196
|
+
await write();
|
|
1197
|
+
}
|
|
1198
|
+
catch (error) {
|
|
1199
|
+
// The document went after the read, and another process made it again: one new try is a rewrite
|
|
1200
|
+
if (session || !isDuplicateKey(error))
|
|
1201
|
+
throw error;
|
|
1202
|
+
await write();
|
|
1203
|
+
}
|
|
1204
|
+
});
|
|
1116
1205
|
return next;
|
|
1117
1206
|
});
|
|
1118
1207
|
}
|