@semics-tech/mongolite 1.3.0 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +79 -21
- package/dist/cjs/collection.js +8 -1
- package/dist/cjs/index.js +108 -1
- package/dist/cjs/mongo-client.js +18 -1
- package/dist/cjs/server.js +164 -0
- package/dist/cjs/sync/commands.js +126 -0
- package/dist/cjs/sync/diff.js +174 -0
- package/dist/cjs/sync/httpSink.js +159 -0
- package/dist/cjs/sync/index.js +40 -0
- package/dist/cjs/sync/mongoExecutor.js +306 -0
- package/dist/cjs/sync/mongoSink.js +161 -0
- package/dist/cjs/sync/outbox.js +358 -0
- package/dist/cjs/sync/protocol.js +177 -0
- package/dist/cjs/sync/replicator.js +747 -0
- package/dist/cjs/sync/shadow.js +178 -0
- package/dist/cjs/sync/types.js +2 -0
- package/dist/cloudflare.d.ts +1 -1
- package/dist/cloudflare.js.map +1 -1
- package/dist/collection.js +8 -1
- package/dist/collection.js.map +1 -1
- package/dist/index.d.ts +68 -2
- package/dist/index.js +96 -1
- package/dist/index.js.map +1 -1
- package/dist/mongo-client.d.ts +13 -0
- package/dist/mongo-client.js +18 -1
- package/dist/mongo-client.js.map +1 -1
- package/dist/server.d.ts +97 -0
- package/dist/server.js +158 -0
- package/dist/server.js.map +1 -0
- package/dist/sync/commands.d.ts +49 -0
- package/dist/sync/commands.js +118 -0
- package/dist/sync/commands.js.map +1 -0
- package/dist/sync/diff.d.ts +55 -0
- package/dist/sync/diff.js +169 -0
- package/dist/sync/diff.js.map +1 -0
- package/dist/sync/httpSink.d.ts +93 -0
- package/dist/sync/httpSink.js +155 -0
- package/dist/sync/httpSink.js.map +1 -0
- package/dist/sync/index.d.ts +17 -0
- package/dist/sync/index.js +10 -0
- package/dist/sync/index.js.map +1 -0
- package/dist/sync/mongoExecutor.d.ts +37 -0
- package/dist/sync/mongoExecutor.js +302 -0
- package/dist/sync/mongoExecutor.js.map +1 -0
- package/dist/sync/mongoSink.d.ts +109 -0
- package/dist/sync/mongoSink.js +123 -0
- package/dist/sync/mongoSink.js.map +1 -0
- package/dist/sync/outbox.d.ts +120 -0
- package/dist/sync/outbox.js +355 -0
- package/dist/sync/outbox.js.map +1 -0
- package/dist/sync/protocol.d.ts +93 -0
- package/dist/sync/protocol.js +168 -0
- package/dist/sync/protocol.js.map +1 -0
- package/dist/sync/replicator.d.ts +177 -0
- package/dist/sync/replicator.js +744 -0
- package/dist/sync/replicator.js.map +1 -0
- package/dist/sync/shadow.d.ts +54 -0
- package/dist/sync/shadow.js +173 -0
- package/dist/sync/shadow.js.map +1 -0
- package/dist/sync/types.d.ts +388 -0
- package/dist/sync/types.js +2 -0
- package/dist/sync/types.js.map +1 -0
- package/package.json +15 -1
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Field-level diffing between the last-known server state and the local document.
|
|
4
|
+
*
|
|
5
|
+
* Replication used to push whole documents, which meant a local edit to one field
|
|
6
|
+
* rewrote every other field too — including fields another writer had just changed,
|
|
7
|
+
* and including fields whose BSON type the local store cannot represent. Diffing
|
|
8
|
+
* narrows each push to `$set`/`$unset` of what genuinely changed, so everything else
|
|
9
|
+
* upstream is left exactly as it was.
|
|
10
|
+
*/
|
|
11
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
+
exports.deepEqual = deepEqual;
|
|
13
|
+
exports.computeDiff = computeDiff;
|
|
14
|
+
exports.isEmptyDiff = isEmptyDiff;
|
|
15
|
+
exports.applyDiff = applyDiff;
|
|
16
|
+
/**
|
|
17
|
+
* True when `value` is a genuine plain object worth recursing into.
|
|
18
|
+
*
|
|
19
|
+
* Class instances are deliberately excluded. A `Date`, `ObjectId` or `Binary` is
|
|
20
|
+
* `typeof 'object'`, and treating one as a bag of fields would copy it into a plain
|
|
21
|
+
* object — destroying exactly the BSON types the shadow exists to preserve.
|
|
22
|
+
*/
|
|
23
|
+
function isPlainObject(value) {
|
|
24
|
+
if (typeof value !== 'object' || value === null || Array.isArray(value))
|
|
25
|
+
return false;
|
|
26
|
+
const proto = Object.getPrototypeOf(value);
|
|
27
|
+
return proto === Object.prototype || proto === null;
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* Structural equality, insensitive to key order.
|
|
31
|
+
*
|
|
32
|
+
* `JSON.stringify` comparison would report `{a:1,b:2}` and `{b:2,a:1}` as different and
|
|
33
|
+
* push a spurious update on every sync.
|
|
34
|
+
*/
|
|
35
|
+
function deepEqual(a, b) {
|
|
36
|
+
if (a === b)
|
|
37
|
+
return true;
|
|
38
|
+
// NaN is the one primitive that isn't equal to itself.
|
|
39
|
+
if (typeof a === 'number' && typeof b === 'number')
|
|
40
|
+
return Number.isNaN(a) && Number.isNaN(b);
|
|
41
|
+
// Two equal instants are separate objects; without this every sync would rewrite
|
|
42
|
+
// any date-valued field.
|
|
43
|
+
if (a instanceof Date || b instanceof Date) {
|
|
44
|
+
return a instanceof Date && b instanceof Date && a.getTime() === b.getTime();
|
|
45
|
+
}
|
|
46
|
+
if (Array.isArray(a) || Array.isArray(b)) {
|
|
47
|
+
if (!Array.isArray(a) || !Array.isArray(b) || a.length !== b.length)
|
|
48
|
+
return false;
|
|
49
|
+
return a.every((item, index) => deepEqual(item, b[index]));
|
|
50
|
+
}
|
|
51
|
+
if (isPlainObject(a) && isPlainObject(b)) {
|
|
52
|
+
const aKeys = Object.keys(a);
|
|
53
|
+
const bKeys = Object.keys(b);
|
|
54
|
+
if (aKeys.length !== bKeys.length)
|
|
55
|
+
return false;
|
|
56
|
+
return aKeys.every((key) => Object.prototype.hasOwnProperty.call(b, key) && deepEqual(a[key], b[key]));
|
|
57
|
+
}
|
|
58
|
+
return false;
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* Computes the minimal update that turns `base` into `local`.
|
|
62
|
+
*
|
|
63
|
+
* `base` must be the **lossy JSON projection** of the server document — the same shape
|
|
64
|
+
* the local store would have produced for it (see `projectToLocalShape` in `shadow.ts`).
|
|
65
|
+
* That is what makes type preservation work: a server `Date` projects to the same ISO
|
|
66
|
+
* string the local store holds, compares equal, and so never appears in the diff — the
|
|
67
|
+
* upstream `Date` survives untouched.
|
|
68
|
+
*
|
|
69
|
+
* Nested objects recurse into dotted paths, so two writers editing different sub-fields
|
|
70
|
+
* of the same document do not collide.
|
|
71
|
+
*
|
|
72
|
+
* **Arrays are atomic.** Any change to an array replaces the whole array. Positional
|
|
73
|
+
* diffing of arrays is famously error-prone once elements are inserted or reordered, and
|
|
74
|
+
* a wrong answer here silently corrupts data — so this deliberately does not attempt it.
|
|
75
|
+
*
|
|
76
|
+
* @param base The last-known server state, projected to local shape. `null` if the
|
|
77
|
+
* document has never been upstream, in which case every field is new.
|
|
78
|
+
* @param local The current local document.
|
|
79
|
+
*/
|
|
80
|
+
function computeDiff(base, local) {
|
|
81
|
+
const diff = { set: {}, unset: [] };
|
|
82
|
+
collect(base ?? {}, local, '', diff);
|
|
83
|
+
return diff;
|
|
84
|
+
}
|
|
85
|
+
function collect(base, local, prefix, diff) {
|
|
86
|
+
const keys = new Set([...Object.keys(base), ...Object.keys(local)]);
|
|
87
|
+
for (const key of keys) {
|
|
88
|
+
// `_id` is the identity, never part of the update.
|
|
89
|
+
if (prefix === '' && key === '_id')
|
|
90
|
+
continue;
|
|
91
|
+
const path = prefix === '' ? key : `${prefix}.${key}`;
|
|
92
|
+
const inLocal = Object.prototype.hasOwnProperty.call(local, key);
|
|
93
|
+
const inBase = Object.prototype.hasOwnProperty.call(base, key);
|
|
94
|
+
if (!inLocal) {
|
|
95
|
+
diff.unset.push(path);
|
|
96
|
+
continue;
|
|
97
|
+
}
|
|
98
|
+
const localValue = local[key];
|
|
99
|
+
const baseValue = base[key];
|
|
100
|
+
if (!inBase) {
|
|
101
|
+
diff.set[path] = localValue;
|
|
102
|
+
continue;
|
|
103
|
+
}
|
|
104
|
+
// Recurse only when both sides are objects; otherwise the shapes differ and the
|
|
105
|
+
// whole subtree is replaced.
|
|
106
|
+
if (isPlainObject(baseValue) && isPlainObject(localValue)) {
|
|
107
|
+
collect(baseValue, localValue, path, diff);
|
|
108
|
+
continue;
|
|
109
|
+
}
|
|
110
|
+
if (!deepEqual(baseValue, localValue)) {
|
|
111
|
+
diff.set[path] = localValue;
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
/** True when the diff would change nothing upstream. */
|
|
116
|
+
function isEmptyDiff(diff) {
|
|
117
|
+
return Object.keys(diff.set).length === 0 && diff.unset.length === 0;
|
|
118
|
+
}
|
|
119
|
+
/**
|
|
120
|
+
* Applies a diff to a copy of `document`, mirroring what the upstream `$set`/`$unset`
|
|
121
|
+
* does.
|
|
122
|
+
*
|
|
123
|
+
* Used to roll the shadow forward after a successful push without a second round trip.
|
|
124
|
+
* Because it edits the *server* document in place, fields the diff never mentions keep
|
|
125
|
+
* their original BSON values — which is the whole point of diffing in the first place.
|
|
126
|
+
*/
|
|
127
|
+
function applyDiff(document, diff) {
|
|
128
|
+
const result = clone(document);
|
|
129
|
+
for (const [path, value] of Object.entries(diff.set)) {
|
|
130
|
+
setPath(result, path.split('.'), value);
|
|
131
|
+
}
|
|
132
|
+
for (const path of diff.unset) {
|
|
133
|
+
unsetPath(result, path.split('.'));
|
|
134
|
+
}
|
|
135
|
+
return result;
|
|
136
|
+
}
|
|
137
|
+
/**
|
|
138
|
+
* Structured clone that leaves class instances (BSON `Date`, `ObjectId`, `Binary`, …)
|
|
139
|
+
* by reference. Copying them field-by-field would destroy exactly the types the shadow
|
|
140
|
+
* exists to preserve.
|
|
141
|
+
*/
|
|
142
|
+
function clone(value) {
|
|
143
|
+
if (Array.isArray(value))
|
|
144
|
+
return value.map(clone);
|
|
145
|
+
if (isPlainObject(value)) {
|
|
146
|
+
const out = {};
|
|
147
|
+
for (const [key, item] of Object.entries(value))
|
|
148
|
+
out[key] = clone(item);
|
|
149
|
+
return out;
|
|
150
|
+
}
|
|
151
|
+
return value;
|
|
152
|
+
}
|
|
153
|
+
function setPath(target, segments, value) {
|
|
154
|
+
const [head, ...rest] = segments;
|
|
155
|
+
if (rest.length === 0) {
|
|
156
|
+
target[head] = value;
|
|
157
|
+
return;
|
|
158
|
+
}
|
|
159
|
+
const next = target[head];
|
|
160
|
+
// Mirror MongoDB: a dotted path creates intermediate objects as needed.
|
|
161
|
+
const container = isPlainObject(next) ? next : {};
|
|
162
|
+
target[head] = container;
|
|
163
|
+
setPath(container, rest, value);
|
|
164
|
+
}
|
|
165
|
+
function unsetPath(target, segments) {
|
|
166
|
+
const [head, ...rest] = segments;
|
|
167
|
+
if (rest.length === 0) {
|
|
168
|
+
delete target[head];
|
|
169
|
+
return;
|
|
170
|
+
}
|
|
171
|
+
const next = target[head];
|
|
172
|
+
if (isPlainObject(next))
|
|
173
|
+
unsetPath(next, rest);
|
|
174
|
+
}
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.SyncRequestRejected = exports.HttpUpstreamSink = void 0;
|
|
4
|
+
/**
|
|
5
|
+
* A {@link SyncSink} that ships changes to a remote HTTP API instead of connecting to
|
|
6
|
+
* MongoDB directly, for deployments where the local instance cannot reach the database.
|
|
7
|
+
*
|
|
8
|
+
* The API receives MongoDB-shaped commands and applies them on the client's behalf —
|
|
9
|
+
* see `@semics-tech/mongolite/server` for a receiver that does exactly that.
|
|
10
|
+
*
|
|
11
|
+
* Version predicates travel with each operation and conflicts come back in the response,
|
|
12
|
+
* so replication behaves the same over HTTP as it does over a direct connection: the
|
|
13
|
+
* upstream stays the source of truth, and a concurrent writer is detected rather than
|
|
14
|
+
* overwritten.
|
|
15
|
+
*/
|
|
16
|
+
const commands_js_1 = require("./commands.js");
|
|
17
|
+
const protocol_js_1 = require("./protocol.js");
|
|
18
|
+
class HttpUpstreamSink {
|
|
19
|
+
constructor(options) {
|
|
20
|
+
this.options = options;
|
|
21
|
+
this.name = 'http';
|
|
22
|
+
this.cookie = null;
|
|
23
|
+
if (!options.baseUrl)
|
|
24
|
+
throw new Error('HttpUpstreamSink requires a `baseUrl`.');
|
|
25
|
+
if (!options.database)
|
|
26
|
+
throw new Error('HttpUpstreamSink requires a `database`.');
|
|
27
|
+
const globalFetch = globalThis.fetch;
|
|
28
|
+
if (!options.fetch && typeof globalFetch !== 'function') {
|
|
29
|
+
throw new Error('HttpUpstreamSink needs a `fetch` implementation. Node 18+ provides one globally; ' +
|
|
30
|
+
'on older runtimes pass `fetch` explicitly.');
|
|
31
|
+
}
|
|
32
|
+
this.fetchImpl = options.fetch ?? globalFetch;
|
|
33
|
+
this.idMapping = options.idMapping ?? 'auto';
|
|
34
|
+
this.versioning = options.versioning ?? true;
|
|
35
|
+
}
|
|
36
|
+
async apply(operations) {
|
|
37
|
+
const includeCommands = this.options.includeCommands ?? true;
|
|
38
|
+
const request = {
|
|
39
|
+
protocol: protocol_js_1.SYNC_PROTOCOL_VERSION,
|
|
40
|
+
replicator: this.options.replicator,
|
|
41
|
+
operations: operations.map((op) => (0, protocol_js_1.toWireOperation)(op, includeCommands
|
|
42
|
+
? (0, commands_js_1.buildWriteCommand)(op, { idMapping: this.idMapping, versioning: this.versioning })
|
|
43
|
+
: undefined)),
|
|
44
|
+
};
|
|
45
|
+
let response;
|
|
46
|
+
try {
|
|
47
|
+
response = await this.send('_sync', request);
|
|
48
|
+
}
|
|
49
|
+
catch (err) {
|
|
50
|
+
// A request the API rejects outright will be rejected again on every retry, so
|
|
51
|
+
// it is reported as a per-operation failure — dead-lettered, not spun on. The
|
|
52
|
+
// sink's contract is that throwing means "transient", and this is not that.
|
|
53
|
+
if (err instanceof SyncRequestRejected) {
|
|
54
|
+
return {
|
|
55
|
+
applied: 0,
|
|
56
|
+
failures: operations.map((_op, index) => ({
|
|
57
|
+
index,
|
|
58
|
+
message: err.message,
|
|
59
|
+
code: err.status,
|
|
60
|
+
})),
|
|
61
|
+
};
|
|
62
|
+
}
|
|
63
|
+
throw err;
|
|
64
|
+
}
|
|
65
|
+
return {
|
|
66
|
+
applied: response.applied ?? 0,
|
|
67
|
+
failures: response.failures,
|
|
68
|
+
conflicts: response.conflicts,
|
|
69
|
+
};
|
|
70
|
+
}
|
|
71
|
+
async fetch(collection, documentIds) {
|
|
72
|
+
if (documentIds.length === 0)
|
|
73
|
+
return [];
|
|
74
|
+
const response = await this.send('_sync/fetch', {
|
|
75
|
+
protocol: protocol_js_1.SYNC_PROTOCOL_VERSION,
|
|
76
|
+
collection,
|
|
77
|
+
documentIds,
|
|
78
|
+
});
|
|
79
|
+
return response.documents ?? [];
|
|
80
|
+
}
|
|
81
|
+
// ---------------------------------------------------------------- internals
|
|
82
|
+
endpoint(path) {
|
|
83
|
+
const base = this.options.baseUrl.replace(/\/+$/, '');
|
|
84
|
+
const prefix = (this.options.pathPrefix ?? 'sync').replace(/^\/+|\/+$/g, '');
|
|
85
|
+
return `${base}/${prefix}/${encodeURIComponent(this.options.database)}/${path}`;
|
|
86
|
+
}
|
|
87
|
+
async requestHeaders() {
|
|
88
|
+
const headers = {
|
|
89
|
+
'content-type': protocol_js_1.SYNC_CONTENT_TYPE,
|
|
90
|
+
accept: protocol_js_1.SYNC_CONTENT_TYPE,
|
|
91
|
+
...this.options.headers,
|
|
92
|
+
...(this.options.getAuthHeaders ? await this.options.getAuthHeaders() : {}),
|
|
93
|
+
};
|
|
94
|
+
if (this.cookie && (this.options.cookies ?? true))
|
|
95
|
+
headers.cookie = this.cookie;
|
|
96
|
+
return headers;
|
|
97
|
+
}
|
|
98
|
+
/**
|
|
99
|
+
* Posts a body and decodes the reply.
|
|
100
|
+
*
|
|
101
|
+
* Retries are deliberately *not* handled here — the replicator already owns backoff,
|
|
102
|
+
* outage buffering and checkpointing. This throws on anything retryable and lets it do
|
|
103
|
+
* its job; throwing twice over would compound the delays.
|
|
104
|
+
*/
|
|
105
|
+
async send(path, body) {
|
|
106
|
+
const timeoutMs = this.options.timeoutMs ?? 30000;
|
|
107
|
+
const controller = new AbortController();
|
|
108
|
+
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
109
|
+
let response;
|
|
110
|
+
try {
|
|
111
|
+
response = await this.fetchImpl(this.endpoint(path), {
|
|
112
|
+
method: 'POST',
|
|
113
|
+
headers: await this.requestHeaders(),
|
|
114
|
+
body: (0, protocol_js_1.encodeBody)(body),
|
|
115
|
+
signal: controller.signal,
|
|
116
|
+
});
|
|
117
|
+
}
|
|
118
|
+
catch (err) {
|
|
119
|
+
// Network failure, DNS, timeout — all transient as far as we can tell.
|
|
120
|
+
throw new Error(`Sync request to ${this.endpoint(path)} failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
121
|
+
}
|
|
122
|
+
finally {
|
|
123
|
+
clearTimeout(timer);
|
|
124
|
+
}
|
|
125
|
+
if (this.options.cookies ?? true) {
|
|
126
|
+
const setCookie = response.headers.get('set-cookie');
|
|
127
|
+
if (setCookie)
|
|
128
|
+
this.cookie = setCookie;
|
|
129
|
+
}
|
|
130
|
+
const text = await response.text();
|
|
131
|
+
if (!response.ok) {
|
|
132
|
+
const detail = text.slice(0, 500);
|
|
133
|
+
// 4xx means the request itself is wrong and will stay wrong; throwing would spin
|
|
134
|
+
// forever. Surfacing it as a per-operation failure dead-letters the batch instead.
|
|
135
|
+
// 408 and 429 are the exceptions — they explicitly invite a retry.
|
|
136
|
+
const retryable = response.status >= 500 || response.status === 408 || response.status === 429;
|
|
137
|
+
if (retryable) {
|
|
138
|
+
throw new Error(`Sync request failed with ${response.status}: ${detail}`);
|
|
139
|
+
}
|
|
140
|
+
throw new SyncRequestRejected(`Sync request rejected with ${response.status}: ${detail}`, response.status);
|
|
141
|
+
}
|
|
142
|
+
return (0, protocol_js_1.decodeBody)(text);
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
exports.HttpUpstreamSink = HttpUpstreamSink;
|
|
146
|
+
/**
|
|
147
|
+
* The remote API refused the request outright.
|
|
148
|
+
*
|
|
149
|
+
* Distinct from a transport failure: retrying will produce the same refusal, so the
|
|
150
|
+
* batch is dead-lettered rather than retried.
|
|
151
|
+
*/
|
|
152
|
+
class SyncRequestRejected extends Error {
|
|
153
|
+
constructor(message, status) {
|
|
154
|
+
super(message);
|
|
155
|
+
this.status = status;
|
|
156
|
+
this.name = 'SyncRequestRejected';
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
exports.SyncRequestRejected = SyncRequestRejected;
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.deepEqual = exports.isEmptyDiff = exports.applyDiff = exports.computeDiff = exports.projectToLocalShape = exports.SHADOW_TABLE = exports.SyncShadow = exports.fromWireOperation = exports.toWireOperation = exports.decodeBody = exports.encodeBody = exports.SyncProtocolError = exports.SYNC_PROTOCOL_VERSION = exports.fetchDocuments = exports.executeWrites = exports.toUpstreamId = exports.buildWriteCommand = exports.SyncRequestRejected = exports.HttpUpstreamSink = exports.UPDATED_AT_FIELD = exports.VERSION_FIELD = exports.MongoUpstreamSink = exports.SYNC_OBJECT_PREFIX = exports.DEAD_LETTER_TABLE = exports.STATE_TABLE = exports.OUTBOX_TABLE = exports.SyncOutbox = exports.SyncReplicator = void 0;
|
|
4
|
+
var replicator_js_1 = require("./replicator.js");
|
|
5
|
+
Object.defineProperty(exports, "SyncReplicator", { enumerable: true, get: function () { return replicator_js_1.SyncReplicator; } });
|
|
6
|
+
var outbox_js_1 = require("./outbox.js");
|
|
7
|
+
Object.defineProperty(exports, "SyncOutbox", { enumerable: true, get: function () { return outbox_js_1.SyncOutbox; } });
|
|
8
|
+
Object.defineProperty(exports, "OUTBOX_TABLE", { enumerable: true, get: function () { return outbox_js_1.OUTBOX_TABLE; } });
|
|
9
|
+
Object.defineProperty(exports, "STATE_TABLE", { enumerable: true, get: function () { return outbox_js_1.STATE_TABLE; } });
|
|
10
|
+
Object.defineProperty(exports, "DEAD_LETTER_TABLE", { enumerable: true, get: function () { return outbox_js_1.DEAD_LETTER_TABLE; } });
|
|
11
|
+
Object.defineProperty(exports, "SYNC_OBJECT_PREFIX", { enumerable: true, get: function () { return outbox_js_1.SYNC_OBJECT_PREFIX; } });
|
|
12
|
+
var mongoSink_js_1 = require("./mongoSink.js");
|
|
13
|
+
Object.defineProperty(exports, "MongoUpstreamSink", { enumerable: true, get: function () { return mongoSink_js_1.MongoUpstreamSink; } });
|
|
14
|
+
Object.defineProperty(exports, "VERSION_FIELD", { enumerable: true, get: function () { return mongoSink_js_1.VERSION_FIELD; } });
|
|
15
|
+
Object.defineProperty(exports, "UPDATED_AT_FIELD", { enumerable: true, get: function () { return mongoSink_js_1.UPDATED_AT_FIELD; } });
|
|
16
|
+
var httpSink_js_1 = require("./httpSink.js");
|
|
17
|
+
Object.defineProperty(exports, "HttpUpstreamSink", { enumerable: true, get: function () { return httpSink_js_1.HttpUpstreamSink; } });
|
|
18
|
+
Object.defineProperty(exports, "SyncRequestRejected", { enumerable: true, get: function () { return httpSink_js_1.SyncRequestRejected; } });
|
|
19
|
+
var commands_js_1 = require("./commands.js");
|
|
20
|
+
Object.defineProperty(exports, "buildWriteCommand", { enumerable: true, get: function () { return commands_js_1.buildWriteCommand; } });
|
|
21
|
+
Object.defineProperty(exports, "toUpstreamId", { enumerable: true, get: function () { return commands_js_1.toUpstreamId; } });
|
|
22
|
+
var mongoExecutor_js_1 = require("./mongoExecutor.js");
|
|
23
|
+
Object.defineProperty(exports, "executeWrites", { enumerable: true, get: function () { return mongoExecutor_js_1.executeWrites; } });
|
|
24
|
+
Object.defineProperty(exports, "fetchDocuments", { enumerable: true, get: function () { return mongoExecutor_js_1.fetchDocuments; } });
|
|
25
|
+
var protocol_js_1 = require("./protocol.js");
|
|
26
|
+
Object.defineProperty(exports, "SYNC_PROTOCOL_VERSION", { enumerable: true, get: function () { return protocol_js_1.SYNC_PROTOCOL_VERSION; } });
|
|
27
|
+
Object.defineProperty(exports, "SyncProtocolError", { enumerable: true, get: function () { return protocol_js_1.SyncProtocolError; } });
|
|
28
|
+
Object.defineProperty(exports, "encodeBody", { enumerable: true, get: function () { return protocol_js_1.encodeBody; } });
|
|
29
|
+
Object.defineProperty(exports, "decodeBody", { enumerable: true, get: function () { return protocol_js_1.decodeBody; } });
|
|
30
|
+
Object.defineProperty(exports, "toWireOperation", { enumerable: true, get: function () { return protocol_js_1.toWireOperation; } });
|
|
31
|
+
Object.defineProperty(exports, "fromWireOperation", { enumerable: true, get: function () { return protocol_js_1.fromWireOperation; } });
|
|
32
|
+
var shadow_js_1 = require("./shadow.js");
|
|
33
|
+
Object.defineProperty(exports, "SyncShadow", { enumerable: true, get: function () { return shadow_js_1.SyncShadow; } });
|
|
34
|
+
Object.defineProperty(exports, "SHADOW_TABLE", { enumerable: true, get: function () { return shadow_js_1.SHADOW_TABLE; } });
|
|
35
|
+
Object.defineProperty(exports, "projectToLocalShape", { enumerable: true, get: function () { return shadow_js_1.projectToLocalShape; } });
|
|
36
|
+
var diff_js_1 = require("./diff.js");
|
|
37
|
+
Object.defineProperty(exports, "computeDiff", { enumerable: true, get: function () { return diff_js_1.computeDiff; } });
|
|
38
|
+
Object.defineProperty(exports, "applyDiff", { enumerable: true, get: function () { return diff_js_1.applyDiff; } });
|
|
39
|
+
Object.defineProperty(exports, "isEmptyDiff", { enumerable: true, get: function () { return diff_js_1.isEmptyDiff; } });
|
|
40
|
+
Object.defineProperty(exports, "deepEqual", { enumerable: true, get: function () { return diff_js_1.deepEqual; } });
|
|
@@ -0,0 +1,306 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.executeWrites = executeWrites;
|
|
4
|
+
exports.classifyBulkError = classifyBulkError;
|
|
5
|
+
exports.fetchDocuments = fetchDocuments;
|
|
6
|
+
/**
|
|
7
|
+
* Applies prepared write commands to MongoDB and classifies the outcome.
|
|
8
|
+
*
|
|
9
|
+
* Shared by the direct sink and by the HTTP receiver, so the subtle part has exactly
|
|
10
|
+
* one implementation: **a lost compare-and-swap raises no error.** It is a silent
|
|
11
|
+
* zero-match in an aggregate count, which means detecting it takes deliberate work —
|
|
12
|
+
* comparing what the batch expected to change against what the server says changed, and
|
|
13
|
+
* then re-reading the candidates to find out which document actually lost.
|
|
14
|
+
*/
|
|
15
|
+
const commands_js_1 = require("./commands.js");
|
|
16
|
+
const diff_js_1 = require("./diff.js");
|
|
17
|
+
/**
|
|
18
|
+
* MongoDB server error codes that mean "this write will never succeed" — retrying them
|
|
19
|
+
* would stall the stream behind one bad document forever, so they are dead-lettered.
|
|
20
|
+
*/
|
|
21
|
+
const PERMANENT_ERROR_CODES = new Set([
|
|
22
|
+
2, // BadValue
|
|
23
|
+
9, // FailedToParse
|
|
24
|
+
10334, // BSONObjectTooLarge
|
|
25
|
+
14, // TypeMismatch
|
|
26
|
+
52, // DollarPrefixedFieldName
|
|
27
|
+
56, // EmptyFieldName
|
|
28
|
+
121, // DocumentValidationFailure
|
|
29
|
+
17280, // KeyTooLong
|
|
30
|
+
]);
|
|
31
|
+
/** MongoDB's duplicate-key code. On an insert-if-absent it means we lost a create race. */
|
|
32
|
+
const DUPLICATE_KEY = 11000;
|
|
33
|
+
/**
|
|
34
|
+
* Runs prepared writes against a database, one `bulkWrite` per collection.
|
|
35
|
+
*
|
|
36
|
+
* Throws on transient failure (network, election, timeout) so the caller retries the
|
|
37
|
+
* whole batch without checkpointing. Returns per-operation failures and conflicts for
|
|
38
|
+
* everything it could classify.
|
|
39
|
+
*/
|
|
40
|
+
async function executeWrites(db, writes, options = {}) {
|
|
41
|
+
const { writeConcern = { w: 'majority' }, versioning = true, idMapping, ObjectId } = options;
|
|
42
|
+
const byCollection = new Map();
|
|
43
|
+
for (const write of writes) {
|
|
44
|
+
const bucket = byCollection.get(write.op.collection);
|
|
45
|
+
if (bucket)
|
|
46
|
+
bucket.push(write);
|
|
47
|
+
else
|
|
48
|
+
byCollection.set(write.op.collection, [write]);
|
|
49
|
+
}
|
|
50
|
+
const failures = [];
|
|
51
|
+
const conflicts = [];
|
|
52
|
+
let applied = 0;
|
|
53
|
+
for (const [collectionName, entries] of byCollection) {
|
|
54
|
+
let result = null;
|
|
55
|
+
let rejected = 0;
|
|
56
|
+
try {
|
|
57
|
+
result = await db.collection(collectionName).bulkWrite(entries.map((entry) => entry.command), {
|
|
58
|
+
// Operations within a batch touch distinct documents, so ordering buys
|
|
59
|
+
// nothing and unordered lets the server parallelise.
|
|
60
|
+
ordered: false,
|
|
61
|
+
writeConcern,
|
|
62
|
+
});
|
|
63
|
+
}
|
|
64
|
+
catch (err) {
|
|
65
|
+
const bulkFailures = classifyBulkError(err, entries.length);
|
|
66
|
+
// A transient failure means the whole batch's fate is unknown — throw so the
|
|
67
|
+
// caller retries it rather than checkpointing work that may not have landed.
|
|
68
|
+
if (bulkFailures === null)
|
|
69
|
+
throw err;
|
|
70
|
+
for (const failure of bulkFailures) {
|
|
71
|
+
const entry = entries[failure.index];
|
|
72
|
+
if (!entry)
|
|
73
|
+
continue;
|
|
74
|
+
// A duplicate `_id` on an insert-if-absent means someone created the document
|
|
75
|
+
// first — a conflict to reconcile, not a poison document. A duplicate on any
|
|
76
|
+
// *other* unique index is a real constraint violation and must be dead-lettered,
|
|
77
|
+
// or it would be retried forever as a phantom conflict.
|
|
78
|
+
if (failure.code === DUPLICATE_KEY && isIdIndexViolation(failure)) {
|
|
79
|
+
conflicts.push({ index: entry.index, reason: 'already-exists' });
|
|
80
|
+
}
|
|
81
|
+
else {
|
|
82
|
+
failures.push({ index: entry.index, message: failure.message, code: failure.code });
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
rejected = bulkFailures.length;
|
|
86
|
+
result = err.result ?? null;
|
|
87
|
+
}
|
|
88
|
+
const landed = entries.length - rejected;
|
|
89
|
+
const suspects = findMisses(entries, result, rejected, versioning);
|
|
90
|
+
// `suspects` are candidates to investigate, not confirmed misses: the counts are
|
|
91
|
+
// aggregate, so one short count implicates every conditional write in the batch.
|
|
92
|
+
// Only the re-read says which of them actually lost, and `applied` must be reduced
|
|
93
|
+
// by that number rather than by the number investigated.
|
|
94
|
+
const confirmed = suspects.length
|
|
95
|
+
? await describeConflicts(db, collectionName, suspects, conflicts, idMapping, ObjectId)
|
|
96
|
+
: 0;
|
|
97
|
+
applied += landed - confirmed;
|
|
98
|
+
}
|
|
99
|
+
return {
|
|
100
|
+
applied,
|
|
101
|
+
failures: failures.length > 0 ? failures : undefined,
|
|
102
|
+
conflicts: conflicts.length > 0 ? conflicts : undefined,
|
|
103
|
+
};
|
|
104
|
+
}
|
|
105
|
+
/**
|
|
106
|
+
* Identifies operations whose conditional write may not have matched.
|
|
107
|
+
*
|
|
108
|
+
* `bulkWrite` reports only aggregate counts, so this compares what the batch expected
|
|
109
|
+
* to change against what the server says changed. When the numbers agree, nothing was
|
|
110
|
+
* missed and no per-document work is needed.
|
|
111
|
+
*/
|
|
112
|
+
function findMisses(entries, result, rejected, versioning) {
|
|
113
|
+
if (!versioning || result === null || rejected === entries.length)
|
|
114
|
+
return [];
|
|
115
|
+
const candidates = entries.filter((entry) => (0, commands_js_1.isConditional)(entry.op));
|
|
116
|
+
if (candidates.length === 0)
|
|
117
|
+
return [];
|
|
118
|
+
const writes = candidates.filter((entry) => entry.op.type !== 'delete');
|
|
119
|
+
const deletes = candidates.filter((entry) => entry.op.type === 'delete');
|
|
120
|
+
const acknowledgedWrites = (result.matchedCount ?? 0) + (result.upsertedCount ?? 0);
|
|
121
|
+
const acknowledgedDeletes = result.deletedCount ?? 0;
|
|
122
|
+
const missed = [];
|
|
123
|
+
// The counts are aggregate, so which specific documents missed is unknown; the
|
|
124
|
+
// caller re-reads the candidates and decides per document.
|
|
125
|
+
if (acknowledgedWrites < writes.length)
|
|
126
|
+
missed.push(...writes);
|
|
127
|
+
if (acknowledgedDeletes < deletes.length)
|
|
128
|
+
missed.push(...deletes);
|
|
129
|
+
return missed;
|
|
130
|
+
}
|
|
131
|
+
/**
|
|
132
|
+
* Re-reads the upstream state of operations that may have missed, and records a
|
|
133
|
+
* conflict for each one whose version really has moved on.
|
|
134
|
+
*
|
|
135
|
+
* The re-read turns an aggregate "something didn't match" into a precise, per-document
|
|
136
|
+
* answer, and supplies the current server document the caller needs to reconcile.
|
|
137
|
+
*
|
|
138
|
+
* @returns How many candidates genuinely lost.
|
|
139
|
+
*/
|
|
140
|
+
async function describeConflicts(db, collectionName, candidates, conflicts, idMapping, ObjectId) {
|
|
141
|
+
const ids = candidates.map((entry) => (0, commands_js_1.toUpstreamId)(entry.op.documentId, idMapping, ObjectId));
|
|
142
|
+
const current = await db
|
|
143
|
+
.collection(collectionName)
|
|
144
|
+
.find({ _id: { $in: ids } })
|
|
145
|
+
.toArray();
|
|
146
|
+
const byId = new Map(current.map((doc) => [String(doc._id), doc]));
|
|
147
|
+
let confirmed = 0;
|
|
148
|
+
for (const { index, op } of candidates) {
|
|
149
|
+
const serverDocument = byId.get(op.documentId);
|
|
150
|
+
const serverVersion = (0, commands_js_1.readVersion)(serverDocument);
|
|
151
|
+
if (op.type === 'delete') {
|
|
152
|
+
// Already gone upstream is the outcome we wanted, however it happened.
|
|
153
|
+
if (!serverDocument)
|
|
154
|
+
continue;
|
|
155
|
+
conflicts.push({ index, reason: 'version-mismatch', serverDocument, serverVersion });
|
|
156
|
+
confirmed += 1;
|
|
157
|
+
continue;
|
|
158
|
+
}
|
|
159
|
+
if (!serverDocument) {
|
|
160
|
+
conflicts.push({ index, reason: 'missing' });
|
|
161
|
+
confirmed += 1;
|
|
162
|
+
continue;
|
|
163
|
+
}
|
|
164
|
+
// The write may have landed after all, with a different operation in the batch
|
|
165
|
+
// being the one that missed.
|
|
166
|
+
//
|
|
167
|
+
// A version one ahead is necessary evidence but nowhere near sufficient: another
|
|
168
|
+
// writer following the same protocol also bumps `_v` by exactly one, so the two
|
|
169
|
+
// cases look identical from the version alone. Confirming that the document
|
|
170
|
+
// actually carries our change is what tells them apart — and getting this wrong
|
|
171
|
+
// silently swallows the very conflicts this machinery exists to catch.
|
|
172
|
+
if ((0, commands_js_1.isConditional)(op) &&
|
|
173
|
+
serverVersion === op.baseVersion + 1 &&
|
|
174
|
+
reflectsWrite(serverDocument, op)) {
|
|
175
|
+
continue;
|
|
176
|
+
}
|
|
177
|
+
conflicts.push({ index, reason: 'version-mismatch', serverDocument, serverVersion });
|
|
178
|
+
confirmed += 1;
|
|
179
|
+
}
|
|
180
|
+
return confirmed;
|
|
181
|
+
}
|
|
182
|
+
/**
|
|
183
|
+
* Splits a driver error into per-operation permanent failures.
|
|
184
|
+
*
|
|
185
|
+
* @returns The failures to dead-letter, or `null` when the error is transient and the
|
|
186
|
+
* whole batch should be retried.
|
|
187
|
+
*/
|
|
188
|
+
function classifyBulkError(err, operationCount) {
|
|
189
|
+
const error = err;
|
|
190
|
+
// The driver retried and still could not reach a writable primary.
|
|
191
|
+
if (error.hasErrorLabel?.('RetryableWriteError'))
|
|
192
|
+
return null;
|
|
193
|
+
const writeErrors = error.writeErrors ?? error.result?.writeErrors;
|
|
194
|
+
if (Array.isArray(writeErrors) && writeErrors.length > 0) {
|
|
195
|
+
const failures = [];
|
|
196
|
+
for (const writeError of writeErrors) {
|
|
197
|
+
const code = writeError.code;
|
|
198
|
+
// An unrecognised write error might be transient (a duplicate key during a
|
|
199
|
+
// failover, say), so the batch is worth retrying whole. A duplicate key is
|
|
200
|
+
// handled separately by the caller, as a create race rather than a defect.
|
|
201
|
+
if (code === undefined || (code !== DUPLICATE_KEY && !PERMANENT_ERROR_CODES.has(code))) {
|
|
202
|
+
return null;
|
|
203
|
+
}
|
|
204
|
+
const index = writeError.index ?? 0;
|
|
205
|
+
if (index >= operationCount)
|
|
206
|
+
return null;
|
|
207
|
+
failures.push({
|
|
208
|
+
index,
|
|
209
|
+
message: writeError.errmsg ?? writeError.message ?? `write error ${code}`,
|
|
210
|
+
code,
|
|
211
|
+
keyPattern: writeError.keyPattern,
|
|
212
|
+
});
|
|
213
|
+
}
|
|
214
|
+
return failures;
|
|
215
|
+
}
|
|
216
|
+
// No per-write detail: a connection, timeout or auth error. Retry it.
|
|
217
|
+
return null;
|
|
218
|
+
}
|
|
219
|
+
/**
|
|
220
|
+
* True when the upstream document carries the change this operation was making.
|
|
221
|
+
*
|
|
222
|
+
* Only the paths the operation itself wrote are checked. That is deliberate: those are
|
|
223
|
+
* the values we would have written in local JSON form, so comparing them against the
|
|
224
|
+
* server is meaningful, whereas fields we never touched may hold BSON types the local
|
|
225
|
+
* store cannot represent and would not compare equal.
|
|
226
|
+
*
|
|
227
|
+
* A false positive is possible if another writer happened to set the identical value —
|
|
228
|
+
* and is harmless, since the upstream then holds what we wanted anyway.
|
|
229
|
+
*/
|
|
230
|
+
function reflectsWrite(serverDocument, op) {
|
|
231
|
+
if (op.diff) {
|
|
232
|
+
for (const [path, value] of Object.entries(op.diff.set)) {
|
|
233
|
+
if ((0, commands_js_1.isReservedPath)(path))
|
|
234
|
+
continue;
|
|
235
|
+
if (!valueAtPathEquals(serverDocument, path, value))
|
|
236
|
+
return false;
|
|
237
|
+
}
|
|
238
|
+
for (const path of op.diff.unset) {
|
|
239
|
+
if ((0, commands_js_1.isReservedPath)(path))
|
|
240
|
+
continue;
|
|
241
|
+
if (readPath(serverDocument, path) !== undefined)
|
|
242
|
+
return false;
|
|
243
|
+
}
|
|
244
|
+
return true;
|
|
245
|
+
}
|
|
246
|
+
// Whole-document write: every field we sent should be present upstream.
|
|
247
|
+
for (const [key, value] of Object.entries((0, commands_js_1.stripReservedFields)(op.document ?? {}))) {
|
|
248
|
+
if (!valueAtPathEquals(serverDocument, key, value))
|
|
249
|
+
return false;
|
|
250
|
+
}
|
|
251
|
+
return true;
|
|
252
|
+
}
|
|
253
|
+
/**
|
|
254
|
+
* Compares a value at a dotted path against what we wrote.
|
|
255
|
+
*
|
|
256
|
+
* The server side is flattened to local JSON shape first, so a `Date` upstream compares
|
|
257
|
+
* equal to the ISO string a local write would have carried.
|
|
258
|
+
*/
|
|
259
|
+
function valueAtPathEquals(document, path, expected) {
|
|
260
|
+
const actual = readPath(document, path);
|
|
261
|
+
if (actual === undefined)
|
|
262
|
+
return false;
|
|
263
|
+
try {
|
|
264
|
+
return (0, diff_js_1.deepEqual)(JSON.parse(JSON.stringify(actual)), expected);
|
|
265
|
+
}
|
|
266
|
+
catch {
|
|
267
|
+
return false;
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
/** Reads a dotted path, returning `undefined` if any segment is missing. */
|
|
271
|
+
function readPath(document, path) {
|
|
272
|
+
let current = document;
|
|
273
|
+
for (const segment of path.split('.')) {
|
|
274
|
+
if (typeof current !== 'object' || current === null)
|
|
275
|
+
return undefined;
|
|
276
|
+
current = current[segment];
|
|
277
|
+
if (current === undefined)
|
|
278
|
+
return undefined;
|
|
279
|
+
}
|
|
280
|
+
return current;
|
|
281
|
+
}
|
|
282
|
+
/**
|
|
283
|
+
* True when a duplicate-key error is about the `_id` index specifically.
|
|
284
|
+
*
|
|
285
|
+
* MongoDB reports the offending index in `keyPattern` (and names it in `errmsg`), which
|
|
286
|
+
* is the only way to tell "someone else created this document" apart from "this document
|
|
287
|
+
* violates a unique constraint on some other field".
|
|
288
|
+
*/
|
|
289
|
+
function isIdIndexViolation(failure) {
|
|
290
|
+
if (failure.keyPattern) {
|
|
291
|
+
const keys = Object.keys(failure.keyPattern);
|
|
292
|
+
return keys.length === 1 && keys[0] === '_id';
|
|
293
|
+
}
|
|
294
|
+
// Older servers and some drivers only give the message; the index name appears in it.
|
|
295
|
+
return /index:\s*_id_/.test(failure.message);
|
|
296
|
+
}
|
|
297
|
+
/** Reads documents back by `_id`, for refreshing a shadow after a conflict. */
|
|
298
|
+
async function fetchDocuments(db, collection, documentIds, idMapping, ObjectId) {
|
|
299
|
+
if (documentIds.length === 0)
|
|
300
|
+
return [];
|
|
301
|
+
const ids = documentIds.map((id) => (0, commands_js_1.toUpstreamId)(id, idMapping, ObjectId));
|
|
302
|
+
return db
|
|
303
|
+
.collection(collection)
|
|
304
|
+
.find({ _id: { $in: ids } })
|
|
305
|
+
.toArray();
|
|
306
|
+
}
|