@semics-tech/mongolite 1.2.0 → 1.4.0

package/dist/cjs/sync/mongoExecutor.js

@@ -0,0 +1,306 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.executeWrites = executeWrites;
+exports.classifyBulkError = classifyBulkError;
+exports.fetchDocuments = fetchDocuments;
+/**
+ * Applies prepared write commands to MongoDB and classifies the outcome.
+ *
+ * Shared by the direct sink and by the HTTP receiver, so the subtle part has exactly
+ * one implementation: **a lost compare-and-swap raises no error.** It is a silent
+ * zero-match in an aggregate count, which means detecting it takes deliberate work —
+ * comparing what the batch expected to change against what the server says changed, and
+ * then re-reading the candidates to find out which document actually lost.
+ */
+const commands_js_1 = require("./commands.js");
+const diff_js_1 = require("./diff.js");
+/**
+ * MongoDB server error codes that mean "this write will never succeed" — retrying them
+ * would stall the stream behind one bad document forever, so they are dead-lettered.
+ */
+const PERMANENT_ERROR_CODES = new Set([
+    2, // BadValue
+    9, // FailedToParse
+    10334, // BSONObjectTooLarge
+    14, // TypeMismatch
+    52, // DollarPrefixedFieldName
+    56, // EmptyFieldName
+    121, // DocumentValidationFailure
+    17280, // KeyTooLong
+]);
+/** MongoDB's duplicate-key code. On an insert-if-absent it means we lost a create race. */
+const DUPLICATE_KEY = 11000;
+/**
+ * Runs prepared writes against a database, one `bulkWrite` per collection.
+ *
+ * Throws on transient failure (network, election, timeout) so the caller retries the
+ * whole batch without checkpointing. Returns per-operation failures and conflicts for
+ * everything it could classify.
+ */
+async function executeWrites(db, writes, options = {}) {
+    const { writeConcern = { w: 'majority' }, versioning = true, idMapping, ObjectId } = options;
+    const byCollection = new Map();
+    for (const write of writes) {
+        const bucket = byCollection.get(write.op.collection);
+        if (bucket)
+            bucket.push(write);
+        else
+            byCollection.set(write.op.collection, [write]);
+    }
+    const failures = [];
+    const conflicts = [];
+    let applied = 0;
+    for (const [collectionName, entries] of byCollection) {
+        let result = null;
+        let rejected = 0;
+        try {
+            result = await db.collection(collectionName).bulkWrite(entries.map((entry) => entry.command), {
+                // Operations within a batch touch distinct documents, so ordering buys
+                // nothing and unordered lets the server parallelise.
+                ordered: false,
+                writeConcern,
+            });
+        }
+        catch (err) {
+            const bulkFailures = classifyBulkError(err, entries.length);
+            // A transient failure means the whole batch's fate is unknown — throw so the
+            // caller retries it rather than checkpointing work that may not have landed.
+            if (bulkFailures === null)
+                throw err;
+            for (const failure of bulkFailures) {
+                const entry = entries[failure.index];
+                if (!entry)
+                    continue;
+                // A duplicate `_id` on an insert-if-absent means someone created the document
+                // first — a conflict to reconcile, not a poison document. A duplicate on any
+                // *other* unique index is a real constraint violation and must be dead-lettered,
+                // or it would be retried forever as a phantom conflict.
+                if (failure.code === DUPLICATE_KEY && isIdIndexViolation(failure)) {
+                    conflicts.push({ index: entry.index, reason: 'already-exists' });
+                }
+                else {
+                    failures.push({ index: entry.index, message: failure.message, code: failure.code });
+                }
+            }
+            rejected = bulkFailures.length;
+            result = err.result ?? null;
+        }
+        const landed = entries.length - rejected;
+        const suspects = findMisses(entries, result, rejected, versioning);
+        // `suspects` are candidates to investigate, not confirmed misses: the counts are
+        // aggregate, so one short count implicates every conditional write in the batch.
+        // Only the re-read says which of them actually lost, and `applied` must be reduced
+        // by that number rather than by the number investigated.
+        const confirmed = suspects.length
+            ? await describeConflicts(db, collectionName, suspects, conflicts, idMapping, ObjectId)
+            : 0;
+        applied += landed - confirmed;
+    }
+    return {
+        applied,
+        failures: failures.length > 0 ? failures : undefined,
+        conflicts: conflicts.length > 0 ? conflicts : undefined,
+    };
+}
+/**
+ * Identifies operations whose conditional write may not have matched.
+ *
+ * `bulkWrite` reports only aggregate counts, so this compares what the batch expected
+ * to change against what the server says changed. When the numbers agree, nothing was
+ * missed and no per-document work is needed.
+ */
+function findMisses(entries, result, rejected, versioning) {
+    if (!versioning || result === null || rejected === entries.length)
+        return [];
+    const candidates = entries.filter((entry) => (0, commands_js_1.isConditional)(entry.op));
+    if (candidates.length === 0)
+        return [];
+    const writes = candidates.filter((entry) => entry.op.type !== 'delete');
+    const deletes = candidates.filter((entry) => entry.op.type === 'delete');
+    const acknowledgedWrites = (result.matchedCount ?? 0) + (result.upsertedCount ?? 0);
+    const acknowledgedDeletes = result.deletedCount ?? 0;
+    const missed = [];
+    // The counts are aggregate, so which specific documents missed is unknown; the
+    // caller re-reads the candidates and decides per document.
+    if (acknowledgedWrites < writes.length)
+        missed.push(...writes);
+    if (acknowledgedDeletes < deletes.length)
+        missed.push(...deletes);
+    return missed;
+}
+/**
+ * Re-reads the upstream state of operations that may have missed, and records a
+ * conflict for each one whose version really has moved on.
+ *
+ * The re-read turns an aggregate "something didn't match" into a precise, per-document
+ * answer, and supplies the current server document the caller needs to reconcile.
+ *
+ * @returns How many candidates genuinely lost.
+ */
+async function describeConflicts(db, collectionName, candidates, conflicts, idMapping, ObjectId) {
+    const ids = candidates.map((entry) => (0, commands_js_1.toUpstreamId)(entry.op.documentId, idMapping, ObjectId));
+    const current = await db
+        .collection(collectionName)
+        .find({ _id: { $in: ids } })
+        .toArray();
+    const byId = new Map(current.map((doc) => [String(doc._id), doc]));
+    let confirmed = 0;
+    for (const { index, op } of candidates) {
+        const serverDocument = byId.get(op.documentId);
+        const serverVersion = (0, commands_js_1.readVersion)(serverDocument);
+        if (op.type === 'delete') {
+            // Already gone upstream is the outcome we wanted, however it happened.
+            if (!serverDocument)
+                continue;
+            conflicts.push({ index, reason: 'version-mismatch', serverDocument, serverVersion });
+            confirmed += 1;
+            continue;
+        }
+        if (!serverDocument) {
+            conflicts.push({ index, reason: 'missing' });
+            confirmed += 1;
+            continue;
+        }
+        // The write may have landed after all, with a different operation in the batch
+        // being the one that missed.
+        //
+        // A version one ahead is necessary evidence but nowhere near sufficient: another
+        // writer following the same protocol also bumps `_v` by exactly one, so the two
+        // cases look identical from the version alone. Confirming that the document
+        // actually carries our change is what tells them apart — and getting this wrong
+        // silently swallows the very conflicts this machinery exists to catch.
+        if ((0, commands_js_1.isConditional)(op) &&
+            serverVersion === op.baseVersion + 1 &&
+            reflectsWrite(serverDocument, op)) {
+            continue;
+        }
+        conflicts.push({ index, reason: 'version-mismatch', serverDocument, serverVersion });
+        confirmed += 1;
+    }
+    return confirmed;
+}
+/**
+ * Splits a driver error into per-operation permanent failures.
+ *
+ * @returns The failures to dead-letter, or `null` when the error is transient and the
+ * whole batch should be retried.
+ */
+function classifyBulkError(err, operationCount) {
+    const error = err;
+    // The driver retried and still could not reach a writable primary.
+    if (error.hasErrorLabel?.('RetryableWriteError'))
+        return null;
+    const writeErrors = error.writeErrors ?? error.result?.writeErrors;
+    if (Array.isArray(writeErrors) && writeErrors.length > 0) {
+        const failures = [];
+        for (const writeError of writeErrors) {
+            const code = writeError.code;
+            // An unrecognised write error might be transient (a duplicate key during a
+            // failover, say), so the batch is worth retrying whole. A duplicate key is
+            // handled separately by the caller, as a create race rather than a defect.
+            if (code === undefined || (code !== DUPLICATE_KEY && !PERMANENT_ERROR_CODES.has(code))) {
+                return null;
+            }
+            const index = writeError.index ?? 0;
+            if (index >= operationCount)
+                return null;
+            failures.push({
+                index,
+                message: writeError.errmsg ?? writeError.message ?? `write error ${code}`,
+                code,
+                keyPattern: writeError.keyPattern,
+            });
+        }
+        return failures;
+    }
+    // No per-write detail: a connection, timeout or auth error. Retry it.
+    return null;
+}
+/**
+ * True when the upstream document carries the change this operation was making.
+ *
+ * Only the paths the operation itself wrote are checked. That is deliberate: those are
+ * the values we would have written in local JSON form, so comparing them against the
+ * server is meaningful, whereas fields we never touched may hold BSON types the local
+ * store cannot represent and would not compare equal.
+ *
+ * A false positive is possible if another writer happened to set the identical value —
+ * and is harmless, since the upstream then holds what we wanted anyway.
+ */
+function reflectsWrite(serverDocument, op) {
+    if (op.diff) {
+        for (const [path, value] of Object.entries(op.diff.set)) {
+            if ((0, commands_js_1.isReservedPath)(path))
+                continue;
+            if (!valueAtPathEquals(serverDocument, path, value))
+                return false;
+        }
+        for (const path of op.diff.unset) {
+            if ((0, commands_js_1.isReservedPath)(path))
+                continue;
+            if (readPath(serverDocument, path) !== undefined)
+                return false;
+        }
+        return true;
+    }
+    // Whole-document write: every field we sent should be present upstream.
+    for (const [key, value] of Object.entries((0, commands_js_1.stripReservedFields)(op.document ?? {}))) {
+        if (!valueAtPathEquals(serverDocument, key, value))
+            return false;
+    }
+    return true;
+}
+/**
+ * Compares a value at a dotted path against what we wrote.
+ *
+ * The server side is flattened to local JSON shape first, so a `Date` upstream compares
+ * equal to the ISO string a local write would have carried.
+ */
+function valueAtPathEquals(document, path, expected) {
+    const actual = readPath(document, path);
+    if (actual === undefined)
+        return false;
+    try {
+        return (0, diff_js_1.deepEqual)(JSON.parse(JSON.stringify(actual)), expected);
+    }
+    catch {
+        return false;
+    }
+}
+/** Reads a dotted path, returning `undefined` if any segment is missing. */
+function readPath(document, path) {
+    let current = document;
+    for (const segment of path.split('.')) {
+        if (typeof current !== 'object' || current === null)
+            return undefined;
+        current = current[segment];
+        if (current === undefined)
+            return undefined;
+    }
+    return current;
+}
+/**
+ * True when a duplicate-key error is about the `_id` index specifically.
+ *
+ * MongoDB reports the offending index in `keyPattern` (and names it in `errmsg`), which
+ * is the only way to tell "someone else created this document" apart from "this document
+ * violates a unique constraint on some other field".
+ */
+function isIdIndexViolation(failure) {
+    if (failure.keyPattern) {
+        const keys = Object.keys(failure.keyPattern);
+        return keys.length === 1 && keys[0] === '_id';
+    }
+    // Older servers and some drivers only give the message; the index name appears in it.
+    return /index:\s*_id_/.test(failure.message);
+}
+/** Reads documents back by `_id`, for refreshing a shadow after a conflict. */
+async function fetchDocuments(db, collection, documentIds, idMapping, ObjectId) {
+    if (documentIds.length === 0)
+        return [];
+    const ids = documentIds.map((id) => (0, commands_js_1.toUpstreamId)(id, idMapping, ObjectId));
+    return db
+        .collection(collection)
+        .find({ _id: { $in: ids } })
+        .toArray();
+}

package/dist/cjs/sync/mongoSink.js

@@ -0,0 +1,161 @@
+"use strict";
+/**
+ * A {@link SyncSink} that writes to a MongoDB deployment using the official
+ * `mongodb` driver.
+ *
+ * The driver is an **optional peer dependency**, imported dynamically the first
+ * time a connection is opened. Projects that never replicate upstream do not
+ * need it installed, and bundlers that never reach this module do not pull it in.
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MongoUpstreamSink = exports.UPDATED_AT_FIELD = exports.VERSION_FIELD = void 0;
+const commands_js_1 = require("./commands.js");
+const mongoExecutor_js_1 = require("./mongoExecutor.js");
+var commands_js_2 = require("./commands.js");
+Object.defineProperty(exports, "VERSION_FIELD", { enumerable: true, get: function () { return commands_js_2.VERSION_FIELD; } });
+Object.defineProperty(exports, "UPDATED_AT_FIELD", { enumerable: true, get: function () { return commands_js_2.UPDATED_AT_FIELD; } });
+class MongoUpstreamSink {
+    constructor(options) {
+        this.options = options;
+        this.name = 'mongodb';
+        this.client = null;
+        this.db = null;
+        this.driver = null;
+        if (!options.connectionString) {
+            throw new Error('MongoUpstreamSink requires a `connectionString`.');
+        }
+        this.idMapping = options.idMapping ?? 'auto';
+        this.versioning = options.versioning ?? true;
+        this.driver = options.driver ?? null;
+    }
+    async connect() {
+        if (this.client && this.db)
+            return;
+        const driver = await this.loadDriver();
+        const client = new driver.MongoClient(this.options.connectionString, this.clientOptions());
+        await client.connect();
+        this.client = client;
+        this.db = client.db(this.options.database);
+    }
+    async apply(operations) {
+        if (!this.db)
+            await this.connect();
+        const db = this.db;
+        if (!db)
+            throw new Error('MongoUpstreamSink is not connected.');
+        const writes = operations.map((op, index) => ({
+            index,
+            op,
+            command: (0, commands_js_1.buildWriteCommand)(op, this.commandOptions()),
+        }));
+        return (0, mongoExecutor_js_1.executeWrites)(db, writes, {
+            writeConcern: this.options.writeConcern ?? { w: 'majority' },
+            versioning: this.versioning,
+            idMapping: this.idMapping,
+            ObjectId: this.driver?.ObjectId,
+        });
+    }
+    /**
+     * Command-building settings. The driver's own `ObjectId` is used when available so
+     * BSON instances match the connection that will serialise them.
+     */
+    commandOptions() {
+        return {
+            idMapping: this.idMapping,
+            versioning: this.versioning,
+            ObjectId: this.driver?.ObjectId,
+        };
+    }
+    async fetch(collection, documentIds) {
+        if (documentIds.length === 0)
+            return [];
+        if (!this.db)
+            await this.connect();
+        const db = this.db;
+        if (!db)
+            throw new Error('MongoUpstreamSink is not connected.');
+        return (0, mongoExecutor_js_1.fetchDocuments)(db, collection, documentIds, this.idMapping, this.driver?.ObjectId);
+    }
+    async close() {
+        const client = this.client;
+        this.client = null;
+        this.db = null;
+        if (client)
+            await client.close();
+    }
+    // ---------------------------------------------------------------- internals
+    async loadDriver() {
+        if (this.driver)
+            return this.driver;
+        try {
+            // Indirect specifier: `mongodb` is optional, and a literal import would
+            // make it a hard build-time dependency for everyone compiling this package.
+            const specifier = 'mongodb';
+            const mod = (await Promise.resolve(`${specifier}`).then(s => __importStar(require(s))));
+            this.driver = mod;
+            return mod;
+        }
+        catch (err) {
+            throw new Error('MongoDB sync requires the `mongodb` driver. Install it with `npm install mongodb`, ' +
+                `or pass a \`driver\` explicitly. Original error: ${err instanceof Error ? err.message : String(err)}`);
+        }
+    }
+    clientOptions() {
+        const { authSource, authMechanism, authMechanismProperties, username, password, tls, tlsCAFile, tlsCertificateKeyFile, tlsCertificateKeyFilePassword, tlsCRLFile, tlsAllowInvalidHostnames, tlsAllowInvalidCertificates, driverOptions, } = this.options;
+        const auth = username !== undefined || password !== undefined ? { username, password } : undefined;
+        // Undefined keys are stripped so they never override a setting that the
+        // connection string already specified.
+        return stripUndefined({
+            authSource,
+            authMechanism,
+            authMechanismProperties,
+            auth,
+            tls,
+            tlsCAFile,
+            tlsCertificateKeyFile,
+            tlsCertificateKeyFilePassword,
+            tlsCRLFile,
+            tlsAllowInvalidHostnames,
+            tlsAllowInvalidCertificates,
+            ...driverOptions,
+        });
+    }
+}
+exports.MongoUpstreamSink = MongoUpstreamSink;
+/** Drops unset keys so they never override what the connection string specified. */
+function stripUndefined(value) {
+    return Object.fromEntries(Object.entries(value).filter(([, v]) => v !== undefined));
+}