@semics-tech/mongolite 1.3.0 → 1.4.1

package/dist/sync/mongoSink.d.ts

@@ -0,0 +1,109 @@
+/**
+ * A {@link SyncSink} that writes to a MongoDB deployment using the official
+ * `mongodb` driver.
+ *
+ * The driver is an **optional peer dependency**, imported dynamically the first
+ * time a connection is opened. Projects that never replicate upstream do not
+ * need it installed, and bundlers that never reach this module do not pull it in.
+ */
+import type { MongoDriverLike, SyncApplyResult, SyncIdMapping, SyncOperation, SyncSink } from './types.js';
+export { VERSION_FIELD, UPDATED_AT_FIELD } from './commands.js';
+/**
+ * Authentication and transport settings for the upstream connection.
+ *
+ * Most deployments need nothing beyond the connection string — it already
+ * carries username, password, `authSource`, `replicaSet` and `tls`. These
+ * options cover the cases a URI cannot express, chiefly the certificate-file
+ * based mechanisms (X.509, mTLS, a private CA).
+ */
+export interface MongoUpstreamAuth {
+    /** Database to authenticate against, when it differs from the target database. */
+    authSource?: string;
+    /**
+     * SASL/auth mechanism, e.g. `'SCRAM-SHA-256'`, `'MONGODB-X509'`,
+     * `'MONGODB-AWS'`, `'MONGODB-OIDC'`, `'GSSAPI'`, `'PLAIN'`.
+     * Omit to let the driver negotiate.
+     */
+    authMechanism?: string;
+    /** Mechanism-specific properties, e.g. `{ AWS_SESSION_TOKEN: '…' }`. */
+    authMechanismProperties?: Record<string, unknown>;
+    /** Username, when not embedded in the connection string. */
+    username?: string;
+    /** Password, when not embedded in the connection string. */
+    password?: string;
+    /** Force TLS on or off. Usually inferred from the connection string. */
+    tls?: boolean;
+    /** Path to a PEM file with the certificate authority chain, for a private CA. */
+    tlsCAFile?: string;
+    /**
+     * Path to a PEM file holding the client certificate **and** its private key —
+     * required for mutual TLS and for `MONGODB-X509` authentication.
+     */
+    tlsCertificateKeyFile?: string;
+    /** Passphrase for an encrypted `tlsCertificateKeyFile`. */
+    tlsCertificateKeyFilePassword?: string;
+    /** Path to a PEM file listing revoked certificates. */
+    tlsCRLFile?: string;
+    /**
+     * Skip hostname verification. Never enable this against a production
+     * deployment — it removes the protection TLS is there to provide.
+     */
+    tlsAllowInvalidHostnames?: boolean;
+    /**
+     * Accept certificates that fail validation. Never enable this against a
+     * production deployment.
+     */
+    tlsAllowInvalidCertificates?: boolean;
+}
+export interface MongoSinkOptions extends MongoUpstreamAuth {
+    /** Upstream connection string, e.g. `mongodb+srv://host/db`. */
+    connectionString: string;
+    /** Target database. Defaults to the one in the connection string's path. */
+    database?: string;
+    /** See {@link SyncIdMapping}. Defaults to `'auto'`. */
+    idMapping?: SyncIdMapping;
+    /**
+     * Guard writes with a conditional update against each document's last-known `_v`.
+     * Defaults to `true`. See {@link SyncOptions.versioning}.
+     */
+    versioning?: boolean;
+    /**
+     * Write concern for replicated batches. Defaults to `{ w: 'majority' }` so an
+     * acknowledged batch has actually survived upstream before it is checkpointed
+     * locally.
+     */
+    writeConcern?: Record<string, unknown>;
+    /**
+     * Extra options passed straight through to `new MongoClient(...)`. Anything
+     * the driver supports — `serverApi`, `proxyHost`, `compressors`,
+     * `autoEncryption`, connection-pool tuning — belongs here.
+     */
+    driverOptions?: Record<string, unknown>;
+    /**
+     * Supply the `mongodb` module yourself instead of having it imported
+     * dynamically. Useful for bundlers that cannot resolve dynamic imports, and
+     * for tests.
+     */
+    driver?: MongoDriverLike;
+}
+export declare class MongoUpstreamSink implements SyncSink {
+    private readonly options;
+    readonly name = "mongodb";
+    private client;
+    private db;
+    private driver;
+    private readonly idMapping;
+    private readonly versioning;
+    constructor(options: MongoSinkOptions);
+    connect(): Promise<void>;
+    apply(operations: SyncOperation[]): Promise<SyncApplyResult>;
+    /**
+     * Command-building settings. The driver's own `ObjectId` is used when available so
+     * BSON instances match the connection that will serialise them.
+     */
+    private commandOptions;
+    fetch(collection: string, documentIds: string[]): Promise<Record<string, unknown>[]>;
+    close(): Promise<void>;
+    private loadDriver;
+    private clientOptions;
+}

package/dist/sync/mongoSink.js

@@ -0,0 +1,123 @@
+/**
+ * A {@link SyncSink} that writes to a MongoDB deployment using the official
+ * `mongodb` driver.
+ *
+ * The driver is an **optional peer dependency**, imported dynamically the first
+ * time a connection is opened. Projects that never replicate upstream do not
+ * need it installed, and bundlers that never reach this module do not pull it in.
+ */
+import { buildWriteCommand } from './commands.js';
+import { executeWrites, fetchDocuments } from './mongoExecutor.js';
+export { VERSION_FIELD, UPDATED_AT_FIELD } from './commands.js';
+export class MongoUpstreamSink {
+    constructor(options) {
+        this.options = options;
+        this.name = 'mongodb';
+        this.client = null;
+        this.db = null;
+        this.driver = null;
+        if (!options.connectionString) {
+            throw new Error('MongoUpstreamSink requires a `connectionString`.');
+        }
+        this.idMapping = options.idMapping ?? 'auto';
+        this.versioning = options.versioning ?? true;
+        this.driver = options.driver ?? null;
+    }
+    async connect() {
+        if (this.client && this.db)
+            return;
+        const driver = await this.loadDriver();
+        const client = new driver.MongoClient(this.options.connectionString, this.clientOptions());
+        await client.connect();
+        this.client = client;
+        this.db = client.db(this.options.database);
+    }
+    async apply(operations) {
+        if (!this.db)
+            await this.connect();
+        const db = this.db;
+        if (!db)
+            throw new Error('MongoUpstreamSink is not connected.');
+        const writes = operations.map((op, index) => ({
+            index,
+            op,
+            command: buildWriteCommand(op, this.commandOptions()),
+        }));
+        return executeWrites(db, writes, {
+            writeConcern: this.options.writeConcern ?? { w: 'majority' },
+            versioning: this.versioning,
+            idMapping: this.idMapping,
+            ObjectId: this.driver?.ObjectId,
+        });
+    }
+    /**
+     * Command-building settings. The driver's own `ObjectId` is used when available so
+     * BSON instances match the connection that will serialise them.
+     */
+    commandOptions() {
+        return {
+            idMapping: this.idMapping,
+            versioning: this.versioning,
+            ObjectId: this.driver?.ObjectId,
+        };
+    }
+    async fetch(collection, documentIds) {
+        if (documentIds.length === 0)
+            return [];
+        if (!this.db)
+            await this.connect();
+        const db = this.db;
+        if (!db)
+            throw new Error('MongoUpstreamSink is not connected.');
+        return fetchDocuments(db, collection, documentIds, this.idMapping, this.driver?.ObjectId);
+    }
+    async close() {
+        const client = this.client;
+        this.client = null;
+        this.db = null;
+        if (client)
+            await client.close();
+    }
+    // ---------------------------------------------------------------- internals
+    async loadDriver() {
+        if (this.driver)
+            return this.driver;
+        try {
+            // Indirect specifier: `mongodb` is optional, and a literal import would
+            // make it a hard build-time dependency for everyone compiling this package.
+            const specifier = 'mongodb';
+            const mod = (await import(specifier));
+            this.driver = mod;
+            return mod;
+        }
+        catch (err) {
+            throw new Error('MongoDB sync requires the `mongodb` driver. Install it with `npm install mongodb`, ' +
+                `or pass a \`driver\` explicitly. Original error: ${err instanceof Error ? err.message : String(err)}`);
+        }
+    }
+    clientOptions() {
+        const { authSource, authMechanism, authMechanismProperties, username, password, tls, tlsCAFile, tlsCertificateKeyFile, tlsCertificateKeyFilePassword, tlsCRLFile, tlsAllowInvalidHostnames, tlsAllowInvalidCertificates, driverOptions, } = this.options;
+        const auth = username !== undefined || password !== undefined ? { username, password } : undefined;
+        // Undefined keys are stripped so they never override a setting that the
+        // connection string already specified.
+        return stripUndefined({
+            authSource,
+            authMechanism,
+            authMechanismProperties,
+            auth,
+            tls,
+            tlsCAFile,
+            tlsCertificateKeyFile,
+            tlsCertificateKeyFilePassword,
+            tlsCRLFile,
+            tlsAllowInvalidHostnames,
+            tlsAllowInvalidCertificates,
+            ...driverOptions,
+        });
+    }
+}
+/** Drops unset keys so they never override what the connection string specified. */
+function stripUndefined(value) {
+    return Object.fromEntries(Object.entries(value).filter(([, v]) => v !== undefined));
+}
+//# sourceMappingURL=mongoSink.js.map

package/dist/sync/mongoSink.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"mongoSink.js","sourceRoot":"","sources":["../../src/sync/mongoSink.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAElD,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAYnE,OAAO,EAAE,aAAa,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAmGhE,MAAM,OAAO,iBAAiB;IAS5B,YAA6B,OAAyB;QAAzB,YAAO,GAAP,OAAO,CAAkB;QAR7C,SAAI,GAAG,SAAS,CAAC;QAElB,WAAM,GAA2B,IAAI,CAAC;QACtC,OAAE,GAAuB,IAAI,CAAC;QAC9B,WAAM,GAA2B,IAAI,CAAC;QAK5C,IAAI,CAAC,OAAO,CAAC,gBAAgB,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAAC;QACtE,CAAC;QACD,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,MAAM,CAAC;QAC7C,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,IAAI,CAAC;QAC7C,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC;IACvC,CAAC;IAED,KAAK,CAAC,OAAO;QACX,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,EAAE;YAAE,OAAO;QAEnC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QACvC,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,gBAAgB,EAAE,IAAI,CAAC,aAAa,EAAE,CAAC,CAAC;QAC3F,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;QAEvB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,EAAE,GAAG,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC7C,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,UAA2B;QACrC,IAAI,CAAC,IAAI,CAAC,EAAE;YAAE,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;QACnC,MAAM,EAAE,GAAG,IAAI,CAAC,EAAE,CAAC;QACnB,IAAI,CAAC,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;QAEhE,MAAM,MAAM,GAAoB,UAAU,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;YAC7D,KAAK;YACL,EAAE;YACF,OAAO,EAAE,iBAAiB,CAAC,EAAE,EAAE,IAAI,CAAC,cAAc,EAAE,CAAC;SACtD,CAAC,CAAC,CAAC;QAEJ,OAAO,aAAa,CAAC,EAAE,EAAE,MAAM,EAAE;YAC/B,YAAY,EAAE,IAAI,CAAC,OAAO,CAAC,YAAY,IAAI,EAAE,CAAC,EAAE,UAAU,EAAE;YAC5D,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,QAAQ,EAAE,IAAI,CAAC,MAAM,EAAE,QAAQ;SAChC,CAAC,CAAC;IACL,CAAC;IAED;;;OAGG;IACK,cAAc;QACpB,OAAO;YACL,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,QAAQ,EAAE,IAAI,CAAC,MAAM,EAAE,QAAQ;SAChC,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,UAAkB,EAAE,WAAqB;QACnD,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QACxC,IAAI,CAAC,IAAI,CAAC,EAAE;YAAE,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;QACnC,MAAM,EAAE,GAAG,IAAI,CAAC,EAAE,CAAC;QACnB,IAAI,CAAC,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;QAEhE,OAAO,cAAc,CAAC,EAAE,EAAE,UAAU,EAAE,WAAW,EAAE,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;IAC5F,CAAC;IAED,KAAK,CAAC,KAAK;QACT,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC3B,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACnB,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC;QACf,IAAI,MAAM;YAAE,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;IACnC,CAAC;IAED,6EAA6E;IAErE,KAAK,CAAC,UAAU;QACtB,IAAI,IAAI,CAAC,MAAM;YAAE,OAAO,IAAI,CAAC,MAAM,CAAC;QAEpC,IAAI,CAAC;YACH,wEAAwE;YACxE,4EAA4E;YAC5E,MAAM,SAAS,GAAG,SAAS,CAAC;YAC5B,MAAM,GAAG,GAAG,CAAC,MAAM,MAAM,CAAC,SAAS,CAAC,CAA+B,CAAC;YACpE,IAAI,CAAC,MAAM,GAAG,GAAG,CAAC;YAClB,OAAO,GAAG,CAAC;QACb,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CACb,qFAAqF;gBACnF,oDACE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CACjD,EAAE,CACL,CAAC;QACJ,CAAC;IACH,CAAC;IAEO,aAAa;QACnB,MAAM,EACJ,UAAU,EACV,aAAa,EACb,uBAAuB,EACvB,QAAQ,EACR,QAAQ,EACR,GAAG,EACH,SAAS,EACT,qBAAqB,EACrB,6BAA6B,EAC7B,UAAU,EACV,wBAAwB,EACxB,2BAA2B,EAC3B,aAAa,GACd,GAAG,IAAI,CAAC,OAAO,CAAC;QAEjB,MAAM,IAAI,GACR,QAAQ,KAAK,SAAS,IAAI,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;QAExF,wEAAwE;QACxE,uCAAuC;QACvC,OAAO,cAAc,CAAC;YACpB,UAAU;YACV,aAAa;YACb,uBAAuB;YACvB,IAAI;YACJ,GAAG;YACH,SAAS;YACT,qBAAqB;YACrB,6BAA6B;YAC7B,UAAU;YACV,wBAAwB;YACxB,2BAA2B;YAC3B,GAAG,aAAa;SACjB,CAAC,CAAC;IACL,CAAC;CACF;AAED,oFAAoF;AACpF,SAAS,cAAc,CAAC,KAA8B;IACpD,OAAO,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,SAAS,CAAC,CAAC,CAAC;AACtF,CAAC"}

package/dist/sync/outbox.d.ts

@@ -0,0 +1,120 @@
+/**
+ * The durable change log ("outbox") that backs MongoDB replication.
+ *
+ * SQLite triggers append a row here inside the very same transaction as the
+ * write that caused it, so a change is either committed locally *and* queued for
+ * replication, or neither. Replicators then drain the log at their own pace and
+ * record a checkpoint; nothing is dropped because the process crashed or the
+ * upstream was unreachable.
+ */
+import type { IDatabaseAdapter } from '../db.js';
+import type { SyncDeadLetter, SyncOutboxRecord } from './types.js';
+export declare const OUTBOX_TABLE = "__mongolite_sync_outbox__";
+export declare const STATE_TABLE = "__mongolite_sync_state__";
+export declare const DEAD_LETTER_TABLE = "__mongolite_sync_deadletter__";
+/** Prefix shared by every table and trigger this module creates. */
+export declare const SYNC_OBJECT_PREFIX = "__mongolite_sync";
+/**
+ * Owns the outbox tables, the capture triggers, and every replicator's
+ * checkpoint. A single outbox serves any number of replicators: each keeps its
+ * own checkpoint row, and rows are pruned only once *every* replicator has
+ * passed them.
+ */
+export declare class SyncOutbox {
+    private readonly db;
+    private schemaReady;
+    private readonly installedTriggers;
+    constructor(db: IDatabaseAdapter);
+    /** Creates the outbox, checkpoint and dead-letter tables. Idempotent. */
+    ensureSchema(): Promise<void>;
+    private createSchema;
+    /**
+     * Installs the insert/update/delete capture triggers for a collection.
+     *
+     * Triggers deliberately outlive the replicator that created them: changes made
+     * while replication is stopped still need to be captured, or restarting would
+     * silently skip them. Use {@link SyncOutbox.removeTriggers} to opt out.
+     */
+    ensureTriggers(collection: string): Promise<void>;
+    /**
+     * Removes the capture triggers for a collection. Changes to it stop being
+     * recorded — existing queued rows are untouched.
+     */
+    removeTriggers(collection: string): Promise<void>;
+    /** Lists user collections, excluding MongoLite's own bookkeeping tables. */
+    listCollections(): Promise<string[]>;
+    /**
+     * Reconciles the in-memory "already installed" cache with what is actually in
+     * the schema.
+     *
+     * Dropping a table drops its triggers with it, so a collection that was
+     * dropped and recreated would otherwise be silently uncaptured — the cache
+     * would insist its triggers were still there.
+     */
+    refreshTriggerCache(): Promise<void>;
+    /** The newest outbox id, or 0 when the log is empty. */
+    currentSequence(): Promise<number>;
+    /** Reads the persisted checkpoint for a replicator, creating its state row if absent. */
+    loadState(name: string): Promise<{
+        checkpoint: number;
+        backfilled: string[];
+    }>;
+    /**
+     * Advances a replicator's checkpoint.
+     *
+     * This is the acknowledgement, and it is a single atomic statement: a crash
+     * either leaves the old checkpoint (the batch is replayed, which is safe
+     * because upstream writes are idempotent) or the new one.
+     */
+    saveCheckpoint(name: string, checkpoint: number): Promise<void>;
+    /** Records that a collection's existing contents have been backfilled. */
+    markBackfilled(name: string, collections: string[]): Promise<void>;
+    /** Forgets a replicator, releasing the outbox rows its checkpoint was pinning. */
+    unregister(name: string): Promise<void>;
+    /** Reads the next batch of changes after `checkpoint`, oldest first. */
+    readBatch(checkpoint: number, limit: number, collections: string[] | null): Promise<SyncOutboxRecord[]>;
+    /** Count of rows a replicator still has to apply. */
+    pendingCount(checkpoint: number, collections: string[] | null): Promise<number>;
+    /**
+     * Deletes rows every registered replicator has already applied.
+     * @returns Number of rows removed.
+     */
+    prune(): Promise<number>;
+    /**
+     * Drops superseded revisions, keeping only the newest row per document.
+     *
+     * Safe because replication carries whole documents rather than deltas: the
+     * newest row alone reproduces the same upstream state. Rows at or below the
+     * slowest replicator's checkpoint are left to {@link SyncOutbox.prune}.
+     *
+     * @returns Number of rows removed.
+     */
+    compact(): Promise<number>;
+    /**
+     * Appends a change back onto the log so it is retried on a later drain.
+     *
+     * Used when a conflict is resolved in favour of the local version: the shadow has
+     * since been refreshed, so the replayed push diffs against current upstream state and
+     * carries the version it actually lost to.
+     */
+    requeue(collection: string, documentId: string, document: Record<string, unknown> | null): Promise<void>;
+    /** Moves a permanently-rejected change out of the stream so it cannot stall replication. */
+    deadLetter(replicator: string, record: {
+        collection: string;
+        documentId: string;
+        operation: string;
+        document: unknown;
+    }, error: string): Promise<void>;
+    /** Reads dead-lettered changes, newest first. */
+    listDeadLetters(replicator?: string, limit?: number): Promise<SyncDeadLetter[]>;
+    /** Removes dead-lettered rows, e.g. after replaying them by hand. */
+    clearDeadLetters(replicator?: string): Promise<number>;
+    /**
+     * Streams a collection's current contents in `_id` order, for the initial
+     * backfill. Keyset pagination keeps memory flat on large collections.
+     */
+    scanCollection(collection: string, chunkSize: number): AsyncGenerator<Array<{
+        documentId: string;
+        document: Record<string, unknown> | null;
+    }>>;
+}