@powersync/service-module-mongodb 0.0.0-dev-20241001150444

package/dist/replication/MongoErrorRateLimiter.js

@@ -0,0 +1,44 @@
+import { setTimeout } from 'timers/promises';
+export class MongoErrorRateLimiter {
+    constructor() {
+        this.nextAllowed = Date.now();
+    }
+    async waitUntilAllowed(options) {
+        const delay = Math.max(0, this.nextAllowed - Date.now());
+        // Minimum delay between connections, even without errors
+        this.setDelay(500);
+        await setTimeout(delay, undefined, { signal: options?.signal });
+    }
+    mayPing() {
+        return Date.now() >= this.nextAllowed;
+    }
+    reportError(e) {
+        // FIXME: Check mongodb-specific requirements
+        const message = e.message ?? '';
+        if (message.includes('password authentication failed')) {
+            // Wait 15 minutes, to avoid triggering Supabase's fail2ban
+            this.setDelay(900000);
+        }
+        else if (message.includes('ENOTFOUND')) {
+            // DNS lookup issue - incorrect URI or deleted instance
+            this.setDelay(120000);
+        }
+        else if (message.includes('ECONNREFUSED')) {
+            // Could be fail2ban or similar
+            this.setDelay(120000);
+        }
+        else if (message.includes('Unable to do postgres query on ended pool') ||
+            message.includes('Postgres unexpectedly closed connection')) {
+            // Connection timed out - ignore / immediately retry
+            // We don't explicitly set the delay to 0, since there could have been another error that
+            // we need to respect.
+        }
+        else {
+            this.setDelay(30000);
+        }
+    }
+    setDelay(delay) {
+        this.nextAllowed = Math.max(this.nextAllowed, Date.now() + delay);
+    }
+}
+//# sourceMappingURL=MongoErrorRateLimiter.js.map

package/dist/replication/MongoErrorRateLimiter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"MongoErrorRateLimiter.js","sourceRoot":"","sources":["../../src/replication/MongoErrorRateLimiter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAG7C,MAAM,OAAO,qBAAqB;IAAlC;QACE,gBAAW,GAAW,IAAI,CAAC,GAAG,EAAE,CAAC;IAwCnC,CAAC;IAtCC,KAAK,CAAC,gBAAgB,CAAC,OAA0D;QAC/E,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;QACzD,yDAAyD;QACzD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;QACnB,MAAM,UAAU,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;IAClE,CAAC;IAED,OAAO;QACL,OAAO,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,WAAW,CAAC;IACxC,CAAC;IAED,WAAW,CAAC,CAAM;QAChB,6CAA6C;QAC7C,MAAM,OAAO,GAAI,CAAC,CAAC,OAAkB,IAAI,EAAE,CAAC;QAC5C,IAAI,OAAO,CAAC,QAAQ,CAAC,gCAAgC,CAAC,EAAE;YACtD,2DAA2D;YAC3D,IAAI,CAAC,QAAQ,CAAC,MAAO,CAAC,CAAC;SACxB;aAAM,IAAI,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE;YACxC,uDAAuD;YACvD,IAAI,CAAC,QAAQ,CAAC,MAAO,CAAC,CAAC;SACxB;aAAM,IAAI,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,EAAE;YAC3C,+BAA+B;YAC/B,IAAI,CAAC,QAAQ,CAAC,MAAO,CAAC,CAAC;SACxB;aAAM,IACL,OAAO,CAAC,QAAQ,CAAC,2CAA2C,CAAC;YAC7D,OAAO,CAAC,QAAQ,CAAC,yCAAyC,CAAC,EAC3D;YACA,oDAAoD;YACpD,yFAAyF;YACzF,sBAAsB;SACvB;aAAM;YACL,IAAI,CAAC,QAAQ,CAAC,KAAM,CAAC,CAAC;SACvB;IACH,CAAC;IAEO,QAAQ,CAAC,KAAa;QAC5B,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,CAAC;IACpE,CAAC;CACF"}

package/dist/replication/MongoManager.d.ts

@@ -0,0 +1,14 @@
+import * as mongo from 'mongodb';
+import { NormalizedMongoConnectionConfig } from '../types/types.js';
+export declare class MongoManager {
+    options: NormalizedMongoConnectionConfig;
+    /**
+     * Do not use this for any transactions.
+     */
+    readonly client: mongo.MongoClient;
+    readonly db: mongo.Db;
+    constructor(options: NormalizedMongoConnectionConfig);
+    get connectionTag(): string;
+    end(): Promise<void>;
+    destroy(): Promise<void>;
+}

package/dist/replication/MongoManager.js

@@ -0,0 +1,36 @@
+import * as mongo from 'mongodb';
+export class MongoManager {
+    constructor(options) {
+        this.options = options;
+        // The pool is lazy - no connections are opened until a query is performed.
+        this.client = new mongo.MongoClient(options.uri, {
+            auth: {
+                username: options.username,
+                password: options.password
+            },
+            // Time for connection to timeout
+            connectTimeoutMS: 5000,
+            // Time for individual requests to timeout
+            socketTimeoutMS: 60000,
+            // How long to wait for new primary selection
+            serverSelectionTimeoutMS: 30000,
+            // Avoid too many connections:
+            // 1. It can overwhelm the source database.
+            // 2. Processing too many queries in parallel can cause the process to run out of memory.
+            maxPoolSize: 8,
+            maxConnecting: 3,
+            maxIdleTimeMS: 60000
+        });
+        this.db = this.client.db(options.database, {});
+    }
+    get connectionTag() {
+        return this.options.tag;
+    }
+    async end() {
+        await this.client.close();
+    }
+    async destroy() {
+        // TODO: Implement?
+    }
+}
+//# sourceMappingURL=MongoManager.js.map

package/dist/replication/MongoManager.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"MongoManager.js","sourceRoot":"","sources":["../../src/replication/MongoManager.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,KAAK,MAAM,SAAS,CAAC;AAGjC,MAAM,OAAO,YAAY;IAOvB,YAAmB,OAAwC;QAAxC,YAAO,GAAP,OAAO,CAAiC;QACzD,2EAA2E;QAC3E,IAAI,CAAC,MAAM,GAAG,IAAI,KAAK,CAAC,WAAW,CAAC,OAAO,CAAC,GAAG,EAAE;YAC/C,IAAI,EAAE;gBACJ,QAAQ,EAAE,OAAO,CAAC,QAAQ;gBAC1B,QAAQ,EAAE,OAAO,CAAC,QAAQ;aAC3B;YACD,iCAAiC;YACjC,gBAAgB,EAAE,IAAK;YACvB,0CAA0C;YAC1C,eAAe,EAAE,KAAM;YACvB,6CAA6C;YAC7C,wBAAwB,EAAE,KAAM;YAEhC,8BAA8B;YAC9B,2CAA2C;YAC3C,yFAAyF;YACzF,WAAW,EAAE,CAAC;YAEd,aAAa,EAAE,CAAC;YAChB,aAAa,EAAE,KAAM;SACtB,CAAC,CAAC;QACH,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;IACjD,CAAC;IAED,IAAW,aAAa;QACtB,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC;IAC1B,CAAC;IAED,KAAK,CAAC,GAAG;QACP,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;IAC5B,CAAC;IAED,KAAK,CAAC,OAAO;QACX,mBAAmB;IACrB,CAAC;CACF"}

package/dist/replication/MongoRelation.d.ts

@@ -0,0 +1,9 @@
+import { storage } from '@powersync/service-core';
+import { SqliteRow, SqliteValue } from '@powersync/service-sync-rules';
+import * as mongo from 'mongodb';
+export declare function getMongoRelation(source: mongo.ChangeStreamNameSpace): storage.SourceEntityDescriptor;
+export declare function getMongoLsn(timestamp: mongo.Timestamp): string;
+export declare function mongoLsnToTimestamp(lsn: string | null): mongo.BSON.Timestamp | null;
+export declare function constructAfterRecord(document: mongo.Document): SqliteRow;
+export declare function toMongoSyncRulesValue(data: any): SqliteValue;
+export declare function createCheckpoint(client: mongo.MongoClient, db: mongo.Db): Promise<string>;

package/dist/replication/MongoRelation.js

@@ -0,0 +1,174 @@
+import * as mongo from 'mongodb';
+import { JSONBig, JsonContainer } from '@powersync/service-jsonbig';
+export function getMongoRelation(source) {
+    return {
+        name: source.coll,
+        schema: source.db,
+        objectId: source.coll,
+        replicationColumns: [{ name: '_id' }]
+    };
+}
+export function getMongoLsn(timestamp) {
+    const a = timestamp.high.toString(16).padStart(8, '0');
+    const b = timestamp.low.toString(16).padStart(8, '0');
+    return a + b;
+}
+export function mongoLsnToTimestamp(lsn) {
+    if (lsn == null) {
+        return null;
+    }
+    const a = parseInt(lsn.substring(0, 8), 16);
+    const b = parseInt(lsn.substring(8, 16), 16);
+    return mongo.Timestamp.fromBits(b, a);
+}
+export function constructAfterRecord(document) {
+    let record = {};
+    for (let key of Object.keys(document)) {
+        record[key] = toMongoSyncRulesValue(document[key]);
+    }
+    return record;
+}
+export function toMongoSyncRulesValue(data) {
+    const autoBigNum = true;
+    if (data == null) {
+        // null or undefined
+        return data;
+    }
+    else if (typeof data == 'string') {
+        return data;
+    }
+    else if (typeof data == 'number') {
+        if (Number.isInteger(data) && autoBigNum) {
+            return BigInt(data);
+        }
+        else {
+            return data;
+        }
+    }
+    else if (typeof data == 'bigint') {
+        return data;
+    }
+    else if (typeof data == 'boolean') {
+        return data ? 1n : 0n;
+    }
+    else if (data instanceof mongo.ObjectId) {
+        return data.toHexString();
+    }
+    else if (data instanceof mongo.UUID) {
+        return data.toHexString();
+    }
+    else if (data instanceof Date) {
+        return data.toISOString().replace('T', ' ');
+    }
+    else if (data instanceof mongo.Binary) {
+        return new Uint8Array(data.buffer);
+    }
+    else if (data instanceof mongo.Long) {
+        return data.toBigInt();
+    }
+    else if (Array.isArray(data)) {
+        // We may be able to avoid some parse + stringify cycles here for JsonSqliteContainer.
+        return JSONBig.stringify(data.map((element) => filterJsonData(element)));
+    }
+    else if (data instanceof Uint8Array) {
+        return data;
+    }
+    else if (data instanceof JsonContainer) {
+        return data.toString();
+    }
+    else if (typeof data == 'object') {
+        let record = {};
+        for (let key of Object.keys(data)) {
+            record[key] = filterJsonData(data[key]);
+        }
+        return JSONBig.stringify(record);
+    }
+    else {
+        return null;
+    }
+}
+const DEPTH_LIMIT = 20;
+function filterJsonData(data, depth = 0) {
+    const autoBigNum = true;
+    if (depth > DEPTH_LIMIT) {
+        // This is primarily to prevent infinite recursion
+        throw new Error(`json nested object depth exceeds the limit of ${DEPTH_LIMIT}`);
+    }
+    if (data == null) {
+        return data; // null or undefined
+    }
+    else if (typeof data == 'string') {
+        return data;
+    }
+    else if (typeof data == 'number') {
+        if (autoBigNum && Number.isInteger(data)) {
+            return BigInt(data);
+        }
+        else {
+            return data;
+        }
+    }
+    else if (typeof data == 'boolean') {
+        return data ? 1n : 0n;
+    }
+    else if (typeof data == 'bigint') {
+        return data;
+    }
+    else if (data instanceof Date) {
+        return data.toISOString().replace('T', ' ');
+    }
+    else if (data instanceof mongo.ObjectId) {
+        return data.toHexString();
+    }
+    else if (data instanceof mongo.UUID) {
+        return data.toHexString();
+    }
+    else if (data instanceof mongo.Binary) {
+        return undefined;
+    }
+    else if (data instanceof mongo.Long) {
+        return data.toBigInt();
+    }
+    else if (Array.isArray(data)) {
+        return data.map((element) => filterJsonData(element, depth + 1));
+    }
+    else if (ArrayBuffer.isView(data)) {
+        return undefined;
+    }
+    else if (data instanceof JsonContainer) {
+        // Can be stringified directly when using our JSONBig implementation
+        return data;
+    }
+    else if (typeof data == 'object') {
+        let record = {};
+        for (let key of Object.keys(data)) {
+            record[key] = filterJsonData(data[key], depth + 1);
+        }
+        return record;
+    }
+    else {
+        return undefined;
+    }
+}
+export async function createCheckpoint(client, db) {
+    const session = client.startSession();
+    try {
+        const result = await db.collection('_powersync_checkpoints').findOneAndUpdate({
+            _id: 'checkpoint'
+        }, {
+            $inc: { i: 1 }
+        }, {
+            upsert: true,
+            returnDocument: 'after',
+            session
+        });
+        const time = session.operationTime;
+        // console.log('marked checkpoint at', time, getMongoLsn(time));
+        // TODO: Use the above when we support custom write checkpoints
+        return getMongoLsn(time);
+    }
+    finally {
+        await session.endSession();
+    }
+}
+//# sourceMappingURL=MongoRelation.js.map

package/dist/replication/MongoRelation.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"MongoRelation.js","sourceRoot":"","sources":["../../src/replication/MongoRelation.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,KAAK,MAAM,SAAS,CAAC;AACjC,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAEpE,MAAM,UAAU,gBAAgB,CAAC,MAAmC;IAClE,OAAO;QACL,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,MAAM,EAAE,MAAM,CAAC,EAAE;QACjB,QAAQ,EAAE,MAAM,CAAC,IAAI;QACrB,kBAAkB,EAAE,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;KACG,CAAC;AAC7C,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,SAA0B;IACpD,MAAM,CAAC,GAAG,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IACvD,MAAM,CAAC,GAAG,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IACtD,OAAO,CAAC,GAAG,CAAC,CAAC;AACf,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,GAAkB;IACpD,IAAI,GAAG,IAAI,IAAI,EAAE;QACf,OAAO,IAAI,CAAC;KACb;IACD,MAAM,CAAC,GAAG,QAAQ,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC5C,MAAM,CAAC,GAAG,QAAQ,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;IAC7C,OAAO,KAAK,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AACxC,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,QAAwB;IAC3D,IAAI,MAAM,GAAc,EAAE,CAAC;IAC3B,KAAK,IAAI,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE;QACrC,MAAM,CAAC,GAAG,CAAC,GAAG,qBAAqB,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;KACpD;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,IAAS;IAC7C,MAAM,UAAU,GAAG,IAAI,CAAC;IACxB,IAAI,IAAI,IAAI,IAAI,EAAE;QAChB,oBAAoB;QACpB,OAAO,IAAI,CAAC;KACb;SAAM,IAAI,OAAO,IAAI,IAAI,QAAQ,EAAE;QAClC,OAAO,IAAI,CAAC;KACb;SAAM,IAAI,OAAO,IAAI,IAAI,QAAQ,EAAE;QAClC,IAAI,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,UAAU,EAAE;YACxC,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC;SACrB;aAAM;YACL,OAAO,IAAI,CAAC;SACb;KACF;SAAM,IAAI,OAAO,IAAI,IAAI,QAAQ,EAAE;QAClC,OAAO,IAAI,CAAC;KACb;SAAM,IAAI,OAAO,IAAI,IAAI,SAAS,EAAE;QACnC,OAAO,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACvB;SAAM,IAAI,IAAI,YAAY,KAAK,CAAC,QAAQ,EAAE;QACzC,OAAO,IAAI,CAAC,WAAW,EAAE,CAAC;KAC3B;SAAM,IAAI,IAAI,YAAY,KAAK,CAAC,IAAI,EAAE;QACrC,OAAO,IAAI,CAAC,WAAW,EAAE,CAAC;KAC3B;SAAM,IAAI,IAAI,YAAY,IAAI,EAAE;QAC/B,OAAO,IAAI,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;KAC7C;SAAM,IAAI,IAAI,YAAY,KAAK,CAAC,MAAM,EAAE;QACvC,OAAO,IAAI,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;KACpC;SAAM,IAAI,IAAI,YAAY,KAAK,CAAC,IAAI,EAAE;QACrC,OAAO,IAAI,CAAC,QAAQ,EAAE,CAAC;KACxB;SAAM,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE;QAC9B,sFAAsF;QACtF,OAAO,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;KAC1E;SAAM,IAAI,IAAI,YAAY,UAAU,EAAE;QACrC,OAAO,IAAI,CAAC;KACb;SAAM,IAAI,IAAI,YAAY,aAAa,EAAE;QACxC,OAAO,IAAI,CAAC,QAAQ,EAAE,CAAC;KACxB;SAAM,IAAI,OAAO,IAAI,IAAI,QAAQ,EAAE;QAClC,IAAI,MAAM,GAAwB,EAAE,CAAC;QACrC,KAAK,IAAI,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;YACjC,MAAM,CAAC,GAAG,CAAC,GAAG,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;SACzC;QACD,OAAO,OAAO,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;KAClC;SAAM;QACL,OAAO,IAAI,CAAC;KACb;AACH,CAAC;AAED,MAAM,WAAW,GAAG,EAAE,CAAC;AAEvB,SAAS,cAAc,CAAC,IAAS,EAAE,KAAK,GAAG,CAAC;IAC1C,MAAM,UAAU,GAAG,IAAI,CAAC;IACxB,IAAI,KAAK,GAAG,WAAW,EAAE;QACvB,kDAAkD;QAClD,MAAM,IAAI,KAAK,CAAC,iDAAiD,WAAW,EAAE,CAAC,CAAC;KACjF;IACD,IAAI,IAAI,IAAI,IAAI,EAAE;QAChB,OAAO,IAAI,CAAC,CAAC,oBAAoB;KAClC;SAAM,IAAI,OAAO,IAAI,IAAI,QAAQ,EAAE;QAClC,OAAO,IAAI,CAAC;KACb;SAAM,IAAI,OAAO,IAAI,IAAI,QAAQ,EAAE;QAClC,IAAI,UAAU,IAAI,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE;YACxC,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC;SACrB;aAAM;YACL,OAAO,IAAI,CAAC;SACb;KACF;SAAM,IAAI,OAAO,IAAI,IAAI,SAAS,EAAE;QACnC,OAAO,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACvB;SAAM,IAAI,OAAO,IAAI,IAAI,QAAQ,EAAE;QAClC,OAAO,IAAI,CAAC;KACb;SAAM,IAAI,IAAI,YAAY,IAAI,EAAE;QAC/B,OAAO,IAAI,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;KAC7C;SAAM,IAAI,IAAI,YAAY,KAAK,CAAC,QAAQ,EAAE;QACzC,OAAO,IAAI,CAAC,WAAW,EAAE,CAAC;KAC3B;SAAM,IAAI,IAAI,YAAY,KAAK,CAAC,IAAI,EAAE;QACrC,OAAO,IAAI,CAAC,WAAW,EAAE,CAAC;KAC3B;SAAM,IAAI,IAAI,YAAY,KAAK,CAAC,MAAM,EAAE;QACvC,OAAO,SAAS,CAAC;KAClB;SAAM,IAAI,IAAI,YAAY,KAAK,CAAC,IAAI,EAAE;QACrC,OAAO,IAAI,CAAC,QAAQ,EAAE,CAAC;KACxB;SAAM,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE;QAC9B,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,cAAc,CAAC,OAAO,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC;KAClE;SAAM,IAAI,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE;QACnC,OAAO,SAAS,CAAC;KAClB;SAAM,IAAI,IAAI,YAAY,aAAa,EAAE;QACxC,oEAAoE;QACpE,OAAO,IAAI,CAAC;KACb;SAAM,IAAI,OAAO,IAAI,IAAI,QAAQ,EAAE;QAClC,IAAI,MAAM,GAAwB,EAAE,CAAC;QACrC,KAAK,IAAI,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;YACjC,MAAM,CAAC,GAAG,CAAC,GAAG,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;SACpD;QACD,OAAO,MAAM,CAAC;KACf;SAAM;QACL,OAAO,SAAS,CAAC;KAClB;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,MAAyB,EAAE,EAAY;IAC5E,MAAM,OAAO,GAAG,MAAM,CAAC,YAAY,EAAE,CAAC;IACtC,IAAI;QACF,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,UAAU,CAAC,wBAAwB,CAAC,CAAC,gBAAgB,CAC3E;YACE,GAAG,EAAE,YAAmB;SACzB,EACD;YACE,IAAI,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE;SACf,EACD;YACE,MAAM,EAAE,IAAI;YACZ,cAAc,EAAE,OAAO;YACvB,OAAO;SACR,CACF,CAAC;QACF,MAAM,IAAI,GAAG,OAAO,CAAC,aAAc,CAAC;QACpC,gEAAgE;QAChE,+DAA+D;QAC/D,OAAO,WAAW,CAAC,IAAI,CAAC,CAAC;KAC1B;YAAS;QACR,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC;KAC5B;AACH,CAAC"}

package/dist/replication/replication-index.d.ts

@@ -0,0 +1,4 @@
+export * from './MongoRelation.js';
+export * from './ChangeStream.js';
+export * from './ChangeStreamReplicator.js';
+export * from './ChangeStreamReplicationJob.js';

package/dist/replication/replication-index.js

@@ -0,0 +1,5 @@
+export * from './MongoRelation.js';
+export * from './ChangeStream.js';
+export * from './ChangeStreamReplicator.js';
+export * from './ChangeStreamReplicationJob.js';
+//# sourceMappingURL=replication-index.js.map

package/dist/replication/replication-index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"replication-index.js","sourceRoot":"","sources":["../../src/replication/replication-index.ts"],"names":[],"mappings":"AAAA,cAAc,oBAAoB,CAAC;AACnC,cAAc,mBAAmB,CAAC;AAClC,cAAc,6BAA6B,CAAC;AAC5C,cAAc,iCAAiC,CAAC"}

package/dist/types/types.d.ts

@@ -0,0 +1,51 @@
+import * as t from 'ts-codec';
+export declare const MONGO_CONNECTION_TYPE: "mongodb";
+export interface NormalizedMongoConnectionConfig {
+    id: string;
+    tag: string;
+    uri: string;
+    database: string;
+    username?: string;
+    password?: string;
+}
+export declare const MongoConnectionConfig: t.Intersection<t.Codec<{
+    type: string;
+    id?: string | undefined;
+    tag?: string | undefined;
+    debug_api?: boolean | undefined;
+}, {
+    type: string;
+    id?: string | undefined;
+    tag?: string | undefined;
+    debug_api?: boolean | undefined;
+}, string, t.CodecProps>, t.ObjectCodec<{
+    type: t.LiteralCodec<"mongodb">;
+    /** Unique identifier for the connection - optional when a single connection is present. */
+    id: t.OptionalCodec<t.Codec<string, string, string, t.CodecProps>>;
+    /** Tag used as reference in sync rules. Defaults to "default". Does not have to be unique. */
+    tag: t.OptionalCodec<t.Codec<string, string, string, t.CodecProps>>;
+    uri: t.IdentityCodec<t.CodecType.String>;
+    username: t.OptionalCodec<t.Codec<string, string, string, t.CodecProps>>;
+    password: t.OptionalCodec<t.Codec<string, string, string, t.CodecProps>>;
+    database: t.OptionalCodec<t.Codec<string, string, string, t.CodecProps>>;
+}>>;
+/**
+ * Config input specified when starting services
+ */
+export type MongoConnectionConfig = t.Decoded<typeof MongoConnectionConfig>;
+/**
+ * Resolved version of {@link MongoConnectionConfig}
+ */
+export type ResolvedConnectionConfig = MongoConnectionConfig & NormalizedMongoConnectionConfig;
+/**
+ * Validate and normalize connection options.
+ *
+ * Returns destructured options.
+ */
+export declare function normalizeConnectionConfig(options: MongoConnectionConfig): NormalizedMongoConnectionConfig;
+/**
+ * Construct a mongodb URI, without username, password or ssl options.
+ *
+ * Only contains hostname, port, database.
+ */
+export declare function baseUri(options: NormalizedMongoConnectionConfig): string;

package/dist/types/types.js

@@ -0,0 +1,37 @@
+import { normalizeMongoConfig } from '@powersync/service-core';
+import * as service_types from '@powersync/service-types';
+import * as t from 'ts-codec';
+export const MONGO_CONNECTION_TYPE = 'mongodb';
+export const MongoConnectionConfig = service_types.configFile.dataSourceConfig.and(t.object({
+    type: t.literal(MONGO_CONNECTION_TYPE),
+    /** Unique identifier for the connection - optional when a single connection is present. */
+    id: t.string.optional(),
+    /** Tag used as reference in sync rules. Defaults to "default". Does not have to be unique. */
+    tag: t.string.optional(),
+    uri: t.string,
+    username: t.string.optional(),
+    password: t.string.optional(),
+    database: t.string.optional()
+}));
+/**
+ * Validate and normalize connection options.
+ *
+ * Returns destructured options.
+ */
+export function normalizeConnectionConfig(options) {
+    const base = normalizeMongoConfig(options);
+    return {
+        id: options.id ?? 'default',
+        tag: options.tag ?? 'default',
+        ...base
+    };
+}
+/**
+ * Construct a mongodb URI, without username, password or ssl options.
+ *
+ * Only contains hostname, port, database.
+ */
+export function baseUri(options) {
+    return options.uri;
+}
+//# sourceMappingURL=types.js.map

package/dist/types/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/types/types.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,KAAK,aAAa,MAAM,0BAA0B,CAAC;AAC1D,OAAO,KAAK,CAAC,MAAM,UAAU,CAAC;AAE9B,MAAM,CAAC,MAAM,qBAAqB,GAAG,SAAkB,CAAC;AAaxD,MAAM,CAAC,MAAM,qBAAqB,GAAG,aAAa,CAAC,UAAU,CAAC,gBAAgB,CAAC,GAAG,CAChF,CAAC,CAAC,MAAM,CAAC;IACP,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,qBAAqB,CAAC;IACtC,2FAA2F;IAC3F,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;IACvB,8FAA8F;IAC9F,GAAG,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;IACxB,GAAG,EAAE,CAAC,CAAC,MAAM;IACb,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;IAC7B,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;IAC7B,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;CAC9B,CAAC,CACH,CAAC;AAYF;;;;GAIG;AACH,MAAM,UAAU,yBAAyB,CAAC,OAA8B;IACtE,MAAM,IAAI,GAAG,oBAAoB,CAAC,OAAO,CAAC,CAAC;IAE3C,OAAO;QACL,EAAE,EAAE,OAAO,CAAC,EAAE,IAAI,SAAS;QAC3B,GAAG,EAAE,OAAO,CAAC,GAAG,IAAI,SAAS;QAE7B,GAAG,IAAI;KACR,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,OAAO,CAAC,OAAwC;IAC9D,OAAO,OAAO,CAAC,GAAG,CAAC;AACrB,CAAC"}

package/package.json

@@ -0,0 +1,47 @@
+{
+  "name": "@powersync/service-module-mongodb",
+  "repository": "https://github.com/powersync-ja/powersync-service",
+  "types": "dist/index.d.ts",
+  "version": "0.0.0-dev-20241001150444",
+  "main": "dist/index.js",
+  "license": "FSL-1.1-Apache-2.0",
+  "type": "module",
+  "publishConfig": {
+    "access": "public"
+  },
+  "exports": {
+    ".": {
+      "import": "./dist/index.js",
+      "require": "./dist/index.js",
+      "default": "./dist/index.js"
+    },
+    "./types": {
+      "import": "./dist/types/types.js",
+      "require": "./dist/types/types.js",
+      "default": "./dist/types/types.js"
+    }
+  },
+  "dependencies": {
+    "mongodb": "^6.7.0",
+    "ts-codec": "^1.2.2",
+    "uuid": "^9.0.1",
+    "uri-js": "^4.4.1",
+    "@powersync/lib-services-framework": "0.0.0-dev-20241001150444",
+    "@powersync/service-core": "0.0.0-dev-20241001150444",
+    "@powersync/service-jsonbig": "0.17.10",
+    "@powersync/service-sync-rules": "0.0.0-dev-20241001150444",
+    "@powersync/service-types": "0.0.0-dev-20241001150444"
+  },
+  "devDependencies": {
+    "@types/uuid": "^9.0.4",
+    "typescript": "^5.2.2",
+    "vitest": "^0.34.6",
+    "vite-tsconfig-paths": "^4.3.2"
+  },
+  "scripts": {
+    "build": "tsc -b",
+    "build:tests": "tsc -b test/tsconfig.json",
+    "clean": "rm -rf ./lib && tsc -b --clean",
+    "test": "vitest --no-threads"
+  }
+}

package/src/api/MongoRouteAPIAdapter.ts

@@ -0,0 +1,86 @@
+import { api, ParseSyncRulesOptions } from '@powersync/service-core';
+import * as mongo from 'mongodb';
+
+import * as sync_rules from '@powersync/service-sync-rules';
+import * as service_types from '@powersync/service-types';
+import * as types from '../types/types.js';
+import { MongoManager } from '../replication/MongoManager.js';
+import { createCheckpoint, getMongoLsn } from '../replication/MongoRelation.js';
+
+export class MongoRouteAPIAdapter implements api.RouteAPI {
+  protected client: mongo.MongoClient;
+  private db: mongo.Db;
+
+  connectionTag: string;
+  defaultSchema: string;
+
+  constructor(protected config: types.ResolvedConnectionConfig) {
+    const manager = new MongoManager(config);
+    this.client = manager.client;
+    this.db = manager.db;
+    this.defaultSchema = manager.db.databaseName;
+    this.connectionTag = config.tag ?? sync_rules.DEFAULT_TAG;
+  }
+
+  getParseSyncRulesOptions(): ParseSyncRulesOptions {
+    return {
+      defaultSchema: this.defaultSchema
+    };
+  }
+
+  async shutdown(): Promise<void> {
+    await this.client.close();
+  }
+
+  async getSourceConfig(): Promise<service_types.configFile.DataSourceConfig> {
+    return this.config;
+  }
+
+  async getConnectionStatus(): Promise<service_types.ConnectionStatusV2> {
+    // TODO: Implement
+    const base = {
+      id: this.config.id,
+      uri: types.baseUri(this.config)
+    };
+    return {
+      ...base,
+      connected: true,
+      errors: []
+    };
+  }
+
+  async executeQuery(query: string, params: any[]): Promise<service_types.internal_routes.ExecuteSqlResponse> {
+    return service_types.internal_routes.ExecuteSqlResponse.encode({
+      results: {
+        columns: [],
+        rows: []
+      },
+      success: false,
+      error: 'SQL querying is not supported for MongoDB'
+    });
+  }
+
+  async getDebugTablesInfo(
+    tablePatterns: sync_rules.TablePattern[],
+    sqlSyncRules: sync_rules.SqlSyncRules
+  ): Promise<api.PatternResult[]> {
+    // TODO: Implement
+    return [];
+  }
+
+  async getReplicationLag(syncRulesId: string): Promise<number> {
+    // TODO: Implement
+
+    return 0;
+  }
+
+  async getReplicationHead(): Promise<string> {
+    return createCheckpoint(this.client, this.db);
+  }
+
+  async getConnectionSchema(): Promise<service_types.DatabaseSchema[]> {
+    // TODO: Implement
+
+    return [];
+  }
+}

package/src/index.ts

@@ -0,0 +1,5 @@
+import { MongoModule } from './module/MongoModule.js';
+
+export const module = new MongoModule();
+
+export default module;

package/src/module/MongoModule.ts

@@ -0,0 +1,52 @@
+import { api, ConfigurationFileSyncRulesProvider, replication, system, TearDownOptions } from '@powersync/service-core';
+import { MongoRouteAPIAdapter } from '../api/MongoRouteAPIAdapter.js';
+import { ConnectionManagerFactory } from '../replication/ConnectionManagerFactory.js';
+import { MongoErrorRateLimiter } from '../replication/MongoErrorRateLimiter.js';
+import { ChangeStreamReplicator } from '../replication/ChangeStreamReplicator.js';
+import * as types from '../types/types.js';
+
+export class MongoModule extends replication.ReplicationModule<types.MongoConnectionConfig> {
+  constructor() {
+    super({
+      name: 'MongoDB',
+      type: types.MONGO_CONNECTION_TYPE,
+      configSchema: types.MongoConnectionConfig
+    });
+  }
+
+  async initialize(context: system.ServiceContextContainer): Promise<void> {
+    await super.initialize(context);
+  }
+
+  protected createRouteAPIAdapter(): api.RouteAPI {
+    return new MongoRouteAPIAdapter(this.resolveConfig(this.decodedConfig!));
+  }
+
+  protected createReplicator(context: system.ServiceContext): replication.AbstractReplicator {
+    const normalisedConfig = this.resolveConfig(this.decodedConfig!);
+    const syncRuleProvider = new ConfigurationFileSyncRulesProvider(context.configuration.sync_rules);
+    const connectionFactory = new ConnectionManagerFactory(normalisedConfig);
+
+    return new ChangeStreamReplicator({
+      id: this.getDefaultId(normalisedConfig.database ?? ''),
+      syncRuleProvider: syncRuleProvider,
+      storageEngine: context.storageEngine,
+      connectionFactory: connectionFactory,
+      rateLimiter: new MongoErrorRateLimiter()
+    });
+  }
+
+  /**
+   * Combines base config with normalized connection settings
+   */
+  private resolveConfig(config: types.MongoConnectionConfig): types.ResolvedConnectionConfig {
+    return {
+      ...config,
+      ...types.normalizeConnectionConfig(config)
+    };
+  }
+
+  async teardown(options: TearDownOptions): Promise<void> {
+    // TODO: Implement?
+  }
+}