@powersync/service-module-mongodb 0.0.0-dev-20241128134723 → 0.0.0-dev-20241219110735

package/dist/storage/implementation/db.js

@@ -0,0 +1,47 @@
+import * as db from '../../db/db-index.js';
+import { BSON_DESERIALIZE_OPTIONS } from './util.js';
+export function createPowerSyncMongo(config) {
+    return new PowerSyncMongo(db.mongo.createMongoClient(config), { database: config.database });
+}
+export class PowerSyncMongo {
+    constructor(client, options) {
+        this.client = client;
+        const db = client.db(options?.database, {
+            ...BSON_DESERIALIZE_OPTIONS
+        });
+        this.db = db;
+        this.current_data = db.collection('current_data');
+        this.bucket_data = db.collection('bucket_data');
+        this.bucket_parameters = db.collection('bucket_parameters');
+        this.op_id_sequence = db.collection('op_id_sequence');
+        this.sync_rules = db.collection('sync_rules');
+        this.source_tables = db.collection('source_tables');
+        this.custom_write_checkpoints = db.collection('custom_write_checkpoints');
+        this.write_checkpoints = db.collection('write_checkpoints');
+        this.instance = db.collection('instance');
+        this.locks = this.db.collection('locks');
+    }
+    /**
+     * Clear all collections.
+     */
+    async clear() {
+        await this.current_data.deleteMany({});
+        await this.bucket_data.deleteMany({});
+        await this.bucket_parameters.deleteMany({});
+        await this.op_id_sequence.deleteMany({});
+        await this.sync_rules.deleteMany({});
+        await this.source_tables.deleteMany({});
+        await this.write_checkpoints.deleteMany({});
+        await this.instance.deleteOne({});
+        await this.locks.deleteMany({});
+    }
+    /**
+     * Drop the entire database.
+     *
+     * Primarily for tests.
+     */
+    async drop() {
+        await this.db.dropDatabase();
+    }
+}
+//# sourceMappingURL=db.js.map

package/dist/storage/implementation/db.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"db.js","sourceRoot":"","sources":["../../../src/storage/implementation/db.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAa3C,OAAO,EAAE,wBAAwB,EAAE,MAAM,WAAW,CAAC;AASrD,MAAM,UAAU,oBAAoB,CAAC,MAAqC;IACxE,OAAO,IAAI,cAAc,CAAC,EAAE,CAAC,KAAK,CAAC,iBAAiB,CAAC,MAAM,CAAC,EAAE,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;AAC/F,CAAC;AAED,MAAM,OAAO,cAAc;IAezB,YAAY,MAAyB,EAAE,OAA+B;QACpE,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QAErB,MAAM,EAAE,GAAG,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,QAAQ,EAAE;YACtC,GAAG,wBAAwB;SAC5B,CAAC,CAAC;QACH,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QAEb,IAAI,CAAC,YAAY,GAAG,EAAE,CAAC,UAAU,CAAsB,cAAc,CAAC,CAAC;QACvE,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;QAChD,IAAI,CAAC,iBAAiB,GAAG,EAAE,CAAC,UAAU,CAAC,mBAAmB,CAAC,CAAC;QAC5D,IAAI,CAAC,cAAc,GAAG,EAAE,CAAC,UAAU,CAAC,gBAAgB,CAAC,CAAC;QACtD,IAAI,CAAC,UAAU,GAAG,EAAE,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC;QAC9C,IAAI,CAAC,aAAa,GAAG,EAAE,CAAC,UAAU,CAAC,eAAe,CAAC,CAAC;QACpD,IAAI,CAAC,wBAAwB,GAAG,EAAE,CAAC,UAAU,CAAC,0BAA0B,CAAC,CAAC;QAC1E,IAAI,CAAC,iBAAiB,GAAG,EAAE,CAAC,UAAU,CAAC,mBAAmB,CAAC,CAAC;QAC5D,IAAI,CAAC,QAAQ,GAAG,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;QAC1C,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;IAC3C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACvC,MAAM,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACtC,MAAM,IAAI,CAAC,iBAAiB,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QAC5C,MAAM,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACzC,MAAM,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACrC,MAAM,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACxC,MAAM,IAAI,CAAC,iBAAiB,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QAC5C,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAClC,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;IAClC,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,IAAI;QACR,MAAM,IAAI,CAAC,EAAE,CAAC,YAAY,EAAE,CAAC;IAC/B,CAAC;CACF"}

package/dist/storage/implementation/models.d.ts

@@ -0,0 +1,139 @@
+import { storage } from '@powersync/service-core';
+import { SqliteJsonValue } from '@powersync/service-sync-rules';
+import * as bson from 'bson';
+/**
+ * Replica id uniquely identifying a row on the source database.
+ *
+ * Can be any value serializable to BSON.
+ *
+ * If the value is an entire document, the data serialized to a v5 UUID may be a good choice here.
+ */
+export type ReplicaId = bson.UUID | bson.Document | any;
+export interface SourceKey {
+    /** group_id */
+    g: number;
+    /** source table id */
+    t: bson.ObjectId;
+    /** source key */
+    k: ReplicaId;
+}
+export interface CurrentDataDocument {
+    _id: SourceKey;
+    data: bson.Binary;
+    buckets: CurrentBucket[];
+    lookups: bson.Binary[];
+}
+export interface CurrentBucket {
+    bucket: string;
+    table: string;
+    id: string;
+}
+export interface BucketParameterDocument {
+    _id: bigint;
+    key: SourceKey;
+    lookup: bson.Binary;
+    bucket_parameters: Record<string, SqliteJsonValue>[];
+}
+export interface BucketDataKey {
+    /** group_id */
+    g: number;
+    /** bucket name */
+    b: string;
+    /** op_id */
+    o: bigint;
+}
+export interface BucketDataDocument {
+    _id: BucketDataKey;
+    op: OpType;
+    source_table?: bson.ObjectId;
+    source_key?: storage.ReplicaId;
+    table?: string;
+    row_id?: string;
+    checksum: number;
+    data: string | null;
+    target_op?: bigint | null;
+}
+export type OpType = 'PUT' | 'REMOVE' | 'MOVE' | 'CLEAR';
+export interface SourceTableDocument {
+    _id: bson.ObjectId;
+    group_id: number;
+    connection_id: number;
+    relation_id: number | string | undefined;
+    schema_name: string;
+    table_name: string;
+    replica_id_columns: string[] | null;
+    replica_id_columns2: {
+        name: string;
+        type_oid?: number;
+        type?: string;
+    }[] | undefined;
+    snapshot_done: boolean | undefined;
+}
+export interface IdSequenceDocument {
+    _id: string;
+    op_id: bigint;
+}
+export interface SyncRuleDocument {
+    _id: number;
+    state: storage.SyncRuleState;
+    /**
+     * True if initial snapshot has been replicated.
+     *
+     * Can only be false if state == PROCESSING.
+     */
+    snapshot_done: boolean;
+    /**
+     * The last consistent checkpoint.
+     *
+     * There may be higher OpIds used in the database if we're in the middle of replicating a large transaction.
+     */
+    last_checkpoint: bigint | null;
+    /**
+     * The LSN associated with the last consistent checkpoint.
+     */
+    last_checkpoint_lsn: string | null;
+    /**
+     * If set, no new checkpoints may be created < this value.
+     */
+    no_checkpoint_before: string | null;
+    /**
+     * Goes together with no_checkpoint_before.
+     *
+     * If a keepalive is triggered that creates the checkpoint > no_checkpoint_before,
+     * then the checkpoint must be equal to this keepalive_op.
+     */
+    keepalive_op: string | null;
+    slot_name: string | null;
+    /**
+     * Last time we persisted a checkpoint.
+     *
+     * This may be old if no data is incoming.
+     */
+    last_checkpoint_ts: Date | null;
+    /**
+     * Last time we persisted a checkpoint or keepalive.
+     *
+     * This should stay fairly current while replicating.
+     */
+    last_keepalive_ts: Date | null;
+    /**
+     * If an error is stopping replication, it will be stored here.
+     */
+    last_fatal_error: string | null;
+    content: string;
+}
+export interface CustomWriteCheckpointDocument {
+    _id: bson.ObjectId;
+    user_id: string;
+    checkpoint: bigint;
+    sync_rules_id: number;
+}
+export interface WriteCheckpointDocument {
+    _id: bson.ObjectId;
+    user_id: string;
+    lsns: Record<string, string>;
+    client_id: bigint;
+}
+export interface InstanceDocument {
+    _id: string;
+}

package/dist/storage/implementation/models.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=models.js.map

package/dist/storage/implementation/models.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"models.js","sourceRoot":"","sources":["../../../src/storage/implementation/models.ts"],"names":[],"mappings":""}

package/dist/storage/implementation/util.d.ts

@@ -0,0 +1,58 @@
+import { storage, utils } from '@powersync/service-core';
+import { SqliteJsonValue } from '@powersync/service-sync-rules';
+import * as bson from 'bson';
+import * as mongo from 'mongodb';
+import { BucketDataDocument } from './models.js';
+/**
+ * Lookup serialization must be number-agnostic. I.e. normalize numbers, instead of preserving numbers.
+ * @param lookup
+ */
+export declare function serializeLookup(lookup: SqliteJsonValue[]): bson.Binary;
+export declare function idPrefixFilter<T>(prefix: Partial<T>, rest: (keyof T)[]): mongo.Condition<T>;
+export declare function generateSlotName(prefix: string, sync_rules_id: number): string;
+/**
+ * Read a single batch of data from a cursor, then close it.
+ *
+ * We do our best to avoid MongoDB fetching any more data than this single batch.
+ *
+ * This is similar to using `singleBatch: true` in find options.
+ * However, that makes `has_more` detection very difficult, since the cursor is always closed
+ * after the first batch. Instead, we do a workaround to only fetch a single batch below.
+ *
+ * For this to be effective, set batchSize = limit in the find command.
+ */
+export declare function readSingleBatch<T>(cursor: mongo.FindCursor<T>): Promise<{
+    data: T[];
+    hasMore: boolean;
+}>;
+export declare const BSON_DESERIALIZE_OPTIONS: bson.DeserializeOptions;
+export declare function mapOpEntry(row: BucketDataDocument): utils.OplogEntry;
+/**
+ * Returns true if two ReplicaId values are the same (serializes to the same BSON value).
+ */
+export declare function replicaIdEquals(a: storage.ReplicaId, b: storage.ReplicaId): boolean;
+export declare function replicaIdToSubkey(table: bson.ObjectId, id: storage.ReplicaId): string;
+/**
+ * True if this is a bson.UUID.
+ *
+ * Works even with multiple copies of the bson package.
+ */
+export declare function isUUID(value: any): value is bson.UUID;
+/**
+ * Helper function for creating a MongoDB client from consumers of this package
+ */
+export declare const createMongoClient: (url: string, options?: mongo.MongoClientOptions) => mongo.MongoClient;
+/**
+ * MongoDB bulkWrite internally splits the operations into batches
+ * so that no batch exceeds 16MB. However, there are cases where
+ * the batch size is very close to 16MB, where additional metadata
+ * on the server pushes it over the limit, resulting in this error
+ * from the server:
+ *
+ * > MongoBulkWriteError: BSONObj size: 16814023 (0x1008FC7) is invalid. Size must be between 0 and 16793600(16MB) First element: insert: "bucket_data"
+ *
+ * We work around the issue by doing our own batching, limiting the
+ * batch size to 15MB. This does add additional overhead with
+ * BSON.calculateObjectSize.
+ */
+export declare function safeBulkWrite<T extends mongo.Document>(collection: mongo.Collection<T>, operations: mongo.AnyBulkWriteOperation<T>[], options: mongo.BulkWriteOptions): Promise<void>;

package/dist/storage/implementation/util.js

@@ -0,0 +1,196 @@
+import { utils } from '@powersync/service-core';
+import * as bson from 'bson';
+import * as crypto from 'crypto';
+import * as mongo from 'mongodb';
+import * as uuid from 'uuid';
+/**
+ * Lookup serialization must be number-agnostic. I.e. normalize numbers, instead of preserving numbers.
+ * @param lookup
+ */
+export function serializeLookup(lookup) {
+    const normalized = lookup.map((value) => {
+        if (typeof value == 'number' && Number.isInteger(value)) {
+            return BigInt(value);
+        }
+        else {
+            return value;
+        }
+    });
+    return new bson.Binary(bson.serialize({ l: normalized }));
+}
+export function idPrefixFilter(prefix, rest) {
+    let filter = {
+        $gte: {
+            ...prefix
+        },
+        $lt: {
+            ...prefix
+        }
+    };
+    for (let key of rest) {
+        filter.$gte[key] = new bson.MinKey();
+        filter.$lt[key] = new bson.MaxKey();
+    }
+    return filter;
+}
+export function generateSlotName(prefix, sync_rules_id) {
+    const slot_suffix = crypto.randomBytes(2).toString('hex');
+    return `${prefix}${sync_rules_id}_${slot_suffix}`;
+}
+/**
+ * Read a single batch of data from a cursor, then close it.
+ *
+ * We do our best to avoid MongoDB fetching any more data than this single batch.
+ *
+ * This is similar to using `singleBatch: true` in find options.
+ * However, that makes `has_more` detection very difficult, since the cursor is always closed
+ * after the first batch. Instead, we do a workaround to only fetch a single batch below.
+ *
+ * For this to be effective, set batchSize = limit in the find command.
+ */
+export async function readSingleBatch(cursor) {
+    try {
+        let data;
+        let hasMore = true;
+        // Let MongoDB load the first batch of data
+        const hasAny = await cursor.hasNext();
+        // Now it's in memory, and we can read it
+        data = cursor.readBufferedDocuments();
+        if (!hasAny || cursor.id?.isZero()) {
+            // A zero id means the cursor is exhaused.
+            // No results (hasAny == false) means even this batch doesn't have data.
+            // This should similar results as `await cursor.hasNext()`, but without
+            // actually fetching the next batch.
+            //
+            // Note that it is safe (but slightly inefficient) to return `hasMore: true`
+            // without there being more data, as long as the next batch
+            // will return `hasMore: false`.
+            hasMore = false;
+        }
+        return { data, hasMore };
+    }
+    finally {
+        // Match the from the cursor iterator logic here:
+        // https://github.com/mongodb/node-mongodb-native/blob/e02534e7d1c627bf50b85ca39f5995dbf165ad44/src/cursor/abstract_cursor.ts#L327-L331
+        if (!cursor.closed) {
+            await cursor.close();
+        }
+    }
+}
+export const BSON_DESERIALIZE_OPTIONS = {
+    // use bigint instead of Long
+    useBigInt64: true
+};
+export function mapOpEntry(row) {
+    if (row.op == 'PUT' || row.op == 'REMOVE') {
+        return {
+            op_id: utils.timestampToOpId(row._id.o),
+            op: row.op,
+            object_type: row.table,
+            object_id: row.row_id,
+            checksum: Number(row.checksum),
+            subkey: replicaIdToSubkey(row.source_table, row.source_key),
+            data: row.data
+        };
+    }
+    else {
+        // MOVE, CLEAR
+        return {
+            op_id: utils.timestampToOpId(row._id.o),
+            op: row.op,
+            checksum: Number(row.checksum)
+        };
+    }
+}
+/**
+ * Returns true if two ReplicaId values are the same (serializes to the same BSON value).
+ */
+export function replicaIdEquals(a, b) {
+    if (a === b) {
+        return true;
+    }
+    else if (typeof a == 'string' && typeof b == 'string') {
+        return a == b;
+    }
+    else if (isUUID(a) && isUUID(b)) {
+        return a.equals(b);
+    }
+    else if (a == null && b == null) {
+        return true;
+    }
+    else if ((b == null && a != null) || (a == null && b != null)) {
+        return false;
+    }
+    else {
+        // There are many possible primitive values, this covers them all
+        return bson.serialize({ id: a }).equals(bson.serialize({ id: b }));
+    }
+}
+export function replicaIdToSubkey(table, id) {
+    if (isUUID(id)) {
+        // Special case for UUID for backwards-compatiblity
+        return `${table.toHexString()}/${id.toHexString()}`;
+    }
+    else {
+        // Hashed UUID from the table and id
+        const repr = bson.serialize({ table, id });
+        return uuid.v5(repr, utils.ID_NAMESPACE);
+    }
+}
+/**
+ * True if this is a bson.UUID.
+ *
+ * Works even with multiple copies of the bson package.
+ */
+export function isUUID(value) {
+    if (value == null || typeof value != 'object') {
+        return false;
+    }
+    const uuid = value;
+    return uuid._bsontype == 'Binary' && uuid.sub_type == bson.Binary.SUBTYPE_UUID;
+}
+/**
+ * Helper function for creating a MongoDB client from consumers of this package
+ */
+export const createMongoClient = (url, options) => {
+    return new mongo.MongoClient(url, options);
+};
+/**
+ * MongoDB bulkWrite internally splits the operations into batches
+ * so that no batch exceeds 16MB. However, there are cases where
+ * the batch size is very close to 16MB, where additional metadata
+ * on the server pushes it over the limit, resulting in this error
+ * from the server:
+ *
+ * > MongoBulkWriteError: BSONObj size: 16814023 (0x1008FC7) is invalid. Size must be between 0 and 16793600(16MB) First element: insert: "bucket_data"
+ *
+ * We work around the issue by doing our own batching, limiting the
+ * batch size to 15MB. This does add additional overhead with
+ * BSON.calculateObjectSize.
+ */
+export async function safeBulkWrite(collection, operations, options) {
+    // Must be below 16MB.
+    // We could probably go a little closer, but 15MB is a safe threshold.
+    const BULK_WRITE_LIMIT = 15 * 1024 * 1024;
+    let batch = [];
+    let currentSize = 0;
+    // Estimated overhead per operation, should be smaller in reality.
+    const keySize = 8;
+    for (let op of operations) {
+        const bsonSize = mongo.BSON.calculateObjectSize(op, {
+            checkKeys: false,
+            ignoreUndefined: true
+        }) + keySize;
+        if (batch.length > 0 && currentSize + bsonSize > BULK_WRITE_LIMIT) {
+            await collection.bulkWrite(batch, options);
+            currentSize = 0;
+            batch = [];
+        }
+        batch.push(op);
+        currentSize += bsonSize;
+    }
+    if (batch.length > 0) {
+        await collection.bulkWrite(batch, options);
+    }
+}
+//# sourceMappingURL=util.js.map

package/dist/storage/implementation/util.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"util.js","sourceRoot":"","sources":["../../../src/storage/implementation/util.ts"],"names":[],"mappings":"AAAA,OAAO,EAAW,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAEzD,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,KAAK,MAAM,MAAM,QAAQ,CAAC;AACjC,OAAO,KAAK,KAAK,MAAM,SAAS,CAAC;AACjC,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAG7B;;;GAGG;AAEH,MAAM,UAAU,eAAe,CAAC,MAAyB;IACvD,MAAM,UAAU,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;QACtC,IAAI,OAAO,KAAK,IAAI,QAAQ,IAAI,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC;YACxD,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;QACvB,CAAC;aAAM,CAAC;YACN,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC,CAAC,CAAC;IACH,OAAO,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC;AAC5D,CAAC;AAED,MAAM,UAAU,cAAc,CAAI,MAAkB,EAAE,IAAiB;IACrE,IAAI,MAAM,GAAG;QACX,IAAI,EAAE;YACJ,GAAG,MAAM;SACH;QACR,GAAG,EAAE;YACH,GAAG,MAAM;SACH;KACT,CAAC;IAEF,KAAK,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;QACrB,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QACrC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;IACtC,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,MAAc,EAAE,aAAqB;IACpE,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IAC1D,OAAO,GAAG,MAAM,GAAG,aAAa,IAAI,WAAW,EAAE,CAAC;AACpD,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CAAI,MAA2B;IAClE,IAAI,CAAC;QACH,IAAI,IAAS,CAAC;QACd,IAAI,OAAO,GAAG,IAAI,CAAC;QACnB,2CAA2C;QAC3C,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;QACtC,yCAAyC;QACzC,IAAI,GAAG,MAAM,CAAC,qBAAqB,EAAE,CAAC;QACtC,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,EAAE,CAAC;YACnC,0CAA0C;YAC1C,wEAAwE;YACxE,uEAAuE;YACvE,oCAAoC;YACpC,EAAE;YACF,4EAA4E;YAC5E,2DAA2D;YAC3D,gCAAgC;YAChC,OAAO,GAAG,KAAK,CAAC;QAClB,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;IAC3B,CAAC;YAAS,CAAC;QACT,iDAAiD;QACjD,uIAAuI;QACvI,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YACnB,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;QACvB,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,CAAC,MAAM,wBAAwB,GAA4B;IAC/D,6BAA6B;IAC7B,WAAW,EAAE,IAAI;CAClB,CAAC;AAEF,MAAM,UAAU,UAAU,CAAC,GAAuB;IAChD,IAAI,GAAG,CAAC,EAAE,IAAI,KAAK,IAAI,GAAG,CAAC,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC1C,OAAO;YACL,KAAK,EAAE,KAAK,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;YACvC,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,WAAW,EAAE,GAAG,CAAC,KAAK;YACtB,SAAS,EAAE,GAAG,CAAC,MAAM;YACrB,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;YAC9B,MAAM,EAAE,iBAAiB,CAAC,GAAG,CAAC,YAAa,EAAE,GAAG,CAAC,UAAW,CAAC;YAC7D,IAAI,EAAE,GAAG,CAAC,IAAI;SACf,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,cAAc;QAEd,OAAO;YACL,KAAK,EAAE,KAAK,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;YACvC,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;SAC/B,CAAC;IACJ,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,eAAe,CAAC,CAAoB,EAAE,CAAoB;IACxE,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACZ,OAAO,IAAI,CAAC;IACd,CAAC;SAAM,IAAI,OAAO,CAAC,IAAI,QAAQ,IAAI,OAAO,CAAC,IAAI,QAAQ,EAAE,CAAC;QACxD,OAAO,CAAC,IAAI,CAAC,CAAC;IAChB,CAAC;SAAM,IAAI,MAAM,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;QAClC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IACrB,CAAC;SAAM,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;QAClC,OAAO,IAAI,CAAC;IACd,CAAC;SAAM,IAAI,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,IAAI,CAAC,EAAE,CAAC;QAChE,OAAO,KAAK,CAAC;IACf,CAAC;SAAM,CAAC;QACN,iEAAiE;QACjE,OAAQ,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAY,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;IACjF,CAAC;AACH,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,KAAoB,EAAE,EAAqB;IAC3E,IAAI,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC;QACf,mDAAmD;QACnD,OAAO,GAAG,KAAK,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC;IACtD,CAAC;SAAM,CAAC;QACN,oCAAoC;QACpC,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC;QAC3C,OAAO,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,KAAK,CAAC,YAAY,CAAC,CAAC;IAC3C,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,MAAM,CAAC,KAAU;IAC/B,IAAI,KAAK,IAAI,IAAI,IAAI,OAAO,KAAK,IAAI,QAAQ,EAAE,CAAC;QAC9C,OAAO,KAAK,CAAC;IACf,CAAC;IACD,MAAM,IAAI,GAAG,KAAkB,CAAC;IAChC,OAAO,IAAI,CAAC,SAAS,IAAI,QAAQ,IAAI,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC;AACjF,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,GAAW,EAAE,OAAkC,EAAE,EAAE;IACnF,OAAO,IAAI,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;AAC7C,CAAC,CAAC;AAEF;;;;;;;;;;;;GAYG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,UAA+B,EAC/B,UAA4C,EAC5C,OAA+B;IAE/B,sBAAsB;IACtB,sEAAsE;IACtE,MAAM,gBAAgB,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC;IAE1C,IAAI,KAAK,GAAqC,EAAE,CAAC;IACjD,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,kEAAkE;IAClE,MAAM,OAAO,GAAG,CAAC,CAAC;IAClB,KAAK,IAAI,EAAE,IAAI,UAAU,EAAE,CAAC;QAC1B,MAAM,QAAQ,GACZ,KAAK,CAAC,IAAI,CAAC,mBAAmB,CAAC,EAAE,EAAE;YACjC,SAAS,EAAE,KAAK;YAChB,eAAe,EAAE,IAAI;SACf,CAAC,GAAG,OAAO,CAAC;QACtB,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,WAAW,GAAG,QAAQ,GAAG,gBAAgB,EAAE,CAAC;YAClE,MAAM,UAAU,CAAC,SAAS,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;YAC3C,WAAW,GAAG,CAAC,CAAC;YAChB,KAAK,GAAG,EAAE,CAAC;QACb,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,WAAW,IAAI,QAAQ,CAAC;IAC1B,CAAC;IACD,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrB,MAAM,UAAU,CAAC,SAAS,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IAC7C,CAAC;AACH,CAAC"}

package/dist/storage/storage-index.d.ts

@@ -0,0 +1,14 @@
+export * from './implementation/config.js';
+export * from './implementation/db.js';
+export * from './implementation/models.js';
+export * from './implementation/MongoBucketBatch.js';
+export * from './implementation/MongoIdSequence.js';
+export * from './implementation/MongoPersistedSyncRules.js';
+export * from './implementation/MongoPersistedSyncRulesContent.js';
+export * from './implementation/MongoStorageProvider.js';
+export * from './implementation/MongoSyncBucketStorage.js';
+export * from './implementation/MongoSyncRulesLock.js';
+export * from './implementation/OperationBatch.js';
+export * from './implementation/PersistedBatch.js';
+export * from './implementation/util.js';
+export * from './MongoBucketStorage.js';

package/dist/storage/storage-index.js

@@ -0,0 +1,15 @@
+export * from './implementation/config.js';
+export * from './implementation/db.js';
+export * from './implementation/models.js';
+export * from './implementation/MongoBucketBatch.js';
+export * from './implementation/MongoIdSequence.js';
+export * from './implementation/MongoPersistedSyncRules.js';
+export * from './implementation/MongoPersistedSyncRulesContent.js';
+export * from './implementation/MongoStorageProvider.js';
+export * from './implementation/MongoSyncBucketStorage.js';
+export * from './implementation/MongoSyncRulesLock.js';
+export * from './implementation/OperationBatch.js';
+export * from './implementation/PersistedBatch.js';
+export * from './implementation/util.js';
+export * from './MongoBucketStorage.js';
+//# sourceMappingURL=storage-index.js.map

package/dist/storage/storage-index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"storage-index.js","sourceRoot":"","sources":["../../src/storage/storage-index.ts"],"names":[],"mappings":"AAAA,cAAc,4BAA4B,CAAC;AAC3C,cAAc,wBAAwB,CAAC;AACvC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,sCAAsC,CAAC;AACrD,cAAc,qCAAqC,CAAC;AACpD,cAAc,6CAA6C,CAAC;AAC5D,cAAc,oDAAoD,CAAC;AACnE,cAAc,0CAA0C,CAAC;AACzD,cAAc,4CAA4C,CAAC;AAC3D,cAAc,wCAAwC,CAAC;AACvD,cAAc,oCAAoC,CAAC;AACnD,cAAc,oCAAoC,CAAC;AACnD,cAAc,0BAA0B,CAAC;AACzC,cAAc,yBAAyB,CAAC"}

package/dist/types/types.d.ts

@@ -1,3 +1,5 @@
+import * as service_types from '@powersync/service-types';
+import { MongoStorageConfig } from '@powersync/service-types/src/config/PowerSyncConfig.js';
 import * as t from 'ts-codec';
 export declare const MONGO_CONNECTION_TYPE: "mongodb";
 export declare enum PostImagesOption {
@@ -84,3 +86,4 @@ export declare function normalizeConnectionConfig(options: MongoConnectionConfig
  * Only contains hostname, port, database.
  */
 export declare function baseUri(options: NormalizedMongoConnectionConfig): string;
+export declare function isMongoStorageConfig(config: service_types.configFile.GenericStorageConfig): config is MongoStorageConfig;

package/dist/types/types.js

@@ -1,6 +1,6 @@
-import { normalizeMongoConfig } from '@powersync/service-core';
 import * as service_types from '@powersync/service-types';
 import * as t from 'ts-codec';
+import { MONGO_STORAGE_TYPE, normalizeMongoConfig } from '../storage/storage-index.js';
 export const MONGO_CONNECTION_TYPE = 'mongodb';
 export var PostImagesOption;
 (function (PostImagesOption) {
@@ -70,4 +70,7 @@ export function normalizeConnectionConfig(options) {
 export function baseUri(options) {
     return options.uri;
 }
+export function isMongoStorageConfig(config) {
+    return config.type == MONGO_STORAGE_TYPE;
+}
 //# sourceMappingURL=types.js.map

package/dist/types/types.js.map

@@ -1 +1 @@
-{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/types/types.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,KAAK,aAAa,MAAM,0BAA0B,CAAC;AAC1D,OAAO,KAAK,CAAC,MAAM,UAAU,CAAC;AAE9B,MAAM,CAAC,MAAM,qBAAqB,GAAG,SAAkB,CAAC;AAExD,MAAM,CAAN,IAAY,gBAkCX;AAlCD,WAAY,gBAAgB;IAC1B;;;;;;;OAOG;IACH,+BAAW,CAAA;IAEX;;;;;;;OAOG;IACH,qDAAiC,CAAA;IAEjC;;;;;;;;;;;OAWG;IACH,2CAAuB,CAAA;AACzB,CAAC,EAlCW,gBAAgB,KAAhB,gBAAgB,QAkC3B;AAeD,MAAM,CAAC,MAAM,qBAAqB,GAAG,aAAa,CAAC,UAAU,CAAC,gBAAgB,CAAC,GAAG,CAChF,CAAC,CAAC,MAAM,CAAC;IACP,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,qBAAqB,CAAC;IACtC,2FAA2F;IAC3F,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;IACvB,8FAA8F;IAC9F,GAAG,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;IACxB,GAAG,EAAE,CAAC,CAAC,MAAM;IACb,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;IAC7B,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;IAC7B,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;IAE7B,WAAW,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,QAAQ,EAAE;CACpG,CAAC,CACH,CAAC;AAYF;;;;GAIG;AACH,MAAM,UAAU,yBAAyB,CAAC,OAA8B;IACtE,MAAM,IAAI,GAAG,oBAAoB,CAAC,OAAO,CAAC,CAAC;IAE3C,OAAO;QACL,GAAG,IAAI;QACP,EAAE,EAAE,OAAO,CAAC,EAAE,IAAI,SAAS;QAC3B,GAAG,EAAE,OAAO,CAAC,GAAG,IAAI,SAAS;QAC7B,UAAU,EAAG,OAAO,CAAC,WAA4C,IAAI,gBAAgB,CAAC,GAAG;KAC1F,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,OAAO,CAAC,OAAwC;IAC9D,OAAO,OAAO,CAAC,GAAG,CAAC;AACrB,CAAC"}
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/types/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,aAAa,MAAM,0BAA0B,CAAC;AAE1D,OAAO,KAAK,CAAC,MAAM,UAAU,CAAC;AAC9B,OAAO,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AAEvF,MAAM,CAAC,MAAM,qBAAqB,GAAG,SAAkB,CAAC;AAExD,MAAM,CAAN,IAAY,gBAkCX;AAlCD,WAAY,gBAAgB;IAC1B;;;;;;;OAOG;IACH,+BAAW,CAAA;IAEX;;;;;;;OAOG;IACH,qDAAiC,CAAA;IAEjC;;;;;;;;;;;OAWG;IACH,2CAAuB,CAAA;AACzB,CAAC,EAlCW,gBAAgB,KAAhB,gBAAgB,QAkC3B;AAeD,MAAM,CAAC,MAAM,qBAAqB,GAAG,aAAa,CAAC,UAAU,CAAC,gBAAgB,CAAC,GAAG,CAChF,CAAC,CAAC,MAAM,CAAC;IACP,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,qBAAqB,CAAC;IACtC,2FAA2F;IAC3F,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;IACvB,8FAA8F;IAC9F,GAAG,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;IACxB,GAAG,EAAE,CAAC,CAAC,MAAM;IACb,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;IAC7B,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;IAC7B,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;IAE7B,WAAW,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,QAAQ,EAAE;CACpG,CAAC,CACH,CAAC;AAYF;;;;GAIG;AACH,MAAM,UAAU,yBAAyB,CAAC,OAA8B;IACtE,MAAM,IAAI,GAAG,oBAAoB,CAAC,OAAO,CAAC,CAAC;IAE3C,OAAO;QACL,GAAG,IAAI;QACP,EAAE,EAAE,OAAO,CAAC,EAAE,IAAI,SAAS;QAC3B,GAAG,EAAE,OAAO,CAAC,GAAG,IAAI,SAAS;QAC7B,UAAU,EAAG,OAAO,CAAC,WAA4C,IAAI,gBAAgB,CAAC,GAAG;KAC1F,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,OAAO,CAAC,OAAwC;IAC9D,OAAO,OAAO,CAAC,GAAG,CAAC;AACrB,CAAC;AAED,MAAM,UAAU,oBAAoB,CAClC,MAAqD;IAErD,OAAO,MAAM,CAAC,IAAI,IAAI,kBAAkB,CAAC;AAC3C,CAAC"}

package/package.json

@@ -2,7 +2,7 @@
   "name": "@powersync/service-module-mongodb",
   "repository": "https://github.com/powersync-ja/powersync-service",
   "types": "dist/index.d.ts",
-  "version": "0.0.0-dev-20241128134723",
+  "version": "0.0.0-dev-20241219110735",
   "main": "dist/index.js",
   "license": "FSL-1.1-Apache-2.0",
   "type": "module",
@@ -22,18 +22,21 @@
     }
   },
   "dependencies": {
-    "mongodb": "^6.7.0",
-    "ts-codec": "^1.2.2",
+    "mongodb": "^6.11.0",
+    "ts-codec": "^1.3.0",
+    "ix": "^5.0.0",
+    "lru-cache": "^10.2.2",
     "uuid": "^9.0.1",
     "uri-js": "^4.4.1",
-    "@powersync/lib-services-framework": "0.2.0",
-    "@powersync/service-core": "0.0.0-dev-20241128134723",
+    "@powersync/lib-services-framework": "0.0.0-dev-20241219110735",
+    "@powersync/service-core": "0.0.0-dev-20241219110735",
     "@powersync/service-jsonbig": "0.17.10",
-    "@powersync/service-sync-rules": "0.21.0",
-    "@powersync/service-types": "0.0.0-dev-20241128134723"
+    "@powersync/service-sync-rules": "0.0.0-dev-20241219110735",
+    "@powersync/service-types": "0.0.0-dev-20241219110735"
   },
   "devDependencies": {
-    "@types/uuid": "^9.0.4"
+    "@types/uuid": "^9.0.4",
+    "@powersync/service-core-tests": "0.0.0-dev-20241219110735"
   },
   "scripts": {
     "build": "tsc -b",

package/src/db/db-index.ts

@@ -0,0 +1 @@
+export * as mongo from './mongo.js';

package/src/db/mongo.ts

@@ -0,0 +1,81 @@
+import * as mongo from 'mongodb';
+import * as timers from 'timers/promises';
+
+import { configFile } from '@powersync/service-types';
+import { normalizeMongoConfig } from '../storage/storage-index.js';
+
+/**
+ * Time for new connection to timeout.
+ */
+export const MONGO_CONNECT_TIMEOUT_MS = 10_000;
+
+/**
+ * Time for individual requests to timeout the socket.
+ */
+export const MONGO_SOCKET_TIMEOUT_MS = 60_000;
+
+/**
+ * Time for individual requests to timeout the operation.
+ *
+ * This is time spent on the cursor, not total time.
+ *
+ * Must be less than MONGO_SOCKET_TIMEOUT_MS to ensure proper error handling.
+ */
+export const MONGO_OPERATION_TIMEOUT_MS = 30_000;
+
+/**
+ * Same as above, but specifically for clear operations.
+ *
+ * These are retried when reaching the timeout.
+ */
+export const MONGO_CLEAR_OPERATION_TIMEOUT_MS = 5_000;
+
+export function createMongoClient(config: configFile.MongoStorageConfig) {
+  const normalized = normalizeMongoConfig(config);
+  return new mongo.MongoClient(normalized.uri, {
+    auth: {
+      username: normalized.username,
+      password: normalized.password
+    },
+    // Time for connection to timeout
+    connectTimeoutMS: MONGO_CONNECT_TIMEOUT_MS,
+    // Time for individual requests to timeout
+    socketTimeoutMS: MONGO_SOCKET_TIMEOUT_MS,
+    // How long to wait for new primary selection
+    serverSelectionTimeoutMS: 30_000,
+
+    // Avoid too many connections:
+    // 1. It can overwhelm the source database.
+    // 2. Processing too many queries in parallel can cause the process to run out of memory.
+    maxPoolSize: 8,
+
+    maxConnecting: 3,
+    maxIdleTimeMS: 60_000
+  });
+}
+
+/**
+ * Wait up to a minute for authentication errors to resolve.
+ *
+ * There can be a delay between an Atlas user being created, and that user being
+ * available on the database cluster. This works around it.
+ *
+ * This is specifically relevant for migrations and teardown - other parts of the stack
+ * can generate handle these failures and just retry or restart.
+ */
+export async function waitForAuth(db: mongo.Db) {
+  const start = Date.now();
+  while (Date.now() - start < 60_000) {
+    try {
+      await db.command({ ping: 1 });
+      // Success
+      break;
+    } catch (e) {
+      if (e.codeName == 'AuthenticationFailed') {
+        await timers.setTimeout(1_000);
+        continue;
+      }
+      throw e;
+    }
+  }
+}

package/src/index.ts

@@ -1 +1,5 @@
 export * from './module/MongoModule.js';
+
+export * as db from './db/db-index.js';
+export * as storage from './storage/storage-index.js';
+