@powersync/service-module-mongodb-storage 0.0.0-dev-20250108073049

package/dist/storage/implementation/db.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"db.js","sourceRoot":"","sources":["../../../src/storage/implementation/db.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,gCAAgC,CAAC;AAC5D,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAuBlD,MAAM,OAAO,cAAc;IAezB,YAAY,MAAyB,EAAE,OAA+B;QACpE,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QAErB,MAAM,EAAE,GAAG,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,QAAQ,EAAE;YACtC,GAAG,OAAO,CAAC,wBAAwB;SACpC,CAAC,CAAC;QACH,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QAEb,IAAI,CAAC,YAAY,GAAG,EAAE,CAAC,UAAU,CAAsB,cAAc,CAAC,CAAC;QACvE,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;QAChD,IAAI,CAAC,iBAAiB,GAAG,EAAE,CAAC,UAAU,CAAC,mBAAmB,CAAC,CAAC;QAC5D,IAAI,CAAC,cAAc,GAAG,EAAE,CAAC,UAAU,CAAC,gBAAgB,CAAC,CAAC;QACtD,IAAI,CAAC,UAAU,GAAG,EAAE,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC;QAC9C,IAAI,CAAC,aAAa,GAAG,EAAE,CAAC,UAAU,CAAC,eAAe,CAAC,CAAC;QACpD,IAAI,CAAC,wBAAwB,GAAG,EAAE,CAAC,UAAU,CAAC,0BAA0B,CAAC,CAAC;QAC1E,IAAI,CAAC,iBAAiB,GAAG,EAAE,CAAC,UAAU,CAAC,mBAAmB,CAAC,CAAC;QAC5D,IAAI,CAAC,QAAQ,GAAG,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;QAC1C,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;IAC3C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACvC,MAAM,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACtC,MAAM,IAAI,CAAC,iBAAiB,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QAC5C,MAAM,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACzC,MAAM,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACrC,MAAM,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACxC,MAAM,IAAI,CAAC,iBAAiB,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QAC5C,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAClC,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;IAClC,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,IAAI;QACR,MAAM,IAAI,CAAC,EAAE,CAAC,YAAY,EAAE,CAAC;IAC/B,CAAC;CACF;AAED,MAAM,UAAU,oBAAoB,CAAC,MAA0B;IAC7D,OAAO,IAAI,cAAc,CAAC,SAAS,CAAC,iBAAiB,CAAC,MAAM,CAAC,EAAE,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;AAChG,CAAC"}

package/dist/storage/implementation/models.d.ts

@@ -0,0 +1,139 @@
+import { storage } from '@powersync/service-core';
+import { SqliteJsonValue } from '@powersync/service-sync-rules';
+import * as bson from 'bson';
+/**
+ * Replica id uniquely identifying a row on the source database.
+ *
+ * Can be any value serializable to BSON.
+ *
+ * If the value is an entire document, the data serialized to a v5 UUID may be a good choice here.
+ */
+export type ReplicaId = bson.UUID | bson.Document | any;
+export interface SourceKey {
+    /** group_id */
+    g: number;
+    /** source table id */
+    t: bson.ObjectId;
+    /** source key */
+    k: ReplicaId;
+}
+export interface BucketDataKey {
+    /** group_id */
+    g: number;
+    /** bucket name */
+    b: string;
+    /** op_id */
+    o: bigint;
+}
+export interface CurrentDataDocument {
+    _id: SourceKey;
+    data: bson.Binary;
+    buckets: CurrentBucket[];
+    lookups: bson.Binary[];
+}
+export interface CurrentBucket {
+    bucket: string;
+    table: string;
+    id: string;
+}
+export interface BucketParameterDocument {
+    _id: bigint;
+    key: SourceKey;
+    lookup: bson.Binary;
+    bucket_parameters: Record<string, SqliteJsonValue>[];
+}
+export interface BucketDataDocument {
+    _id: BucketDataKey;
+    op: OpType;
+    source_table?: bson.ObjectId;
+    source_key?: ReplicaId;
+    table?: string;
+    row_id?: string;
+    checksum: number;
+    data: string | null;
+    target_op?: bigint | null;
+}
+export type OpType = 'PUT' | 'REMOVE' | 'MOVE' | 'CLEAR';
+export interface SourceTableDocument {
+    _id: bson.ObjectId;
+    group_id: number;
+    connection_id: number;
+    relation_id: number | string | undefined;
+    schema_name: string;
+    table_name: string;
+    replica_id_columns: string[] | null;
+    replica_id_columns2: {
+        name: string;
+        type_oid?: number;
+        type?: string;
+    }[] | undefined;
+    snapshot_done: boolean | undefined;
+}
+export interface IdSequenceDocument {
+    _id: string;
+    op_id: bigint;
+}
+export interface SyncRuleDocument {
+    _id: number;
+    state: storage.SyncRuleState;
+    /**
+     * True if initial snapshot has been replicated.
+     *
+     * Can only be false if state == PROCESSING.
+     */
+    snapshot_done: boolean;
+    /**
+     * The last consistent checkpoint.
+     *
+     * There may be higher OpIds used in the database if we're in the middle of replicating a large transaction.
+     */
+    last_checkpoint: bigint | null;
+    /**
+     * The LSN associated with the last consistent checkpoint.
+     */
+    last_checkpoint_lsn: string | null;
+    /**
+     * If set, no new checkpoints may be created < this value.
+     */
+    no_checkpoint_before: string | null;
+    /**
+     * Goes together with no_checkpoint_before.
+     *
+     * If a keepalive is triggered that creates the checkpoint > no_checkpoint_before,
+     * then the checkpoint must be equal to this keepalive_op.
+     */
+    keepalive_op: string | null;
+    slot_name: string | null;
+    /**
+     * Last time we persisted a checkpoint.
+     *
+     * This may be old if no data is incoming.
+     */
+    last_checkpoint_ts: Date | null;
+    /**
+     * Last time we persisted a checkpoint or keepalive.
+     *
+     * This should stay fairly current while replicating.
+     */
+    last_keepalive_ts: Date | null;
+    /**
+     * If an error is stopping replication, it will be stored here.
+     */
+    last_fatal_error: string | null;
+    content: string;
+}
+export interface CustomWriteCheckpointDocument {
+    _id: bson.ObjectId;
+    user_id: string;
+    checkpoint: bigint;
+    sync_rules_id: number;
+}
+export interface WriteCheckpointDocument {
+    _id: bson.ObjectId;
+    user_id: string;
+    lsns: Record<string, string>;
+    client_id: bigint;
+}
+export interface InstanceDocument {
+    _id: string;
+}

package/dist/storage/implementation/models.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=models.js.map

package/dist/storage/implementation/models.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"models.js","sourceRoot":"","sources":["../../../src/storage/implementation/models.ts"],"names":[],"mappings":""}

package/dist/storage/implementation/util.d.ts

@@ -0,0 +1,46 @@
+import { storage, utils } from '@powersync/service-core';
+import * as bson from 'bson';
+import * as mongo from 'mongodb';
+import { PowerSyncMongo } from './db.js';
+import { BucketDataDocument } from './models.js';
+export declare function idPrefixFilter<T>(prefix: Partial<T>, rest: (keyof T)[]): mongo.Condition<T>;
+export declare function generateSlotName(prefix: string, sync_rules_id: number): string;
+/**
+ * Read a single batch of data from a cursor, then close it.
+ *
+ * We do our best to avoid MongoDB fetching any more data than this single batch.
+ *
+ * This is similar to using `singleBatch: true` in find options.
+ * However, that makes `has_more` detection very difficult, since the cursor is always closed
+ * after the first batch. Instead, we do a workaround to only fetch a single batch below.
+ *
+ * For this to be effective, set batchSize = limit in the find command.
+ */
+export declare function readSingleBatch<T>(cursor: mongo.FindCursor<T>): Promise<{
+    data: T[];
+    hasMore: boolean;
+}>;
+export declare function mapOpEntry(row: BucketDataDocument): utils.OplogEntry;
+export declare function replicaIdToSubkey(table: bson.ObjectId, id: storage.ReplicaId): string;
+/**
+ * Helper function for creating a MongoDB client from consumers of this package
+ */
+export declare const createMongoClient: (url: string, options?: mongo.MongoClientOptions) => mongo.MongoClient;
+/**
+ * Helper for unit tests
+ */
+export declare const connectMongoForTests: (url: string, isCI: boolean) => PowerSyncMongo;
+/**
+ * MongoDB bulkWrite internally splits the operations into batches
+ * so that no batch exceeds 16MB. However, there are cases where
+ * the batch size is very close to 16MB, where additional metadata
+ * on the server pushes it over the limit, resulting in this error
+ * from the server:
+ *
+ * > MongoBulkWriteError: BSONObj size: 16814023 (0x1008FC7) is invalid. Size must be between 0 and 16793600(16MB) First element: insert: "bucket_data"
+ *
+ * We work around the issue by doing our own batching, limiting the
+ * batch size to 15MB. This does add additional overhead with
+ * BSON.calculateObjectSize.
+ */
+export declare function safeBulkWrite<T extends mongo.Document>(collection: mongo.Collection<T>, operations: mongo.AnyBulkWriteOperation<T>[], options: mongo.BulkWriteOptions): Promise<void>;

package/dist/storage/implementation/util.js

@@ -0,0 +1,155 @@
+import { storage, utils } from '@powersync/service-core';
+import * as bson from 'bson';
+import * as crypto from 'crypto';
+import * as mongo from 'mongodb';
+import * as uuid from 'uuid';
+import { PowerSyncMongo } from './db.js';
+export function idPrefixFilter(prefix, rest) {
+    let filter = {
+        $gte: {
+            ...prefix
+        },
+        $lt: {
+            ...prefix
+        }
+    };
+    for (let key of rest) {
+        filter.$gte[key] = new bson.MinKey();
+        filter.$lt[key] = new bson.MaxKey();
+    }
+    return filter;
+}
+export function generateSlotName(prefix, sync_rules_id) {
+    const slot_suffix = crypto.randomBytes(2).toString('hex');
+    return `${prefix}${sync_rules_id}_${slot_suffix}`;
+}
+/**
+ * Read a single batch of data from a cursor, then close it.
+ *
+ * We do our best to avoid MongoDB fetching any more data than this single batch.
+ *
+ * This is similar to using `singleBatch: true` in find options.
+ * However, that makes `has_more` detection very difficult, since the cursor is always closed
+ * after the first batch. Instead, we do a workaround to only fetch a single batch below.
+ *
+ * For this to be effective, set batchSize = limit in the find command.
+ */
+export async function readSingleBatch(cursor) {
+    try {
+        let data;
+        let hasMore = true;
+        // Let MongoDB load the first batch of data
+        const hasAny = await cursor.hasNext();
+        // Now it's in memory, and we can read it
+        data = cursor.readBufferedDocuments();
+        if (!hasAny || cursor.id?.isZero()) {
+            // A zero id means the cursor is exhaused.
+            // No results (hasAny == false) means even this batch doesn't have data.
+            // This should similar results as `await cursor.hasNext()`, but without
+            // actually fetching the next batch.
+            //
+            // Note that it is safe (but slightly inefficient) to return `hasMore: true`
+            // without there being more data, as long as the next batch
+            // will return `hasMore: false`.
+            hasMore = false;
+        }
+        return { data, hasMore };
+    }
+    finally {
+        // Match the from the cursor iterator logic here:
+        // https://github.com/mongodb/node-mongodb-native/blob/e02534e7d1c627bf50b85ca39f5995dbf165ad44/src/cursor/abstract_cursor.ts#L327-L331
+        if (!cursor.closed) {
+            await cursor.close();
+        }
+    }
+}
+export function mapOpEntry(row) {
+    if (row.op == 'PUT' || row.op == 'REMOVE') {
+        return {
+            op_id: utils.timestampToOpId(row._id.o),
+            op: row.op,
+            object_type: row.table,
+            object_id: row.row_id,
+            checksum: Number(row.checksum),
+            subkey: replicaIdToSubkey(row.source_table, row.source_key),
+            data: row.data
+        };
+    }
+    else {
+        // MOVE, CLEAR
+        return {
+            op_id: utils.timestampToOpId(row._id.o),
+            op: row.op,
+            checksum: Number(row.checksum)
+        };
+    }
+}
+export function replicaIdToSubkey(table, id) {
+    if (storage.isUUID(id)) {
+        // Special case for UUID for backwards-compatiblity
+        return `${table.toHexString()}/${id.toHexString()}`;
+    }
+    else {
+        // Hashed UUID from the table and id
+        const repr = bson.serialize({ table, id });
+        return uuid.v5(repr, utils.ID_NAMESPACE);
+    }
+}
+/**
+ * Helper function for creating a MongoDB client from consumers of this package
+ */
+export const createMongoClient = (url, options) => {
+    return new mongo.MongoClient(url, options);
+};
+/**
+ * Helper for unit tests
+ */
+export const connectMongoForTests = (url, isCI) => {
+    // Short timeout for tests, to fail fast when the server is not available.
+    // Slightly longer timeouts for CI, to avoid arbitrary test failures
+    const client = createMongoClient(url, {
+        connectTimeoutMS: isCI ? 15000 : 5000,
+        socketTimeoutMS: isCI ? 15000 : 5000,
+        serverSelectionTimeoutMS: isCI ? 15000 : 2500
+    });
+    return new PowerSyncMongo(client);
+};
+/**
+ * MongoDB bulkWrite internally splits the operations into batches
+ * so that no batch exceeds 16MB. However, there are cases where
+ * the batch size is very close to 16MB, where additional metadata
+ * on the server pushes it over the limit, resulting in this error
+ * from the server:
+ *
+ * > MongoBulkWriteError: BSONObj size: 16814023 (0x1008FC7) is invalid. Size must be between 0 and 16793600(16MB) First element: insert: "bucket_data"
+ *
+ * We work around the issue by doing our own batching, limiting the
+ * batch size to 15MB. This does add additional overhead with
+ * BSON.calculateObjectSize.
+ */
+export async function safeBulkWrite(collection, operations, options) {
+    // Must be below 16MB.
+    // We could probably go a little closer, but 15MB is a safe threshold.
+    const BULK_WRITE_LIMIT = 15 * 1024 * 1024;
+    let batch = [];
+    let currentSize = 0;
+    // Estimated overhead per operation, should be smaller in reality.
+    const keySize = 8;
+    for (let op of operations) {
+        const bsonSize = mongo.BSON.calculateObjectSize(op, {
+            checkKeys: false,
+            ignoreUndefined: true
+        }) + keySize;
+        if (batch.length > 0 && currentSize + bsonSize > BULK_WRITE_LIMIT) {
+            await collection.bulkWrite(batch, options);
+            currentSize = 0;
+            batch = [];
+        }
+        batch.push(op);
+        currentSize += bsonSize;
+    }
+    if (batch.length > 0) {
+        await collection.bulkWrite(batch, options);
+    }
+}
+//# sourceMappingURL=util.js.map

package/dist/storage/implementation/util.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"util.js","sourceRoot":"","sources":["../../../src/storage/implementation/util.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AACzD,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,KAAK,MAAM,MAAM,QAAQ,CAAC;AACjC,OAAO,KAAK,KAAK,MAAM,SAAS,CAAC;AACjC,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAGzC,MAAM,UAAU,cAAc,CAAI,MAAkB,EAAE,IAAiB;IACrE,IAAI,MAAM,GAAG;QACX,IAAI,EAAE;YACJ,GAAG,MAAM;SACH;QACR,GAAG,EAAE;YACH,GAAG,MAAM;SACH;KACT,CAAC;IAEF,KAAK,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;QACrB,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QACrC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;IACtC,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,MAAc,EAAE,aAAqB;IACpE,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IAC1D,OAAO,GAAG,MAAM,GAAG,aAAa,IAAI,WAAW,EAAE,CAAC;AACpD,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CAAI,MAA2B;IAClE,IAAI,CAAC;QACH,IAAI,IAAS,CAAC;QACd,IAAI,OAAO,GAAG,IAAI,CAAC;QACnB,2CAA2C;QAC3C,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;QACtC,yCAAyC;QACzC,IAAI,GAAG,MAAM,CAAC,qBAAqB,EAAE,CAAC;QACtC,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,EAAE,CAAC;YACnC,0CAA0C;YAC1C,wEAAwE;YACxE,uEAAuE;YACvE,oCAAoC;YACpC,EAAE;YACF,4EAA4E;YAC5E,2DAA2D;YAC3D,gCAAgC;YAChC,OAAO,GAAG,KAAK,CAAC;QAClB,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;IAC3B,CAAC;YAAS,CAAC;QACT,iDAAiD;QACjD,uIAAuI;QACvI,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YACnB,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;QACvB,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,GAAuB;IAChD,IAAI,GAAG,CAAC,EAAE,IAAI,KAAK,IAAI,GAAG,CAAC,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC1C,OAAO;YACL,KAAK,EAAE,KAAK,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;YACvC,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,WAAW,EAAE,GAAG,CAAC,KAAK;YACtB,SAAS,EAAE,GAAG,CAAC,MAAM;YACrB,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;YAC9B,MAAM,EAAE,iBAAiB,CAAC,GAAG,CAAC,YAAa,EAAE,GAAG,CAAC,UAAW,CAAC;YAC7D,IAAI,EAAE,GAAG,CAAC,IAAI;SACf,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,cAAc;QAEd,OAAO;YACL,KAAK,EAAE,KAAK,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;YACvC,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;SAC/B,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,KAAoB,EAAE,EAAqB;IAC3E,IAAI,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC;QACvB,mDAAmD;QACnD,OAAO,GAAG,KAAK,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC;IACtD,CAAC;SAAM,CAAC;QACN,oCAAoC;QACpC,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC;QAC3C,OAAO,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,KAAK,CAAC,YAAY,CAAC,CAAC;IAC3C,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,GAAW,EAAE,OAAkC,EAAE,EAAE;IACnF,OAAO,IAAI,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;AAC7C,CAAC,CAAC;AAEF;;GAEG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,GAAW,EAAE,IAAa,EAAE,EAAE;IACjE,0EAA0E;IAC1E,oEAAoE;IACpE,MAAM,MAAM,GAAG,iBAAiB,CAAC,GAAG,EAAE;QACpC,gBAAgB,EAAE,IAAI,CAAC,CAAC,CAAC,KAAM,CAAC,CAAC,CAAC,IAAK;QACvC,eAAe,EAAE,IAAI,CAAC,CAAC,CAAC,KAAM,CAAC,CAAC,CAAC,IAAK;QACtC,wBAAwB,EAAE,IAAI,CAAC,CAAC,CAAC,KAAM,CAAC,CAAC,CAAC,IAAK;KAChD,CAAC,CAAC;IACH,OAAO,IAAI,cAAc,CAAC,MAAM,CAAC,CAAC;AACpC,CAAC,CAAC;AAEF;;;;;;;;;;;;GAYG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,UAA+B,EAC/B,UAA4C,EAC5C,OAA+B;IAE/B,sBAAsB;IACtB,sEAAsE;IACtE,MAAM,gBAAgB,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC;IAE1C,IAAI,KAAK,GAAqC,EAAE,CAAC;IACjD,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,kEAAkE;IAClE,MAAM,OAAO,GAAG,CAAC,CAAC;IAClB,KAAK,IAAI,EAAE,IAAI,UAAU,EAAE,CAAC;QAC1B,MAAM,QAAQ,GACZ,KAAK,CAAC,IAAI,CAAC,mBAAmB,CAAC,EAAE,EAAE;YACjC,SAAS,EAAE,KAAK;YAChB,eAAe,EAAE,IAAI;SACf,CAAC,GAAG,OAAO,CAAC;QACtB,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,WAAW,GAAG,QAAQ,GAAG,gBAAgB,EAAE,CAAC;YAClE,MAAM,UAAU,CAAC,SAAS,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;YAC3C,WAAW,GAAG,CAAC,CAAC;YAChB,KAAK,GAAG,EAAE,CAAC;QACb,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,WAAW,IAAI,QAAQ,CAAC;IAC1B,CAAC;IACD,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrB,MAAM,UAAU,CAAC,SAAS,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IAC7C,CAAC;AACH,CAAC"}

package/dist/storage/storage-index.d.ts

@@ -0,0 +1,14 @@
+export * from './implementation/db.js';
+export * from './implementation/models.js';
+export * from './implementation/MongoBucketBatch.js';
+export * from './implementation/MongoIdSequence.js';
+export * from './implementation/MongoPersistedSyncRules.js';
+export * from './implementation/MongoPersistedSyncRulesContent.js';
+export * from './implementation/MongoStorageProvider.js';
+export * from './implementation/MongoSyncBucketStorage.js';
+export * from './implementation/MongoSyncRulesLock.js';
+export * from './implementation/MongoTestStorageFactoryGenerator.js';
+export * from './implementation/OperationBatch.js';
+export * from './implementation/PersistedBatch.js';
+export * from './implementation/util.js';
+export * from './MongoBucketStorage.js';

package/dist/storage/storage-index.js

@@ -0,0 +1,15 @@
+export * from './implementation/db.js';
+export * from './implementation/models.js';
+export * from './implementation/MongoBucketBatch.js';
+export * from './implementation/MongoIdSequence.js';
+export * from './implementation/MongoPersistedSyncRules.js';
+export * from './implementation/MongoPersistedSyncRulesContent.js';
+export * from './implementation/MongoStorageProvider.js';
+export * from './implementation/MongoSyncBucketStorage.js';
+export * from './implementation/MongoSyncRulesLock.js';
+export * from './implementation/MongoTestStorageFactoryGenerator.js';
+export * from './implementation/OperationBatch.js';
+export * from './implementation/PersistedBatch.js';
+export * from './implementation/util.js';
+export * from './MongoBucketStorage.js';
+//# sourceMappingURL=storage-index.js.map

package/dist/storage/storage-index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"storage-index.js","sourceRoot":"","sources":["../../src/storage/storage-index.ts"],"names":[],"mappings":"AAAA,cAAc,wBAAwB,CAAC;AACvC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,sCAAsC,CAAC;AACrD,cAAc,qCAAqC,CAAC;AACpD,cAAc,6CAA6C,CAAC;AAC5D,cAAc,oDAAoD,CAAC;AACnE,cAAc,0CAA0C,CAAC;AACzD,cAAc,4CAA4C,CAAC;AAC3D,cAAc,wCAAwC,CAAC;AACvD,cAAc,sDAAsD,CAAC;AACrE,cAAc,oCAAoC,CAAC;AACnD,cAAc,oCAAoC,CAAC;AACnD,cAAc,0BAA0B,CAAC;AACzC,cAAc,yBAAyB,CAAC"}

package/dist/types/types.d.ts

@@ -0,0 +1,18 @@
+import * as service_types from '@powersync/service-types';
+import * as t from 'ts-codec';
+export declare const MongoStorageConfig: t.Intersection<t.Codec<{
+    uri: string;
+    type: "mongodb";
+    database?: string | undefined;
+    username?: string | undefined;
+    password?: string | undefined;
+}, {
+    uri: string;
+    type: "mongodb";
+    database?: string | undefined;
+    username?: string | undefined;
+    password?: string | undefined;
+}, string, t.CodecProps>, t.ObjectCodec<{}>>;
+export type MongoStorageConfig = t.Encoded<typeof MongoStorageConfig>;
+export type MongoStorageConfigDecoded = t.Decoded<typeof MongoStorageConfig>;
+export declare function isMongoStorageConfig(config: service_types.configFile.GenericStorageConfig): config is MongoStorageConfig;

package/dist/types/types.js

@@ -0,0 +1,9 @@
+import * as lib_mongo from '@powersync/lib-service-mongodb';
+import * as t from 'ts-codec';
+export const MongoStorageConfig = lib_mongo.BaseMongoConfig.and(t.object({
+// Add any mongo specific storage settings here in future
+}));
+export function isMongoStorageConfig(config) {
+    return config.type == lib_mongo.MONGO_CONNECTION_TYPE;
+}
+//# sourceMappingURL=types.js.map

package/dist/types/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/types/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,gCAAgC,CAAC;AAE5D,OAAO,KAAK,CAAC,MAAM,UAAU,CAAC;AAE9B,MAAM,CAAC,MAAM,kBAAkB,GAAG,SAAS,CAAC,eAAe,CAAC,GAAG,CAC7D,CAAC,CAAC,MAAM,CAAC;AACP,yDAAyD;CAC1D,CAAC,CACH,CAAC;AAKF,MAAM,UAAU,oBAAoB,CAClC,MAAqD;IAErD,OAAO,MAAM,CAAC,IAAI,IAAI,SAAS,CAAC,qBAAqB,CAAC;AACxD,CAAC"}

package/package.json

@@ -0,0 +1,48 @@
+{
+  "name": "@powersync/service-module-mongodb-storage",
+  "repository": "https://github.com/powersync-ja/powersync-service",
+  "types": "dist/index.d.ts",
+  "version": "0.0.0-dev-20250108073049",
+  "main": "dist/index.js",
+  "license": "FSL-1.1-Apache-2.0",
+  "type": "module",
+  "publishConfig": {
+    "access": "public"
+  },
+  "exports": {
+    ".": {
+      "import": "./dist/index.js",
+      "require": "./dist/index.js",
+      "default": "./dist/index.js"
+    },
+    "./types": {
+      "import": "./dist/types/types.js",
+      "require": "./dist/types/types.js",
+      "default": "./dist/types/types.js"
+    }
+  },
+  "dependencies": {
+    "mongodb": "^6.11.0",
+    "bson": "^6.8.0",
+    "ts-codec": "^1.3.0",
+    "ix": "^5.0.0",
+    "lru-cache": "^10.2.2",
+    "uuid": "^9.0.1",
+    "@powersync/lib-services-framework": "0.0.0-dev-20250108073049",
+    "@powersync/service-core": "0.0.0-dev-20250108073049",
+    "@powersync/service-jsonbig": "0.17.10",
+    "@powersync/service-sync-rules": "0.0.0-dev-20250108073049",
+    "@powersync/service-types": "0.0.0-dev-20250108073049",
+    "@powersync/lib-service-mongodb": "0.0.0-dev-20250108073049"
+  },
+  "devDependencies": {
+    "@types/uuid": "^9.0.4",
+    "@powersync/service-core-tests": "0.0.0-dev-20250108073049"
+  },
+  "scripts": {
+    "build": "tsc -b",
+    "build:tests": "tsc -b test/tsconfig.json",
+    "clean": "rm -rf ./dist && tsc -b --clean",
+    "test": "vitest"
+  }
+}

package/src/index.ts

@@ -0,0 +1,7 @@
+export * from './module/MongoStorageModule.js';
+
+export * from './storage/storage-index.js';
+export * as storage from './storage/storage-index.js';
+
+export * from './types/types.js';
+export * as types from './types/types.js';

package/src/migrations/MongoMigrationAgent.ts

@@ -0,0 +1,39 @@
+import * as framework from '@powersync/lib-services-framework';
+
+import * as lib_mongo from '@powersync/lib-service-mongodb';
+import { migrations } from '@powersync/service-core';
+import * as path from 'path';
+import { fileURLToPath } from 'url';
+import { createPowerSyncMongo, PowerSyncMongo } from '../storage/storage-index.js';
+import { MongoStorageConfig } from '../types/types.js';
+import { createMongoMigrationStore } from './mongo-migration-store.js';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+const MONGO_LOCK_PROCESS = 'migrations';
+const MIGRATIONS_DIR = path.join(__dirname, '/db/migrations');
+
+export class MongoMigrationAgent extends migrations.AbstractPowerSyncMigrationAgent {
+  store: framework.MigrationStore;
+  locks: framework.LockManager;
+
+  protected client: PowerSyncMongo;
+
+  constructor(mongoConfig: MongoStorageConfig) {
+    super();
+
+    this.client = createPowerSyncMongo(mongoConfig);
+
+    this.store = createMongoMigrationStore(this.client.db);
+    this.locks = new lib_mongo.locks.MongoLockManager({ collection: this.client.locks, name: MONGO_LOCK_PROCESS });
+  }
+
+  getInternalScriptsDir(): string {
+    return MIGRATIONS_DIR;
+  }
+
+  async [Symbol.asyncDispose](): Promise<void> {
+    await this.client.client.close();
+  }
+}

package/src/migrations/db/migrations/1684951997326-init.ts

@@ -0,0 +1,39 @@
+import * as lib_mongo from '@powersync/lib-service-mongodb';
+import { migrations } from '@powersync/service-core';
+import * as storage from '../../../storage/storage-index.js';
+import { MongoStorageConfig } from '../../../types/types.js';
+
+export const up: migrations.PowerSyncMigrationFunction = async (context) => {
+  const {
+    service_context: { configuration }
+  } = context;
+  const database = storage.createPowerSyncMongo(configuration.storage as MongoStorageConfig);
+  await lib_mongo.waitForAuth(database.db);
+  try {
+    await database.bucket_parameters.createIndex(
+      {
+        'key.g': 1,
+        lookup: 1,
+        _id: 1
+      },
+      { name: 'lookup1' }
+    );
+  } finally {
+    await database.client.close();
+  }
+};
+
+export const down: migrations.PowerSyncMigrationFunction = async (context) => {
+  const {
+    service_context: { configuration }
+  } = context;
+
+  const database = storage.createPowerSyncMongo(configuration.storage as MongoStorageConfig);
+  try {
+    if (await database.bucket_parameters.indexExists('lookup')) {
+      await database.bucket_parameters.dropIndex('lookup1');
+    }
+  } finally {
+    await database.client.close();
+  }
+};

package/src/migrations/db/migrations/1688556755264-initial-sync-rules.ts

@@ -0,0 +1,5 @@
+export const up = async () => {
+  // No-op - we don't auto-create sync rules anymore
+};
+
+export const down = async () => {};