@powersync/service-core 0.0.2

package/dist/storage/mongo/OperationBatch.js

@@ -0,0 +1,101 @@
+import * as util from '../../util/util-index.js';
+/**
+ * Maximum number of operations in a batch.
+ */
+const MAX_BATCH_COUNT = 2000;
+/**
+ * Maximum size of operations in the batch (estimated).
+ */
+const MAX_RECORD_BATCH_SIZE = 14000000;
+/**
+ * Maximum size of size of current_data documents we lookup at a time.
+ */
+const MAX_CURRENT_DATA_BATCH_SIZE = 16000000;
+/**
+ * Batch of input operations.
+ *
+ * We accumulate operations up to MAX_RECORD_BATCH_SIZE,
+ * then further split into sub-batches if MAX_CURRENT_DATA_BATCH_SIZE is exceeded.
+ */
+export class OperationBatch {
+    constructor() {
+        this.batch = [];
+        this.currentSize = 0;
+    }
+    get length() {
+        return this.batch.length;
+    }
+    push(op) {
+        this.batch.push(op);
+        this.currentSize += op.estimatedSize;
+    }
+    shouldFlush() {
+        return this.batch.length >= MAX_BATCH_COUNT || this.currentSize > MAX_RECORD_BATCH_SIZE;
+    }
+    *batched(sizes) {
+        let currentBatch = [];
+        let currentBatchSize = 0;
+        for (let op of this.batch) {
+            const key = op.internalBeforeKey;
+            const size = sizes.get(key) ?? 0;
+            if (currentBatchSize + size > MAX_CURRENT_DATA_BATCH_SIZE && currentBatch.length > 0) {
+                yield currentBatch;
+                currentBatch = [];
+                currentBatchSize = 0;
+            }
+            currentBatchSize += size;
+            currentBatch.push(op);
+        }
+        if (currentBatch.length > 0) {
+            yield currentBatch;
+        }
+    }
+}
+export class RecordOperation {
+    constructor(record) {
+        this.record = record;
+        const after = record.after;
+        const afterId = after ? util.getUuidReplicaIdentityBson(after, record.sourceTable.replicaIdColumns) : null;
+        const beforeId = record.before
+            ? util.getUuidReplicaIdentityBson(record.before, record.sourceTable.replicaIdColumns)
+            : afterId;
+        this.afterId = afterId;
+        this.beforeId = beforeId;
+        this.internalBeforeKey = cacheKey(record.sourceTable.id, beforeId);
+        this.internalAfterKey = afterId ? cacheKey(record.sourceTable.id, afterId) : null;
+        this.estimatedSize = estimateRowSize(record.before) + estimateRowSize(record.after);
+    }
+}
+export function cacheKey(table, id) {
+    return `${table.toHexString()}.${id.toHexString()}`;
+}
+/**
+ * Estimate in-memory size of row.
+ */
+function estimateRowSize(record) {
+    if (record == null) {
+        return 12;
+    }
+    let size = 0;
+    for (let [key, value] of Object.entries(record)) {
+        size += 12 + key.length;
+        // number | string | null | bigint | Uint8Array
+        if (value == null) {
+            size += 4;
+        }
+        else if (typeof value == 'number') {
+            size += 8;
+        }
+        else if (typeof value == 'bigint') {
+            size += 8;
+        }
+        else if (typeof value == 'string') {
+            size += value.length;
+        }
+        else if (value instanceof Uint8Array) {
+            size += value.byteLength;
+        }
+    }
+    return size;
+}
+//# sourceMappingURL=OperationBatch.js.map

package/dist/storage/mongo/OperationBatch.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"OperationBatch.js","sourceRoot":"","sources":["../../../src/storage/mongo/OperationBatch.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,IAAI,MAAM,sBAAsB,CAAC;AAG7C;;GAEG;AACH,MAAM,eAAe,GAAG,IAAI,CAAC;AAE7B;;GAEG;AACH,MAAM,qBAAqB,GAAG,QAAU,CAAC;AAEzC;;GAEG;AACH,MAAM,2BAA2B,GAAG,QAAU,CAAC;AAE/C;;;;;GAKG;AACH,MAAM,OAAO,cAAc;IAA3B;QACE,UAAK,GAAsB,EAAE,CAAC;QAC9B,gBAAW,GAAW,CAAC,CAAC;IAiC1B,CAAC;IA/BC,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;IAC3B,CAAC;IAED,IAAI,CAAC,EAAmB;QACtB,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACpB,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC,aAAa,CAAC;IACvC,CAAC;IAED,WAAW;QACT,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,IAAI,eAAe,IAAI,IAAI,CAAC,WAAW,GAAG,qBAAqB,CAAC;IAC1F,CAAC;IAED,CAAC,OAAO,CAAC,KAA0B;QACjC,IAAI,YAAY,GAAsB,EAAE,CAAC;QACzC,IAAI,gBAAgB,GAAG,CAAC,CAAC;QACzB,KAAK,IAAI,EAAE,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YAC1B,MAAM,GAAG,GAAG,EAAE,CAAC,iBAAiB,CAAC;YACjC,MAAM,IAAI,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACjC,IAAI,gBAAgB,GAAG,IAAI,GAAG,2BAA2B,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACrF,MAAM,YAAY,CAAC;gBACnB,YAAY,GAAG,EAAE,CAAC;gBAClB,gBAAgB,GAAG,CAAC,CAAC;YACvB,CAAC;YACD,gBAAgB,IAAI,IAAI,CAAC;YACzB,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACxB,CAAC;QACD,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5B,MAAM,YAAY,CAAC;QACrB,CAAC;IACH,CAAC;CACF;AAED,MAAM,OAAO,eAAe;IAO1B,YAA4B,MAAmB;QAAnB,WAAM,GAAN,MAAM,CAAa;QAC7C,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;QAC3B,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,0BAA0B,CAAC,KAAK,EAAE,MAAM,CAAC,WAAW,CAAC,gBAAiB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAC5G,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM;YAC5B,CAAC,CAAC,IAAI,CAAC,0BAA0B,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,WAAW,CAAC,gBAAiB,CAAC;YACtF,CAAC,CAAC,OAAQ,CAAC;QACb,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,iBAAiB,GAAG,QAAQ,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;QACnE,IAAI,CAAC,gBAAgB,GAAG,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAElF,IAAI,CAAC,aAAa,GAAG,eAAe,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,eAAe,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACtF,CAAC;CACF;AAED,MAAM,UAAU,QAAQ,CAAC,KAAoB,EAAE,EAAa;IAC1D,OAAO,GAAG,KAAK,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC;AACtD,CAAC;AAED;;GAEG;AACH,SAAS,eAAe,CAAC,MAAsC;IAC7D,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;QACnB,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAChD,IAAI,IAAI,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC;QACxB,+CAA+C;QAC/C,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;YAClB,IAAI,IAAI,CAAC,CAAC;QACZ,CAAC;aAAM,IAAI,OAAO,KAAK,IAAI,QAAQ,EAAE,CAAC;YACpC,IAAI,IAAI,CAAC,CAAC;QACZ,CAAC;aAAM,IAAI,OAAO,KAAK,IAAI,QAAQ,EAAE,CAAC;YACpC,IAAI,IAAI,CAAC,CAAC;QACZ,CAAC;aAAM,IAAI,OAAO,KAAK,IAAI,QAAQ,EAAE,CAAC;YACpC,IAAI,IAAI,KAAK,CAAC,MAAM,CAAC;QACvB,CAAC;aAAM,IAAI,KAAK,YAAY,UAAU,EAAE,CAAC;YACvC,IAAI,IAAI,KAAK,CAAC,UAAU,CAAC;QAC3B,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC"}

package/dist/storage/mongo/PersistedBatch.d.ts

@@ -0,0 +1,42 @@
+import { EvaluatedParameters, EvaluatedRow } from '@powersync/service-sync-rules';
+import * as bson from 'bson';
+import * as mongo from 'mongodb';
+import { SourceTable } from '../SourceTable.js';
+import { MongoIdSequence } from './MongoIdSequence.js';
+import { PowerSyncMongo } from './db.js';
+import { BucketDataDocument, BucketParameterDocument, CurrentBucket, CurrentDataDocument, SourceKey } from './models.js';
+/**
+ * Keeps track of bulkwrite operations within a transaction.
+ *
+ * There may be multiple of these batches per transaction, but it may not span
+ * multiple transactions.
+ */
+export declare class PersistedBatch {
+    private group_id;
+    bucketData: mongo.AnyBulkWriteOperation<BucketDataDocument>[];
+    bucketParameters: mongo.AnyBulkWriteOperation<BucketParameterDocument>[];
+    currentData: mongo.AnyBulkWriteOperation<CurrentDataDocument>[];
+    /**
+     * Very rough estimate of transaction size.
+     */
+    currentSize: number;
+    constructor(group_id: number, writtenSize: number);
+    saveBucketData(options: {
+        op_seq: MongoIdSequence;
+        sourceKey: bson.UUID;
+        table: SourceTable;
+        evaluated: EvaluatedRow[];
+        before_buckets: CurrentBucket[];
+    }): void;
+    saveParameterData(data: {
+        op_seq: MongoIdSequence;
+        sourceKey: bson.UUID;
+        sourceTable: SourceTable;
+        evaluated: EvaluatedParameters[];
+        existing_lookups: bson.Binary[];
+    }): void;
+    deleteCurrentData(id: SourceKey): void;
+    upsertCurrentData(id: SourceKey, values: Partial<CurrentDataDocument>): void;
+    shouldFlushTransaction(): boolean;
+    flush(db: PowerSyncMongo, session: mongo.ClientSession): Promise<void>;
+}

package/dist/storage/mongo/PersistedBatch.js

@@ -0,0 +1,200 @@
+import { JSONBig } from '@powersync/service-jsonbig';
+import * as util from '../../util/util-index.js';
+import { currentBucketKey } from './MongoBucketBatch.js';
+import { serializeLookup } from './util.js';
+/**
+ * Maximum size of operations we write in a single transaction.
+ *
+ * It's tricky to find the exact limit, but from experience, over 100MB
+ * can cause an error:
+ * > transaction is too large and will not fit in the storage engine cache
+ *
+ * Additionally, unbounded size here can balloon our memory usage in some edge
+ * cases.
+ *
+ * When we reach this threshold, we commit the transaction and start a new one.
+ */
+const MAX_TRANSACTION_BATCH_SIZE = 30000000;
+/**
+ * Keeps track of bulkwrite operations within a transaction.
+ *
+ * There may be multiple of these batches per transaction, but it may not span
+ * multiple transactions.
+ */
+export class PersistedBatch {
+    constructor(group_id, writtenSize) {
+        this.group_id = group_id;
+        this.bucketData = [];
+        this.bucketParameters = [];
+        this.currentData = [];
+        /**
+         * Very rough estimate of transaction size.
+         */
+        this.currentSize = 0;
+        this.currentSize = writtenSize;
+    }
+    saveBucketData(options) {
+        const remaining_buckets = new Map();
+        for (let b of options.before_buckets) {
+            const key = currentBucketKey(b);
+            remaining_buckets.set(key, b);
+        }
+        const dchecksum = util.hashDelete(`${options.table.id}/${options.sourceKey}`);
+        for (let k of options.evaluated) {
+            const key = currentBucketKey(k);
+            remaining_buckets.delete(key);
+            // INSERT
+            const recordData = JSONBig.stringify(k.data);
+            const checksum = util.hashData(k.table, k.id, recordData);
+            this.currentSize += recordData.length + 200;
+            this.bucketData.push({
+                insertOne: {
+                    document: {
+                        _id: {
+                            g: this.group_id,
+                            b: k.bucket,
+                            o: options.op_seq.next()
+                        },
+                        op: 'PUT',
+                        source_table: options.table.id,
+                        source_key: options.sourceKey,
+                        table: k.table,
+                        row_id: k.id,
+                        checksum: checksum,
+                        data: recordData
+                    }
+                }
+            });
+        }
+        for (let bd of remaining_buckets.values()) {
+            // REMOVE
+            this.bucketData.push({
+                insertOne: {
+                    document: {
+                        _id: {
+                            g: this.group_id,
+                            b: bd.bucket,
+                            o: options.op_seq.next()
+                        },
+                        op: 'REMOVE',
+                        source_table: options.table.id,
+                        source_key: options.sourceKey,
+                        table: bd.table,
+                        row_id: bd.id,
+                        checksum: dchecksum,
+                        data: null
+                    }
+                }
+            });
+            this.currentSize += 200;
+        }
+    }
+    saveParameterData(data) {
+        // This is similar to saving bucket data.
+        // A key difference is that we don't need to keep the history intact.
+        // We do need to keep track of recent history though - enough that we can get consistent data for any specific checkpoint.
+        // Instead of storing per bucket id, we store per "lookup".
+        // A key difference is that we don't need to store or keep track of anything per-bucket - the entire record is
+        // either persisted or removed.
+        // We also don't need to keep history intact.
+        const { sourceTable, sourceKey, evaluated } = data;
+        const remaining_lookups = new Map();
+        for (let l of data.existing_lookups) {
+            remaining_lookups.set(l.toString('base64'), l);
+        }
+        // 1. Insert new entries
+        for (let result of evaluated) {
+            const binLookup = serializeLookup(result.lookup);
+            const hex = binLookup.toString('base64');
+            remaining_lookups.delete(hex);
+            const op_id = data.op_seq.next();
+            this.bucketParameters.push({
+                insertOne: {
+                    document: {
+                        _id: op_id,
+                        key: {
+                            g: this.group_id,
+                            t: sourceTable.id,
+                            k: sourceKey
+                        },
+                        lookup: binLookup,
+                        bucket_parameters: result.bucket_parameters
+                    }
+                }
+            });
+            this.currentSize += 200;
+        }
+        // 2. "REMOVE" entries for any lookup not touched.
+        for (let lookup of remaining_lookups.values()) {
+            const op_id = data.op_seq.next();
+            this.bucketParameters.push({
+                insertOne: {
+                    document: {
+                        _id: op_id,
+                        key: {
+                            g: this.group_id,
+                            t: sourceTable.id,
+                            k: sourceKey
+                        },
+                        lookup: lookup,
+                        bucket_parameters: []
+                    }
+                }
+            });
+            this.currentSize += 200;
+        }
+    }
+    deleteCurrentData(id) {
+        const op = {
+            deleteOne: {
+                filter: { _id: id }
+            }
+        };
+        this.currentData.push(op);
+        this.currentSize += 50;
+    }
+    upsertCurrentData(id, values) {
+        const op = {
+            updateOne: {
+                filter: { _id: id },
+                update: {
+                    $set: values
+                },
+                upsert: true
+            }
+        };
+        this.currentData.push(op);
+        this.currentSize += (values.data?.length() ?? 0) + 100;
+    }
+    shouldFlushTransaction() {
+        return this.currentSize >= MAX_TRANSACTION_BATCH_SIZE;
+    }
+    async flush(db, session) {
+        if (this.bucketData.length > 0) {
+            await db.bucket_data.bulkWrite(this.bucketData, {
+                session,
+                // inserts only - order doesn't matter
+                ordered: false
+            });
+        }
+        if (this.bucketParameters.length > 0) {
+            await db.bucket_parameters.bulkWrite(this.bucketParameters, {
+                session,
+                // inserts only - order doesn't matter
+                ordered: false
+            });
+        }
+        if (this.currentData.length > 0) {
+            await db.current_data.bulkWrite(this.currentData, {
+                session,
+                // may update and delete data within the same batch - order matters
+                ordered: true
+            });
+        }
+        this.bucketData = [];
+        this.bucketParameters = [];
+        this.currentData = [];
+        this.currentSize = 0;
+    }
+}
+//# sourceMappingURL=PersistedBatch.js.map

package/dist/storage/mongo/PersistedBatch.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"PersistedBatch.js","sourceRoot":"","sources":["../../../src/storage/mongo/PersistedBatch.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAC;AAKrD,OAAO,KAAK,IAAI,MAAM,sBAAsB,CAAC;AAE7C,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAUzD,OAAO,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAE5C;;;;;;;;;;;GAWG;AACH,MAAM,0BAA0B,GAAG,QAAU,CAAC;AAE9C;;;;;GAKG;AACH,MAAM,OAAO,cAAc;IAUzB,YAAoB,QAAgB,EAAE,WAAmB;QAArC,aAAQ,GAAR,QAAQ,CAAQ;QATpC,eAAU,GAAsD,EAAE,CAAC;QACnE,qBAAgB,GAA2D,EAAE,CAAC;QAC9E,gBAAW,GAAuD,EAAE,CAAC;QAErE;;WAEG;QACH,gBAAW,GAAG,CAAC,CAAC;QAGd,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;IACjC,CAAC;IAED,cAAc,CAAC,OAMd;QACC,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAyB,CAAC;QAC3D,KAAK,IAAI,CAAC,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;YACrC,MAAM,GAAG,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC;YAChC,iBAAiB,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;QAChC,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,EAAE,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC;QAE9E,KAAK,IAAI,CAAC,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;YAChC,MAAM,GAAG,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC;YAChC,iBAAiB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAE9B,SAAS;YACT,MAAM,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YAC7C,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,EAAE,EAAE,UAAU,CAAC,CAAC;YAC1D,IAAI,CAAC,WAAW,IAAI,UAAU,CAAC,MAAM,GAAG,GAAG,CAAC;YAE5C,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;gBACnB,SAAS,EAAE;oBACT,QAAQ,EAAE;wBACR,GAAG,EAAE;4BACH,CAAC,EAAE,IAAI,CAAC,QAAQ;4BAChB,CAAC,EAAE,CAAC,CAAC,MAAM;4BACX,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE;yBACzB;wBACD,EAAE,EAAE,KAAK;wBACT,YAAY,EAAE,OAAO,CAAC,KAAK,CAAC,EAAE;wBAC9B,UAAU,EAAE,OAAO,CAAC,SAAS;wBAC7B,KAAK,EAAE,CAAC,CAAC,KAAK;wBACd,MAAM,EAAE,CAAC,CAAC,EAAE;wBACZ,QAAQ,EAAE,QAAQ;wBAClB,IAAI,EAAE,UAAU;qBACjB;iBACF;aACF,CAAC,CAAC;QACL,CAAC;QAED,KAAK,IAAI,EAAE,IAAI,iBAAiB,CAAC,MAAM,EAAE,EAAE,CAAC;YAC1C,SAAS;YACT,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;gBACnB,SAAS,EAAE;oBACT,QAAQ,EAAE;wBACR,GAAG,EAAE;4BACH,CAAC,EAAE,IAAI,CAAC,QAAQ;4BAChB,CAAC,EAAE,EAAE,CAAC,MAAM;4BACZ,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE;yBACzB;wBACD,EAAE,EAAE,QAAQ;wBACZ,YAAY,EAAE,OAAO,CAAC,KAAK,CAAC,EAAE;wBAC9B,UAAU,EAAE,OAAO,CAAC,SAAS;wBAC7B,KAAK,EAAE,EAAE,CAAC,KAAK;wBACf,MAAM,EAAE,EAAE,CAAC,EAAE;wBACb,QAAQ,EAAE,SAAS;wBACnB,IAAI,EAAE,IAAI;qBACX;iBACF;aACF,CAAC,CAAC;YACH,IAAI,CAAC,WAAW,IAAI,GAAG,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,iBAAiB,CAAC,IAMjB;QACC,yCAAyC;QACzC,qEAAqE;QACrE,0HAA0H;QAC1H,2DAA2D;QAC3D,8GAA8G;QAC9G,+BAA+B;QAC/B,6CAA6C;QAC7C,MAAM,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC;QAEnD,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAuB,CAAC;QACzD,KAAK,IAAI,CAAC,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACpC,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;QACjD,CAAC;QAED,wBAAwB;QACxB,KAAK,IAAI,MAAM,IAAI,SAAS,EAAE,CAAC;YAC7B,MAAM,SAAS,GAAG,eAAe,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YACjD,MAAM,GAAG,GAAG,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;YACzC,iBAAiB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC9B,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACjC,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC;gBACzB,SAAS,EAAE;oBACT,QAAQ,EAAE;wBACR,GAAG,EAAE,KAAK;wBACV,GAAG,EAAE;4BACH,CAAC,EAAE,IAAI,CAAC,QAAQ;4BAChB,CAAC,EAAE,WAAW,CAAC,EAAE;4BACjB,CAAC,EAAE,SAAS;yBACb;wBACD,MAAM,EAAE,SAAS;wBACjB,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;qBAC5C;iBACF;aACF,CAAC,CAAC;YAEH,IAAI,CAAC,WAAW,IAAI,GAAG,CAAC;QAC1B,CAAC;QAED,kDAAkD;QAClD,KAAK,IAAI,MAAM,IAAI,iBAAiB,CAAC,MAAM,EAAE,EAAE,CAAC;YAC9C,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACjC,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC;gBACzB,SAAS,EAAE;oBACT,QAAQ,EAAE;wBACR,GAAG,EAAE,KAAK;wBACV,GAAG,EAAE;4BACH,CAAC,EAAE,IAAI,CAAC,QAAQ;4BAChB,CAAC,EAAE,WAAW,CAAC,EAAE;4BACjB,CAAC,EAAE,SAAS;yBACb;wBACD,MAAM,EAAE,MAAM;wBACd,iBAAiB,EAAE,EAAE;qBACtB;iBACF;aACF,CAAC,CAAC;YAEH,IAAI,CAAC,WAAW,IAAI,GAAG,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,iBAAiB,CAAC,EAAa;QAC7B,MAAM,EAAE,GAAqD;YAC3D,SAAS,EAAE;gBACT,MAAM,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE;aACpB;SACF,CAAC;QACF,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC1B,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC;IACzB,CAAC;IAED,iBAAiB,CAAC,EAAa,EAAE,MAAoC;QACnE,MAAM,EAAE,GAAqD;YAC3D,SAAS,EAAE;gBACT,MAAM,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE;gBACnB,MAAM,EAAE;oBACN,IAAI,EAAE,MAAM;iBACb;gBACD,MAAM,EAAE,IAAI;aACb;SACF,CAAC;QACF,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC1B,IAAI,CAAC,WAAW,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC;IACzD,CAAC;IAED,sBAAsB;QACpB,OAAO,IAAI,CAAC,WAAW,IAAI,0BAA0B,CAAC;IACxD,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,EAAkB,EAAE,OAA4B;QAC1D,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC/B,MAAM,EAAE,CAAC,WAAW,CAAC,SAAS,CAAC,IAAI,CAAC,UAAU,EAAE;gBAC9C,OAAO;gBACP,sCAAsC;gBACtC,OAAO,EAAE,KAAK;aACf,CAAC,CAAC;QACL,CAAC;QACD,IAAI,IAAI,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrC,MAAM,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC,IAAI,CAAC,gBAAgB,EAAE;gBAC1D,OAAO;gBACP,sCAAsC;gBACtC,OAAO,EAAE,KAAK;aACf,CAAC,CAAC;QACL,CAAC;QACD,IAAI,IAAI,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAChC,MAAM,EAAE,CAAC,YAAY,CAAC,SAAS,CAAC,IAAI,CAAC,WAAW,EAAE;gBAChD,OAAO;gBACP,mEAAmE;gBACnE,OAAO,EAAE,IAAI;aACd,CAAC,CAAC;QACL,CAAC;QAED,IAAI,CAAC,UAAU,GAAG,EAAE,CAAC;QACrB,IAAI,CAAC,gBAAgB,GAAG,EAAE,CAAC;QAC3B,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC;QACtB,IAAI,CAAC,WAAW,GAAG,CAAC,CAAC;IACvB,CAAC;CACF"}

package/dist/storage/mongo/db.d.ts

@@ -0,0 +1,23 @@
+import * as mongo from 'mongodb';
+import { BucketDataDocument, BucketParameterDocument, CurrentDataDocument, IdSequenceDocument, SourceTableDocument, SyncRuleDocument, WriteCheckpointDocument } from './models.js';
+import { configFile } from '@powersync/service-types';
+export interface PowerSyncMongoOptions {
+    /**
+     * Optional - uses the database from the MongoClient connection URI if not specified.
+     */
+    database?: string;
+}
+export declare function createPowerSyncMongo(config: configFile.PowerSyncConfig['storage']): PowerSyncMongo;
+export declare class PowerSyncMongo {
+    readonly current_data: mongo.Collection<CurrentDataDocument>;
+    readonly bucket_data: mongo.Collection<BucketDataDocument>;
+    readonly bucket_parameters: mongo.Collection<BucketParameterDocument>;
+    readonly op_id_sequence: mongo.Collection<IdSequenceDocument>;
+    readonly sync_rules: mongo.Collection<SyncRuleDocument>;
+    readonly source_tables: mongo.Collection<SourceTableDocument>;
+    readonly write_checkpoints: mongo.Collection<WriteCheckpointDocument>;
+    readonly client: mongo.MongoClient;
+    readonly db: mongo.Db;
+    constructor(client: mongo.MongoClient, options?: PowerSyncMongoOptions);
+    clear(): Promise<void>;
+}

package/dist/storage/mongo/db.js

@@ -0,0 +1,34 @@
+import * as db from '../../db/db-index.js';
+import { BSON_DESERIALIZE_OPTIONS } from './util.js';
+export function createPowerSyncMongo(config) {
+    return new PowerSyncMongo(db.mongo.createMongoClient(config), { database: config.database });
+}
+export class PowerSyncMongo {
+    constructor(client, options) {
+        this.client = client;
+        const db = client.db(options?.database, {
+            // Note this issue with useBigInt64: https://jira.mongodb.org/browse/NODE-6165
+            // Unfortunately there is no workaround if we want to continue using bigint.
+            // We selectively disable this in individual queries where we don't need that.
+            ...BSON_DESERIALIZE_OPTIONS
+        });
+        this.db = db;
+        this.current_data = db.collection('current_data');
+        this.bucket_data = db.collection('bucket_data');
+        this.bucket_parameters = db.collection('bucket_parameters');
+        this.op_id_sequence = db.collection('op_id_sequence');
+        this.sync_rules = db.collection('sync_rules');
+        this.source_tables = db.collection('source_tables');
+        this.write_checkpoints = db.collection('write_checkpoints');
+    }
+    async clear() {
+        await this.current_data.deleteMany({});
+        await this.bucket_data.deleteMany({});
+        await this.bucket_parameters.deleteMany({});
+        await this.op_id_sequence.deleteMany({});
+        await this.sync_rules.deleteMany({});
+        await this.source_tables.deleteMany({});
+        await this.write_checkpoints.deleteMany({});
+    }
+}
+//# sourceMappingURL=db.js.map

package/dist/storage/mongo/db.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"db.js","sourceRoot":"","sources":["../../../src/storage/mongo/db.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAWvC,OAAO,EAAE,wBAAwB,EAAE,MAAM,WAAW,CAAC;AAUrD,MAAM,UAAU,oBAAoB,CAAC,MAA6C;IAChF,OAAO,IAAI,cAAc,CAAC,EAAE,CAAC,KAAK,CAAC,iBAAiB,CAAC,MAAM,CAAC,EAAE,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;AAC/F,CAAC;AAED,MAAM,OAAO,cAAc;IAYzB,YAAY,MAAyB,EAAE,OAA+B;QACpE,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QAErB,MAAM,EAAE,GAAG,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,QAAQ,EAAE;YACtC,8EAA8E;YAC9E,4EAA4E;YAC5E,8EAA8E;YAC9E,GAAG,wBAAwB;SAC5B,CAAC,CAAC;QACH,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QAEb,IAAI,CAAC,YAAY,GAAG,EAAE,CAAC,UAAU,CAAsB,cAAc,CAAC,CAAC;QACvE,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;QAChD,IAAI,CAAC,iBAAiB,GAAG,EAAE,CAAC,UAAU,CAAC,mBAAmB,CAAC,CAAC;QAC5D,IAAI,CAAC,cAAc,GAAG,EAAE,CAAC,UAAU,CAAC,gBAAgB,CAAC,CAAC;QACtD,IAAI,CAAC,UAAU,GAAG,EAAE,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC;QAC9C,IAAI,CAAC,aAAa,GAAG,EAAE,CAAC,UAAU,CAAC,eAAe,CAAC,CAAC;QACpD,IAAI,CAAC,iBAAiB,GAAG,EAAE,CAAC,UAAU,CAAC,mBAAmB,CAAC,CAAC;IAC9D,CAAC;IAED,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACvC,MAAM,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACtC,MAAM,IAAI,CAAC,iBAAiB,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QAC5C,MAAM,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACzC,MAAM,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACrC,MAAM,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACxC,MAAM,IAAI,CAAC,iBAAiB,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;IAC9C,CAAC;CACF"}

package/dist/storage/mongo/models.d.ts

@@ -0,0 +1,137 @@
+import * as bson from 'bson';
+import { SqliteJsonValue } from '@powersync/service-sync-rules';
+export interface SourceKey {
+    /** group_id */
+    g: number;
+    /** source table id */
+    t: bson.ObjectId;
+    /** source key */
+    k: bson.UUID;
+}
+export interface BucketDataKey {
+    /** group_id */
+    g: number;
+    /** bucket name */
+    b: string;
+    /** op_id */
+    o: bigint;
+}
+export interface CurrentDataDocument {
+    _id: SourceKey;
+    data: bson.Binary;
+    buckets: CurrentBucket[];
+    lookups: bson.Binary[];
+}
+export interface CurrentBucket {
+    bucket: string;
+    table: string;
+    id: string;
+}
+export interface BucketParameterDocument {
+    _id: bigint;
+    key: SourceKey;
+    lookup: bson.Binary;
+    bucket_parameters: Record<string, SqliteJsonValue>[];
+}
+export interface BucketDataDocument {
+    _id: BucketDataKey;
+    op: OpType;
+    source_table: bson.ObjectId;
+    source_key: bson.UUID;
+    table: string;
+    row_id: string;
+    checksum: number;
+    data: string | null;
+}
+export type OpType = 'PUT' | 'REMOVE' | 'MOVE' | 'CLEAR';
+export interface SourceTableDocument {
+    _id: bson.ObjectId;
+    group_id: number;
+    connection_id: number;
+    relation_id: number | undefined;
+    schema_name: string;
+    table_name: string;
+    replica_id_columns: string[] | null;
+    replica_id_columns2: {
+        name: string;
+        type_oid: number;
+    }[] | undefined;
+    snapshot_done: boolean | undefined;
+}
+export interface IdSequenceDocument {
+    _id: string;
+    op_id: bigint;
+}
+export declare enum SyncRuleState {
+    /**
+     * New sync rules - needs to be processed (initial replication).
+     *
+     * While multiple sets of sync rules _can_ be in PROCESSING,
+     * it's generally pointless, so we only keep one in that state.
+     */
+    PROCESSING = "PROCESSING",
+    /**
+     * Sync rule processing is done, and can be used for sync.
+     *
+     * Only one set of sync rules should be in ACTIVE state.
+     */
+    ACTIVE = "ACTIVE",
+    /**
+     * This state is used when the sync rules has been replaced,
+     * and replication is or should be stopped.
+     */
+    STOP = "STOP",
+    /**
+     * After sync rules have been stopped, the data needs to be
+     * deleted. Once deleted, the state is TERMINATED.
+     */
+    TERMINATED = "TERMINATED"
+}
+export interface SyncRuleDocument {
+    _id: number;
+    state: SyncRuleState;
+    /**
+     * True if initial snapshot has been replicated.
+     *
+     * Can only be false if state == PROCESSING.
+     */
+    snapshot_done: boolean;
+    /**
+     * The last consistent checkpoint.
+     *
+     * There may be higher OpIds used in the database if we're in the middle of replicating a large transaction.
+     */
+    last_checkpoint: bigint | null;
+    /**
+     * The LSN associated with the last consistent checkpoint.
+     */
+    last_checkpoint_lsn: string | null;
+    /**
+     * If set, no new checkpoints may be created < this value.
+     */
+    no_checkpoint_before: string | null;
+    slot_name: string | null;
+    /**
+     * Last time we persisted a checkpoint.
+     *
+     * This may be old if no data is incoming.
+     */
+    last_checkpoint_ts: Date | null;
+    /**
+     * Last time we persisted a checkpoint or keepalive.
+     *
+     * This should stay fairly current while replicating.
+     */
+    last_keepalive_ts: Date | null;
+    /**
+     * If an error is stopping replication, it will be stored here.
+     */
+    last_fatal_error: string | null;
+    content: string;
+}
+export interface WriteCheckpointDocument {
+    _id: bson.ObjectId;
+    user_id: string;
+    lsns: Record<string, string>;
+    client_id: bigint;
+}

package/dist/storage/mongo/models.js

@@ -0,0 +1,27 @@
+export var SyncRuleState;
+(function (SyncRuleState) {
+    /**
+     * New sync rules - needs to be processed (initial replication).
+     *
+     * While multiple sets of sync rules _can_ be in PROCESSING,
+     * it's generally pointless, so we only keep one in that state.
+     */
+    SyncRuleState["PROCESSING"] = "PROCESSING";
+    /**
+     * Sync rule processing is done, and can be used for sync.
+     *
+     * Only one set of sync rules should be in ACTIVE state.
+     */
+    SyncRuleState["ACTIVE"] = "ACTIVE";
+    /**
+     * This state is used when the sync rules has been replaced,
+     * and replication is or should be stopped.
+     */
+    SyncRuleState["STOP"] = "STOP";
+    /**
+     * After sync rules have been stopped, the data needs to be
+     * deleted. Once deleted, the state is TERMINATED.
+     */
+    SyncRuleState["TERMINATED"] = "TERMINATED";
+})(SyncRuleState || (SyncRuleState = {}));
+//# sourceMappingURL=models.js.map

package/dist/storage/mongo/models.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"models.js","sourceRoot":"","sources":["../../../src/storage/mongo/models.ts"],"names":[],"mappings":"AAuEA,MAAM,CAAN,IAAY,aAyBX;AAzBD,WAAY,aAAa;IACvB;;;;;OAKG;IACH,0CAAyB,CAAA;IAEzB;;;;OAIG;IACH,kCAAiB,CAAA;IACjB;;;OAGG;IACH,8BAAa,CAAA;IACb;;;OAGG;IACH,0CAAyB,CAAA;AAC3B,CAAC,EAzBW,aAAa,KAAb,aAAa,QAyBxB"}

package/dist/storage/mongo/util.d.ts

@@ -0,0 +1,26 @@
+import { SqliteJsonValue } from '@powersync/service-sync-rules';
+import * as bson from 'bson';
+import * as mongo from 'mongodb';
+/**
+ * Lookup serialization must be number-agnostic. I.e. normalize numbers, instead of preserving numbers.
+ * @param lookup
+ */
+export declare function serializeLookup(lookup: SqliteJsonValue[]): bson.Binary;
+export declare function idPrefixFilter<T>(prefix: Partial<T>, rest: (keyof T)[]): mongo.Condition<T>;
+export declare function generateSlotName(prefix: string, sync_rules_id: number): string;
+/**
+ * Read a single batch of data from a cursor, then close it.
+ *
+ * We do our best to avoid MongoDB fetching any more data than this single batch.
+ *
+ * This is similar to using `singleBatch: true` in find options.
+ * However, that makes `has_more` detection very difficult, since the cursor is always closed
+ * after the first batch. Instead, we do a workaround to only fetch a single batch below.
+ *
+ * For this to be effective, set batchSize = limit in the find command.
+ */
+export declare function readSingleBatch<T>(cursor: mongo.FindCursor<T>): Promise<{
+    data: T[];
+    hasMore: boolean;
+}>;
+export declare const BSON_DESERIALIZE_OPTIONS: bson.DeserializeOptions;