@powersync/service-module-mongodb-storage 0.15.4 → 0.16.0

package/dist/storage/implementation/v3/PersistedBatchV3.js

@@ -0,0 +1,259 @@
+import { ReplicationAssertionError } from '@powersync/lib-services-framework';
+import * as bson from 'bson';
+import { mongoTableId } from '../../../utils/util.js';
+import { PersistedBatch } from '../common/PersistedBatch.js';
+import { serializeBucketDataV3, taggedBucketParameterDocumentToV3 } from './models.js';
+import { serializeParameterLookupV3 } from './MongoParameterLookupV3.js';
+export class PersistedBatchV3 extends PersistedBatch {
+    currentData = [];
+    sourceTablePendingDeletes = new Map();
+    checkDefinitionId(definitionId) {
+        if (definitionId == null) {
+            // This is required for V3 storage.
+            throw new ReplicationAssertionError('Expected v3 bucket when incrementalReprocessing is enabled');
+        }
+        return definitionId;
+    }
+    getBucketDefinitionId(bucketSource) {
+        return this.mapping.bucketSourceId(bucketSource);
+    }
+    saveParameterData(data) {
+        const { sourceTable, sourceKey, evaluated } = data;
+        const remaining_lookups = new Map();
+        for (let lookup of data.existing_lookups) {
+            if (lookup.indexId == null) {
+                throw new ReplicationAssertionError('Expected v3 lookup when incrementalReprocessing is enabled');
+            }
+            remaining_lookups.set(`${lookup.indexId}.${lookup.lookup.toString('base64')}`, lookup);
+        }
+        for (let result of evaluated) {
+            const sourceDefinitionId = this.mapping.parameterLookupId(result.lookup.source);
+            const binLookup = serializeParameterLookupV3(result.lookup);
+            remaining_lookups.delete(`${sourceDefinitionId}.${binLookup.toString('base64')}`);
+            const op_id = data.op_seq.next();
+            this.debugLastOpId = op_id;
+            const values = {
+                _id: op_id,
+                key: {
+                    t: mongoTableId(sourceTable.id),
+                    k: sourceKey
+                },
+                lookup: binLookup,
+                bucket_parameters: result.bucketParameters
+            };
+            this.bucketParameters.push({
+                ...values,
+                index: sourceDefinitionId
+            });
+            this.currentSize += 200;
+        }
+        for (let lookup of remaining_lookups.values()) {
+            const op_id = data.op_seq.next();
+            this.debugLastOpId = op_id;
+            const indexId = lookup.indexId;
+            if (indexId == null) {
+                throw new ReplicationAssertionError('Expected v3 lookup when incrementalReprocessing is enabled');
+            }
+            const values = {
+                _id: op_id,
+                key: {
+                    t: mongoTableId(sourceTable.id),
+                    k: sourceKey
+                },
+                lookup: lookup.lookup,
+                bucket_parameters: []
+            };
+            this.bucketParameters.push({
+                ...values,
+                index: indexId
+            });
+            this.currentSize += 200;
+        }
+    }
+    hardDeleteCurrentData(sourceTableId, replicaId) {
+        this.currentData.push({
+            sourceTableId,
+            operation: {
+                deleteOne: {
+                    filter: { _id: replicaId }
+                }
+            }
+        });
+        this.currentSize += 50;
+    }
+    softDeleteCurrentData(sourceTableId, replicaId, checkpointGreaterThan) {
+        this.currentData.push({
+            sourceTableId,
+            operation: {
+                updateOne: {
+                    filter: { _id: replicaId },
+                    update: {
+                        $set: {
+                            data: null,
+                            buckets: [],
+                            lookups: [],
+                            pending_delete: checkpointGreaterThan
+                        }
+                    },
+                    upsert: true
+                }
+            }
+        });
+        const sourceTableKey = sourceTableId.toHexString();
+        const existingPendingDelete = this.sourceTablePendingDeletes.get(sourceTableKey);
+        if (existingPendingDelete == null || checkpointGreaterThan > existingPendingDelete) {
+            this.sourceTablePendingDeletes.set(sourceTableKey, checkpointGreaterThan);
+        }
+        this.currentSize += 50;
+    }
+    upsertCurrentData(values) {
+        const buckets = values.buckets.map((bucket) => {
+            if (bucket.definitionId == null) {
+                throw new ReplicationAssertionError('Expected v3 bucket when incrementalReprocessing is enabled');
+            }
+            return {
+                def: bucket.definitionId,
+                bucket: bucket.bucket,
+                table: bucket.table,
+                id: bucket.id
+            };
+        });
+        const lookups = values.lookups.map((lookup) => {
+            if (lookup.indexId == null) {
+                throw new ReplicationAssertionError('Expected v3 lookup when incrementalReprocessing is enabled');
+            }
+            return {
+                i: lookup.indexId,
+                l: lookup.lookup
+            };
+        });
+        this.currentData.push({
+            sourceTableId: values.sourceTableId,
+            operation: {
+                updateOne: {
+                    filter: { _id: values.replicaId },
+                    update: {
+                        $set: {
+                            data: values.data,
+                            buckets,
+                            lookups
+                        },
+                        $unset: { pending_delete: 1 }
+                    },
+                    upsert: true
+                }
+            }
+        });
+        this.currentSize += (values.data?.length() ?? 0) + 100;
+    }
+    get currentDataCount() {
+        return this.currentData.length;
+    }
+    async flushBucketData(session) {
+        const operationsByDefinition = new Map();
+        for (const document of this.bucketData) {
+            const existing = operationsByDefinition.get(document.bucketKey.definitionId) ?? [];
+            existing.push(document);
+            operationsByDefinition.set(document.bucketKey.definitionId, existing);
+        }
+        for (const [definitionId, documents] of operationsByDefinition.entries()) {
+            await this.db.bucketDataV3(this.group_id, definitionId).bulkWrite(documents.map((document) => ({
+                insertOne: {
+                    document: serializeBucketDataV3(document)
+                }
+            })), {
+                session,
+                ordered: false
+            });
+        }
+    }
+    async flushBucketParameters(session) {
+        const operationsByIndex = new Map();
+        for (const document of this.bucketParameters) {
+            const existing = operationsByIndex.get(document.index) ?? [];
+            existing.push(document);
+            operationsByIndex.set(document.index, existing);
+        }
+        for (const [indexId, documents] of operationsByIndex.entries()) {
+            await this.db.parameterIndexV3(this.group_id, indexId).bulkWrite(documents.map((document) => ({
+                insertOne: {
+                    document: taggedBucketParameterDocumentToV3(document)
+                }
+            })), {
+                session,
+                ordered: false
+            });
+        }
+    }
+    async flushCurrentData(session) {
+        const operationsBySourceTable = new Map();
+        for (const operation of this.currentData) {
+            const sourceTableId = operation.sourceTableId.toHexString();
+            const existing = operationsBySourceTable.get(sourceTableId) ?? [];
+            existing.push(operation);
+            operationsBySourceTable.set(sourceTableId, existing);
+        }
+        const sourceTableUpdates = [
+            ...this.sourceTablePendingDeletes.entries()
+        ].map(([key, value]) => {
+            return {
+                updateOne: {
+                    filter: { _id: new bson.ObjectId(key) },
+                    update: {
+                        $max: {
+                            latest_pending_delete: value
+                        }
+                    }
+                }
+            };
+        });
+        if (sourceTableUpdates.length > 0) {
+            await this.db.sourceTablesV3(this.group_id).bulkWrite(sourceTableUpdates, { session, ordered: false });
+        }
+        for (const operations of operationsBySourceTable.values()) {
+            const sourceTableId = operations[0].sourceTableId;
+            await this.db.sourceRecordsV3(this.group_id, sourceTableId).bulkWrite(operations.map((entry) => entry.operation), {
+                session,
+                ordered: true
+            });
+        }
+    }
+    async flushBucketStates(session) {
+        await this.db.bucketStateV3(this.group_id).bulkWrite(this.getBucketStateUpdates(), {
+            session,
+            ordered: false
+        });
+    }
+    resetCurrentData() {
+        this.currentData = [];
+        this.sourceTablePendingDeletes.clear();
+    }
+    getBucketStateUpdates() {
+        return Array.from(this.bucketStates.values()).map((state) => {
+            if (state.definitionId == null) {
+                throw new ReplicationAssertionError('Expected bucket definition id when incrementalReprocessing is enabled');
+            }
+            return {
+                updateOne: {
+                    filter: {
+                        _id: {
+                            d: state.definitionId,
+                            b: state.bucket
+                        }
+                    },
+                    update: {
+                        $set: {
+                            last_op: state.lastOp
+                        },
+                        $inc: {
+                            'estimate_since_compact.count': state.incrementCount,
+                            'estimate_since_compact.bytes': state.incrementBytes
+                        }
+                    },
+                    upsert: true
+                }
+            };
+        });
+    }
+}
+//# sourceMappingURL=PersistedBatchV3.js.map

package/dist/storage/implementation/v3/PersistedBatchV3.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"PersistedBatchV3.js","sourceRoot":"","sources":["../../../../src/storage/implementation/v3/PersistedBatchV3.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,yBAAyB,EAAE,MAAM,mCAAmC,CAAC;AAG9E,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAEtD,OAAO,EAEL,cAAc,EAGf,MAAM,6BAA6B,CAAC;AAErC,OAAO,EAIL,qBAAqB,EAErB,iCAAiC,EAClC,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,0BAA0B,EAAE,MAAM,6BAA6B,CAAC;AAGzE,MAAM,OAAO,gBAAiB,SAAQ,cAAc;IAGlD,WAAW,GAAsG,EAAE,CAAC;IACpH,yBAAyB,GAAG,IAAI,GAAG,EAAwB,CAAC;IAElD,iBAAiB,CAAC,YAAuC;QACjE,IAAI,YAAY,IAAI,IAAI,EAAE,CAAC;YACzB,mCAAmC;YACnC,MAAM,IAAI,yBAAyB,CAAC,4DAA4D,CAAC,CAAC;QACpG,CAAC;QACD,OAAO,YAAY,CAAC;IACtB,CAAC;IAES,qBAAqB,CAAC,YAA8B;QAC5D,OAAO,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,YAAY,CAAC,CAAC;IACnD,CAAC;IAED,iBAAiB,CAAC,IAA8B;QAC9C,MAAM,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC;QACnD,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAgE,CAAC;QAElG,KAAK,IAAI,MAAM,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACzC,IAAI,MAAM,CAAC,OAAO,IAAI,IAAI,EAAE,CAAC;gBAC3B,MAAM,IAAI,yBAAyB,CAAC,4DAA4D,CAAC,CAAC;YACpG,CAAC;YACD,iBAAiB,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;QACzF,CAAC;QAED,KAAK,IAAI,MAAM,IAAI,SAAS,EAAE,CAAC;YAC7B,MAAM,kBAAkB,GAAG,IAAI,CAAC,OAAO,CAAC,iBAAiB,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAChF,MAAM,SAAS,GAAG,0BAA0B,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAC5D,iBAAiB,CAAC,MAAM,CAAC,GAAG,kBAAkB,IAAI,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;YAElF,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACjC,IAAI,CAAC,aAAa,GAAG,KAAK,CAAC;YAC3B,MAAM,MAAM,GAA8B;gBACxC,GAAG,EAAE,KAAK;gBACV,GAAG,EAAE;oBACH,CAAC,EAAE,YAAY,CAAC,WAAW,CAAC,EAAE,CAAC;oBAC/B,CAAC,EAAE,SAAS;iBACY;gBAC1B,MAAM,EAAE,SAAS;gBACjB,iBAAiB,EAAE,MAAM,CAAC,gBAAgB;aAC3C,CAAC;YACF,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC;gBACzB,GAAG,MAAM;gBACT,KAAK,EAAE,kBAAkB;aAC1B,CAAC,CAAC;YAEH,IAAI,CAAC,WAAW,IAAI,GAAG,CAAC;QAC1B,CAAC;QAED,KAAK,IAAI,MAAM,IAAI,iBAAiB,CAAC,MAAM,EAAE,EAAE,CAAC;YAC9C,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACjC,IAAI,CAAC,aAAa,GAAG,KAAK,CAAC;YAC3B,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;YAC/B,IAAI,OAAO,IAAI,IAAI,EAAE,CAAC;gBACpB,MAAM,IAAI,yBAAyB,CAAC,4DAA4D,CAAC,CAAC;YACpG,CAAC;YACD,MAAM,MAAM,GAA8B;gBACxC,GAAG,EAAE,KAAK;gBACV,GAAG,EAAE;oBACH,CAAC,EAAE,YAAY,CAAC,WAAW,CAAC,EAAE,CAAC;oBAC/B,CAAC,EAAE,SAAS;iBACY;gBAC1B,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,iBAAiB,EAAE,EAAE;aACtB,CAAC;YACF,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC;gBACzB,GAAG,MAAM;gBACT,KAAK,EAAE,OAAO;aACf,CAAC,CAAC;YAEH,IAAI,CAAC,WAAW,IAAI,GAAG,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,qBAAqB,CAAC,aAA4B,EAAE,SAA4B;QAC9E,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC;YACpB,aAAa;YACb,SAAS,EAAE;gBACT,SAAS,EAAE;oBACT,MAAM,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE;iBAC3B;aACF;SACF,CAAC,CAAC;QACH,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC;IACzB,CAAC;IAED,qBAAqB,CACnB,aAA4B,EAC5B,SAA4B,EAC5B,qBAAmC;QAEnC,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC;YACpB,aAAa;YACb,SAAS,EAAE;gBACT,SAAS,EAAE;oBACT,MAAM,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE;oBAC1B,MAAM,EAAE;wBACN,IAAI,EAAE;4BACJ,IAAI,EAAE,IAAI;4BACV,OAAO,EAAE,EAAsC;4BAC/C,OAAO,EAAE,EAAsC;4BAC/C,cAAc,EAAE,qBAAqB;yBACtC;qBACF;oBACD,MAAM,EAAE,IAAI;iBACb;aACF;SACF,CAAC,CAAC;QACH,MAAM,cAAc,GAAG,aAAa,CAAC,WAAW,EAAE,CAAC;QACnD,MAAM,qBAAqB,GAAG,IAAI,CAAC,yBAAyB,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;QACjF,IAAI,qBAAqB,IAAI,IAAI,IAAI,qBAAqB,GAAG,qBAAqB,EAAE,CAAC;YACnF,IAAI,CAAC,yBAAyB,CAAC,GAAG,CAAC,cAAc,EAAE,qBAAqB,CAAC,CAAC;QAC5E,CAAC;QAED,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC;IACzB,CAAC;IAED,iBAAiB,CAAC,MAAgC;QAChD,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;YAC5C,IAAI,MAAM,CAAC,YAAY,IAAI,IAAI,EAAE,CAAC;gBAChC,MAAM,IAAI,yBAAyB,CAAC,4DAA4D,CAAC,CAAC;YACpG,CAAC;YACD,OAAO;gBACL,GAAG,EAAE,MAAM,CAAC,YAAY;gBACxB,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,EAAE,EAAE,MAAM,CAAC,EAAE;aACd,CAAC;QACJ,CAAC,CAAC,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;YAC5C,IAAI,MAAM,CAAC,OAAO,IAAI,IAAI,EAAE,CAAC;gBAC3B,MAAM,IAAI,yBAAyB,CAAC,4DAA4D,CAAC,CAAC;YACpG,CAAC;YACD,OAAO;gBACL,CAAC,EAAE,MAAM,CAAC,OAAO;gBACjB,CAAC,EAAE,MAAM,CAAC,MAAM;aACjB,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC;YACpB,aAAa,EAAE,MAAM,CAAC,aAAa;YACnC,SAAS,EAAE;gBACT,SAAS,EAAE;oBACT,MAAM,EAAE,EAAE,GAAG,EAAE,MAAM,CAAC,SAAS,EAAE;oBACjC,MAAM,EAAE;wBACN,IAAI,EAAE;4BACJ,IAAI,EAAE,MAAM,CAAC,IAAI;4BACjB,OAAO;4BACP,OAAO;yBACR;wBACD,MAAM,EAAE,EAAE,cAAc,EAAE,CAAC,EAAE;qBAC9B;oBACD,MAAM,EAAE,IAAI;iBACb;aACF;SACF,CAAC,CAAC;QACH,IAAI,CAAC,WAAW,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC;IACzD,CAAC;IAED,IAAc,gBAAgB;QAC5B,OAAO,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC;IACjC,CAAC;IAES,KAAK,CAAC,eAAe,CAAC,OAA4B;QAC1D,MAAM,sBAAsB,GAAG,IAAI,GAAG,EAA8C,CAAC;QACrF,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACvC,MAAM,QAAQ,GAAG,sBAAsB,CAAC,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC;YACnF,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACxB,sBAAsB,CAAC,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAC;QACxE,CAAC;QAED,KAAK,MAAM,CAAC,YAAY,EAAE,SAAS,CAAC,IAAI,sBAAsB,CAAC,OAAO,EAAE,EAAE,CAAC;YACzE,MAAM,IAAI,CAAC,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC,SAAS,CAC/D,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;gBAC3B,SAAS,EAAE;oBACT,QAAQ,EAAE,qBAAqB,CAAC,QAAQ,CAAC;iBAC1C;aACF,CAAC,CAAC,EACH;gBACE,OAAO;gBACP,OAAO,EAAE,KAAK;aACf,CACF,CAAC;QACJ,CAAC;IACH,CAAC;IAES,KAAK,CAAC,qBAAqB,CAAC,OAA4B;QAChE,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAwC,CAAC;QAC1E,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC7C,MAAM,QAAQ,GAAG,iBAAiB,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;YAC7D,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACxB,iBAAiB,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;QAClD,CAAC;QAED,KAAK,MAAM,CAAC,OAAO,EAAE,SAAS,CAAC,IAAI,iBAAiB,CAAC,OAAO,EAAE,EAAE,CAAC;YAC/D,MAAM,IAAI,CAAC,EAAE,CAAC,gBAAgB,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,SAAS,CAC9D,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;gBAC3B,SAAS,EAAE;oBACT,QAAQ,EAAE,iCAAiC,CAAC,QAAQ,CAAC;iBACtD;aACF,CAAC,CAAC,EACH;gBACE,OAAO;gBACP,OAAO,EAAE,KAAK;aACf,CACF,CAAC;QACJ,CAAC;IACH,CAAC;IAES,KAAK,CAAC,gBAAgB,CAAC,OAA4B;QAC3D,MAAM,uBAAuB,GAAG,IAAI,GAAG,EAAmC,CAAC;QAC3E,KAAK,MAAM,SAAS,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACzC,MAAM,aAAa,GAAG,SAAS,CAAC,aAAa,CAAC,WAAW,EAAE,CAAC;YAC5D,MAAM,QAAQ,GAAG,uBAAuB,CAAC,GAAG,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC;YAClE,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACzB,uBAAuB,CAAC,GAAG,CAAC,aAAa,EAAE,QAAQ,CAAC,CAAC;QACvD,CAAC;QAED,MAAM,kBAAkB,GAAyD;YAC/E,GAAG,IAAI,CAAC,yBAAyB,CAAC,OAAO,EAAE;SAC5C,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE;YACrB,OAAO;gBACL,SAAS,EAAE;oBACT,MAAM,EAAE,EAAE,GAAG,EAAE,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE;oBACvC,MAAM,EAAE;wBACN,IAAI,EAAE;4BACJ,qBAAqB,EAAE,KAAK;yBAC7B;qBACF;iBACF;aACF,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,IAAI,kBAAkB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAClC,MAAM,IAAI,CAAC,EAAE,CAAC,cAAc,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,SAAS,CAAC,kBAAkB,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;QACzG,CAAC;QAED,KAAK,MAAM,UAAU,IAAI,uBAAuB,CAAC,MAAM,EAAE,EAAE,CAAC;YAC1D,MAAM,aAAa,GAAG,UAAU,CAAC,CAAC,CAAE,CAAC,aAAa,CAAC;YACnD,MAAM,IAAI,CAAC,EAAE,CAAC,eAAe,CAAC,IAAI,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC,SAAS,CACnE,UAAU,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,EAC1C;gBACE,OAAO;gBACP,OAAO,EAAE,IAAI;aACd,CACF,CAAC;QACJ,CAAC;IACH,CAAC;IAES,KAAK,CAAC,iBAAiB,CAAC,OAA4B;QAC5D,MAAM,IAAI,CAAC,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,qBAAqB,EAAE,EAAE;YACjF,OAAO;YACP,OAAO,EAAE,KAAK;SACf,CAAC,CAAC;IACL,CAAC;IAES,gBAAgB;QACxB,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC;QACtB,IAAI,CAAC,yBAAyB,CAAC,KAAK,EAAE,CAAC;IACzC,CAAC;IAEO,qBAAqB;QAC3B,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAwB,EAAE,EAAE;YAC7E,IAAI,KAAK,CAAC,YAAY,IAAI,IAAI,EAAE,CAAC;gBAC/B,MAAM,IAAI,yBAAyB,CAAC,uEAAuE,CAAC,CAAC;YAC/G,CAAC;YACD,OAAO;gBACL,SAAS,EAAE;oBACT,MAAM,EAAE;wBACN,GAAG,EAAE;4BACH,CAAC,EAAE,KAAK,CAAC,YAAY;4BACrB,CAAC,EAAE,KAAK,CAAC,MAAM;yBAChB;qBACF;oBACD,MAAM,EAAE;wBACN,IAAI,EAAE;4BACJ,OAAO,EAAE,KAAK,CAAC,MAAM;yBACtB;wBACD,IAAI,EAAE;4BACJ,8BAA8B,EAAE,KAAK,CAAC,cAAc;4BACpD,8BAA8B,EAAE,KAAK,CAAC,cAAc;yBACrD;qBACF;oBACD,MAAM,EAAE,IAAI;iBACb;aAC2D,CAAC;QACjE,CAAC,CAAC,CAAC;IACL,CAAC;CACF"}

package/dist/storage/implementation/v3/SingleBucketStoreV3.d.ts

@@ -0,0 +1,18 @@
+import { mongo } from '@powersync/lib-service-mongodb';
+import { InternalOpId } from '@powersync/service-core';
+import { BucketDataDoc, BucketKey } from '../common/BucketDataDoc.js';
+import { BucketDataDocumentGeneric, BucketDataDocumentGenericId, SingleBucketStore } from '../common/SingleBucketStore.js';
+import { BucketDataProperties } from '../models.js';
+import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js';
+export declare class SingleBucketStoreV3 implements SingleBucketStore {
+    private db;
+    readonly key: BucketKey;
+    readonly collection: mongo.Collection<BucketDataDocumentGeneric>;
+    constructor(db: VersionedPowerSyncMongoV3, key: BucketKey);
+    docId(o: InternalOpId): BucketDataDocumentGenericId;
+    get minId(): BucketDataDocumentGenericId;
+    get maxId(): BucketDataDocumentGenericId;
+    toPersistedDocument(source: Omit<BucketDataDoc, 'bucketKey'>): BucketDataDocumentGeneric;
+    fromPersistedDocument(doc: BucketDataDocumentGeneric): BucketDataDoc;
+    fromPartialPersistedDocument<T extends keyof BucketDataProperties>(doc: Pick<BucketDataDocumentGeneric, '_id' | T>): Pick<BucketDataDoc, 'bucketKey' | 'o' | T>;
+}

package/dist/storage/implementation/v3/SingleBucketStoreV3.js

@@ -0,0 +1,48 @@
+import { mongo } from '@powersync/lib-service-mongodb';
+import { loadBucketDataDocumentV3, serializeBucketDataV3 } from './models.js';
+export class SingleBucketStoreV3 {
+    db;
+    key;
+    collection;
+    constructor(db, key) {
+        this.db = db;
+        this.key = key;
+        this.collection = db.bucketDataV3(key.replicationStreamId, key.definitionId);
+    }
+    docId(o) {
+        // `satisfies BucketDataKeyV3` checks that we use the correct type for V3 storage
+        // `as BucketDataDocumentGenericId` does a cast to get the interface virtual type
+        return {
+            b: this.key.bucket,
+            o
+        };
+    }
+    get minId() {
+        return {
+            b: this.key.bucket,
+            o: new mongo.MinKey()
+        }; // No way to properly type this
+    }
+    get maxId() {
+        return {
+            b: this.key.bucket,
+            o: new mongo.MaxKey()
+        }; // No way to properly type this
+    }
+    toPersistedDocument(source) {
+        return serializeBucketDataV3({ bucketKey: this.key, ...source });
+    }
+    fromPersistedDocument(doc) {
+        return loadBucketDataDocumentV3(this.key, doc);
+    }
+    fromPartialPersistedDocument(doc) {
+        const document = doc;
+        const { _id, ...rest } = document;
+        return {
+            bucketKey: this.key,
+            o: _id.o,
+            ...rest
+        };
+    }
+}
+//# sourceMappingURL=SingleBucketStoreV3.js.map

package/dist/storage/implementation/v3/SingleBucketStoreV3.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"SingleBucketStoreV3.js","sourceRoot":"","sources":["../../../../src/storage/implementation/v3/SingleBucketStoreV3.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,gCAAgC,CAAC;AAUvD,OAAO,EAAyC,wBAAwB,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AAErH,MAAM,OAAO,mBAAmB;IAIpB;IACQ;IAJF,UAAU,CAA8C;IAExE,YACU,EAA6B,EACrB,GAAc;QADtB,OAAE,GAAF,EAAE,CAA2B;QACrB,QAAG,GAAH,GAAG,CAAW;QAE9B,IAAI,CAAC,UAAU,GAAG,EAAE,CAAC,YAAY,CAC/B,GAAG,CAAC,mBAAmB,EACvB,GAAG,CAAC,YAAY,CACyC,CAAC;IAC9D,CAAC;IAED,KAAK,CAAC,CAAe;QACnB,iFAAiF;QACjF,iFAAiF;QACjF,OAAO;YACL,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,MAAM;YAClB,CAAC;SACuD,CAAC;IAC7D,CAAC;IAED,IAAI,KAAK;QACP,OAAO;YACL,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,MAAM;YAClB,CAAC,EAAE,IAAI,KAAK,CAAC,MAAM,EAAE;SACf,CAAC,CAAC,+BAA+B;IAC3C,CAAC;IAED,IAAI,KAAK;QACP,OAAO;YACL,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,MAAM;YAClB,CAAC,EAAE,IAAI,KAAK,CAAC,MAAM,EAAE;SACf,CAAC,CAAC,+BAA+B;IAC3C,CAAC;IAED,mBAAmB,CAAC,MAAwC;QAC1D,OAAO,qBAAqB,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,EAAE,CAA8B,CAAC;IAChG,CAAC;IAED,qBAAqB,CAAC,GAA8B;QAClD,OAAO,wBAAwB,CAAC,IAAI,CAAC,GAAG,EAAE,GAA2B,CAAC,CAAC;IACzE,CAAC;IAED,4BAA4B,CAC1B,GAA+C;QAE/C,MAAM,QAAQ,GAAG,GAA4C,CAAC;QAC9D,MAAM,EAAE,GAAG,EAAE,GAAG,IAAI,EAAE,GAAG,QAAQ,CAAC;QAClC,OAAO;YACL,SAAS,EAAE,IAAI,CAAC,GAAG;YACnB,CAAC,EAAE,GAAG,CAAC,CAAC;YACR,GAAG,IAAI;SACsC,CAAC;IAClD,CAAC;CACF"}

package/dist/storage/implementation/v3/SourceRecordStoreV3.d.ts

@@ -0,0 +1,22 @@
+import { mongo } from '@powersync/lib-service-mongodb';
+import { Logger } from '@powersync/lib-services-framework';
+import { EvaluatedParameters, EvaluatedRow } from '@powersync/service-sync-rules';
+import * as bson from 'bson';
+import { BucketDefinitionMapping } from '../BucketDefinitionMapping.js';
+import { LoadedSourceRecord, SourceRecordLookupEntry, SourceRecordStore } from '../common/SourceRecordStore.js';
+import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js';
+export declare class SourceRecordStoreV3 implements SourceRecordStore {
+    private readonly db;
+    private readonly groupId;
+    private readonly mapping;
+    constructor(db: VersionedPowerSyncMongoV3, groupId: number, mapping: BucketDefinitionMapping);
+    mapEvaluatedBuckets(evaluated: EvaluatedRow[]): LoadedSourceRecord['buckets'];
+    mapParameterLookups(paramEvaluated: EvaluatedParameters[]): LoadedSourceRecord['lookups'];
+    private createLoadedDocument;
+    loadSizes(session: mongo.ClientSession, entries: SourceRecordLookupEntry[]): Promise<Map<string, number>>;
+    loadDocuments(session: mongo.ClientSession, entries: SourceRecordLookupEntry[], idsOnly: boolean): Promise<Map<string, LoadedSourceRecord>>;
+    loadTruncateBatch(session: mongo.ClientSession, sourceTableId: bson.ObjectId, limit: number): Promise<LoadedSourceRecord[]>;
+    postCommitCleanup(lastCheckpoint: bigint, logger: Logger): Promise<void>;
+    private deletePendingDeletes;
+    private groupEntries;
+}

package/dist/storage/implementation/v3/SourceRecordStoreV3.js

@@ -0,0 +1,164 @@
+import * as lib_mongo from '@powersync/lib-service-mongodb';
+import { retryOnMongoMaxTimeMSExpired } from '../../../utils/util.js';
+import { cacheKey } from '../OperationBatch.js';
+import { serializeParameterLookupV3 } from './MongoParameterLookupV3.js';
+export class SourceRecordStoreV3 {
+    db;
+    groupId;
+    mapping;
+    constructor(db, groupId, mapping) {
+        this.db = db;
+        this.groupId = groupId;
+        this.mapping = mapping;
+    }
+    mapEvaluatedBuckets(evaluated) {
+        return evaluated.map((entry) => ({
+            definitionId: this.mapping.bucketSourceId(entry.source),
+            bucket: entry.bucket,
+            table: entry.table,
+            id: entry.id
+        }));
+    }
+    mapParameterLookups(paramEvaluated) {
+        return paramEvaluated.map((entry) => ({
+            indexId: this.mapping.parameterLookupId(entry.lookup.source),
+            lookup: serializeParameterLookupV3(entry.lookup)
+        }));
+    }
+    createLoadedDocument(sourceTableId, id, data, buckets, lookups) {
+        return {
+            sourceTableId,
+            replicaId: id,
+            data,
+            buckets: buckets.map((bucket) => ({
+                definitionId: bucket.def,
+                bucket: bucket.bucket,
+                table: bucket.table,
+                id: bucket.id
+            })),
+            lookups: lookups.map((lookup) => ({
+                indexId: lookup.i,
+                lookup: lookup.l
+            })),
+            cacheKey: cacheKey(sourceTableId, id)
+        };
+    }
+    async loadSizes(session, entries) {
+        const sizes = new Map();
+        for (const [sourceTableId, replicaIds] of this.groupEntries(entries)) {
+            const filter = {
+                _id: { $in: replicaIds }
+            };
+            const sizeCursor = this.db
+                .sourceRecordsV3(this.groupId, sourceTableId)
+                .aggregate([
+                {
+                    $match: filter
+                },
+                {
+                    $project: {
+                        _id: 1,
+                        size: { $bsonSize: '$$ROOT' }
+                    }
+                }
+            ], { session });
+            for await (const doc of sizeCursor.stream()) {
+                sizes.set(cacheKey(sourceTableId, doc._id), doc.size);
+            }
+        }
+        return sizes;
+    }
+    async loadDocuments(session, entries, idsOnly) {
+        const documents = new Map();
+        const projection = idsOnly ? { _id: 1 } : undefined;
+        for (const [sourceTableId, replicaIds] of this.groupEntries(entries)) {
+            const filter = {
+                _id: { $in: replicaIds }
+            };
+            const cursor = this.db.sourceRecordsV3(this.groupId, sourceTableId).find(filter, { session, projection });
+            for await (const doc of cursor.stream()) {
+                const loaded = this.createLoadedDocument(sourceTableId, doc._id, idsOnly ? null : doc.data, idsOnly ? [] : doc.buckets, idsOnly ? [] : doc.lookups);
+                documents.set(loaded.cacheKey, loaded);
+            }
+        }
+        return documents;
+    }
+    async loadTruncateBatch(session, sourceTableId, limit) {
+        const cursor = this.db.sourceRecordsV3(this.groupId, sourceTableId).find({
+            pending_delete: { $exists: false }
+        }, {
+            projection: {
+                _id: 1,
+                buckets: 1,
+                lookups: 1
+            },
+            limit,
+            session
+        });
+        return (await cursor.toArray()).map((doc) => this.createLoadedDocument(sourceTableId, doc._id, null, doc.buckets, doc.lookups));
+    }
+    async postCommitCleanup(lastCheckpoint, logger) {
+        // This cleans up soft deletes in source_records collections.
+        // Since there may be a lot (100+) of these collections in some cases, we track which
+        // ones have dirty deletes in source_tables.
+        const dirtySourceTables = await this.db
+            .sourceTablesV3(this.groupId)
+            .find({
+            latest_pending_delete: { $exists: true }
+        }, {
+            projection: { _id: 1, latest_pending_delete: 1 }
+        })
+            .toArray();
+        let deletedCount = 0;
+        const sourceTableUpdates = [];
+        for (const sourceTable of dirtySourceTables) {
+            const collection = this.db.sourceRecordsV3(this.groupId, sourceTable._id);
+            const result = await this.deletePendingDeletes(collection, sourceTable._id, lastCheckpoint, logger);
+            deletedCount += result.deletedCount;
+            if (sourceTable.latest_pending_delete != null && sourceTable.latest_pending_delete <= lastCheckpoint) {
+                sourceTableUpdates.push({
+                    updateOne: {
+                        filter: {
+                            _id: sourceTable._id,
+                            // If the source table received more writes in the meantime, this will filter it out
+                            latest_pending_delete: sourceTable.latest_pending_delete
+                        },
+                        update: {
+                            $unset: {
+                                latest_pending_delete: 1
+                            }
+                        }
+                    }
+                });
+            }
+        }
+        if (sourceTableUpdates.length > 0) {
+            await this.db.sourceTablesV3(this.groupId).bulkWrite(sourceTableUpdates, { ordered: false });
+        }
+        if (deletedCount > 0) {
+            logger.info(`Cleaned up ${deletedCount} pending delete current_data records for checkpoint ${lastCheckpoint}`);
+        }
+    }
+    async deletePendingDeletes(collection, sourceTableId, lastCheckpoint, logger) {
+        return retryOnMongoMaxTimeMSExpired(() => collection.deleteMany({
+            pending_delete: { $exists: true, $lte: lastCheckpoint }
+        }, {
+            maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS
+        }), {
+            retryDelayMs: lib_mongo.db.MONGO_OPERATION_TIMEOUT_MS / 5,
+            onRetry: (n) => {
+                logger.warn(`Cleared batch ${n} of pending deletes for source table ${sourceTableId}, continuing...`);
+            }
+        });
+    }
+    groupEntries(entries) {
+        const grouped = new Map();
+        for (const entry of entries) {
+            const existing = grouped.get(entry.sourceTableId) ?? [];
+            existing.push(entry.replicaId);
+            grouped.set(entry.sourceTableId, existing);
+        }
+        return grouped;
+    }
+}
+//# sourceMappingURL=SourceRecordStoreV3.js.map

package/dist/storage/implementation/v3/SourceRecordStoreV3.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"SourceRecordStoreV3.js","sourceRoot":"","sources":["../../../../src/storage/implementation/v3/SourceRecordStoreV3.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,gCAAgC,CAAC;AAM5D,OAAO,EAAE,4BAA4B,EAAE,MAAM,wBAAwB,CAAC;AAEtE,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAEhD,OAAO,EAAE,0BAA0B,EAAE,MAAM,6BAA6B,CAAC;AAIzE,MAAM,OAAO,mBAAmB;IAEX;IACA;IACA;IAHnB,YACmB,EAA6B,EAC7B,OAAe,EACf,OAAgC;QAFhC,OAAE,GAAF,EAAE,CAA2B;QAC7B,YAAO,GAAP,OAAO,CAAQ;QACf,YAAO,GAAP,OAAO,CAAyB;IAChD,CAAC;IAEJ,mBAAmB,CAAC,SAAyB;QAC3C,OAAO,SAAS,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;YAC/B,YAAY,EAAE,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,KAAK,CAAC,MAAM,CAAC;YACvD,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,EAAE,EAAE,KAAK,CAAC,EAAE;SACb,CAAC,CAAC,CAAC;IACN,CAAC;IAED,mBAAmB,CAAC,cAAqC;QACvD,OAAO,cAAc,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;YACpC,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,iBAAiB,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC;YAC5D,MAAM,EAAE,0BAA0B,CAAC,KAAK,CAAC,MAAM,CAAC;SACjD,CAAC,CAAC,CAAC;IACN,CAAC;IAEO,oBAAoB,CAC1B,aAA4B,EAC5B,EAAqB,EACrB,IAAwB,EACxB,OAAyC,EACzC,OAAyC;QAEzC,OAAO;YACL,aAAa;YACb,SAAS,EAAE,EAAE;YACb,IAAI;YACJ,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;gBAChC,YAAY,EAAE,MAAM,CAAC,GAAG;gBACxB,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,EAAE,EAAE,MAAM,CAAC,EAAE;aACd,CAAC,CAAC;YACH,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;gBAChC,OAAO,EAAE,MAAM,CAAC,CAAC;gBACjB,MAAM,EAAE,MAAM,CAAC,CAAC;aACjB,CAAC,CAAC;YACH,QAAQ,EAAE,QAAQ,CAAC,aAAa,EAAE,EAAE,CAAC;SACtC,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,OAA4B,EAAE,OAAkC;QAC9E,MAAM,KAAK,GAAG,IAAI,GAAG,EAAkB,CAAC;QACxC,KAAK,MAAM,CAAC,aAAa,EAAE,UAAU,CAAC,IAAI,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,EAAE,CAAC;YACrE,MAAM,MAAM,GAAG;gBACb,GAAG,EAAE,EAAE,GAAG,EAAE,UAAmB,EAAE;aACgB,CAAC;YACpD,MAAM,UAAU,GAAsE,IAAI,CAAC,EAAE;iBAC1F,eAAe,CAAC,IAAI,CAAC,OAAO,EAAE,aAAa,CAAC;iBAC5C,SAAS,CACR;gBACE;oBACE,MAAM,EAAE,MAAM;iBACf;gBACD;oBACE,QAAQ,EAAE;wBACR,GAAG,EAAE,CAAC;wBACN,IAAI,EAAE,EAAE,SAAS,EAAE,QAAQ,EAAE;qBAC9B;iBACF;aACF,EACD,EAAE,OAAO,EAAE,CACZ,CAAC;YACJ,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,UAAU,CAAC,MAAM,EAAE,EAAE,CAAC;gBAC5C,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,aAAa,EAAE,GAAG,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC;YACxD,CAAC;QACH,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAED,KAAK,CAAC,aAAa,CACjB,OAA4B,EAC5B,OAAkC,EAClC,OAAgB;QAEhB,MAAM,SAAS,GAAG,IAAI,GAAG,EAA8B,CAAC;QACxD,MAAM,UAAU,GAAG,OAAO,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;QACpD,KAAK,MAAM,CAAC,aAAa,EAAE,UAAU,CAAC,IAAI,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,EAAE,CAAC;YACrE,MAAM,MAAM,GAAG;gBACb,GAAG,EAAE,EAAE,GAAG,EAAE,UAAmB,EAAE;aACgB,CAAC;YACpD,MAAM,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC,eAAe,CAAC,IAAI,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC,CAAC;YAC1G,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC;gBACxC,MAAM,MAAM,GAAG,IAAI,CAAC,oBAAoB,CACtC,aAAa,EACb,GAAG,CAAC,GAAG,EACP,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,EACzB,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,EAC1B,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAC3B,CAAC;gBACF,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;YACzC,CAAC;QACH,CAAC;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,KAAK,CAAC,iBAAiB,CACrB,OAA4B,EAC5B,aAA4B,EAC5B,KAAa;QAEb,MAAM,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC,eAAe,CAAC,IAAI,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC,IAAI,CACtE;YACE,cAAc,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE;SACnC,EACD;YACE,UAAU,EAAE;gBACV,GAAG,EAAE,CAAC;gBACN,OAAO,EAAE,CAAC;gBACV,OAAO,EAAE,CAAC;aACX;YACD,KAAK;YACL,OAAO;SACR,CACF,CAAC;QACF,OAAO,CAAC,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAC1C,IAAI,CAAC,oBAAoB,CAAC,aAAa,EAAE,GAAG,CAAC,GAAG,EAAE,IAAI,EAAE,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,OAAO,CAAC,CAClF,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,iBAAiB,CAAC,cAAsB,EAAE,MAAc;QAC5D,6DAA6D;QAC7D,qFAAqF;QACrF,4CAA4C;QAE5C,MAAM,iBAAiB,GAAG,MAAM,IAAI,CAAC,EAAE;aACpC,cAAc,CAAC,IAAI,CAAC,OAAO,CAAC;aAC5B,IAAI,CACH;YACE,qBAAqB,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE;SACzC,EACD;YACE,UAAU,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,qBAAqB,EAAE,CAAC,EAAE;SACjD,CACF;aACA,OAAO,EAAE,CAAC;QAEb,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,MAAM,kBAAkB,GAAyD,EAAE,CAAC;QACpF,KAAK,MAAM,WAAW,IAAI,iBAAiB,EAAE,CAAC;YAC5C,MAAM,UAAU,GAAG,IAAI,CAAC,EAAE,CAAC,eAAe,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,GAAG,CAAC,CAAC;YAC1E,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,oBAAoB,CAAC,UAAU,EAAE,WAAW,CAAC,GAAG,EAAE,cAAc,EAAE,MAAM,CAAC,CAAC;YACpG,YAAY,IAAI,MAAM,CAAC,YAAY,CAAC;YAEpC,IAAI,WAAW,CAAC,qBAAqB,IAAI,IAAI,IAAI,WAAW,CAAC,qBAAqB,IAAI,cAAc,EAAE,CAAC;gBACrG,kBAAkB,CAAC,IAAI,CAAC;oBACtB,SAAS,EAAE;wBACT,MAAM,EAAE;4BACN,GAAG,EAAE,WAAW,CAAC,GAAG;4BACpB,oFAAoF;4BACpF,qBAAqB,EAAE,WAAW,CAAC,qBAAqB;yBACzD;wBACD,MAAM,EAAE;4BACN,MAAM,EAAE;gCACN,qBAAqB,EAAE,CAAC;6BACzB;yBACF;qBACF;iBACF,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,IAAI,kBAAkB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAClC,MAAM,IAAI,CAAC,EAAE,CAAC,cAAc,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,kBAAkB,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;QAC/F,CAAC;QACD,IAAI,YAAY,GAAG,CAAC,EAAE,CAAC;YACrB,MAAM,CAAC,IAAI,CAAC,cAAc,YAAY,uDAAuD,cAAc,EAAE,CAAC,CAAC;QACjH,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,oBAAoB,CAChC,UAAmD,EACnD,aAA4B,EAC5B,cAAsB,EACtB,MAAc;QAEd,OAAO,4BAA4B,CACjC,GAAG,EAAE,CACH,UAAU,CAAC,UAAU,CACnB;YACE,cAAc,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,cAAc,EAAE;SACxD,EACD;YACE,SAAS,EAAE,SAAS,CAAC,EAAE,CAAC,gCAAgC;SACzD,CACF,EACH;YACE,YAAY,EAAE,SAAS,CAAC,EAAE,CAAC,0BAA0B,GAAG,CAAC;YACzD,OAAO,EAAE,CAAC,CAAS,EAAE,EAAE;gBACrB,MAAM,CAAC,IAAI,CAAC,iBAAiB,CAAC,wCAAwC,aAAa,iBAAiB,CAAC,CAAC;YACxG,CAAC;SACF,CACF,CAAC;IACJ,CAAC;IAEO,YAAY,CAAC,OAAkC;QACrD,MAAM,OAAO,GAAG,IAAI,GAAG,EAAsC,CAAC;QAC9D,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC;YACxD,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;YAC/B,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,aAAa,EAAE,QAAQ,CAAC,CAAC;QAC7C,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;CACF"}

package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.d.ts

@@ -0,0 +1,21 @@
+import { mongo } from '@powersync/lib-service-mongodb';
+import { BucketDefinitionId, ParameterIndexId } from '../BucketDefinitionMapping.js';
+import { BaseVersionedPowerSyncMongo } from '../common/VersionedPowerSyncMongoBase.js';
+import { CommonSourceTableDocument } from '../models.js';
+import { BucketDataDocumentV3, BucketParameterDocumentV3, BucketStateDocumentV3, CurrentDataDocumentV3, SourceTableDocumentV3 } from './models.js';
+export declare class VersionedPowerSyncMongoV3 extends BaseVersionedPowerSyncMongo {
+    sourceRecordsV3(replicationStreamId: number, sourceTableId: mongo.ObjectId): mongo.Collection<CurrentDataDocumentV3>;
+    listSourceRecordCollectionsV3(replicationStreamId: number): Promise<mongo.Collection<CurrentDataDocumentV3>[]>;
+    initializeSourceRecordsCollection(replicationStreamId: number, sourceTableId: mongo.ObjectId): Promise<void>;
+    commonSourceTables(replicationStreamId: number): mongo.Collection<CommonSourceTableDocument>;
+    bucketStateV3(replicationStreamId: number): mongo.Collection<BucketStateDocumentV3>;
+    parameterIndexV3(replicationStreamId: number, indexId: ParameterIndexId): mongo.Collection<BucketParameterDocumentV3>;
+    sourceTablesV3(replicationStreamId: number): mongo.Collection<SourceTableDocumentV3>;
+    initializeStreamStorage(replicationStreamId: number): Promise<void>;
+    bucketDataV3(replicationStreamId: number, definitionId: BucketDefinitionId): mongo.Collection<BucketDataDocumentV3>;
+    listBucketDataCollectionsV3(replicationStreamId: number): Promise<mongo.Collection<BucketDataDocumentV3>[]>;
+    listParameterIndexCollectionsV3(replicationStreamId: number): Promise<{
+        collection: mongo.Collection<BucketParameterDocumentV3>;
+        indexId: ParameterIndexId;
+    }[]>;
+}

package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js

@@ -0,0 +1,71 @@
+import { BaseVersionedPowerSyncMongo } from '../common/VersionedPowerSyncMongoBase.js';
+export class VersionedPowerSyncMongoV3 extends BaseVersionedPowerSyncMongo {
+    sourceRecordsV3(replicationStreamId, sourceTableId) {
+        const collectionName = this.sourceRecordsCollectionName(replicationStreamId, sourceTableId);
+        return this.db.collection(collectionName);
+    }
+    async listSourceRecordCollectionsV3(replicationStreamId) {
+        return this.listCollectionsByPrefix(`source_records_${replicationStreamId}_`);
+    }
+    async initializeSourceRecordsCollection(replicationStreamId, sourceTableId) {
+        await this.sourceRecordsV3(replicationStreamId, sourceTableId).createIndex({
+            pending_delete: 1
+        }, {
+            partialFilterExpression: { pending_delete: { $exists: true } },
+            name: 'pending_delete'
+        });
+    }
+    commonSourceTables(replicationStreamId) {
+        return this.sourceTablesV3(replicationStreamId);
+    }
+    bucketStateV3(replicationStreamId) {
+        return this.db.collection(`bucket_state_${replicationStreamId}`);
+    }
+    parameterIndexV3(replicationStreamId, indexId) {
+        return this.db.collection(`parameter_index_${replicationStreamId}_${indexId}`);
+    }
+    sourceTablesV3(replicationStreamId) {
+        return this.db.collection(this.sourceTableCollectionName(replicationStreamId));
+    }
+    async initializeStreamStorage(replicationStreamId) {
+        const sourceTables = this.sourceTablesV3(replicationStreamId);
+        const bucketState = this.bucketStateV3(replicationStreamId);
+        await sourceTables.createIndex({
+            connection_id: 1,
+            schema_name: 1,
+            table_name: 1,
+            relation_id: 1
+        }, {
+            name: 'source_lookup'
+        });
+        await sourceTables.createIndex({
+            latest_pending_delete: 1
+        }, {
+            partialFilterExpression: { latest_pending_delete: { $exists: true } },
+            name: 'latest_pending_delete'
+        });
+        await bucketState.createIndex({
+            last_op: 1
+        }, { name: 'bucket_updates', unique: true });
+        await bucketState.createIndex({
+            'estimate_since_compact.count': -1
+        }, { name: 'dirty_count' });
+    }
+    bucketDataV3(replicationStreamId, definitionId) {
+        return this.db.collection(`bucket_data_${replicationStreamId}_${definitionId}`);
+    }
+    listBucketDataCollectionsV3(replicationStreamId) {
+        return this.upstream.listBucketDataCollectionsV3(replicationStreamId);
+    }
+    async listParameterIndexCollectionsV3(replicationStreamId) {
+        const prefix = `parameter_index_${replicationStreamId}_`;
+        const collections = await this.db.listCollections({ name: new RegExp(`^${prefix}`) }, { nameOnly: true }).toArray();
+        return collections
+            .filter((collection) => collection.name.startsWith(prefix))
+            .map((collection) => ({
+            collection: this.db.collection(collection.name),
+            indexId: collection.name.slice(prefix.length)
+        }));
+    }
+}
+//# sourceMappingURL=VersionedPowerSyncMongoV3.js.map

package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"VersionedPowerSyncMongoV3.js","sourceRoot":"","sources":["../../../../src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,2BAA2B,EAAE,MAAM,0CAA0C,CAAC;AAUvF,MAAM,OAAO,yBAA0B,SAAQ,2BAA2B;IACxE,eAAe,CAAC,mBAA2B,EAAE,aAA6B;QACxE,MAAM,cAAc,GAAG,IAAI,CAAC,2BAA2B,CAAC,mBAAmB,EAAE,aAAa,CAAC,CAAC;QAC5F,OAAO,IAAI,CAAC,EAAE,CAAC,UAAU,CAAwB,cAAc,CAAC,CAAC;IACnE,CAAC;IAED,KAAK,CAAC,6BAA6B,CAAC,mBAA2B;QAC7D,OAAO,IAAI,CAAC,uBAAuB,CAAwB,kBAAkB,mBAAmB,GAAG,CAAC,CAAC;IACvG,CAAC;IAED,KAAK,CAAC,iCAAiC,CAAC,mBAA2B,EAAE,aAA6B;QAChG,MAAM,IAAI,CAAC,eAAe,CAAC,mBAAmB,EAAE,aAAa,CAAC,CAAC,WAAW,CACxE;YACE,cAAc,EAAE,CAAC;SAClB,EACD;YACE,uBAAuB,EAAE,EAAE,cAAc,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE;YAC9D,IAAI,EAAE,gBAAgB;SACvB,CACF,CAAC;IACJ,CAAC;IAED,kBAAkB,CAAC,mBAA2B;QAC5C,OAAO,IAAI,CAAC,cAAc,CAAC,mBAAmB,CAAgD,CAAC;IACjG,CAAC;IAED,aAAa,CAAC,mBAA2B;QACvC,OAAO,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,gBAAgB,mBAAmB,EAAE,CAAC,CAAC;IACnE,CAAC;IAED,gBAAgB,CACd,mBAA2B,EAC3B,OAAyB;QAEzB,OAAO,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,mBAAmB,mBAAmB,IAAI,OAAO,EAAE,CAAC,CAAC;IACjF,CAAC;IAED,cAAc,CAAC,mBAA2B;QACxC,OAAO,IAAI,CAAC,EAAE,CAAC,UAAU,CAAwB,IAAI,CAAC,yBAAyB,CAAC,mBAAmB,CAAC,CAAC,CAAC;IACxG,CAAC;IAED,KAAK,CAAC,uBAAuB,CAAC,mBAA2B;QACvD,MAAM,YAAY,GAAG,IAAI,CAAC,cAAc,CAAC,mBAAmB,CAAC,CAAC;QAC9D,MAAM,WAAW,GAAG,IAAI,CAAC,aAAa,CAAC,mBAAmB,CAAC,CAAC;QAC5D,MAAM,YAAY,CAAC,WAAW,CAC5B;YACE,aAAa,EAAE,CAAC;YAChB,WAAW,EAAE,CAAC;YACd,UAAU,EAAE,CAAC;YACb,WAAW,EAAE,CAAC;SACf,EACD;YACE,IAAI,EAAE,eAAe;SACtB,CACF,CAAC;QACF,MAAM,YAAY,CAAC,WAAW,CAC5B;YACE,qBAAqB,EAAE,CAAC;SACzB,EACD;YACE,uBAAuB,EAAE,EAAE,qBAAqB,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE;YACrE,IAAI,EAAE,uBAAuB;SAC9B,CACF,CAAC;QACF,MAAM,WAAW,CAAC,WAAW,CAC3B;YACE,OAAO,EAAE,CAAC;SACX,EACD,EAAE,IAAI,EAAE,gBAAgB,EAAE,MAAM,EAAE,IAAI,EAAE,CACzC,CAAC;QACF,MAAM,WAAW,CAAC,WAAW,CAC3B;YACE,8BAA8B,EAAE,CAAC,CAAC;SACnC,EACD,EAAE,IAAI,EAAE,aAAa,EAAE,CACxB,CAAC;IACJ,CAAC;IAED,YAAY,CAAC,mBAA2B,EAAE,YAAgC;QACxE,OAAO,IAAI,CAAC,EAAE,CAAC,UAAU,CAAuB,eAAe,mBAAmB,IAAI,YAAY,EAAE,CAAC,CAAC;IACxG,CAAC;IAED,2BAA2B,CAAC,mBAA2B;QACrD,OAAO,IAAI,CAAC,QAAQ,CAAC,2BAA2B,CAAC,mBAAmB,CAAC,CAAC;IACxE,CAAC;IAED,KAAK,CAAC,+BAA+B,CACnC,mBAA2B;QAE3B,MAAM,MAAM,GAAG,mBAAmB,mBAAmB,GAAG,CAAC;QACzD,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,eAAe,CAAC,EAAE,IAAI,EAAE,IAAI,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC,EAAE,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC;QAEpH,OAAO,WAAW;aACf,MAAM,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;aAC1D,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;YACpB,UAAU,EAAE,IAAI,CAAC,EAAE,CAAC,UAAU,CAA4B,UAAU,CAAC,IAAI,CAAC;YAC1E,OAAO,EAAE,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC;SAC9C,CAAC,CAAC,CAAC;IACR,CAAC;CACF"}