@powersync/service-module-mongodb-storage 0.15.4 → 0.17.0

package/dist/storage/implementation/v3/MongoCompactorV3.js

@@ -0,0 +1,68 @@
+import { mongo } from '@powersync/lib-service-mongodb';
+import { ReplicationAssertionError, ServiceAssertionError } from '@powersync/lib-services-framework';
+import { MongoCompactor } from '../MongoCompactor.js';
+import { SingleBucketStoreV3 } from './SingleBucketStoreV3.js';
+export class MongoCompactorV3 extends MongoCompactor {
+    async *dirtyBucketBatches(options) {
+        if (options.minBucketChanges <= 0) {
+            throw new ReplicationAssertionError('minBucketChanges must be >= 1');
+        }
+        // Same scan strategy as V1, but with the V3 bucket_state key shape.
+        yield* this.dirtyBucketBatchesForCollection(this.db.bucketStateV3(this.group_id), { d: new mongo.MinKey(), b: new mongo.MinKey() }, { d: new mongo.MaxKey(), b: new mongo.MaxKey() }, options, (bucketState) => bucketState._id.d);
+    }
+    async dirtyBucketBatchForChecksums(options) {
+        if (options.minBucketChanges <= 0) {
+            throw new ReplicationAssertionError('minBucketChanges must be >= 1');
+        }
+        return this.dirtyBucketBatchForChecksumsForCollection(this.db.bucketStateV3(this.group_id), {
+            'estimate_since_compact.count': { $gte: options.minBucketChanges }
+        }, (bucketState) => bucketState._id.d);
+    }
+    async writeBucketStateUpdates() {
+        await this.db
+            .bucketStateV3(this.group_id)
+            .bulkWrite(this.bucketStateUpdates, { ordered: false });
+    }
+    async computeChecksumsForBuckets(buckets) {
+        return this.storage.checksums.computePartialChecksumsDirectByDefinition(buckets.map(({ bucket, definitionId }) => {
+            if (definitionId == null) {
+                throw new ServiceAssertionError(`Missing definitionId for V3 bucket checksum update on bucket ${bucket}`);
+            }
+            return {
+                bucket,
+                definitionId,
+                end: this.maxOpId
+            };
+        }));
+    }
+    bucketStateFilter(bucket, definitionId) {
+        if (definitionId == null) {
+            throw new ServiceAssertionError(`Missing definitionId for V3 bucket state filter on bucket ${bucket}`);
+        }
+        return {
+            _id: {
+                d: definitionId,
+                b: bucket
+            }
+        };
+    }
+    async getBucketDataContext(bucket, definitionId) {
+        if (definitionId == null) {
+            // Not the _most_ efficient approach, but this is not used often
+            const allDefinitionIds = this.storage.mapping.allBucketDefinitionIds();
+            if (allDefinitionIds.length == 0) {
+                return null;
+            }
+            const potentialIds = allDefinitionIds.map((definitionId) => ({ d: definitionId, b: bucket }));
+            const bucketState = await this.db.bucketStateV3(this.group_id).findOne({
+                _id: { $in: potentialIds }
+            });
+            if (bucketState == null) {
+                return null;
+            }
+            definitionId = bucketState._id.d;
+        }
+        return new SingleBucketStoreV3(this.db, { bucket, definitionId, replicationStreamId: this.group_id });
+    }
+}
+//# sourceMappingURL=MongoCompactorV3.js.map

package/dist/storage/implementation/v3/MongoCompactorV3.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"MongoCompactorV3.js","sourceRoot":"","sources":["../../../../src/storage/implementation/v3/MongoCompactorV3.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,gCAAgC,CAAC;AACvD,OAAO,EAAE,yBAAyB,EAAE,qBAAqB,EAAE,MAAM,mCAAmC,CAAC;AAKrG,OAAO,EAAe,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAGnE,OAAO,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AAG/D,MAAM,OAAO,gBAAiB,SAAQ,cAAc;IAI3C,KAAK,CAAC,CAAC,kBAAkB,CAAC,OAGhC;QACC,IAAI,OAAO,CAAC,gBAAgB,IAAI,CAAC,EAAE,CAAC;YAClC,MAAM,IAAI,yBAAyB,CAAC,+BAA+B,CAAC,CAAC;QACvE,CAAC;QACD,oEAAoE;QACpE,KAAK,CAAC,CAAC,IAAI,CAAC,+BAA+B,CACzC,IAAI,CAAC,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,QAAQ,CAAC,EACpC,EAAE,CAAC,EAAE,IAAI,KAAK,CAAC,MAAM,EAAS,EAAE,CAAC,EAAE,IAAI,KAAK,CAAC,MAAM,EAAS,EAAE,EAC9D,EAAE,CAAC,EAAE,IAAI,KAAK,CAAC,MAAM,EAAS,EAAE,CAAC,EAAE,IAAI,KAAK,CAAC,MAAM,EAAS,EAAE,EAC9D,OAAO,EACP,CAAC,WAAW,EAAE,EAAE,CAAE,WAAqC,CAAC,GAAG,CAAC,CAAC,CAC9D,CAAC;IACJ,CAAC;IAEM,KAAK,CAAC,4BAA4B,CAAC,OAAqC;QAC7E,IAAI,OAAO,CAAC,gBAAgB,IAAI,CAAC,EAAE,CAAC;YAClC,MAAM,IAAI,yBAAyB,CAAC,+BAA+B,CAAC,CAAC;QACvE,CAAC;QACD,OAAO,IAAI,CAAC,yCAAyC,CACnD,IAAI,CAAC,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,QAAQ,CAAC,EACpC;YACE,8BAA8B,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,gBAAgB,EAAE;SACnE,EACD,CAAC,WAAW,EAAE,EAAE,CAAE,WAAqC,CAAC,GAAG,CAAC,CAAC,CAC9D,CAAC;IACJ,CAAC;IAES,KAAK,CAAC,uBAAuB;QACrC,MAAM,IAAI,CAAC,EAAE;aACV,aAAa,CAAC,IAAI,CAAC,QAAQ,CAAC;aAC5B,SAAS,CAAC,IAAI,CAAC,kBAA0E,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;IACpH,CAAC;IAES,KAAK,CAAC,0BAA0B,CACxC,OAAuD;QAEvD,OAAO,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,yCAAyC,CACrE,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,EAAE,YAAY,EAAE,EAAE,EAAE;YACvC,IAAI,YAAY,IAAI,IAAI,EAAE,CAAC;gBACzB,MAAM,IAAI,qBAAqB,CAAC,gEAAgE,MAAM,EAAE,CAAC,CAAC;YAC5G,CAAC;YACD,OAAO;gBACL,MAAM;gBACN,YAAY;gBACZ,GAAG,EAAE,IAAI,CAAC,OAAO;aAClB,CAAC;QACJ,CAAC,CAAC,CACH,CAAC;IACJ,CAAC;IAES,iBAAiB,CACzB,MAAc,EACd,YAAuC;QAEvC,IAAI,YAAY,IAAI,IAAI,EAAE,CAAC;YACzB,MAAM,IAAI,qBAAqB,CAAC,6DAA6D,MAAM,EAAE,CAAC,CAAC;QACzG,CAAC;QACD,OAAO;YACL,GAAG,EAAE;gBACH,CAAC,EAAE,YAAY;gBACf,CAAC,EAAE,MAAM;aACV;SACF,CAAC;IACJ,CAAC;IAES,KAAK,CAAC,oBAAoB,CAClC,MAAc,EACd,YAAuC;QAEvC,IAAI,YAAY,IAAI,IAAI,EAAE,CAAC;YACzB,gEAAgE;YAChE,MAAM,gBAAgB,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,sBAAsB,EAAE,CAAC;YACvE,IAAI,gBAAgB,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;gBACjC,OAAO,IAAI,CAAC;YACd,CAAC;YACD,MAAM,YAAY,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;YAC9F,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC;gBACrE,GAAG,EAAE,EAAE,GAAG,EAAE,YAAY,EAAE;aAC3B,CAAC,CAAC;YACH,IAAI,WAAW,IAAI,IAAI,EAAE,CAAC;gBACxB,OAAO,IAAI,CAAC;YACd,CAAC;YACD,YAAY,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC;QACnC,CAAC;QAED,OAAO,IAAI,mBAAmB,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,YAAY,EAAE,mBAAmB,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;IACxG,CAAC;CACF"}

package/dist/storage/implementation/v3/MongoParameterCompactorV3.d.ts

@@ -0,0 +1,9 @@
+import { mongo } from '@powersync/lib-service-mongodb';
+import { MongoParameterCompactor } from '../MongoParameterCompactor.js';
+import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js';
+export declare class MongoParameterCompactorV3 extends MongoParameterCompactor {
+    protected readonly db: VersionedPowerSyncMongoV3;
+    protected getCollections(): Promise<mongo.Collection<mongo.Document>[]>;
+    protected collectionFilter(): mongo.Document;
+    protected deleteFilter(doc: mongo.Document): mongo.Document;
+}

package/dist/storage/implementation/v3/MongoParameterCompactorV3.js

@@ -0,0 +1,18 @@
+import { MongoParameterCompactor } from '../MongoParameterCompactor.js';
+export class MongoParameterCompactorV3 extends MongoParameterCompactor {
+    async getCollections() {
+        const collections = await this.db.listParameterIndexCollectionsV3(this.group_id);
+        return collections.map((collection) => collection.collection);
+    }
+    collectionFilter() {
+        return {};
+    }
+    deleteFilter(doc) {
+        return {
+            lookup: doc.lookup,
+            _id: { $lte: doc._id },
+            key: doc.key
+        };
+    }
+}
+//# sourceMappingURL=MongoParameterCompactorV3.js.map

package/dist/storage/implementation/v3/MongoParameterCompactorV3.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"MongoParameterCompactorV3.js","sourceRoot":"","sources":["../../../../src/storage/implementation/v3/MongoParameterCompactorV3.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,uBAAuB,EAAE,MAAM,+BAA+B,CAAC;AAGxE,MAAM,OAAO,yBAA0B,SAAQ,uBAAuB;IAG1D,KAAK,CAAC,cAAc;QAC5B,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,+BAA+B,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACjF,OAAO,WAAW,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,UAAU,CAAC,UAAyD,CAAC,CAAC;IAC/G,CAAC;IAES,gBAAgB;QACxB,OAAO,EAAE,CAAC;IACZ,CAAC;IAES,YAAY,CAAC,GAAmB;QACxC,OAAO;YACL,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,GAAG,EAAE,EAAE,IAAI,EAAE,GAAG,CAAC,GAAG,EAAE;YACtB,GAAG,EAAE,GAAG,CAAC,GAAG;SACb,CAAC;IACJ,CAAC;CACF"}

package/dist/storage/implementation/v3/MongoParameterLookupV3.d.ts

@@ -0,0 +1,4 @@
+import { ParameterIndexId, ScopedParameterLookup, SqliteJsonValue } from '@powersync/service-sync-rules';
+import * as bson from 'bson';
+export declare function serializeParameterLookupV3(lookup: ScopedParameterLookup): bson.Binary;
+export declare function deserializeParameterLookupV3(lookup: bson.Binary, indexId: ParameterIndexId): SqliteJsonValue[];

package/dist/storage/implementation/v3/MongoParameterLookupV3.js

@@ -0,0 +1,9 @@
+import { deserializeParameterLookup } from '@powersync/service-core';
+import * as bson from 'bson';
+export function serializeParameterLookupV3(lookup) {
+    return new bson.Binary(bson.serialize({ l: lookup.values.slice(2) }));
+}
+export function deserializeParameterLookupV3(lookup, indexId) {
+    return [indexId, '', ...deserializeParameterLookup(lookup)];
+}
+//# sourceMappingURL=MongoParameterLookupV3.js.map

package/dist/storage/implementation/v3/MongoParameterLookupV3.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"MongoParameterLookupV3.js","sourceRoot":"","sources":["../../../../src/storage/implementation/v3/MongoParameterLookupV3.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,0BAA0B,EAAE,MAAM,yBAAyB,CAAC;AAErE,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAE7B,MAAM,UAAU,0BAA0B,CAAC,MAA6B;IACtE,OAAO,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AACxE,CAAC;AAED,MAAM,UAAU,4BAA4B,CAAC,MAAmB,EAAE,OAAyB;IACzF,OAAO,CAAC,OAAO,EAAE,EAAE,EAAE,GAAG,0BAA0B,CAAC,MAAM,CAAC,CAAC,CAAC;AAC9D,CAAC"}

package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts

@@ -0,0 +1,63 @@
+import * as lib_mongo from '@powersync/lib-service-mongodb';
+import { mongo } from '@powersync/lib-service-mongodb';
+import { CheckpointChanges, GetCheckpointChangesOptions, InternalOpId, storage, utils } from '@powersync/service-core';
+import { ParameterLookupRows, ScopedParameterLookup } from '@powersync/service-sync-rules';
+import { MongoBucketStorage } from '../../MongoBucketStorage.js';
+import { MongoSyncBucketStorageCheckpoint, MongoSyncBucketStorageContext } from '../common/MongoSyncBucketStorageContext.js';
+import { MongoBucketBatchOptions } from '../MongoBucketBatch.js';
+import { MongoChecksums } from '../MongoChecksums.js';
+import { MongoCompactOptions, MongoCompactor } from '../MongoCompactor.js';
+import { MongoParameterCompactor } from '../MongoParameterCompactor.js';
+import { MongoPersistedSyncRulesContentV1 } from '../MongoPersistedSyncRulesContent.js';
+import { MongoSyncBucketStorage, MongoSyncBucketStorageOptions } from '../MongoSyncBucketStorage.js';
+import { MongoChecksumsV3 } from './MongoChecksumsV3.js';
+import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js';
+export declare class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage {
+    readonly db: VersionedPowerSyncMongoV3;
+    readonly checksums: MongoChecksumsV3;
+    private readonly syncRulesV3;
+    constructor(factory: MongoBucketStorage, group_id: number, sync_rules: MongoPersistedSyncRulesContentV1, slot_name: string, writeCheckpointMode: storage.WriteCheckpointMode | undefined, options: MongoSyncBucketStorageOptions);
+    private get syncConfigId();
+    private get syncRulesCollection();
+    private syncConfigMatch;
+    private syncConfigProjection;
+    private syncConfigArrayFilters;
+    /**
+     * For now, we only support a single sync config per replication stream.
+     *
+     * In the future we'll add support for multiple.
+     */
+    private selectedSyncConfig;
+    protected initializeVersionStorage(): Promise<void>;
+    protected createMongoChecksums(options: MongoSyncBucketStorageOptions): MongoChecksums;
+    createMongoCompactor(options: MongoCompactOptions): MongoCompactor;
+    protected createMongoParameterCompactor(checkpoint: InternalOpId, options: storage.CompactOptions): MongoParameterCompactor;
+    protected createWriterImpl(batchOptions: MongoBucketBatchOptions): storage.BucketStorageBatch;
+    protected fetchCheckpointState(session: mongo.ClientSession): Promise<{
+        checkpoint: bigint;
+        lsn: string | null;
+    } | null>;
+    protected getWriterSyncState(): Promise<{
+        lastCheckpointLsn: string | null;
+        resumeFromLsn: string | null;
+        keepaliveOp: bigint | null;
+        syncConfigId: lib_mongo.mongo.BSON.ObjectId;
+    }>;
+    protected terminateSyncRuleState(): Promise<void>;
+    protected getStatusImpl(): Promise<storage.SyncRuleStatus>;
+    protected clearSyncRuleState(): Promise<void>;
+    protected get versionContext(): MongoSyncBucketStorageContext<VersionedPowerSyncMongoV3>;
+    protected getParameterSetsImpl(checkpoint: MongoSyncBucketStorageCheckpoint, lookups: ScopedParameterLookup[], limit: number): Promise<ParameterLookupRows[]>;
+    protected getBucketDataBatchImpl(checkpoint: utils.InternalOpId, dataBuckets: storage.BucketDataRequest[], options?: storage.BucketDataBatchOptions): AsyncIterable<storage.SyncBucketDataChunk>;
+    protected clearBucketData(_signal?: AbortSignal): Promise<void>;
+    protected clearParameterIndexes(_signal?: AbortSignal): Promise<void>;
+    protected clearSourceRecords(_signal?: AbortSignal): Promise<void>;
+    protected clearBucketState(_signal?: AbortSignal): Promise<void>;
+    protected clearSourceTables(_signal?: AbortSignal): Promise<void>;
+    protected getDataBucketChangesImpl(options: GetCheckpointChangesOptions): Promise<Pick<CheckpointChanges, 'updatedDataBuckets' | 'invalidateDataBuckets'>>;
+    protected getParameterBucketChangesImpl(options: GetCheckpointChangesOptions): Promise<Pick<CheckpointChanges, 'updatedParameterLookups' | 'invalidateParameterBuckets'>>;
+}
+export declare function getParameterSetsV3(ctx: MongoSyncBucketStorageContext<VersionedPowerSyncMongoV3>, checkpoint: MongoSyncBucketStorageCheckpoint, lookups: ScopedParameterLookup[], limit: number): Promise<ParameterLookupRows[]>;
+export declare function getBucketDataBatchV3(ctx: MongoSyncBucketStorageContext<VersionedPowerSyncMongoV3>, checkpoint: utils.InternalOpId, dataBuckets: storage.BucketDataRequest[], options?: storage.BucketDataBatchOptions): AsyncIterable<storage.SyncBucketDataChunk>;
+export declare function getDataBucketChangesV3(ctx: MongoSyncBucketStorageContext<VersionedPowerSyncMongoV3>, options: GetCheckpointChangesOptions): Promise<Pick<CheckpointChanges, 'updatedDataBuckets' | 'invalidateDataBuckets'>>;
+export declare function getParameterBucketChangesV3(ctx: MongoSyncBucketStorageContext<VersionedPowerSyncMongoV3>, options: GetCheckpointChangesOptions): Promise<Pick<CheckpointChanges, 'updatedParameterLookups' | 'invalidateParameterBuckets'>>;