@powersync/service-module-mongodb-storage 0.15.3 → 0.16.0

package/src/storage/implementation/v3/MongoChecksumsV3.ts

@@ -0,0 +1,120 @@
+import {
+  bson,
+  BucketChecksum,
+  FetchPartialBucketChecksum,
+  InternalOpId,
+  PartialChecksumMap,
+  PartialOrFullChecksum
+} from '@powersync/service-core';
+import { BucketDefinitionMapping } from '../BucketDefinitionMapping.js';
+import {
+  emptyChecksumForRequest,
+  FetchPartialBucketChecksumV3,
+  MongoChecksumOptions,
+  MongoChecksums
+} from '../MongoChecksums.js';
+import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js';
+
+export class MongoChecksumsV3 extends MongoChecksums {
+  declare protected readonly db: VersionedPowerSyncMongoV3;
+  private readonly mapping: BucketDefinitionMapping;
+
+  constructor(db: VersionedPowerSyncMongoV3, group_id: number, options: MongoChecksumOptions) {
+    super(db, group_id, options);
+    this.mapping = options.mapping!;
+  }
+
+  private normalizeBatch(batch: FetchPartialBucketChecksum[]): FetchPartialBucketChecksumV3[] {
+    return batch.map((request) => ({
+      bucket: request.bucket,
+      definitionId: this.mapping.bucketSourceId(request.source),
+      start: request.start,
+      end: request.end
+    }));
+  }
+
+  async computePartialChecksumsDirectByDefinition(batch: FetchPartialBucketChecksumV3[]): Promise<PartialChecksumMap> {
+    const results = new Map<string, PartialOrFullChecksum>();
+    const requestsByDefinition = new Map<string, FetchPartialBucketChecksumV3[]>();
+
+    for (const request of batch) {
+      const existing = requestsByDefinition.get(request.definitionId) ?? [];
+      existing.push(request);
+      requestsByDefinition.set(request.definitionId, existing);
+    }
+
+    for (const [definitionId, requests] of requestsByDefinition.entries()) {
+      const groupResults = await this.computePartialChecksumsForCollection(
+        requests,
+        this.db.bucketDataV3(this.group_id, definitionId),
+        createV3BucketFilter
+      );
+      for (const checksum of groupResults.values()) {
+        results.set(checksum.bucket, checksum);
+      }
+    }
+
+    return new Map<string, PartialOrFullChecksum>(
+      batch.map((request) => [request.bucket, results.get(request.bucket) ?? emptyChecksumForRequest(request)])
+    );
+  }
+
+  protected async fetchPreStates(
+    batch: FetchPartialBucketChecksum[]
+  ): Promise<Map<string, { opId: InternalOpId; checksum: BucketChecksum }>> {
+    const preFilters = this.normalizeBatch(batch)
+      .filter((request) => request.start == null)
+      .map((request) => ({
+        _id: {
+          d: request.definitionId,
+          b: request.bucket
+        },
+        'compacted_state.op_id': { $exists: true, $lte: request.end }
+      }));
+
+    const preStates = new Map<string, { opId: InternalOpId; checksum: BucketChecksum }>();
+    if (preFilters.length == 0) {
+      return preStates;
+    }
+
+    const states = await this.db
+      .bucketStateV3(this.group_id)
+      .find({
+        $or: preFilters
+      })
+      .toArray();
+
+    for (const state of states) {
+      const compactedState = state.compacted_state!;
+      preStates.set(state._id.b, {
+        opId: compactedState.op_id,
+        checksum: {
+          bucket: state._id.b,
+          checksum: Number(compactedState.checksum),
+          count: compactedState.count
+        }
+      });
+    }
+
+    return preStates;
+  }
+
+  protected async computePartialChecksumsInternal(batch: FetchPartialBucketChecksum[]): Promise<PartialChecksumMap> {
+    return this.computePartialChecksumsDirectByDefinition(this.normalizeBatch(batch));
+  }
+}
+
+function createV3BucketFilter(request: Pick<FetchPartialBucketChecksumV3, 'bucket' | 'start' | 'end'>) {
+  return {
+    _id: {
+      $gt: {
+        b: request.bucket,
+        o: request.start ?? new bson.MinKey()
+      },
+      $lte: {
+        b: request.bucket,
+        o: request.end
+      }
+    }
+  };
+}

package/src/storage/implementation/v3/MongoCompactorV3.ts

@@ -0,0 +1,107 @@
+import { mongo } from '@powersync/lib-service-mongodb';
+import { ReplicationAssertionError, ServiceAssertionError } from '@powersync/lib-services-framework';
+import { storage } from '@powersync/service-core';
+import { BucketDefinitionId } from '../BucketDefinitionMapping.js';
+import { SingleBucketStore } from '../common/SingleBucketStore.js';
+import { BucketStateDocumentBase } from '../models.js';
+import { DirtyBucket, MongoCompactor } from '../MongoCompactor.js';
+import { BucketStateDocumentV3 } from './models.js';
+import type { MongoSyncBucketStorageV3 } from './MongoSyncBucketStorageV3.js';
+import { SingleBucketStoreV3 } from './SingleBucketStoreV3.js';
+import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js';
+
+export class MongoCompactorV3 extends MongoCompactor {
+  declare protected readonly db: VersionedPowerSyncMongoV3;
+  declare protected readonly storage: MongoSyncBucketStorageV3;
+
+  public async *dirtyBucketBatches(options: {
+    minBucketChanges: number;
+    minChangeRatio: number;
+  }): AsyncGenerator<DirtyBucket[]> {
+    if (options.minBucketChanges <= 0) {
+      throw new ReplicationAssertionError('minBucketChanges must be >= 1');
+    }
+    // Same scan strategy as V1, but with the V3 bucket_state key shape.
+    yield* this.dirtyBucketBatchesForCollection(
+      this.db.bucketStateV3(this.group_id),
+      { d: new mongo.MinKey() as any, b: new mongo.MinKey() as any },
+      { d: new mongo.MaxKey() as any, b: new mongo.MaxKey() as any },
+      options,
+      (bucketState) => (bucketState as BucketStateDocumentV3)._id.d
+    );
+  }
+
+  public async dirtyBucketBatchForChecksums(options: { minBucketChanges: number }): Promise<DirtyBucket[]> {
+    if (options.minBucketChanges <= 0) {
+      throw new ReplicationAssertionError('minBucketChanges must be >= 1');
+    }
+    return this.dirtyBucketBatchForChecksumsForCollection(
+      this.db.bucketStateV3(this.group_id),
+      {
+        'estimate_since_compact.count': { $gte: options.minBucketChanges }
+      },
+      (bucketState) => (bucketState as BucketStateDocumentV3)._id.d
+    );
+  }
+
+  protected async writeBucketStateUpdates(): Promise<void> {
+    await this.db
+      .bucketStateV3(this.group_id)
+      .bulkWrite(this.bucketStateUpdates as mongo.AnyBulkWriteOperation<BucketStateDocumentV3>[], { ordered: false });
+  }
+
+  protected async computeChecksumsForBuckets(
+    buckets: Pick<DirtyBucket, 'bucket' | 'definitionId'>[]
+  ): Promise<storage.PartialChecksumMap> {
+    return this.storage.checksums.computePartialChecksumsDirectByDefinition(
+      buckets.map(({ bucket, definitionId }) => {
+        if (definitionId == null) {
+          throw new ServiceAssertionError(`Missing definitionId for V3 bucket checksum update on bucket ${bucket}`);
+        }
+        return {
+          bucket,
+          definitionId,
+          end: this.maxOpId
+        };
+      })
+    );
+  }
+
+  protected bucketStateFilter(
+    bucket: string,
+    definitionId: BucketDefinitionId | null
+  ): mongo.Filter<BucketStateDocumentBase> {
+    if (definitionId == null) {
+      throw new ServiceAssertionError(`Missing definitionId for V3 bucket state filter on bucket ${bucket}`);
+    }
+    return {
+      _id: {
+        d: definitionId,
+        b: bucket
+      }
+    };
+  }
+
+  protected async getBucketDataContext(
+    bucket: string,
+    definitionId: BucketDefinitionId | null
+  ): Promise<SingleBucketStore | null> {
+    if (definitionId == null) {
+      // Not the _most_ efficient approach, but this is not used often
+      const allDefinitionIds = this.storage.mapping.allBucketDefinitionIds();
+      if (allDefinitionIds.length == 0) {
+        return null;
+      }
+      const potentialIds = allDefinitionIds.map((definitionId) => ({ d: definitionId, b: bucket }));
+      const bucketState = await this.db.bucketStateV3(this.group_id).findOne({
+        _id: { $in: potentialIds }
+      });
+      if (bucketState == null) {
+        return null;
+      }
+      definitionId = bucketState._id.d;
+    }
+
+    return new SingleBucketStoreV3(this.db, { bucket, definitionId, replicationStreamId: this.group_id });
+  }
+}

package/src/storage/implementation/v3/MongoParameterCompactorV3.ts

@@ -0,0 +1,24 @@
+import { mongo } from '@powersync/lib-service-mongodb';
+import { MongoParameterCompactor } from '../MongoParameterCompactor.js';
+import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js';
+
+export class MongoParameterCompactorV3 extends MongoParameterCompactor {
+  declare protected readonly db: VersionedPowerSyncMongoV3;
+
+  protected async getCollections(): Promise<mongo.Collection<mongo.Document>[]> {
+    const collections = await this.db.listParameterIndexCollectionsV3(this.group_id);
+    return collections.map((collection) => collection.collection as unknown as mongo.Collection<mongo.Document>);
+  }
+
+  protected collectionFilter(): mongo.Document {
+    return {};
+  }
+
+  protected deleteFilter(doc: mongo.Document): mongo.Document {
+    return {
+      lookup: doc.lookup,
+      _id: { $lte: doc._id },
+      key: doc.key
+    };
+  }
+}

package/src/storage/implementation/v3/MongoParameterLookupV3.ts

@@ -0,0 +1,12 @@
+import { deserializeParameterLookup } from '@powersync/service-core';
+import { ScopedParameterLookup, SqliteJsonValue } from '@powersync/service-sync-rules';
+import * as bson from 'bson';
+import { ParameterIndexId } from '../BucketDefinitionMapping.js';
+
+export function serializeParameterLookupV3(lookup: ScopedParameterLookup): bson.Binary {
+  return new bson.Binary(bson.serialize({ l: lookup.values.slice(2) }));
+}
+
+export function deserializeParameterLookupV3(lookup: bson.Binary, indexId: ParameterIndexId): SqliteJsonValue[] {
+  return [indexId, '', ...deserializeParameterLookup(lookup)];
+}