@powersync/service-module-mongodb-storage 0.15.4 → 0.16.0

package/src/storage/implementation/v1/PersistedBatchV1.ts

@@ -0,0 +1,230 @@
+import { mongo } from '@powersync/lib-service-mongodb';
+import { ReplicationAssertionError } from '@powersync/lib-services-framework';
+import { storage } from '@powersync/service-core';
+import * as bson from 'bson';
+
+import { BucketDataSource } from '@powersync/service-sync-rules';
+import { mongoTableId } from '../../../utils/util.js';
+import { BucketDefinitionId } from '../BucketDefinitionMapping.js';
+import { EMPTY_DATA } from '../MongoBucketBatchShared.js';
+import {
+  BucketStateUpdate,
+  PersistedBatch,
+  SaveParameterDataOptions,
+  UpsertCurrentDataOptions
+} from '../common/PersistedBatch.js';
+import { LEGACY_BUCKET_DATA_DEFINITION_ID, LEGACY_BUCKET_PARAMETER_INDEX_ID, SourceKey } from '../models.js';
+import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js';
+import {
+  BucketParameterDocument,
+  BucketStateDocumentV1,
+  CurrentDataDocument,
+  serializeBucketDataV1,
+  taggedBucketParameterDocumentToV1
+} from './models.js';
+
+export class PersistedBatchV1 extends PersistedBatch {
+  declare protected readonly db: VersionedPowerSyncMongoV1;
+
+  currentData: mongo.AnyBulkWriteOperation<CurrentDataDocument>[] = [];
+
+  protected checkDefinitionId(_definitionId: BucketDefinitionId | null): BucketDefinitionId {
+    // V1 storage doesn't persist the id, and we don't use it.
+    return LEGACY_BUCKET_DATA_DEFINITION_ID;
+  }
+
+  protected getBucketDefinitionId(_bucketSource: BucketDataSource): BucketDefinitionId {
+    return LEGACY_BUCKET_DATA_DEFINITION_ID;
+  }
+
+  saveParameterData(data: SaveParameterDataOptions) {
+    const { sourceTable, sourceKey, evaluated } = data;
+    const remaining_lookups = new Map<string, bson.Binary>();
+
+    for (let lookup of data.existing_lookups) {
+      if (lookup.indexId != null) {
+        throw new ReplicationAssertionError('Unexpected v3 lookup when incrementalReprocessing is disabled');
+      }
+      remaining_lookups.set(lookup.lookup.toString('base64'), lookup.lookup);
+    }
+
+    for (let result of evaluated) {
+      const binLookup = storage.serializeLookup(result.lookup);
+      remaining_lookups.delete(binLookup.toString('base64'));
+
+      const op_id = data.op_seq.next();
+      this.debugLastOpId = op_id;
+      const values: BucketParameterDocument = {
+        _id: op_id,
+        key: {
+          g: this.group_id,
+          t: mongoTableId(sourceTable.id),
+          k: sourceKey
+        },
+        lookup: binLookup,
+        bucket_parameters: result.bucketParameters
+      };
+      this.bucketParameters.push({
+        ...values,
+        index: LEGACY_BUCKET_PARAMETER_INDEX_ID
+      });
+
+      this.currentSize += 200;
+    }
+
+    for (let lookup of remaining_lookups.values()) {
+      const op_id = data.op_seq.next();
+      this.debugLastOpId = op_id;
+      const values: BucketParameterDocument = {
+        _id: op_id,
+        key: {
+          g: this.group_id,
+          t: mongoTableId(sourceTable.id),
+          k: sourceKey
+        },
+        lookup,
+        bucket_parameters: []
+      };
+      this.bucketParameters.push({
+        ...values,
+        index: LEGACY_BUCKET_PARAMETER_INDEX_ID
+      });
+
+      this.currentSize += 200;
+    }
+  }
+
+  hardDeleteCurrentData(sourceTableId: bson.ObjectId, replicaId: storage.ReplicaId) {
+    this.currentData.push({
+      deleteOne: {
+        filter: { _id: this.currentDataId(sourceTableId, replicaId) }
+      }
+    });
+    this.currentSize += 50;
+  }
+
+  softDeleteCurrentData(sourceTableId: bson.ObjectId, replicaId: storage.ReplicaId, _checkpointGreaterThan: bigint) {
+    this.hardDeleteCurrentData(sourceTableId, replicaId);
+  }
+
+  upsertCurrentData(values: UpsertCurrentDataOptions) {
+    const buckets = values.buckets.map((bucket) => {
+      if (bucket.definitionId != null) {
+        throw new ReplicationAssertionError('Unexpected v3 bucket when incrementalReprocessing is disabled');
+      }
+      return {
+        bucket: bucket.bucket,
+        table: bucket.table,
+        id: bucket.id
+      };
+    });
+    const lookups = values.lookups.map((lookup) => {
+      if (lookup.indexId != null) {
+        throw new ReplicationAssertionError('Unexpected v3 lookup when incrementalReprocessing is disabled');
+      }
+      return lookup.lookup;
+    });
+
+    this.currentData.push({
+      updateOne: {
+        filter: { _id: this.currentDataId(values.sourceTableId, values.replicaId) },
+        update: {
+          $set: {
+            data: values.data ?? EMPTY_DATA,
+            buckets,
+            lookups
+          }
+        },
+        upsert: true
+      }
+    });
+    this.currentSize += (values.data?.length() ?? 0) + 100;
+  }
+
+  protected get currentDataCount() {
+    return this.currentData.length;
+  }
+
+  protected async flushBucketData(session: mongo.ClientSession) {
+    await this.db.bucketDataV1.bulkWrite(
+      this.bucketData.map((document) => ({
+        insertOne: {
+          document: serializeBucketDataV1(document)
+        }
+      })),
+      {
+        session,
+        ordered: false
+      }
+    );
+  }
+
+  protected async flushBucketParameters(session: mongo.ClientSession) {
+    await this.db.parameterIndexV1.bulkWrite(
+      this.bucketParameters.map((document) => ({
+        insertOne: {
+          document: taggedBucketParameterDocumentToV1(document)
+        }
+      })),
+      {
+        session,
+        ordered: false
+      }
+    );
+  }
+
+  protected async flushCurrentData(session: mongo.ClientSession) {
+    if (this.currentData.length == 0) {
+      return;
+    }
+
+    await this.db.sourceRecordsV1.bulkWrite(this.currentData, {
+      session,
+      ordered: true
+    });
+  }
+
+  protected async flushBucketStates(session: mongo.ClientSession) {
+    await this.db.bucketStateV1.bulkWrite(this.getBucketStateUpdates(), {
+      session,
+      ordered: false
+    });
+  }
+
+  protected resetCurrentData() {
+    this.currentData = [];
+  }
+
+  private getBucketStateUpdates(): mongo.AnyBulkWriteOperation<BucketStateDocumentV1>[] {
+    return Array.from(this.bucketStates.values()).map((state: BucketStateUpdate) => {
+      return {
+        updateOne: {
+          filter: {
+            _id: {
+              g: this.group_id,
+              b: state.bucket
+            }
+          },
+          update: {
+            $set: {
+              last_op: state.lastOp
+            },
+            $inc: {
+              'estimate_since_compact.count': state.incrementCount,
+              'estimate_since_compact.bytes': state.incrementBytes
+            }
+          },
+          upsert: true
+        }
+      } satisfies mongo.AnyBulkWriteOperation<BucketStateDocumentV1>;
+    });
+  }
+
+  private currentDataId(sourceTableId: bson.ObjectId, replicaId: storage.ReplicaId): SourceKey {
+    return {
+      g: this.group_id,
+      t: sourceTableId,
+      k: replicaId
+    };
+  }
+}

package/src/storage/implementation/v1/SingleBucketStoreV1.ts

@@ -0,0 +1,74 @@
+import { mongo } from '@powersync/lib-service-mongodb';
+import { InternalOpId } from '@powersync/service-core';
+import { BucketDataDoc, BucketKey } from '../common/BucketDataDoc.js';
+import {
+  BucketDataDocumentGeneric,
+  BucketDataDocumentGenericId,
+  SingleBucketStore
+} from '../common/SingleBucketStore.js';
+import { BucketDataProperties } from '../models.js';
+import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js';
+import { BucketDataDocumentV1, BucketDataKeyV1, serializeBucketDataV1 } from './models.js';
+
+export class SingleBucketStoreV1 implements SingleBucketStore {
+  public readonly collection: mongo.Collection<BucketDataDocumentGeneric>;
+
+  constructor(
+    private db: VersionedPowerSyncMongoV1,
+    public readonly key: BucketKey
+  ) {
+    this.collection = db.bucketDataV1 as unknown as mongo.Collection<BucketDataDocumentGeneric>;
+  }
+
+  docId(o: InternalOpId): BucketDataDocumentGenericId {
+    // `satisfies BucketDataKeyV1` checks that we use the correct type for V1 storage
+    // `as anyt` is to allow casting to the interface virtual type
+    return {
+      g: this.key.replicationStreamId,
+      b: this.key.bucket,
+      o
+    } satisfies BucketDataKeyV1 as any;
+  }
+
+  get minId(): BucketDataDocumentGenericId {
+    return {
+      g: this.key.replicationStreamId,
+      b: this.key.bucket,
+      o: new mongo.MinKey()
+    } as any;
+  }
+
+  get maxId(): BucketDataDocumentGenericId {
+    return {
+      g: this.key.replicationStreamId,
+      b: this.key.bucket,
+      o: new mongo.MaxKey()
+    } as any;
+  }
+
+  toPersistedDocument(source: Omit<BucketDataDoc, 'bucketKey'>): BucketDataDocumentGeneric {
+    return serializeBucketDataV1({ bucketKey: this.key, ...source }) as unknown as BucketDataDocumentGeneric;
+  }
+
+  fromPersistedDocument(doc: BucketDataDocumentGeneric): BucketDataDoc {
+    const document = doc as unknown as BucketDataDocumentV1;
+    const { _id, ...rest } = document;
+    return {
+      bucketKey: this.key,
+      o: _id.o,
+      ...rest
+    };
+  }
+
+  fromPartialPersistedDocument<T extends keyof BucketDataProperties>(
+    doc: Pick<BucketDataDocumentGeneric, '_id' | T>
+  ): Pick<BucketDataDoc, 'bucketKey' | 'o' | T> {
+    const document = doc as Pick<BucketDataDocumentV1, '_id' | T>;
+    const { _id, ...rest } = document;
+    return {
+      bucketKey: this.key,
+      o: _id.o,
+      ...rest
+    } as Pick<BucketDataDoc, 'bucketKey' | 'o' | T>;
+  }
+}

package/src/storage/implementation/v1/SourceRecordStoreV1.ts

@@ -0,0 +1,156 @@
+import { mongo } from '@powersync/lib-service-mongodb';
+import { Logger } from '@powersync/lib-services-framework';
+import { storage } from '@powersync/service-core';
+import { EvaluatedParameters, EvaluatedRow } from '@powersync/service-sync-rules';
+import * as bson from 'bson';
+import { idPrefixFilter } from '../../../utils/util.js';
+import { cacheKey } from '../OperationBatch.js';
+import {
+  LoadedSourceRecord,
+  SourceRecordLookupEntry,
+  SourceRecordLookupState,
+  SourceRecordStore
+} from '../common/SourceRecordStore.js';
+import { SourceKey } from '../models.js';
+import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js';
+import { CurrentDataDocument } from './models.js';
+
+export class SourceRecordStoreV1 implements SourceRecordStore {
+  constructor(
+    private readonly db: VersionedPowerSyncMongoV1,
+    private readonly groupId: number
+  ) {}
+
+  mapEvaluatedBuckets(evaluated: EvaluatedRow[]): LoadedSourceRecord['buckets'] {
+    return evaluated.map((entry) => ({
+      definitionId: null,
+      bucket: entry.bucket,
+      table: entry.table,
+      id: entry.id
+    }));
+  }
+
+  mapParameterLookups(paramEvaluated: EvaluatedParameters[]): SourceRecordLookupState[] {
+    return paramEvaluated.map((entry) => ({
+      indexId: null,
+      lookup: storage.serializeLookup(entry.lookup)
+    }));
+  }
+
+  private createId(sourceTableId: bson.ObjectId, replicaId: storage.ReplicaId): SourceKey {
+    return {
+      g: this.groupId,
+      t: sourceTableId,
+      k: replicaId
+    } satisfies SourceKey;
+  }
+
+  private createLoadedDocument(
+    sourceTableId: bson.ObjectId,
+    id: SourceKey,
+    data: bson.Binary | null,
+    buckets: CurrentDataDocument['buckets'],
+    lookups: CurrentDataDocument['lookups']
+  ): LoadedSourceRecord {
+    return {
+      sourceTableId,
+      replicaId: id.k,
+      data,
+      buckets: buckets.map((bucket) => ({
+        definitionId: null,
+        bucket: bucket.bucket,
+        table: bucket.table,
+        id: bucket.id
+      })),
+      lookups: lookups.map((lookup) => ({
+        indexId: null,
+        lookup
+      })),
+      cacheKey: cacheKey(sourceTableId, id.k)
+    };
+  }
+
+  async loadSizes(session: mongo.ClientSession, entries: SourceRecordLookupEntry[]): Promise<Map<string, number>> {
+    const sizes = new Map<string, number>();
+    const sizeCursor: mongo.AggregationCursor<CurrentDataDocument & { size: number }> =
+      this.db.sourceRecordsV1.aggregate(
+        [
+          {
+            $match: {
+              _id: {
+                $in: entries.map((entry) => this.createId(entry.sourceTableId, entry.replicaId) as SourceKey)
+              }
+            }
+          },
+          {
+            $project: {
+              _id: 1,
+              size: { $bsonSize: '$$ROOT' }
+            }
+          }
+        ],
+        { session }
+      );
+    for await (const doc of sizeCursor.stream()) {
+      sizes.set(cacheKey(doc._id.t, doc._id.k), doc.size);
+    }
+    return sizes;
+  }
+
+  async loadDocuments(
+    session: mongo.ClientSession,
+    entries: SourceRecordLookupEntry[],
+    idsOnly: boolean
+  ): Promise<Map<string, LoadedSourceRecord>> {
+    const documents = new Map<string, LoadedSourceRecord>();
+    const projection = idsOnly ? { _id: 1 } : undefined;
+    const cursor = this.db.sourceRecordsV1.find(
+      {
+        _id: {
+          $in: entries.map((entry) => this.createId(entry.sourceTableId, entry.replicaId) as SourceKey)
+        }
+      },
+      { session, projection }
+    );
+    for await (const doc of cursor.stream()) {
+      const loaded = this.createLoadedDocument(
+        doc._id.t,
+        doc._id,
+        idsOnly ? null : doc.data,
+        idsOnly ? [] : doc.buckets,
+        idsOnly ? [] : doc.lookups
+      );
+      documents.set(loaded.cacheKey, loaded);
+    }
+    return documents;
+  }
+
+  async loadTruncateBatch(
+    session: mongo.ClientSession,
+    sourceTableId: bson.ObjectId,
+    limit: number
+  ): Promise<LoadedSourceRecord[]> {
+    const cursor = this.db.sourceRecordsV1.find(
+      {
+        _id: idPrefixFilter<SourceKey>({ g: this.groupId, t: sourceTableId }, ['k']),
+        pending_delete: { $exists: false }
+      },
+      {
+        projection: {
+          _id: 1,
+          buckets: 1,
+          lookups: 1
+        },
+        limit,
+        session
+      }
+    );
+    return (await cursor.toArray()).map((doc) =>
+      this.createLoadedDocument(sourceTableId, doc._id, null, doc.buckets, doc.lookups)
+    );
+  }
+
+  async postCommitCleanup(_lastCheckpoint: bigint, _logger: Logger): Promise<void> {
+    // No-op for V1.
+  }
+}

package/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts

@@ -0,0 +1,28 @@
+import { mongo } from '@powersync/lib-service-mongodb';
+import { BaseVersionedPowerSyncMongo } from '../common/VersionedPowerSyncMongoBase.js';
+import { CommonSourceTableDocument } from '../models.js';
+import { BucketDataDocumentV1, BucketParameterDocument, BucketStateDocumentV1, CurrentDataDocument } from './models.js';
+
+export class VersionedPowerSyncMongoV1 extends BaseVersionedPowerSyncMongo {
+  get sourceRecordsV1(): mongo.Collection<CurrentDataDocument> {
+    return this.upstream.current_data;
+  }
+
+  get bucketStateV1(): mongo.Collection<BucketStateDocumentV1> {
+    return this.upstream.bucket_state;
+  }
+
+  commonSourceTables(_replicationStreamId: number): mongo.Collection<CommonSourceTableDocument> {
+    return this.upstream.source_tables as any as mongo.Collection<CommonSourceTableDocument>;
+  }
+
+  async initializeStreamStorage(_replicationStreamId: number): Promise<void> {}
+
+  get bucketDataV1(): mongo.Collection<BucketDataDocumentV1> {
+    return this.upstream.bucket_data;
+  }
+
+  get parameterIndexV1(): mongo.Collection<BucketParameterDocument> {
+    return this.upstream.bucket_parameters;
+  }
+}

package/src/storage/implementation/v1/models.ts

@@ -0,0 +1,84 @@
+import * as bson from 'bson';
+import { BucketDataDoc } from '../common/BucketDataDoc.js';
+import {
+  BucketDataDocumentBase,
+  BucketParameterDocumentBase,
+  BucketStateDocumentBase,
+  CurrentBucket,
+  LEGACY_BUCKET_DATA_DEFINITION_ID,
+  SourceKey,
+  SourceTableDocument,
+  TaggedBucketParameterDocument
+} from '../models.js';
+
+export interface BucketDataKeyV1 {
+  /** group_id */
+  g: number;
+  /** bucket name */
+  b: string;
+  /** op_id */
+  o: bigint;
+}
+
+export interface CurrentDataDocument {
+  _id: SourceKey;
+  data: bson.Binary;
+  buckets: CurrentBucket[];
+  lookups: bson.Binary[];
+}
+
+export interface BucketParameterDocument extends BucketParameterDocumentBase<SourceKey> {}
+
+export interface BucketDataDocumentV1 extends BucketDataDocumentBase {
+  _id: BucketDataKeyV1;
+}
+
+export function serializeBucketDataV1(document: BucketDataDoc): BucketDataDocumentV1 {
+  const { bucketKey, o } = document;
+  return {
+    _id: {
+      g: bucketKey.replicationStreamId,
+      b: bucketKey.bucket,
+      o: o
+    },
+    // List fields directly, so that we don't accidentally persist any unknown fields
+    op: document.op,
+    source_table: document.source_table,
+    source_key: document.source_key,
+    table: document.table,
+    row_id: document.row_id,
+    checksum: document.checksum,
+    data: document.data,
+    target_op: document.target_op
+  };
+}
+
+export function loadBucketDataDocumentV1(doc: BucketDataDocumentV1): BucketDataDoc {
+  const { _id, ...rest } = doc;
+  return {
+    bucketKey: {
+      replicationStreamId: _id.g,
+      definitionId: LEGACY_BUCKET_DATA_DEFINITION_ID,
+      bucket: _id.b
+    },
+    o: _id.o,
+    ...rest
+  };
+}
+
+export function taggedBucketParameterDocumentToV1(document: TaggedBucketParameterDocument): BucketParameterDocument {
+  const { index: _index, ...rest } = document;
+  return rest as BucketParameterDocument;
+}
+
+export interface SourceTableDocumentV1 extends SourceTableDocument {
+  group_id: number;
+}
+
+export interface BucketStateDocumentV1 extends BucketStateDocumentBase {
+  _id: BucketStateDocumentBase['_id'] & {
+    g: number;
+  };
+}
+
+export type BucketStateDocument = BucketStateDocumentV1;

package/src/storage/implementation/v3/MongoBucketBatchV3.ts

@@ -0,0 +1,44 @@
+import * as lib_mongo from '@powersync/lib-service-mongodb';
+import { storage } from '@powersync/service-core';
+import { mongoTableId } from '../../../utils/util.js';
+import { MongoBucketBatch, MongoBucketBatchOptions } from '../MongoBucketBatch.js';
+import { PersistedBatch } from '../common/PersistedBatch.js';
+import { SourceRecordStore } from '../common/SourceRecordStore.js';
+import { PersistedBatchV3 } from './PersistedBatchV3.js';
+import { SourceRecordStoreV3 } from './SourceRecordStoreV3.js';
+import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js';
+
+export class MongoBucketBatchV3 extends MongoBucketBatch {
+  declare public readonly db: VersionedPowerSyncMongoV3;
+
+  private readonly store: SourceRecordStore;
+
+  constructor(options: MongoBucketBatchOptions) {
+    super(options);
+    this.store = new SourceRecordStoreV3(this.db, this.group_id, this.mapping);
+  }
+
+  protected createPersistedBatch(writtenSize: number): PersistedBatch {
+    return new PersistedBatchV3(this.db, this.group_id, this.mapping, writtenSize, {
+      logger: this.logger
+    });
+  }
+
+  protected get sourceRecordStore(): SourceRecordStore {
+    return this.store;
+  }
+
+  protected async cleanupDroppedSourceTables(sourceTables: storage.SourceTable[]) {
+    for (const table of sourceTables) {
+      await this.db
+        .sourceRecordsV3(this.group_id, mongoTableId(table.id))
+        .drop()
+        .catch((error) => {
+          if (lib_mongo.isMongoServerError(error) && error.codeName === 'NamespaceNotFound') {
+            return;
+          }
+          throw error;
+        });
+    }
+  }
+}